Documentation CI Migration (#26584)

* Docs CI (#26153)

* feat: test with FF instead of config

* feat: update gitlab ci file

* chore: match prod gitlab for typesync

* fix: use latest pull config

* chore: test with leaving config in

* fix: update script to pull from 0 for permissions

* fix: pull from correct position

* feat: update pull_config

* feat: add allowlist and check (#26197)

* Move build scripts and setup local dev (#26506)

* feat: update make file for sourcing, remove build files

* feat: naming and comments

* feat: better named vars

* feat: add command to preserve build files

* feat: build in ci

* fix: image pathing

* chore: image version

* chore: FF for image URL

* feat: point at new build image with docs files

* add better feedback when missing vars (#26592)

* add better feedback when missing vars

* fix

* feat: remove cache check

* Move more build scripts (#26602)

* feat: remove more scripts

* feat: create script to handle file moving

* fix: script header

* fix: pathing

* feat: update local start, update git ignore

* chore: debug ci

* feat: udpate pathing

* feat: add output so we know the script is working

* chore: remove debugging lines

* feat: move more files

* chore: remove files

* chore: debug

* chore: revert changes

* fix: remove script

* fix: local sourcing and ignores

* feat: pin to latest image

---------

Co-authored-by: David Jones <[email protected]>

.gitignore

@@ -48,6 +48,7 @@ content/en/logs/guide/forwarder.md
 
 # Generated by build script
 package-lock.json
+.gitmodules
 
 # Ignoring all integrations file
 content/en/integrations/*.md
@@ -253,3 +254,18 @@ assets/jsconfig.json
 !.yarn/releases
 !.yarn/sdks
 !.yarn/versions
+
+# local build files
+.backup_path
+local/bin/py/build/*
+local/bin/py/sh
+local/bin/py/js
+local/bin/js/
+local/bin/py/translations
+local/bin/py/add_notranslate_tag.py
+local/bin/py/placehold_translations.py
+local/bin/py/missing_metrics.py
+local/bin/py/submit_github_status_check.py
+!local/bin/py/build/configuration/pull_config_preview.yaml
+!local/bin/py/build/configuration/pull_config.yaml
+!local/bin/py/build/configuration/integration_merge.yaml

.gitlab-ci.yml

@@ -55,12 +55,18 @@ variables:
   PIP_CACHE_DIR: "${CI_PROJECT_DIR}/.pipcache/"
   URGENT_FIX: "false" # set to true if the branch is an urgent fix to skip the build_live queue
 
+  # Feature Flag Vars
+  FF_PREVIEW_PULL_CONFIG_CACHE: "true" # set to false to disable caching of pull_config.yaml in preview builds
+  FF_BUILD_IMAGE_URL: "registry.ddbuild.io/ci/websites/webops-site-build:v50389622-96d5f437"
+
 
 # ================== copy scripts =============== #
 .common_script: &common_script
   - "[ -d local/bin ] && find local/bin/ -type f -exec cp {} /usr/local/bin \\;"  # load scripts
   - "[ -d local/etc ] && find local/etc/ -type f -exec cp {} /etc \\;"  # load configs
   - "[ -f /usr/local/bin/helpers.sh ] && source /usr/local/bin/helpers.sh"  # source helpers so they are available in the container
+  - chmod +x /usr/local/bin/docs-ci/sh/merge-build-files.sh
+  - /usr/local/bin/docs-ci/sh/merge-build-files.sh
   - chmod +x /usr/local/bin/*.py # make python scripts executable by the runner
   - "[ -f /etc/profile ] && source /etc/profile" # for golang on path
   - mkdir -p logs
@@ -164,7 +170,7 @@ before_script:
       when: on_success
 # ================== templates ================== #
 .base_template: &base_template
-  image: registry.ddbuild.io/ci/websites/webops-site-build:v49904560-30b69972
+  image: ${FF_BUILD_IMAGE_URL}
   tags:
     - "arch:amd64"
   rules:
@@ -427,6 +433,7 @@ post_build_failure_status_to_spec_repo:
     LOCAL: "False"
     RESOURCE_GROUP_ORDER: oldest_first
     RESOURCE_GROUP_LIMIT: 1
+    FF_PREVIEW_PULL_CONFIG_CACHE: "false"
   script:
     - dog --config "$HOME/.dogrc" event post "documentation deploy ${CI_COMMIT_REF_NAME} started" "${CI_PROJECT_URL}/pipelines/${CI_PIPELINE_ID}" --alert_type "info" --tags="${DEFAULT_TAGS}"
     - notify_slack.py "<https://github.com/DataDog/documentation/commit/${CI_COMMIT_SHA}|${CI_COMMIT_SHA:0:8}> sent to gitlab for production deployment. <${CI_PROJECT_URL}/pipelines/${CI_PIPELINE_ID}|details>" --status="#FFD700"
@@ -558,18 +565,6 @@ link_checks:on-schedule:
   interruptible: true
   allow_failure: true
 
-# We are relying on the preview version of this job due to the length of time it takes
-# test_missing_tms_live:
-#   <<: *base_template
-#   <<: *live_rules
-#   stage: post-deploy
-#   cache: {}
-#   environment: "live"
-#   dependencies:
-#     - build_live
-#   script:
-#     - check_missing_tms
-
 # template for what runs in pa11y to avoid duplication
 .pa11y_live_template: &pa11y_live_template
   image: registry.ddbuild.io/ci/websites/pa11y:v48852638-b2768bdb

Makefile

@@ -2,7 +2,7 @@
 SHELL = /bin/bash
 # MAKEFLAGS := --jobs=$(shell nproc)
 # MAKEFLAGS += --output-sync --no-print-directory
-.PHONY: help clean-all clean dependencies server start start-no-pre-build start-docker stop-docker all-examples clean-examples placeholders update_pre_build config derefs source-dd-source vector_data
+.PHONY: help clean-all clean start-preserve-build dependencies server start start-no-pre-build start-docker stop-docker all-examples clean-examples placeholders update_pre_build config derefs source-dd-source vector_data
 .DEFAULT_GOAL := help
 PY3=$(shell if [ `which pyenv` ]; then \
 				if [ `pyenv which python3` ]; then \
@@ -39,7 +39,7 @@ CONFIGURATION_FILE ?= "./local/bin/py/build/configurations/pull_config_preview.y
 help:
 	@perl -nle'print $& if m{^[a-zA-Z_-]+:.*?## .*$$}' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-25s\033[0m %s\n", $$1, $$2}'
 
-clean-all: clean clean-examples clean-dependent-repos ## Clean everything (environment, sourced repos, generated files)
+clean-all: clean clean-examples clean-dependent-repos clean-build-scripts ## Clean everything (environment, sourced repos, generated files, build scripts)
 	rm -rf ./node_modules ./hugpython ./public
 
 clean-dependent-repos:
@@ -67,14 +67,21 @@ server:
 	fi;
 
 # Download all dependencies and run the site
-start: dependencies ## Build and run docs including external content.
+start: setup-build-scripts ## Build and run docs including external content.
+	@make dependencies
 	@make update_websites_sources_module
 	@make server
 
 # Skip downloading any dependencies and run the site (hugo needs at the least node)
 start-no-pre-build: node_modules  ## Build and run docs excluding external content.
 	@make server
 
+# Leave build scripts as is for local testing
+# This is useful for testing changes to the build scripts locally
+start-preserve-build: dependencies
+	@make update_websites_sources_module
+	@make server
+
 start-docker: clean  ## Build and run docs including external content via docker
 	@export REPO_PATH=$(PWD) && \
 	export GITHUB_TOKEN=$(GITHUB_TOKEN) && \
@@ -199,6 +206,94 @@ all-examples: $(foreach repo,$(EXAMPLES_REPOS),$(addprefix examples/, $(patsubst
 clean-examples: $(foreach repo,$(EXAMPLES_REPOS),$(addprefix examples/, $(patsubst datadog-api-client-%,clean-%-examples,$(repo))))
 	@rm -rf examples
 
+# Local build setup
+PY_PATH := local/bin/py
+JS_PATH := local/bin/js
+TRANSLATIONS_PATH := $(PY_PATH)/translations
+BUILD_SCRIPTS_PATH := $(PY_PATH)/build
+
+.PHONY: setup-build-scripts clean-build-scripts backup-config restore-config
+
+# Save specific config files
+backup-config:
+	@tmp_backup=$$(mktemp -d) && \
+	mkdir -p $$tmp_backup/build_backup && \
+	if [ -d "$(BUILD_SCRIPTS_PATH)/configurations" ]; then \
+		for config in pull_config_preview.yaml pull_config.yaml integration_merge.yaml; do \
+			if [ -f "$(BUILD_SCRIPTS_PATH)/configurations/$$config" ]; then \
+				cp "$(BUILD_SCRIPTS_PATH)/configurations/$$config" "$$tmp_backup/build_backup/$$config"; \
+			fi \
+		done; \
+	fi && \
+	if [ -d "$(PY_PATH)" ]; then \
+	    mkdir -p $$tmp_backup/py_backup && \
+	    find $(PY_PATH) -mindepth 1 -maxdepth 1 \
+	        ! -name "build" \
+	        ! -name "translations" \
+	        ! -name "add_notranslate_tag.py" \
+	        ! -name "missing_metrics.py" \
+	        ! -name "placehold_translations.py" \
+	        ! -name "submit_github_status_check.py" \
+	        -exec cp -r {} $$tmp_backup/py_backup/ \; ; \
+	fi && \
+	echo "$$tmp_backup" > .backup_path
+
+# Restore specific config files
+restore-config:
+	@if [ -f .backup_path ]; then \
+		backup_dir=$$(cat .backup_path) && \
+		if [ -d "$$backup_dir/py_backup" ]; then \
+			cp -r $$backup_dir/py_backup/* $(PY_PATH)/; \
+		fi && \
+		if [ -d "$$backup_dir/build_backup" ]; then \
+			mkdir -p $(BUILD_SCRIPTS_PATH)/configurations && \
+			for config in pull_config_preview.yaml pull_config.yaml integration_merge.yaml; do \
+				if [ -f "$$backup_dir/build_backup/$$config" ]; then \
+					cp "$$backup_dir/build_backup/$$config" "$(BUILD_SCRIPTS_PATH)/configurations/$$config"; \
+				fi \
+			done; \
+		fi && \
+		rm -rf $$backup_dir && \
+		rm .backup_path; \
+	fi
+
+$(PY_PATH):
+	@mkdir -p $(PY_PATH)
+
+# Clean build scripts
+clean-build-scripts:
+	@echo "Cleaning build files..."
+	@$(MAKE) backup-config
+	@rm -rf $(PY_PATH)
+	@rm -rf $(JS_PATH)
+	@mkdir -p $(PY_PATH)
+	@mkdir -p $(BUILD_SCRIPTS_PATH)
+	@$(MAKE) restore-config
+
+# Source the build scripts and maintain file structure
+setup-build-scripts: $(PY_PATH) backup-config clean-build-scripts
+	@echo "Fetching latest build scripts..."; \
+	if [ -z "$(BUILD_SCRIPT_BRANCH)" ] || [ -z "$(BUILD_SCRIPT_REPO_URL)" ] || [ -z "$(BUILD_SCRIPT_SOURCE_DIR)" ]; then \
+		echo -e "\033[0;31mone or more build-script env vars are undefined, check your makefile.config \033[0m"; \
+		exit 1; \
+	fi;
+	@tmp_dir=$$(mktemp -d) && \
+	git clone --depth 1 -b $(BUILD_SCRIPT_BRANCH) $(BUILD_SCRIPT_REPO_URL) $$tmp_dir && \
+	if [ -d "$$tmp_dir/$(BUILD_SCRIPT_SOURCE_DIR)" ]; then \
+		echo "Moving files to python directory..." && \
+		cp -r $$tmp_dir/$(BUILD_SCRIPT_SOURCE_DIR)/* $(PY_PATH)/ && \
+		if [ -d "$(PY_PATH)/services" ]; then \
+			echo "Cleaning up directory structure..." && \
+			rm -rf $(PY_PATH)/services; \
+		fi \
+	fi && \
+	rm -rf $$tmp_dir
+	@$(MAKE) restore-config
+	mkdir local/bin/js
+	cp -r $(PY_PATH)/js/* $(JS_PATH)/
+	rm -rf local/bin/py/sh local/bin/py/js
+	@echo "Build scripts updated successfully!"
+
 # Function that will clone a repo or sparse clone a repo
 # If the dir already exists it will attempt to update it instead
 #

Makefile.config.example

@@ -6,3 +6,8 @@ DATADOG_API_KEY := false
 DATADOG_APP_KEY := false
 FULL_BUILD := false
 CONFIGURATION_FILE := "./local/bin/py/build/configurations/pull_config_preview.yaml"
+
+#build-script env vars
+BUILD_SCRIPT_BRANCH := main
+BUILD_SCRIPT_REPO_URL := 
+BUILD_SCRIPT_SOURCE_DIR :=