From d159353e66cd5c6ef4c1648bcbb7a285e13d12bc Mon Sep 17 00:00:00 2001 From: Brian Deutsch Date: Wed, 6 Nov 2024 13:39:49 -0500 Subject: [PATCH] cleanup algolia references (#26150) Signed-off-by: Brian Deutsch --- .gitlab-ci.yml | 53 +------- local/bin/js/algolia-index-sync.js | 195 ----------------------------- local/etc/requirements3.txt | 1 - package.json | 2 - yarn.lock | 155 ----------------------- 5 files changed, 1 insertion(+), 405 deletions(-) delete mode 100644 local/bin/js/algolia-index-sync.js diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8891137defeb0..6edf6fcc39d39 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -257,39 +257,6 @@ missing_tms_preview: - check_missing_tms interruptible: true -# algolia_sync_preview:manual: -# <<: *base_template -# stage: post-deploy -# environment: "preview" -# allow_failure: true -# interruptible: true -# timeout: 2h -# cache: -# - *hugo_cache -# - *yarn_cache -# - *pip_cache -# dependencies: -# - build_preview -# script: -# # build the site to get algolia.json ondemand -# - touch Makefile.config preview_error.log -# - make dependencies -# - make config -# - yarn run prebuild -# - build_site -# # end build -# - yarn add atomic-algolia@https://s3.amazonaws.com/origin-static-assets/corp-node-packages/master/atomic-algolia-v1.0.2.tgz -# - > -# ALGOLIA_APP_ID=$(get_secret 'algolia_preview_application_id') -# ALGOLIA_ADMIN_KEY=$(get_secret 'algolia_preview_api_key') -# CI_PIPELINE_SOURCE=$CI_PIPELINE_SOURCE -# yarn run algolia:sync -# rules: -# - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' -# when: never -# - if: '$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH && $CI_COMMIT_TAG =~ "/^$/"' -# when: manual - typesense_sync_preview:manual: <<: *base_template stage: post-deploy @@ -304,7 +271,7 @@ typesense_sync_preview:manual: dependencies: - build_preview script: - # build the site to get algolia.json ondemand + # build the site to get search.json ondemand - touch Makefile.config preview_error.log - make dependencies - make config @@ -321,24 +288,6 @@ typesense_sync_preview:manual: - if: '$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH && $CI_COMMIT_TAG =~ "/^$/"' when: manual -# algolia_sync_live: -# <<: *base_template -# <<: *live_rules -# stage: post-deploy -# environment: "live" -# allow_failure: true -# interruptible: true -# timeout: 2h -# dependencies: -# - build_live -# script: -# - yarn add atomic-algolia@https://s3.amazonaws.com/origin-static-assets/corp-node-packages/master/atomic-algolia-v1.0.2.tgz -# - > -# ALGOLIA_APP_ID=$(get_secret 'algolia_docsearch_application_id') -# ALGOLIA_ADMIN_KEY=$(get_secret 'algolia_docsearch_api_key') -# CI_PIPELINE_SOURCE=$CI_PIPELINE_SOURCE -# yarn run algolia:sync - typesense_sync_live: <<: *base_template <<: *live_rules diff --git a/local/bin/js/algolia-index-sync.js b/local/bin/js/algolia-index-sync.js deleted file mode 100644 index 31fdd34cc3d6f..0000000000000 --- a/local/bin/js/algolia-index-sync.js +++ /dev/null @@ -1,195 +0,0 @@ -const algoliasearch = require('algoliasearch'); -const atomicalgolia = require('atomic-algolia'); - -const getIndexName = () => { - let algoliaIndexName = process.env.ALGOLIA_INDEX_NAME || ''; - const configDocs = require('../../../assets/scripts/config/config-docs'); - - switch (process.env.CI_ENVIRONMENT_NAME) { - case 'live': - algoliaIndexName = configDocs['live'].algoliaConfig.index; - break; - case 'preview': - algoliaIndexName = configDocs['preview'].algoliaConfig.index; - break; - default: - algoliaIndexName = ''; - break; - } - - return algoliaIndexName; -}; - -const updateSettings = (index) => { - // Distinct: true allows us to return a single result for each page despite indexing each section inidivdually. - // We use the page's base_url (without the anchor) to group together multiple results for one page using "attributeForDistinct" - // Custom ranking by order allows us to return the full page result whenever possible (we assign -1 order to full page results) - const settings = { - searchableAttributes: [ - 'unordered(tags)', - 'unordered(title)', - 'unordered(section_header)', - 'unordered(content)' - ], - ranking: ['words', 'filters', 'typo', 'attribute', 'proximity', 'exact', 'custom'], - customRanking: ['desc(rank)', 'asc(order)'], - attributesToHighlight: ['title', 'section_header', 'content', 'tags'], - attributesForFaceting: ['language', 'searchable(tags)'], - attributesToSnippet: ['content:20'], - indexLanguages: ['ja', 'en', 'fr', 'ko'], - queryLanguages: ['ja', 'en', 'fr', 'ko'], - attributeForDistinct: 'distinct_base_url', - distinct: true, - minWordSizefor1Typo: 3, - minWordSizefor2Typos: 7, - ignorePlurals: true, - optionalWords: ['the', 'without'], - separatorsToIndex: '_@.#', - advancedSyntax: true - }; - - return index.setSettings(settings, { forwardToReplicas: true }); -}; - -const updateSynonyms = (index) => { - const synonyms = [ - { - objectID: 'agent', - type: 'synonym', - synonyms: ['agent', 'datadog agent'] - }, - { - objectID: 'azure', - type: 'synonym', - synonyms: ['microsoft azure', 'azure'] - }, - { - objectID: 'gcp', - type: 'synonym', - synonyms: ['gcp', 'google cloud platform'] - }, - { - objectID: 'aws', - type: 'synonym', - synonyms: ['aws', 'amazon web service'] - }, - { - objectID: 'rum', - type: 'synonym', - synonyms: ['RUM', 'rum', 'real user monitoring'] - }, - { - objectID: 'permissions', - type: 'synonym', - synonyms: ['permissions', 'Role Permissions'] - }, - { - objectID: 'dbm', - type: 'synonym', - synonyms: ['dbm', 'database monitoring'] - }, - { - objectID: 'npm', - type: 'synonym', - synonyms: ['npm', 'network performance monitoring'] - }, - { - objectID: 'ksm', - type: 'synonym', - synonyms: ['ksm', 'ksm core', 'kubernetes state metrics'] - }, - { - objectID: 'sast', - type: 'synonym', - synonyms: ['sast', 'static analysis', 'static application security testing'] - }, - { - objectID: 'sca', - type: 'synonym', - synonyms: ['sca', 'software composition analysis'] - }, - { - objectID: 'otel', - type: 'synonym', - synonyms: ['otel', 'opentelemetry'] - } - ]; - - return index.saveSynonyms(synonyms, { - forwardToReplicas: true, - replaceExistingSynonyms: true - }); -}; - -const updateReplicas = (client, indexName) => { - const replicas = {}; - - replicas[`${indexName}_api`] = { - customRanking: ['asc(rank)'] - }; - - Object.entries(replicas).forEach(([replicaIndexName, replicaSettings]) => { - console.log(`Updating replica ${replicaIndexName}..`); - const index = client.initIndex(replicaIndexName); - index.setSettings(replicaSettings).then((response) => { - console.log(`Index ${replicaIndexName} configuration update complete...`); - }); - }); -}; - -const updateIndex = (indexName) => { - console.info('Syncing local index with Algolia...') - const fullLocalAlogliaSearchIndex = require('../../../public/algolia.json'); - let localAlgoliaSearchIndex - let filterLanguage = '' - - // Only the full nightly build re-indexes all language pages in Algolia. - // Master/preview pipelines will re-index only English pages automatically. - // This is done to improve performance as Docs continues scaling. - if (process.env.CI_PIPELINE_SOURCE.toLowerCase() !== 'schedule') { - localAlgoliaSearchIndex = fullLocalAlogliaSearchIndex.filter(record => record.language === "en") - filterLanguage = 'en' - } else { - localAlgoliaSearchIndex = fullLocalAlogliaSearchIndex - } - - const cb = (error, result) => { - if (error) { - console.error(error); - throw error; - } - - console.log(result); - }; - - atomicalgolia(indexName, localAlgoliaSearchIndex, filterLanguage, cb); -}; - -const sync = () => { - const appId = process.env.ALGOLIA_APP_ID || ''; - const adminKey = process.env.ALGOLIA_ADMIN_KEY || ''; - const indexName = getIndexName(); - - if (appId === '' || adminKey === '' || indexName === '') { - console.error('Missing Algolia App Id, API Key, or Index name. Exiting...'); - process.exit(1); - } - - const client = algoliasearch(appId, adminKey); - const index = client.initIndex(indexName); - - updateSettings(index) - .then(() => { - console.log(`${indexName} settings update complete`); - updateReplicas(client, indexName); - }) - .catch((err) => console.error(err)); - - updateSynonyms(index) - .then(() => console.log(`${indexName} synonyms update complete`)) - .catch((err) => console.error(err)); - - updateIndex(indexName); -}; - -sync(); diff --git a/local/etc/requirements3.txt b/local/etc/requirements3.txt index 8a8b36a8e2656..37d40d50616aa 100644 --- a/local/etc/requirements3.txt +++ b/local/etc/requirements3.txt @@ -1,7 +1,6 @@ setuptools==60.10.0 PyGithub==1.47 markdown -algoliasearch>=2.2.0 beautifulsoup4==4.5.1 cffi==1.15.0 cssutils>=1.0.0 diff --git a/package.json b/package.json index 168ad7baaa12c..80605b4e46644 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,6 @@ "description": "Datadog Docs built on HUGO", "private": true, "scripts": { - "algolia:sync": "node ./local/bin/js/algolia-index-sync.js", "typesense:sync:preview": "NODE_ENV=preview node ./local/bin/js/typesense_sync.cjs", "typesense:sync:production": "NODE_ENV=production node ./local/bin/js/typesense_sync.cjs", "preinstall": "bash local/bin/sh/preinstall.sh", @@ -46,7 +45,6 @@ "@datadog/browser-logs": "^5.27.0", "@datadog/browser-rum": "^5.27.0", "@popperjs/core": "^2.11.8", - "algoliasearch": "4.22.1", "alpinejs": "^3.13.7", "bootstrap": "^5.2", "del": "4.1.1", diff --git a/yarn.lock b/yarn.lock index fc65e566a00e5..62143fd8ff763 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5,86 +5,6 @@ __metadata: version: 6 cacheKey: 8 -"@algolia/cache-browser-local-storage@npm:4.22.1": - version: 4.22.1 - resolution: "@algolia/cache-browser-local-storage@npm:4.22.1" - dependencies: - "@algolia/cache-common": 4.22.1 - checksum: 82e65c0dbc015d55bf17842757d21c3769fde95c10235d038062ccb41f2f64b3b1efd953df0f1b4892f352d83cdf2b8374a8f1b4e06b4ba42b35c3a449d316e7 - languageName: node - linkType: hard - -"@algolia/cache-common@npm:4.22.1": - version: 4.22.1 - resolution: "@algolia/cache-common@npm:4.22.1" - checksum: b57b195fdf75ca53417541fd03b48fa2351c18261f21ddc462ca4e76adef4750a35df9db707e9acc9f7a67fb465757d7f254423b4f8b0661056e4d2ec07392c1 - languageName: node - linkType: hard - -"@algolia/cache-in-memory@npm:4.22.1": - version: 4.22.1 - resolution: "@algolia/cache-in-memory@npm:4.22.1" - dependencies: - "@algolia/cache-common": 4.22.1 - checksum: 83dfe0e3360f5dd03ead8165f6e92e5a414d9e43eee2dd2fb682d418ddcf8c2cb176d040f57ac75018f62ab805518991157bf8572625f1420515f1959f4fdcaa - languageName: node - linkType: hard - -"@algolia/client-account@npm:4.22.1": - version: 4.22.1 - resolution: "@algolia/client-account@npm:4.22.1" - dependencies: - "@algolia/client-common": 4.22.1 - "@algolia/client-search": 4.22.1 - "@algolia/transporter": 4.22.1 - checksum: 85f3f7f9fa8e9d5b723e128f3b801583d73e4dc529086d57adfc1ac1718c3e13c0660c0d3f3a43a033d5aa231962ed405912826ae74a5c996929943fc575e7ed - languageName: node - linkType: hard - -"@algolia/client-analytics@npm:4.22.1": - version: 4.22.1 - resolution: "@algolia/client-analytics@npm:4.22.1" - dependencies: - "@algolia/client-common": 4.22.1 - "@algolia/client-search": 4.22.1 - "@algolia/requester-common": 4.22.1 - "@algolia/transporter": 4.22.1 - checksum: 8a352ae6bbddeba86fbf12de6bdd4f48d7331c5d8cdf803af803f5957a10d0a09ead58e75e364d25a7becdf68e9d5252377e307c055cbfbd6bc2048557bd75f6 - languageName: node - linkType: hard - -"@algolia/client-common@npm:4.22.1": - version: 4.22.1 - resolution: "@algolia/client-common@npm:4.22.1" - dependencies: - "@algolia/requester-common": 4.22.1 - "@algolia/transporter": 4.22.1 - checksum: 848225464bf62972eee80faed400b6e9666678e724c5ddd3ecedc6fb57db1cd5c47c4a06a4cba90f83db38353ea8dcbf53b51d1423164a0258bce7bbe417e7f8 - languageName: node - linkType: hard - -"@algolia/client-personalization@npm:4.22.1": - version: 4.22.1 - resolution: "@algolia/client-personalization@npm:4.22.1" - dependencies: - "@algolia/client-common": 4.22.1 - "@algolia/requester-common": 4.22.1 - "@algolia/transporter": 4.22.1 - checksum: 64c359c12d2722dfcc821a3bacfed2c49f94159060776fba871b66e4961757732d9696840f415b45bbe5ad1dbd39e2c512a81851a9cfeeec8511975396dad245 - languageName: node - linkType: hard - -"@algolia/client-search@npm:4.22.1": - version: 4.22.1 - resolution: "@algolia/client-search@npm:4.22.1" - dependencies: - "@algolia/client-common": 4.22.1 - "@algolia/requester-common": 4.22.1 - "@algolia/transporter": 4.22.1 - checksum: 0477f003c19cf1dbb6190fd491136927bc7174fa9d5a27dead218a51807da9c519be114000bd035265c018682fd8654d110cab07f74004c8c8d069db19800c3d - languageName: node - linkType: hard - "@algolia/events@npm:^4.0.1": version: 4.0.1 resolution: "@algolia/events@npm:4.0.1" @@ -92,58 +12,6 @@ __metadata: languageName: node linkType: hard -"@algolia/logger-common@npm:4.22.1": - version: 4.22.1 - resolution: "@algolia/logger-common@npm:4.22.1" - checksum: 3ac5430f73e8eabb4e7561b271d38151fb7f128491437c202dac3d54f7c3a83ebc96818532746422ea4abdf9d68a6ccb716dc8b97f69101ff642afaff12057e5 - languageName: node - linkType: hard - -"@algolia/logger-console@npm:4.22.1": - version: 4.22.1 - resolution: "@algolia/logger-console@npm:4.22.1" - dependencies: - "@algolia/logger-common": 4.22.1 - checksum: fc6ea0623b257420f4e10ca1a78875dfb4c55841a0db5712150344d742ca457038f209b63c4e25848338c652e5ca5ea052a4143c87c3dc1203eedc5bff0c54f3 - languageName: node - linkType: hard - -"@algolia/requester-browser-xhr@npm:4.22.1": - version: 4.22.1 - resolution: "@algolia/requester-browser-xhr@npm:4.22.1" - dependencies: - "@algolia/requester-common": 4.22.1 - checksum: 825cf73fdc6aa8b159cd35ebb1facbeccb9fe27c4360661b7c9287d830d92409baaa38ad78f6c6f72bcdebc6e9d6ae8a5c8648e998fd34617b7f1eb7a59ea83b - languageName: node - linkType: hard - -"@algolia/requester-common@npm:4.22.1": - version: 4.22.1 - resolution: "@algolia/requester-common@npm:4.22.1" - checksum: 7caae4924efccabefd6b1a1d4e7090ed2f6dd4ab53dedf2f2095d5c1ef016c841129331c79791f7ded8072e174204503814f11119ac8bc75f5e10ae2eb42a85b - languageName: node - linkType: hard - -"@algolia/requester-node-http@npm:4.22.1": - version: 4.22.1 - resolution: "@algolia/requester-node-http@npm:4.22.1" - dependencies: - "@algolia/requester-common": 4.22.1 - checksum: 511348954b7747006875132ed0bc922ec3cfcf0187f41a665fc45426982479dd5cd55fab1de592ac9a71180539ff2e4c7457eea3bdab0e56bce27de2de1ba677 - languageName: node - linkType: hard - -"@algolia/transporter@npm:4.22.1": - version: 4.22.1 - resolution: "@algolia/transporter@npm:4.22.1" - dependencies: - "@algolia/cache-common": 4.22.1 - "@algolia/logger-common": 4.22.1 - "@algolia/requester-common": 4.22.1 - checksum: 737e787ac77215f30db54ebab3431e06cfee1790ab7cf45222546470546ecb276eedfaa0aedf78b2c95efa9d5550ea7d47c947535f143d7002a22f781d8721ce - languageName: node - linkType: hard - "@apidevtools/json-schema-ref-parser@npm:^8.0.0": version: 8.0.0 resolution: "@apidevtools/json-schema-ref-parser@npm:8.0.0" @@ -4484,28 +4352,6 @@ __metadata: languageName: node linkType: hard -"algoliasearch@npm:4.22.1": - version: 4.22.1 - resolution: "algoliasearch@npm:4.22.1" - dependencies: - "@algolia/cache-browser-local-storage": 4.22.1 - "@algolia/cache-common": 4.22.1 - "@algolia/cache-in-memory": 4.22.1 - "@algolia/client-account": 4.22.1 - "@algolia/client-analytics": 4.22.1 - "@algolia/client-common": 4.22.1 - "@algolia/client-personalization": 4.22.1 - "@algolia/client-search": 4.22.1 - "@algolia/logger-common": 4.22.1 - "@algolia/logger-console": 4.22.1 - "@algolia/requester-browser-xhr": 4.22.1 - "@algolia/requester-common": 4.22.1 - "@algolia/requester-node-http": 4.22.1 - "@algolia/transporter": 4.22.1 - checksum: 65226e7ac081fd2dccd2a949b211a67010d933e86572be7cc714b3f5ca07edf099e8a2fd328e8c4a772d971d06ec49779c2b8670196c811faede5b78765cb5bf - languageName: node - linkType: hard - "alpinejs@npm:^3.13.7": version: 3.13.7 resolution: "alpinejs@npm:3.13.7" @@ -6502,7 +6348,6 @@ __metadata: "@datadog/datadog-ci": ^2.36.0 "@popperjs/core": ^2.11.8 acorn: ^7.4.1 - algoliasearch: 4.22.1 alpinejs: ^3.13.7 bootstrap: ^5.2 cross-env: ^5.2.1