From dd0d3ee9f3e4d07a26625ec290f2e90b3f22d506 Mon Sep 17 00:00:00 2001 From: Ansh Goyal Date: Wed, 8 Jan 2025 13:26:39 +0000 Subject: [PATCH 01/13] add script --- config/edit-page-config.json | 4 +- package.json | 1 + scripts/markdown/check-editlinks.js | 170 ++++++++++++++++++++++++++++ 3 files changed, 173 insertions(+), 2 deletions(-) create mode 100644 scripts/markdown/check-editlinks.js diff --git a/config/edit-page-config.json b/config/edit-page-config.json index 32921f145d4e..9a66d9cd3f9f 100644 --- a/config/edit-page-config.json +++ b/config/edit-page-config.json @@ -1,7 +1,7 @@ [ { "value": "/tools/generator", - "href": "https://github.com/asyncapi/generator/tree/master/docs" + "href": "https://github.com/asyncapi/generator/tree/master/apps/generator/docs" }, { "value": "reference/specification/", @@ -19,4 +19,4 @@ "value": "reference/extensions/", "href": "https://github.com/asyncapi/extensions-catalog/tree/master/extensions" } -] \ No newline at end of file +] diff --git a/package.json b/package.json index f9cc8e225374..11532ad2eed4 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ "generate:tools": "node scripts/build-tools.js", "test:netlify": "deno test --allow-env --trace-ops netlify/**/*.test.ts", "test:md": "node scripts/markdown/check-markdown.js", + "test:editlinks": "node scripts/markdown/check-editlinks.js", "dev:storybook": "storybook dev -p 6006", "build:storybook": "storybook build" }, diff --git a/scripts/markdown/check-editlinks.js b/scripts/markdown/check-editlinks.js new file mode 100644 index 000000000000..d73b322486da --- /dev/null +++ b/scripts/markdown/check-editlinks.js @@ -0,0 +1,170 @@ +const fs = require('fs').promises; +const path = require('path'); +const fetch = require('node-fetch-2'); +const editUrls = require('../../config/edit-page-config.json'); + +const ignoreFiles = [ + 'reference/specification/v2.x.md', + 'reference/specification/v3.0.0-explorer.md', + 'reference/specification/v3.0.0.md' +]; + +/** + * Introduces a delay in the execution flow + * @param {number} ms - The number of milliseconds to pause + */ +async function pause(ms) { + return new Promise((res) => { + setTimeout(res, ms); + }); +} + +/** + * Process a batch of URLs to check for 404s + * @param {object[]} batch - Array of path objects to check + * @returns {Promise} Array of URLs that returned 404 + */ +async function processBatch(batch) { + return Promise.all( + batch.map(async ({ filePath, urlPath, editLink }) => { + try { + if (!editLink || ignoreFiles.some((ignorePath) => filePath.endsWith(ignorePath))) return null; + + const response = await fetch(editLink, { method: 'HEAD' }); + if (response.status === 404) { + return { filePath, urlPath, editLink }; + } + return null; + } catch (error) { + console.error(`Error checking ${editLink}:`, error.message); + return editLink; + } + }) + ); +} + +/** + * Check all URLs in batches + * @param {object[]} paths - Array of all path objects to check + * @returns {Promise} Array of URLs that returned 404 + */ +async function checkUrls(paths) { + const result = []; + const batchSize = 5; + + for (let i = 0; i < paths.length; i += batchSize) { + console.log(`Processing batch ${Math.floor(i / batchSize) + 1}/${Math.ceil(paths.length / batchSize)}`); + const batch = paths.slice(i, i + batchSize); + const batchResults = await processBatch(batch); + await pause(1000); + + // Filter out null results and add valid URLs to results + result.push(...batchResults.filter((url) => url !== null)); + } + + return result; +} + +/** + * Determines the appropriate edit link based on the URL path and file path + * @param {string} urlPath - The URL path to generate an edit link for + * @param {string} filePath - The actual file path + * @param {object[]} editOptions - Array of edit link options + * @returns {string|null} The generated edit link or null if no match + */ +function determineEditLink(urlPath, filePath, editOptions) { + // Remove leading 'docs/' if present for matching + const pathForMatching = urlPath.startsWith('docs/') ? urlPath.slice(5) : urlPath; + + const target = + editOptions.find((edit) => pathForMatching.includes(edit.value)) || editOptions.find((edit) => edit.value === ''); + + if (!target) return null; + + // Handle the empty value case (fallback) + if (target.value === '') { + return `${target.href}/docs/${urlPath}.md`; + } + + // For other cases with specific targets + return `${target.href}/${path.basename(filePath)}`; +} + +/** + * Recursively processes markdown files in a directory to generate paths and edit links + * @param {string} folderPath - The path to the folder to process + * @param {object[]} editOptions - Array of edit link options + * @param {string} [relativePath=''] - The relative path for URL generation + * @param {object[]} [result=[]] - Accumulator for results + * @returns {Promise} Array of objects containing file paths and edit links + */ +async function generatePaths(folderPath, editOptions, relativePath = '', result = []) { + try { + const files = await fs.readdir(folderPath); + + await Promise.all( + files.map(async (file) => { + const filePath = path.join(folderPath, file); + const relativeFilePath = path.join(relativePath, file); + + // Skip _section.md files + if (file === '_section.md') { + return; + } + + const stats = await fs.stat(filePath); + + if (stats.isDirectory()) { + // Process directory + await generatePaths(filePath, editOptions, relativeFilePath, result); + } else if (stats.isFile() && file.endsWith('.md')) { + // Process all markdown files (including index.md) + const urlPath = relativeFilePath.split(path.sep).join('/').replace('.md', ''); + result.push({ + filePath, + urlPath, + editLink: determineEditLink(urlPath, filePath, editOptions) + }); + } + }) + ); + + return result; + } catch (err) { + console.error(`Error processing directory ${folderPath}:`, err); + throw err; + } +} + +async function main() { + const editOptions = editUrls; + + try { + const docsFolderPath = path.resolve(__dirname, '../../markdown/docs'); + const paths = await generatePaths(docsFolderPath, editOptions); + console.log('Starting URL checks...'); + const invalidUrls = await checkUrls(paths); + + if (invalidUrls.length === 0) { + console.log('All URLs are valid.'); + process.exit(0); + } + + console.log('\nURLs returning 404:\n'); + invalidUrls.forEach((url) => console.log(`- ${url.editLink} generated from ${url.filePath}\n`)); + console.log(`\nTotal invalid URLs found: ${invalidUrls.length}`); + + if (invalidUrls.length > 0) { + process.exit(1); + } + } catch (error) { + console.error('Failed to check edit links:', error); + process.exit(1); + } +} + +if (require.main === module) { + main(); +} + +module.exports = { generatePaths, determineEditLink, main }; From 05a2fe48bc5710437c0199fd76f50e72fea00ab3 Mon Sep 17 00:00:00 2001 From: Ansh Goyal Date: Wed, 8 Jan 2025 13:55:34 +0000 Subject: [PATCH 02/13] add workflow --- .github/workflows/check-edit-links.yml | 47 ++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 .github/workflows/check-edit-links.yml diff --git a/.github/workflows/check-edit-links.yml b/.github/workflows/check-edit-links.yml new file mode 100644 index 000000000000..2c695ef85977 --- /dev/null +++ b/.github/workflows/check-edit-links.yml @@ -0,0 +1,47 @@ +name: Weekly Link Checker + +on: + schedule: + - cron: "0 0 * * 0" # Runs every week at midnight on Sunday + workflow_dispatch: + +jobs: + check-links: + name: Run Link Checker and Notify Slack + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + + - name: Install dependencies + run: npm install + + - name: Run link checker + id: linkcheck + run: | + npm run test:checklinks | tee output.log + + - name: Extract 404 URLs from output + id: extract-404 + run: | + ERRORS=$(sed -n '/URLs returning 404:/,$p' output.log) + echo "errors<> $GITHUB_OUTPUT + echo "$ERRORS" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + + - name: Notify Slack + if: ${{ steps.extract-404.outputs.errors != '' }} + uses: rtCamp/action-slack-notify@v2 + with: + webhook_url: ${{ secrets.WEBSITE_SLACK_WEBHOOK }} + message: | + 🚨 The following URLs returned 404 during the link check: + ``` + ${{ steps.extract-404.outputs.errors }} + \ No newline at end of file From 6cf369a5f8c7a66751ed8c8d9a7126ea2a08125b Mon Sep 17 00:00:00 2001 From: Ansh Goyal Date: Wed, 8 Jan 2025 14:40:28 +0000 Subject: [PATCH 03/13] fix mdx --- components/layout/DocsLayout.tsx | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/components/layout/DocsLayout.tsx b/components/layout/DocsLayout.tsx index b395c9445edf..061124f5cda6 100644 --- a/components/layout/DocsLayout.tsx +++ b/components/layout/DocsLayout.tsx @@ -33,6 +33,10 @@ interface IDocsLayoutProps { */ function generateEditLink(post: IPost) { let last = post.id.substring(post.id.lastIndexOf('/') + 1); + + if (last.endsWith('.mdx')) { + last = last.replace('.mdx', '.md'); + } const target = editOptions.find((edit) => { return post.slug.includes(edit.value); }); From 9675b464a7d21b87e630faa6388ebfe0851cb29b Mon Sep 17 00:00:00 2001 From: Ansh Goyal Date: Wed, 8 Jan 2025 14:46:54 +0000 Subject: [PATCH 04/13] fix wf --- .github/workflows/check-edit-links.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check-edit-links.yml b/.github/workflows/check-edit-links.yml index 2c695ef85977..6db4d831996c 100644 --- a/.github/workflows/check-edit-links.yml +++ b/.github/workflows/check-edit-links.yml @@ -25,7 +25,7 @@ jobs: - name: Run link checker id: linkcheck run: | - npm run test:checklinks | tee output.log + npm run test:editlinks | tee output.log - name: Extract 404 URLs from output id: extract-404 From c3c8266daa1e44710584a1e316e3a4a432f202df Mon Sep 17 00:00:00 2001 From: Ansh Goyal Date: Wed, 8 Jan 2025 14:56:33 +0000 Subject: [PATCH 05/13] fix wf --- .github/workflows/check-edit-links.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/check-edit-links.yml b/.github/workflows/check-edit-links.yml index 6db4d831996c..670582577b5f 100644 --- a/.github/workflows/check-edit-links.yml +++ b/.github/workflows/check-edit-links.yml @@ -2,7 +2,7 @@ name: Weekly Link Checker on: schedule: - - cron: "0 0 * * 0" # Runs every week at midnight on Sunday + - cron: '0 0 * * 0' # Runs every week at midnight on Sunday workflow_dispatch: jobs: @@ -39,9 +39,11 @@ jobs: if: ${{ steps.extract-404.outputs.errors != '' }} uses: rtCamp/action-slack-notify@v2 with: - webhook_url: ${{ secrets.WEBSITE_SLACK_WEBHOOK }} - message: | + SLACK_WEBHOOK: ${{ secrets.WEBSITE_SLACK_WEBHOOK }} + SLACK_TITLE: 'Edit Links Checker Errors Report' + SLACK_MESSAGE: | 🚨 The following URLs returned 404 during the link check: ``` ${{ steps.extract-404.outputs.errors }} - \ No newline at end of file + ``` + MSG_MINIMAL: true From 689a8615b692fbb2634048439a93a5b30d5c1804 Mon Sep 17 00:00:00 2001 From: Ansh Goyal Date: Wed, 8 Jan 2025 15:00:36 +0000 Subject: [PATCH 06/13] fix wf --- .github/workflows/check-edit-links.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check-edit-links.yml b/.github/workflows/check-edit-links.yml index 670582577b5f..7ccca35199ac 100644 --- a/.github/workflows/check-edit-links.yml +++ b/.github/workflows/check-edit-links.yml @@ -38,7 +38,7 @@ jobs: - name: Notify Slack if: ${{ steps.extract-404.outputs.errors != '' }} uses: rtCamp/action-slack-notify@v2 - with: + env: SLACK_WEBHOOK: ${{ secrets.WEBSITE_SLACK_WEBHOOK }} SLACK_TITLE: 'Edit Links Checker Errors Report' SLACK_MESSAGE: | From c13eb87fb957a71268ae0325ac48bd6f836b88cc Mon Sep 17 00:00:00 2001 From: Ansh Goyal Date: Wed, 8 Jan 2025 15:18:46 +0000 Subject: [PATCH 07/13] fix cov --- jest.config.js | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/jest.config.js b/jest.config.js index 4a32c5b1cc04..9aeaa34bed3f 100644 --- a/jest.config.js +++ b/jest.config.js @@ -4,7 +4,12 @@ module.exports = { coverageReporters: ['text', 'lcov', 'json-summary'], coverageDirectory: 'coverage', collectCoverageFrom: ['scripts/**/*.js'], - coveragePathIgnorePatterns: ['scripts/compose.js', 'scripts/tools/categorylist.js', 'scripts/tools/tags-color.js'], + coveragePathIgnorePatterns: [ + 'scripts/compose.js', + 'scripts/tools/categorylist.js', + 'scripts/tools/tags-color.js', + 'scripts/markdown/check-editlinks.js' + ], // To disallow netlify edge function tests from running - testMatch: ['**/tests/**/*.test.*', '!**/netlify/**/*.test.*'], -}; \ No newline at end of file + testMatch: ['**/tests/**/*.test.*', '!**/netlify/**/*.test.*'] +}; From a79a06a0af5985a6697bcf6363fc6e1a9aa6ff06 Mon Sep 17 00:00:00 2001 From: Ansh Goyal Date: Tue, 14 Jan 2025 06:15:45 +0000 Subject: [PATCH 08/13] fix comments --- scripts/markdown/check-editlinks.js | 36 ++++++++++++++--------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/scripts/markdown/check-editlinks.js b/scripts/markdown/check-editlinks.js index d73b322486da..c38111d6c005 100644 --- a/scripts/markdown/check-editlinks.js +++ b/scripts/markdown/check-editlinks.js @@ -52,16 +52,22 @@ async function checkUrls(paths) { const result = []; const batchSize = 5; + const batches = []; for (let i = 0; i < paths.length; i += batchSize) { - console.log(`Processing batch ${Math.floor(i / batchSize) + 1}/${Math.ceil(paths.length / batchSize)}`); const batch = paths.slice(i, i + batchSize); - const batchResults = await processBatch(batch); - await pause(1000); - - // Filter out null results and add valid URLs to results - result.push(...batchResults.filter((url) => url !== null)); + batches.push(batch); } + console.log(`Processing ${batches.length} batches concurrently...`); + const batchResultsArray = await Promise.all( + batches.map(async (batch) => { + const batchResults = await processBatch(batch); + await pause(1000); + return batchResults.filter((url) => url !== null); + }) + ); + + result.push(...batchResultsArray.flat()); return result; } @@ -115,10 +121,8 @@ async function generatePaths(folderPath, editOptions, relativePath = '', result const stats = await fs.stat(filePath); if (stats.isDirectory()) { - // Process directory await generatePaths(filePath, editOptions, relativeFilePath, result); } else if (stats.isFile() && file.endsWith('.md')) { - // Process all markdown files (including index.md) const urlPath = relativeFilePath.split(path.sep).join('/').replace('.md', ''); result.push({ filePath, @@ -145,21 +149,15 @@ async function main() { console.log('Starting URL checks...'); const invalidUrls = await checkUrls(paths); - if (invalidUrls.length === 0) { - console.log('All URLs are valid.'); - process.exit(0); - } - - console.log('\nURLs returning 404:\n'); - invalidUrls.forEach((url) => console.log(`- ${url.editLink} generated from ${url.filePath}\n`)); - console.log(`\nTotal invalid URLs found: ${invalidUrls.length}`); - if (invalidUrls.length > 0) { - process.exit(1); + console.log('\nURLs returning 404:\n'); + invalidUrls.forEach((url) => console.log(`- ${url.editLink} generated from ${url.filePath}\n`)); + console.log(`\nTotal invalid URLs found: ${invalidUrls.length}`); + } else { + console.log('All URLs are valid.'); } } catch (error) { console.error('Failed to check edit links:', error); - process.exit(1); } } From ceae634669da800dd40244b1b586ebe91e4fda92 Mon Sep 17 00:00:00 2001 From: Ansh Goyal Date: Tue, 14 Jan 2025 08:37:53 +0000 Subject: [PATCH 09/13] add tests --- jest.config.js | 7 +- scripts/markdown/check-editlinks.js | 16 +- .../fixtures/markdown/check-editlinks-data.js | 85 ++++++++++ tests/fixtures/markdown/edit-page-config.json | 22 +++ tests/markdown/check-editlinks.test.js | 157 ++++++++++++++++++ 5 files changed, 271 insertions(+), 16 deletions(-) create mode 100644 tests/fixtures/markdown/check-editlinks-data.js create mode 100644 tests/fixtures/markdown/edit-page-config.json create mode 100644 tests/markdown/check-editlinks.test.js diff --git a/jest.config.js b/jest.config.js index 9aeaa34bed3f..1e24444efd95 100644 --- a/jest.config.js +++ b/jest.config.js @@ -4,12 +4,7 @@ module.exports = { coverageReporters: ['text', 'lcov', 'json-summary'], coverageDirectory: 'coverage', collectCoverageFrom: ['scripts/**/*.js'], - coveragePathIgnorePatterns: [ - 'scripts/compose.js', - 'scripts/tools/categorylist.js', - 'scripts/tools/tags-color.js', - 'scripts/markdown/check-editlinks.js' - ], + coveragePathIgnorePatterns: ['scripts/compose.js', 'scripts/tools/categorylist.js', 'scripts/tools/tags-color.js'], // To disallow netlify edge function tests from running testMatch: ['**/tests/**/*.test.*', '!**/netlify/**/*.test.*'] }; diff --git a/scripts/markdown/check-editlinks.js b/scripts/markdown/check-editlinks.js index c38111d6c005..f3195f116b0d 100644 --- a/scripts/markdown/check-editlinks.js +++ b/scripts/markdown/check-editlinks.js @@ -36,8 +36,7 @@ async function processBatch(batch) { } return null; } catch (error) { - console.error(`Error checking ${editLink}:`, error.message); - return editLink; + return Promise.reject(new Error(`Error checking ${editLink}:`, error.message)); } }) ); @@ -82,10 +81,7 @@ function determineEditLink(urlPath, filePath, editOptions) { // Remove leading 'docs/' if present for matching const pathForMatching = urlPath.startsWith('docs/') ? urlPath.slice(5) : urlPath; - const target = - editOptions.find((edit) => pathForMatching.includes(edit.value)) || editOptions.find((edit) => edit.value === ''); - - if (!target) return null; + const target = editOptions.find((edit) => pathForMatching.includes(edit.value)); // Handle the empty value case (fallback) if (target.value === '') { @@ -135,8 +131,7 @@ async function generatePaths(folderPath, editOptions, relativePath = '', result return result; } catch (err) { - console.error(`Error processing directory ${folderPath}:`, err); - throw err; + throw new Error(`Error processing directory ${folderPath}:`, err); } } @@ -157,12 +152,13 @@ async function main() { console.log('All URLs are valid.'); } } catch (error) { - console.error('Failed to check edit links:', error); + throw new Error('Failed to check edit links:', error); } } +/* istanbul ignore next */ if (require.main === module) { main(); } -module.exports = { generatePaths, determineEditLink, main }; +module.exports = { generatePaths, processBatch, checkUrls, determineEditLink, main }; diff --git a/tests/fixtures/markdown/check-editlinks-data.js b/tests/fixtures/markdown/check-editlinks-data.js new file mode 100644 index 000000000000..9b012daeed07 --- /dev/null +++ b/tests/fixtures/markdown/check-editlinks-data.js @@ -0,0 +1,85 @@ +const determineEditLinkData = [ + { + urlPath: 'docs/concepts/application', + filePath: 'markdown/docs/concepts/application.md', + editLink: 'https://github.com/asyncapi/website/blob/master/markdown/docs/docs/concepts/application.md' + }, + { + urlPath: 'concepts/application', + filePath: 'markdown/docs/concepts/application.md', + editLink: 'https://github.com/asyncapi/website/blob/master/markdown/docs/concepts/application.md' + }, + { + urlPath: '/tools/cli', + filePath: 'markdown/docs/tools/cli/index.md', + editLink: 'https://github.com/asyncapi/cli/tree/master/docs/index.md' + } +]; + +const processBatchData = [ + { + filePath: '/markdown/docs/tutorials/generate-code.md', + urlPath: 'tutorials/generate-code', + editLink: 'https://github.com/asyncapi/website/blob/master/markdown/docs/tutorials/generate-code.md' + }, + { + filePath: '/markdown/docs/tutorials/index.md', + urlPath: 'tutorials/index', + editLink: 'https://github.com/asyncapi/website/blob/master/markdown/docs/tutorials/index.md' + } +]; + +const testPaths = [ + { + filePath: '/markdown/docs/guides/index.md', + urlPath: 'guides/index', + editLink: 'https://github.com/asyncapi/website/blob/master/markdown/docs/guides/index.md' + }, + { + filePath: '/markdown/docs/guides/message-validation.md', + urlPath: 'guides/message-validation', + editLink: 'https://github.com/asyncapi/website/blob/master/markdown/docs/guides/message-validation.md' + }, + { + filePath: '/markdown/docs/guides/validate.md', + urlPath: 'guides/validate', + editLink: 'https://github.com/asyncapi/website/blob/master/markdown/docs/guides/validate.md' + }, + { + filePath: '/markdown/docs/reference/index.md', + urlPath: 'reference/index', + editLink: 'https://github.com/asyncapi/website/blob/master/markdown/docs/reference/index.md' + }, + { + filePath: '/markdown/docs/tools/index.md', + urlPath: 'tools/index', + editLink: 'https://github.com/asyncapi/website/blob/master/markdown/docs/tools/index.md' + }, + { + filePath: '/markdown/docs/migration/index.md', + urlPath: 'migration/index', + editLink: 'https://github.com/asyncapi/website/blob/master/markdown/docs/migration/index.md' + }, + { + filePath: '/markdown/docs/migration/migrating-to-v3.md', + urlPath: 'migration/migrating-to-v3', + editLink: 'https://github.com/asyncapi/website/blob/master/markdown/docs/migration/migrating-to-v3.md' + }, + { + filePath: '/markdown/docs/tutorials/create-asyncapi-document.md', + urlPath: 'tutorials/create-asyncapi-document', + editLink: 'https://github.com/asyncapi/website/blob/master/markdown/docs/tutorials/create-asyncapi-document.md' + }, + { + filePath: '/markdown/docs/tutorials/generate-code.md', + urlPath: 'tutorials/generate-code', + editLink: 'https://github.com/asyncapi/website/blob/master/markdown/docs/tutorials/generate-code.md' + }, + { + filePath: '/markdown/docs/tutorials/index.md', + urlPath: 'tutorials/index', + editLink: 'https://github.com/asyncapi/website/blob/master/markdown/docs/tutorials/index.md' + } +]; + +module.exports = { determineEditLinkData, processBatchData, testPaths }; diff --git a/tests/fixtures/markdown/edit-page-config.json b/tests/fixtures/markdown/edit-page-config.json new file mode 100644 index 000000000000..9a66d9cd3f9f --- /dev/null +++ b/tests/fixtures/markdown/edit-page-config.json @@ -0,0 +1,22 @@ +[ + { + "value": "/tools/generator", + "href": "https://github.com/asyncapi/generator/tree/master/apps/generator/docs" + }, + { + "value": "reference/specification/", + "href": "https://github.com/asyncapi/spec/blob/master/spec/asyncapi.md" + }, + { + "value": "/tools/cli", + "href": "https://github.com/asyncapi/cli/tree/master/docs" + }, + { + "value": "", + "href": "https://github.com/asyncapi/website/blob/master/markdown" + }, + { + "value": "reference/extensions/", + "href": "https://github.com/asyncapi/extensions-catalog/tree/master/extensions" + } +] diff --git a/tests/markdown/check-editlinks.test.js b/tests/markdown/check-editlinks.test.js new file mode 100644 index 000000000000..c0e1431337e1 --- /dev/null +++ b/tests/markdown/check-editlinks.test.js @@ -0,0 +1,157 @@ +const path = require('path'); +const fetch = require('node-fetch-2'); +const editOptions = require('../fixtures/markdown/edit-page-config.json'); +const { + generatePaths, + processBatch, + checkUrls, + determineEditLink, + main +} = require('../../scripts/markdown/check-editlinks'); +const { determineEditLinkData, processBatchData, testPaths } = require('../fixtures/markdown/check-editlinks-data'); + +jest.mock('node-fetch-2', () => jest.fn()); + +describe('URL Checker Tests', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + describe('determineEditLink', () => { + it('should generate correct edit link for docs with /docs prefix', () => { + const result = determineEditLink( + determineEditLinkData[0].urlPath, + determineEditLinkData[0].filePath, + editOptions + ); + expect(result).toBe(determineEditLinkData[0].editLink); + }); + + it('should generate correct edit link for docs without /docs prefix', () => { + const result = determineEditLink( + determineEditLinkData[1].urlPath, + determineEditLinkData[1].filePath, + editOptions + ); + expect(result).toBe(determineEditLinkData[1].editLink); + }); + it('should generate correct edit link for docs with a config', () => { + const result = determineEditLink( + determineEditLinkData[2].urlPath, + determineEditLinkData[2].filePath, + editOptions + ); + expect(result).toBe(determineEditLinkData[2].editLink); + }); + }); + + describe('generatePaths', () => { + const testDir = path.resolve(__dirname, '../../markdown/docs'); + + it('should generate correct paths for markdown files', async () => { + const paths = await generatePaths(testDir, editOptions); + expect(Array.isArray(paths)).toBe(true); + paths.forEach((pathObj) => { + expect(pathObj).toHaveProperty('filePath'); + expect(pathObj).toHaveProperty('urlPath'); + expect(pathObj).toHaveProperty('editLink'); + }); + }); + + it('should skip _section.md files', async () => { + const paths = await generatePaths(testDir, editOptions); + const sectionFiles = paths.filter((p) => p.filePath.endsWith('_section.md')); + expect(sectionFiles.length).toBe(0); + }); + + it('should handle errors gracefully', async () => { + const invalidDir = path.join(__dirname, 'nonexistent'); + await expect(generatePaths(invalidDir, editOptions)).rejects.toThrow(); + }); + }); + + describe('processBatch', () => { + const testBatch = processBatchData; + + it('should process valid URLs correctly', async () => { + fetch.mockImplementation(() => Promise.resolve({ status: 200 })); + const results = await processBatch(testBatch); + expect(results.filter((r) => r !== null).length).toBe(0); + }); + + it('should detect 404 URLs', async () => { + fetch.mockImplementation(() => Promise.resolve({ status: 404 })); + const results = await processBatch(testBatch); + const validResults = results.filter((r) => r !== null); + expect(validResults.length).toBe(2); + expect(validResults[0].editLink).toBe(testBatch[0].editLink); + }); + + it('should handle network errors', async () => { + fetch.mockImplementation(() => Promise.reject(new Error('Network error'))); + await expect(processBatch(testBatch)).rejects.toThrow(); + }); + + it('should ignore files in ignoreFiles list', async () => { + const batchWithIgnored = [ + ...testBatch, + { + filePath: 'reference/specification/v2.x.md', + urlPath: 'docs/reference/specification/v2.x', + editLink: 'https://github.com/org/repo/edit/main/v2.x.md' + } + ]; + fetch.mockImplementation(() => Promise.resolve({ status: 404 })); + const results = await processBatch(batchWithIgnored); + const validResults = results.filter((r) => r !== null); + expect(validResults.length).toBe(2); + }); + }); + + describe('checkUrls', () => { + it('should process all URLs in batches', async () => { + fetch.mockImplementation(() => Promise.resolve({ status: 200 })); + const results = await checkUrls(testPaths); + expect(results.length).toBe(0); + expect(fetch).toHaveBeenCalledTimes(10); + }); + + it('should handle mixed responses correctly', async () => { + fetch.mockImplementation((url) => { + return Promise.resolve({ + status: url.includes('migration') ? 404 : 200 + }); + }); + const results = await checkUrls(testPaths); + expect(results.length).toBe(2); + }); + }); + + describe('main', () => { + it('should run successfully when all URLs are valid', async () => { + fetch.mockImplementation(() => Promise.resolve({ status: 200 })); + const consoleSpy = jest.spyOn(console, 'log'); + + await main(); + + expect(consoleSpy).toHaveBeenCalledWith('All URLs are valid.'); + consoleSpy.mockRestore(); + }); + + it('should report invalid URLs when found', async () => { + fetch.mockImplementation(() => Promise.resolve({ status: 404 })); + const consoleSpy = jest.spyOn(console, 'log'); + + await main(); + + expect(consoleSpy).toHaveBeenCalledWith(expect.stringContaining('URLs returning 404:')); + consoleSpy.mockRestore(); + }); + + it('should handle errors gracefully', async () => { + fetch.mockImplementation(() => Promise.reject(new Error('Network error'))); + + await expect(main()).rejects.toThrow(); + }); + }); +}); From c101b6612c036ffcb91efd7e64e597039eea8fa7 Mon Sep 17 00:00:00 2001 From: Ansh Goyal Date: Tue, 14 Jan 2025 10:09:06 +0000 Subject: [PATCH 10/13] add timeout thing --- scripts/markdown/check-editlinks.js | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/scripts/markdown/check-editlinks.js b/scripts/markdown/check-editlinks.js index f3195f116b0d..7f5de4449f46 100644 --- a/scripts/markdown/check-editlinks.js +++ b/scripts/markdown/check-editlinks.js @@ -25,12 +25,20 @@ async function pause(ms) { * @returns {Promise} Array of URLs that returned 404 */ async function processBatch(batch) { + const TIMEOUT_MS = 5000; return Promise.all( batch.map(async ({ filePath, urlPath, editLink }) => { try { if (!editLink || ignoreFiles.some((ignorePath) => filePath.endsWith(ignorePath))) return null; - const response = await fetch(editLink, { method: 'HEAD' }); + const controller = new AbortController(); + /* istanbul ignore next */ + const timeout = setTimeout(() => controller.abort(), TIMEOUT_MS); + const response = await fetch(editLink, { + method: 'HEAD', + signal: controller.signal + }); + clearTimeout(timeout); if (response.status === 404) { return { filePath, urlPath, editLink }; } From b035fc4dc545638539b30e6de24f2eb0af66ab29 Mon Sep 17 00:00:00 2001 From: Ansh Goyal Date: Tue, 14 Jan 2025 14:43:56 +0000 Subject: [PATCH 11/13] fix comments --- .github/workflows/check-edit-links.yml | 6 ++--- package.json | 2 +- scripts/dashboard/build-dashboard.js | 2 +- ...check-editlinks.js => check-edit-links.js} | 11 +--------- tests/fixtures/markdown/edit-page-config.json | 22 ------------------- ...links.test.js => check-edit-links.test.js} | 4 ++-- 6 files changed, 8 insertions(+), 39 deletions(-) rename scripts/markdown/{check-editlinks.js => check-edit-links.js} (96%) delete mode 100644 tests/fixtures/markdown/edit-page-config.json rename tests/markdown/{check-editlinks.test.js => check-edit-links.test.js} (97%) diff --git a/.github/workflows/check-edit-links.yml b/.github/workflows/check-edit-links.yml index 7ccca35199ac..26483f025b08 100644 --- a/.github/workflows/check-edit-links.yml +++ b/.github/workflows/check-edit-links.yml @@ -1,4 +1,4 @@ -name: Weekly Link Checker +name: Weekly Docs Link Checker on: schedule: @@ -17,7 +17,7 @@ jobs: - name: Set up Node.js uses: actions/setup-node@v4 with: - node-version: '20' + node-version-file: '.nvmrc' - name: Install dependencies run: npm install @@ -40,7 +40,7 @@ jobs: uses: rtCamp/action-slack-notify@v2 env: SLACK_WEBHOOK: ${{ secrets.WEBSITE_SLACK_WEBHOOK }} - SLACK_TITLE: 'Edit Links Checker Errors Report' + SLACK_TITLE: 'Docs Edit Link Checker Errors Report' SLACK_MESSAGE: | 🚨 The following URLs returned 404 during the link check: ``` diff --git a/package.json b/package.json index a34585322891..22c6192d66fa 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,7 @@ "generate:tools": "node scripts/build-tools.js", "test:netlify": "deno test --allow-env --trace-ops netlify/**/*.test.ts", "test:md": "node scripts/markdown/check-markdown.js", - "test:editlinks": "node scripts/markdown/check-editlinks.js", + "test:editlinks": "node scripts/markdown/check-edit-links.js", "dev:storybook": "storybook dev -p 6006", "build:storybook": "storybook build" }, diff --git a/scripts/dashboard/build-dashboard.js b/scripts/dashboard/build-dashboard.js index c20be204e87b..066cf8b84c94 100644 --- a/scripts/dashboard/build-dashboard.js +++ b/scripts/dashboard/build-dashboard.js @@ -181,4 +181,4 @@ if (require.main === module) { start(resolve(__dirname, '..', '..', 'dashboard.json')); } -module.exports = { getLabel, monthsSince, mapGoodFirstIssues, getHotDiscussions, getDiscussionByID, getDiscussions, writeToFile, start, processHotDiscussions }; +module.exports = { getLabel, monthsSince, mapGoodFirstIssues, getHotDiscussions, getDiscussionByID, getDiscussions, writeToFile, start, processHotDiscussions, pause }; diff --git a/scripts/markdown/check-editlinks.js b/scripts/markdown/check-edit-links.js similarity index 96% rename from scripts/markdown/check-editlinks.js rename to scripts/markdown/check-edit-links.js index 7f5de4449f46..cf40dfb9ee38 100644 --- a/scripts/markdown/check-editlinks.js +++ b/scripts/markdown/check-edit-links.js @@ -2,6 +2,7 @@ const fs = require('fs').promises; const path = require('path'); const fetch = require('node-fetch-2'); const editUrls = require('../../config/edit-page-config.json'); +const { pause } = require('../dashboard/build-dashboard'); const ignoreFiles = [ 'reference/specification/v2.x.md', @@ -9,16 +10,6 @@ const ignoreFiles = [ 'reference/specification/v3.0.0.md' ]; -/** - * Introduces a delay in the execution flow - * @param {number} ms - The number of milliseconds to pause - */ -async function pause(ms) { - return new Promise((res) => { - setTimeout(res, ms); - }); -} - /** * Process a batch of URLs to check for 404s * @param {object[]} batch - Array of path objects to check diff --git a/tests/fixtures/markdown/edit-page-config.json b/tests/fixtures/markdown/edit-page-config.json deleted file mode 100644 index 9a66d9cd3f9f..000000000000 --- a/tests/fixtures/markdown/edit-page-config.json +++ /dev/null @@ -1,22 +0,0 @@ -[ - { - "value": "/tools/generator", - "href": "https://github.com/asyncapi/generator/tree/master/apps/generator/docs" - }, - { - "value": "reference/specification/", - "href": "https://github.com/asyncapi/spec/blob/master/spec/asyncapi.md" - }, - { - "value": "/tools/cli", - "href": "https://github.com/asyncapi/cli/tree/master/docs" - }, - { - "value": "", - "href": "https://github.com/asyncapi/website/blob/master/markdown" - }, - { - "value": "reference/extensions/", - "href": "https://github.com/asyncapi/extensions-catalog/tree/master/extensions" - } -] diff --git a/tests/markdown/check-editlinks.test.js b/tests/markdown/check-edit-links.test.js similarity index 97% rename from tests/markdown/check-editlinks.test.js rename to tests/markdown/check-edit-links.test.js index c0e1431337e1..6de5be05df4c 100644 --- a/tests/markdown/check-editlinks.test.js +++ b/tests/markdown/check-edit-links.test.js @@ -1,13 +1,13 @@ const path = require('path'); const fetch = require('node-fetch-2'); -const editOptions = require('../fixtures/markdown/edit-page-config.json'); +const editOptions = require('../../config/edit-page-config.json'); const { generatePaths, processBatch, checkUrls, determineEditLink, main -} = require('../../scripts/markdown/check-editlinks'); +} = require('../../scripts/markdown/check-edit-links'); const { determineEditLinkData, processBatchData, testPaths } = require('../fixtures/markdown/check-editlinks-data'); jest.mock('node-fetch-2', () => jest.fn()); From afa5a3360ae4d5fcf6ad60e5be611fbfaa98af2d Mon Sep 17 00:00:00 2001 From: Ansh Goyal Date: Tue, 14 Jan 2025 15:02:46 +0000 Subject: [PATCH 12/13] fix comments --- scripts/markdown/check-edit-links.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/markdown/check-edit-links.js b/scripts/markdown/check-edit-links.js index cf40dfb9ee38..9c9582942b79 100644 --- a/scripts/markdown/check-edit-links.js +++ b/scripts/markdown/check-edit-links.js @@ -35,7 +35,7 @@ async function processBatch(batch) { } return null; } catch (error) { - return Promise.reject(new Error(`Error checking ${editLink}:`, error.message)); + return Promise.reject(new Error(`Error checking ${editLink}: ${error.message}`)); } }) ); @@ -130,7 +130,7 @@ async function generatePaths(folderPath, editOptions, relativePath = '', result return result; } catch (err) { - throw new Error(`Error processing directory ${folderPath}:`, err); + throw new Error(`Error processing directory ${folderPath}: ${err.message}`); } } @@ -151,7 +151,7 @@ async function main() { console.log('All URLs are valid.'); } } catch (error) { - throw new Error('Failed to check edit links:', error); + throw new Error(`Failed to check edit links: ${error.message}`); } } From 22a31ce1532b10bfb5fde4ad1fe4cc40953a6725 Mon Sep 17 00:00:00 2001 From: Ansh Goyal Date: Tue, 14 Jan 2025 16:38:01 +0000 Subject: [PATCH 13/13] add env var support --- scripts/markdown/check-edit-links.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/markdown/check-edit-links.js b/scripts/markdown/check-edit-links.js index 9c9582942b79..b9789c56ac7e 100644 --- a/scripts/markdown/check-edit-links.js +++ b/scripts/markdown/check-edit-links.js @@ -16,7 +16,7 @@ const ignoreFiles = [ * @returns {Promise} Array of URLs that returned 404 */ async function processBatch(batch) { - const TIMEOUT_MS = 5000; + const TIMEOUT_MS = process.env.TIMEOUT_MS || 5000; return Promise.all( batch.map(async ({ filePath, urlPath, editLink }) => { try { @@ -48,7 +48,7 @@ async function processBatch(batch) { */ async function checkUrls(paths) { const result = []; - const batchSize = 5; + const batchSize = process.env.BATCH_SIZE || 5; const batches = []; for (let i = 0; i < paths.length; i += batchSize) {