From a4329d338896ea68817cc72be8ceaab4801afeb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Lorber?= Date: Fri, 30 Aug 2024 08:02:26 +0200 Subject: [PATCH] refactor(mdx-loader): read metadata from memory (loaded content) instead of fs (#10457) * mdx loader shouldn't read metadata from file system but from memory * comments * refactor: apply lint autofix * apply same for blog * apply same for blog * refactor: apply lint autofix * apply same for pages --- packages/docusaurus-mdx-loader/src/loader.ts | 55 +++++++++++++------ packages/docusaurus-mdx-loader/src/utils.ts | 4 +- .../src/contentHelpers.ts | 35 ++++++++++++ .../src/index.ts | 46 +++++----------- .../src/contentHelpers.ts | 34 ++++++++++++ .../src/index.ts | 41 +++++--------- .../src/contentHelpers.ts | 33 +++++++++++ .../src/index.ts | 14 ++++- 8 files changed, 182 insertions(+), 80 deletions(-) create mode 100644 packages/docusaurus-plugin-content-blog/src/contentHelpers.ts create mode 100644 packages/docusaurus-plugin-content-docs/src/contentHelpers.ts create mode 100644 packages/docusaurus-plugin-content-pages/src/contentHelpers.ts diff --git a/packages/docusaurus-mdx-loader/src/loader.ts b/packages/docusaurus-mdx-loader/src/loader.ts index 71a9fe992eb6..380a1b2a8e14 100644 --- a/packages/docusaurus-mdx-loader/src/loader.ts +++ b/packages/docusaurus-mdx-loader/src/loader.ts @@ -35,6 +35,12 @@ type Pluggable = any; // TODO fix this asap export type MDXPlugin = Pluggable; +// This represents the path to the mdx metadata bundle path + its loaded content +export type LoadedMetadata = { + metadataPath: string; + metadataContent: unknown; +}; + export type Options = Partial & { markdownConfig: MarkdownConfig; staticDirs: string[]; @@ -42,10 +48,13 @@ export type Options = Partial & { isMDXPartial?: (filePath: string) => boolean; isMDXPartialFrontMatterWarningDisabled?: boolean; removeContentTitle?: boolean; - metadataPath?: string | ((filePath: string) => string); + + // TODO Docusaurus v4: rename to just "metadata"? + // We kept retro-compatibility in v3 in case plugins/sites use mdx loader + metadataPath?: string | ((filePath: string) => string | LoadedMetadata); createAssets?: (metadata: { frontMatter: {[key: string]: unknown}; - metadata: {[key: string]: unknown}; + metadata: unknown; }) => {[key: string]: unknown}; resolveMarkdownLink?: ResolveMarkdownLink; @@ -103,32 +112,40 @@ ${JSON.stringify(frontMatter, null, 2)}`; } } - function getMetadataPath(): string | undefined { + async function loadMetadata(): Promise { if (!isMDXPartial) { // Read metadata for this MDX and export it. if (options.metadataPath && typeof options.metadataPath === 'function') { - return options.metadataPath(filePath); + const metadata = options.metadataPath(filePath); + if (!metadata) { + return undefined; + } + if (typeof metadata === 'string') { + return { + metadataPath: metadata, + metadataContent: await readMetadataPath(metadata), + }; + } + if (!metadata.metadataPath) { + throw new Error(`Metadata path missing for file ${filePath}`); + } + if (!metadata.metadataContent) { + throw new Error(`Metadata content missing for file ${filePath}`); + } + return metadata; } } return undefined; } - const metadataPath = getMetadataPath(); - if (metadataPath) { - this.addDependency(metadataPath); + const metadata = await loadMetadata(); + if (metadata) { + this.addDependency(metadata.metadataPath); } - const metadataJsonString = metadataPath - ? await readMetadataPath(metadataPath) - : undefined; - - const metadata = metadataJsonString - ? (JSON.parse(metadataJsonString) as {[key: string]: unknown}) - : undefined; - const assets = options.createAssets && metadata - ? options.createAssets({frontMatter, metadata}) + ? options.createAssets({frontMatter, metadata: metadata.metadataContent}) : undefined; const fileLoaderUtils = getFileLoaderUtils(compilerName === 'server'); @@ -138,7 +155,11 @@ ${JSON.stringify(frontMatter, null, 2)}`; const exportsCode = ` export const frontMatter = ${stringifyObject(frontMatter)}; export const contentTitle = ${stringifyObject(contentTitle)}; -${metadataJsonString ? `export const metadata = ${metadataJsonString};` : ''} +${ + metadata + ? `export const metadata = ${JSON.stringify(metadata.metadataContent)};` + : '' +} ${ assets ? `export const assets = ${createAssetsExportCode({ diff --git a/packages/docusaurus-mdx-loader/src/utils.ts b/packages/docusaurus-mdx-loader/src/utils.ts index 94a17f6d0954..90b8e77a71f9 100644 --- a/packages/docusaurus-mdx-loader/src/utils.ts +++ b/packages/docusaurus-mdx-loader/src/utils.ts @@ -19,9 +19,9 @@ import type {Options} from './loader'; * starting with _). That's why it's important to provide the `isMDXPartial` * function in config */ -export async function readMetadataPath(metadataPath: string): Promise { +export async function readMetadataPath(metadataPath: string): Promise { try { - return await fs.readFile(metadataPath, 'utf8'); + return await fs.readJSON(metadataPath, 'utf8'); } catch (error) { throw new Error( logger.interpolate`MDX loader can't read MDX metadata file path=${metadataPath}. Maybe the isMDXPartial option function was not provided?`, diff --git a/packages/docusaurus-plugin-content-blog/src/contentHelpers.ts b/packages/docusaurus-plugin-content-blog/src/contentHelpers.ts new file mode 100644 index 000000000000..5cfb028d2278 --- /dev/null +++ b/packages/docusaurus-plugin-content-blog/src/contentHelpers.ts @@ -0,0 +1,35 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +import type {BlogContent, BlogPost} from '@docusaurus/plugin-content-blog'; + +function indexBlogPostsBySource(content: BlogContent): Map { + return new Map( + content.blogPosts.map((blogPost) => [blogPost.metadata.source, blogPost]), + ); +} + +// TODO this is bad, we should have a better way to do this (new lifecycle?) +// The source to blog/permalink is a mutable map passed to the mdx loader +// See https://github.com/facebook/docusaurus/pull/10457 +// See https://github.com/facebook/docusaurus/pull/10185 +export function createContentHelpers() { + const sourceToBlogPost = new Map(); + const sourceToPermalink = new Map(); + + // Mutable map update :/ + function updateContent(content: BlogContent): void { + sourceToBlogPost.clear(); + sourceToPermalink.clear(); + indexBlogPostsBySource(content).forEach((value, key) => { + sourceToBlogPost.set(key, value); + sourceToPermalink.set(key, value.metadata.permalink); + }); + } + + return {updateContent, sourceToBlogPost, sourceToPermalink}; +} diff --git a/packages/docusaurus-plugin-content-blog/src/index.ts b/packages/docusaurus-plugin-content-blog/src/index.ts index 8325cc06d302..0f7e4b3dcd36 100644 --- a/packages/docusaurus-plugin-content-blog/src/index.ts +++ b/packages/docusaurus-plugin-content-blog/src/index.ts @@ -19,7 +19,6 @@ import { getDataFilePath, DEFAULT_PLUGIN_ID, resolveMarkdownLinkPathname, - type SourceToPermalink, } from '@docusaurus/utils'; import {getTagsFilePathsToWatch} from '@docusaurus/utils-validation'; import { @@ -40,6 +39,7 @@ import {createBlogFeedFiles, createFeedHtmlHeadTags} from './feed'; import {createAllRoutes} from './routes'; import {checkAuthorsMapPermalinkCollisions, getAuthorsMap} from './authorsMap'; +import {createContentHelpers} from './contentHelpers'; import type {BlogContentPaths, BlogMarkdownLoaderOptions} from './types'; import type {LoadContext, Plugin} from '@docusaurus/types'; import type { @@ -55,33 +55,6 @@ import type {RuleSetRule, RuleSetUseItem} from 'webpack'; const PluginName = 'docusaurus-plugin-content-blog'; -// TODO this is bad, we should have a better way to do this (new lifecycle?) -// The source to permalink is currently a mutable map passed to the mdx loader -// for link resolution -// see https://github.com/facebook/docusaurus/pull/10185 -function createSourceToPermalinkHelper() { - const sourceToPermalink: SourceToPermalink = new Map(); - - function computeSourceToPermalink(content: BlogContent): SourceToPermalink { - return new Map( - content.blogPosts.map(({metadata: {source, permalink}}) => [ - source, - permalink, - ]), - ); - } - - // Mutable map update :/ - function update(content: BlogContent): void { - sourceToPermalink.clear(); - computeSourceToPermalink(content).forEach((value, key) => { - sourceToPermalink.set(key, value); - }); - } - - return {get: () => sourceToPermalink, update}; -} - export default async function pluginContentBlog( context: LoadContext, options: PluginOptions, @@ -128,7 +101,7 @@ export default async function pluginContentBlog( contentPaths, }); - const sourceToPermalinkHelper = createSourceToPermalinkHelper(); + const contentHelpers = createContentHelpers(); async function createBlogMDXLoaderRule(): Promise { const { @@ -162,7 +135,16 @@ export default async function pluginContentBlog( // Note that metadataPath must be the same/in-sync as // the path from createData for each MDX. const aliasedPath = aliasedSitePath(mdxPath, siteDir); - return path.join(dataDir, `${docuHash(aliasedPath)}.json`); + const metadataPath = path.join( + dataDir, + `${docuHash(aliasedPath)}.json`, + ); + const metadataContent = + contentHelpers.sourceToBlogPost.get(aliasedPath)!.metadata; + return { + metadataPath, + metadataContent, + }; }, // For blog posts a title in markdown is always removed // Blog posts title are rendered separately @@ -184,7 +166,7 @@ export default async function pluginContentBlog( resolveMarkdownLink: ({linkPathname, sourceFilePath}) => { const permalink = resolveMarkdownLinkPathname(linkPathname, { sourceFilePath, - sourceToPermalink: sourceToPermalinkHelper.get(), + sourceToPermalink: contentHelpers.sourceToPermalink, siteDir, contentPaths, }); @@ -352,7 +334,7 @@ export default async function pluginContentBlog( }, async contentLoaded({content, actions}) { - sourceToPermalinkHelper.update(content); + contentHelpers.updateContent(content); await createAllRoutes({ baseUrl, diff --git a/packages/docusaurus-plugin-content-docs/src/contentHelpers.ts b/packages/docusaurus-plugin-content-docs/src/contentHelpers.ts new file mode 100644 index 000000000000..75eddef4cc31 --- /dev/null +++ b/packages/docusaurus-plugin-content-docs/src/contentHelpers.ts @@ -0,0 +1,34 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +import type {DocMetadata, LoadedContent} from '@docusaurus/plugin-content-docs'; + +function indexDocsBySource(content: LoadedContent): Map { + const allDocs = content.loadedVersions.flatMap((v) => v.docs); + return new Map(allDocs.map((doc) => [doc.source, doc])); +} + +// TODO this is bad, we should have a better way to do this (new lifecycle?) +// The source to doc/permalink is a mutable map passed to the mdx loader +// See https://github.com/facebook/docusaurus/pull/10457 +// See https://github.com/facebook/docusaurus/pull/10185 +export function createContentHelpers() { + const sourceToDoc = new Map(); + const sourceToPermalink = new Map(); + + // Mutable map update :/ + function updateContent(content: LoadedContent): void { + sourceToDoc.clear(); + sourceToPermalink.clear(); + indexDocsBySource(content).forEach((value, key) => { + sourceToDoc.set(key, value); + sourceToPermalink.set(key, value.permalink); + }); + } + + return {updateContent, sourceToDoc, sourceToPermalink}; +} diff --git a/packages/docusaurus-plugin-content-docs/src/index.ts b/packages/docusaurus-plugin-content-docs/src/index.ts index 5ccb28a33177..ffa4ac5311a1 100644 --- a/packages/docusaurus-plugin-content-docs/src/index.ts +++ b/packages/docusaurus-plugin-content-docs/src/index.ts @@ -19,7 +19,6 @@ import { createSlugger, resolveMarkdownLinkPathname, DEFAULT_PLUGIN_ID, - type SourceToPermalink, type TagsFile, } from '@docusaurus/utils'; import { @@ -54,6 +53,7 @@ import { import {createAllRoutes} from './routes'; import {createSidebarsUtils} from './sidebars/utils'; +import {createContentHelpers} from './contentHelpers'; import type { PluginOptions, DocMetadataBase, @@ -66,29 +66,6 @@ import type {LoadContext, Plugin} from '@docusaurus/types'; import type {DocFile, FullVersion} from './types'; import type {RuleSetRule} from 'webpack'; -// TODO this is bad, we should have a better way to do this (new lifecycle?) -// The source to permalink is currently a mutable map passed to the mdx loader -// for link resolution -// see https://github.com/facebook/docusaurus/pull/10185 -function createSourceToPermalinkHelper() { - const sourceToPermalink: SourceToPermalink = new Map(); - - function computeSourceToPermalink(content: LoadedContent): SourceToPermalink { - const allDocs = content.loadedVersions.flatMap((v) => v.docs); - return new Map(allDocs.map(({source, permalink}) => [source, permalink])); - } - - // Mutable map update :/ - function update(content: LoadedContent): void { - sourceToPermalink.clear(); - computeSourceToPermalink(content).forEach((value, key) => { - sourceToPermalink.set(key, value); - }); - } - - return {get: () => sourceToPermalink, update}; -} - export default async function pluginContentDocs( context: LoadContext, options: PluginOptions, @@ -115,7 +92,7 @@ export default async function pluginContentDocs( // TODO env should be injected into all plugins const env = process.env.NODE_ENV as DocEnv; - const sourceToPermalinkHelper = createSourceToPermalinkHelper(); + const contentHelpers = createContentHelpers(); async function createDocsMDXLoaderRule(): Promise { const { @@ -146,7 +123,15 @@ export default async function pluginContentDocs( // Note that metadataPath must be the same/in-sync as // the path from createData for each MDX. const aliasedPath = aliasedSitePath(mdxPath, siteDir); - return path.join(dataDir, `${docuHash(aliasedPath)}.json`); + const metadataPath = path.join( + dataDir, + `${docuHash(aliasedPath)}.json`, + ); + const metadataContent = contentHelpers.sourceToDoc.get(aliasedPath); + return { + metadataPath, + metadataContent, + }; }, // Assets allow to convert some relative images paths to // require(...) calls @@ -161,7 +146,7 @@ export default async function pluginContentDocs( ); const permalink = resolveMarkdownLinkPathname(linkPathname, { sourceFilePath, - sourceToPermalink: sourceToPermalinkHelper.get(), + sourceToPermalink: contentHelpers.sourceToPermalink, siteDir, contentPaths: version, }); @@ -335,7 +320,7 @@ export default async function pluginContentDocs( }, async contentLoaded({content, actions}) { - sourceToPermalinkHelper.update(content); + contentHelpers.updateContent(content); const versions: FullVersion[] = content.loadedVersions.map(toFullVersion); diff --git a/packages/docusaurus-plugin-content-pages/src/contentHelpers.ts b/packages/docusaurus-plugin-content-pages/src/contentHelpers.ts new file mode 100644 index 000000000000..766cab0b5106 --- /dev/null +++ b/packages/docusaurus-plugin-content-pages/src/contentHelpers.ts @@ -0,0 +1,33 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +import type {LoadedContent, Metadata} from '@docusaurus/plugin-content-pages'; + +function indexPagesBySource(content: LoadedContent): Map { + return new Map(content.map((page) => [page.source, page])); +} + +// TODO this is bad, we should have a better way to do this (new lifecycle?) +// The source to page/permalink is a mutable map passed to the mdx loader +// See https://github.com/facebook/docusaurus/pull/10457 +// See https://github.com/facebook/docusaurus/pull/10185 +export function createContentHelpers() { + const sourceToPage = new Map(); + // const sourceToPermalink = new Map(); + + // Mutable map update :/ + function updateContent(content: LoadedContent): void { + sourceToPage.clear(); + // sourceToPermalink.clear(); + indexPagesBySource(content).forEach((value, key) => { + sourceToPage.set(key, value); + // sourceToPermalink.set(key, value.metadata.permalink); + }); + } + + return {updateContent, sourceToPage}; +} diff --git a/packages/docusaurus-plugin-content-pages/src/index.ts b/packages/docusaurus-plugin-content-pages/src/index.ts index 82c2a975ce30..d9f6b2ff8cd2 100644 --- a/packages/docusaurus-plugin-content-pages/src/index.ts +++ b/packages/docusaurus-plugin-content-pages/src/index.ts @@ -24,6 +24,7 @@ import { getContentPathList, loadPagesContent, } from './content'; +import {createContentHelpers} from './contentHelpers'; import type {LoadContext, Plugin} from '@docusaurus/types'; import type { PluginOptions, @@ -46,6 +47,8 @@ export default async function pluginContentPages( ); const dataDir = path.join(pluginDataDirRoot, options.id ?? DEFAULT_PLUGIN_ID); + const contentHelpers = createContentHelpers(); + async function createPagesMDXLoaderRule(): Promise { const { admonitions, @@ -73,7 +76,15 @@ export default async function pluginContentPages( // Note that metadataPath must be the same/in-sync as // the path from createData for each MDX. const aliasedSource = aliasedSitePath(mdxPath, siteDir); - return path.join(dataDir, `${docuHash(aliasedSource)}.json`); + const metadataPath = path.join( + dataDir, + `${docuHash(aliasedSource)}.json`, + ); + const metadataContent = contentHelpers.sourceToPage.get(aliasedSource); + return { + metadataPath, + metadataContent, + }; }, // Assets allow to convert some relative images paths to // require(...) calls @@ -114,6 +125,7 @@ export default async function pluginContentPages( if (!content) { return; } + contentHelpers.updateContent(content); await createAllRoutes({content, options, actions}); },