From 33707b956909468c2f36cc9ad6acb1beefaccdaf Mon Sep 17 00:00:00 2001 From: Florian PAUL Date: Thu, 8 Aug 2024 15:57:35 +0200 Subject: [PATCH 1/2] fix: copy specs local file refs --- .../helpers/copy-referenced-files.spec.ts | 42 ++++++++ .../core/helpers/copy-referenced-files.ts | 96 +++++++++++++++++++ .../schematics/typescript/core/index.ts | 10 ++ .../schematics/testing/mocks/spec-chunk2.yaml | 9 ++ .../mocks/spec-chunk3/spec-chunk3.yaml | 9 ++ .../mocks/spec-chunk4/spec-chunk4.yaml | 12 +++ .../testing/mocks/split-spec/spec-chunk1.yaml | 9 ++ .../testing/mocks/split-spec/split-spec.yaml | 45 +++++++++ 8 files changed, 232 insertions(+) create mode 100644 packages/@ama-sdk/schematics/schematics/typescript/core/helpers/copy-referenced-files.spec.ts create mode 100644 packages/@ama-sdk/schematics/schematics/typescript/core/helpers/copy-referenced-files.ts create mode 100644 packages/@ama-sdk/schematics/testing/mocks/spec-chunk2.yaml create mode 100644 packages/@ama-sdk/schematics/testing/mocks/spec-chunk3/spec-chunk3.yaml create mode 100644 packages/@ama-sdk/schematics/testing/mocks/spec-chunk4/spec-chunk4.yaml create mode 100644 packages/@ama-sdk/schematics/testing/mocks/split-spec/spec-chunk1.yaml create mode 100644 packages/@ama-sdk/schematics/testing/mocks/split-spec/split-spec.yaml diff --git a/packages/@ama-sdk/schematics/schematics/typescript/core/helpers/copy-referenced-files.spec.ts b/packages/@ama-sdk/schematics/schematics/typescript/core/helpers/copy-referenced-files.spec.ts new file mode 100644 index 0000000000..b9f91fe7a8 --- /dev/null +++ b/packages/@ama-sdk/schematics/schematics/typescript/core/helpers/copy-referenced-files.spec.ts @@ -0,0 +1,42 @@ +import { cleanVirtualFileSystem, useVirtualFileSystem } from '@o3r/test-helpers'; +import { readFile } from 'node:fs/promises'; +import { dirname, join } from 'node:path'; + +describe('Copy Referenced Files', () => { + const virtualFileSystem = useVirtualFileSystem(); + const copyReferencedFiles = require('./copy-referenced-files').copyReferencedFiles; + + const migrationScriptMocksPath = join(__dirname, '../../../../testing/mocks'); + const specFilePath = '../models/split-spec/split-spec.yaml'; + const outputDirectory = './local-references'; + + const copyMockFile = async (virtualPath: string, realPath: string) => { + if (!virtualFileSystem.existsSync(dirname(virtualPath))) { + await virtualFileSystem.promises.mkdir(dirname(virtualPath), {recursive: true}); + } + await virtualFileSystem.promises.writeFile(virtualPath, await readFile(join(migrationScriptMocksPath, realPath), {encoding: 'utf8'})); + }; + + beforeAll(async () => { + await virtualFileSystem.promises.mkdir(dirname(specFilePath), {recursive: true}); + await copyMockFile(specFilePath, 'split-spec/split-spec.yaml'); + await copyMockFile('../models/split-spec/spec-chunk1.yaml', 'split-spec/spec-chunk1.yaml'); + await copyMockFile('../models/spec-chunk2.yaml', 'spec-chunk2.yaml'); + await copyMockFile('../models/spec-chunk3/spec-chunk3.yaml', 'spec-chunk3/spec-chunk3.yaml'); + await copyMockFile('../models/spec-chunk4/spec-chunk4.yaml', 'spec-chunk4/spec-chunk4.yaml'); + }); + + afterAll(() => { + cleanVirtualFileSystem(); + }); + + it('should copy the local files referenced in the spec', async () => { + const baseRelativePath = await copyReferencedFiles(specFilePath, outputDirectory); + expect(baseRelativePath).toMatch(/^local-references[\\/]split-spec$/); + expect(virtualFileSystem.existsSync(join(outputDirectory, 'split-spec/split-spec.yaml'))).toBe(true); + expect(virtualFileSystem.existsSync(join(outputDirectory, 'split-spec/spec-chunk1.yaml'))).toBe(true); + expect(virtualFileSystem.existsSync(join(outputDirectory, 'spec-chunk2.yaml'))).toBe(true); + expect(virtualFileSystem.existsSync(join(outputDirectory, 'spec-chunk3/spec-chunk3.yaml'))).toBe(true); + expect(virtualFileSystem.existsSync(join(outputDirectory, 'spec-chunk4/spec-chunk4.yaml'))).toBe(true); + }); +}); diff --git a/packages/@ama-sdk/schematics/schematics/typescript/core/helpers/copy-referenced-files.ts b/packages/@ama-sdk/schematics/schematics/typescript/core/helpers/copy-referenced-files.ts new file mode 100644 index 0000000000..ac73243b3d --- /dev/null +++ b/packages/@ama-sdk/schematics/schematics/typescript/core/helpers/copy-referenced-files.ts @@ -0,0 +1,96 @@ +import { existsSync } from 'node:fs'; +import { copyFile, mkdir, readFile, rm } from 'node:fs/promises'; +import { dirname, join, normalize, posix, relative, resolve } from 'node:path'; + +const refMatcher = /\B['"]?[$]ref['"]?\s*:\s*([^#\n]+)/g; + +/** + * Extract the list of local references from a single spec file content + * @param specContent + * @param basePath + */ +function extractRefPaths(specContent: string, basePath: string): string[] { + const refs = specContent.match(refMatcher); + return refs ? + refs + .map((capture) => capture.replace(refMatcher, '$1').replace(/['"]/g, '')) + .filter((refPath) => refPath.startsWith('.')) + .map((refPath) => join(basePath, refPath)) + : []; +} + +/** + * Recursively extract the list of local references starting from the input spec file + * @param specFilePath + * @param referenceFilePath + * @param visited + */ +async function extractRefPathRecursive(specFilePath: string, referenceFilePath: string, visited: Set): Promise { + const resolvedFilePath = resolve(specFilePath); + if (!visited.has(resolvedFilePath)) { + visited.add(resolvedFilePath); + + const specContent = await readFile(specFilePath, {encoding: 'utf8'}); + const refPaths = extractRefPaths(specContent, relative(dirname(referenceFilePath), dirname(specFilePath))); + const recursiveRefPaths = await Promise.all( + refPaths.map((refPath) => extractRefPathRecursive(join(dirname(referenceFilePath), refPath), referenceFilePath, visited)) + ); + return [ + ...refPaths, + ...recursiveRefPaths.flat() + ]; + } + return []; +} + +/** + * Replace all the local relative references using the new base relative path + * @param specContent + * @param newBaseRelativePath + */ +export function updateLocalRelativeRefs(specContent: string, newBaseRelativePath: string) { + const formatPath = (inputPath:string) => (inputPath.startsWith('.') ? inputPath : `./${inputPath}`).replace(/\\+/g, '/'); + return specContent.replace(refMatcher, (match, ref: string) => { + const refPath = ref.replace(/['"]/g, ''); + return refPath.startsWith('.') ? + match.replace(refPath, formatPath(normalize(posix.join(newBaseRelativePath, refPath)))) + : match; + }); +} + +/** + * Copy the local files referenced in the input spec file to the output directory + * @param specFilePath + * @param outputDirectory + */ +export async function copyReferencedFiles(specFilePath: string, outputDirectory: string) { + const dedupe = (paths: string[]) => + paths.filter((refPath, index) => { + const actualPath = join(dirname(specFilePath), refPath); + return paths.findIndex((otherRefPath) => join(dirname(specFilePath), otherRefPath) === actualPath) === index; + }); + const refPaths = dedupe(await extractRefPathRecursive(specFilePath, specFilePath, new Set())); + if (refPaths.length) { + if (existsSync(outputDirectory)) { + await rm(outputDirectory, { recursive: true }); + } + + // Calculate the lowest level base path to keep the same directory structure + const maxDepth = Math.max(...refPaths.map((refPath) => refPath.split('..').length)); + const basePath = join(specFilePath, '../'.repeat(maxDepth)); + const baseRelativePath = relative(basePath, dirname(specFilePath)); + + // Copy the files + await Promise.all(refPaths.map(async (refPath) => { + const sourcePath = join(dirname(specFilePath), refPath); + const destPath = join(outputDirectory, baseRelativePath, refPath); + if (!existsSync(dirname(destPath))) { + await mkdir(dirname(destPath), { recursive: true }); + } + await copyFile(sourcePath, destPath); + })); + + return join(outputDirectory, baseRelativePath); + } + return ''; +} diff --git a/packages/@ama-sdk/schematics/schematics/typescript/core/index.ts b/packages/@ama-sdk/schematics/schematics/typescript/core/index.ts index cb3a112be0..da50794267 100644 --- a/packages/@ama-sdk/schematics/schematics/typescript/core/index.ts +++ b/packages/@ama-sdk/schematics/schematics/typescript/core/index.ts @@ -24,6 +24,7 @@ import { OpenApiCliOptions } from '../../code-generator/open-api-cli-generator/o import { treeGlob } from '../../helpers/tree-glob'; import { NgGenerateTypescriptSDKCoreSchematicsSchema } from './schema'; import { OpenApiCliGenerator } from '../../code-generator/open-api-cli-generator/open-api-cli.generator'; +import { copyReferencedFiles, updateLocalRelativeRefs } from './helpers/copy-referenced-files'; import { generateOperationFinderFromSingleFile } from './helpers/path-extractor'; const JAVA_OPTIONS = ['specPath', 'specConfigPath', 'globalProperty', 'outputPath']; @@ -153,10 +154,19 @@ function ngGenerateTypescriptSDKFn(options: NgGenerateTypescriptSDKCoreSchematic let specContent!: string; if (URL.canParse(generatorOptions.specPath) && (new URL(generatorOptions.specPath)).protocol.startsWith('http')) { specContent = await (await fetch(generatorOptions.specPath)).text(); + specContent = updateLocalRelativeRefs(specContent, path.dirname(generatorOptions.specPath)); } else { const specPath = path.isAbsolute(generatorOptions.specPath) || !options.directory ? generatorOptions.specPath : path.join(options.directory, generatorOptions.specPath); specContent = readFileSync(specPath, {encoding: 'utf-8'}).toString(); + + if (path.relative(process.cwd(), specPath).startsWith('..')) { + // TODO would be better to create files on tree instead of FS + const newRelativePath = await copyReferencedFiles(specPath, './spec-local-references'); + if (newRelativePath) { + specContent = updateLocalRelativeRefs(specContent, newRelativePath); + } + } } try { diff --git a/packages/@ama-sdk/schematics/testing/mocks/spec-chunk2.yaml b/packages/@ama-sdk/schematics/testing/mocks/spec-chunk2.yaml new file mode 100644 index 0000000000..bb5b30a0db --- /dev/null +++ b/packages/@ama-sdk/schematics/testing/mocks/spec-chunk2.yaml @@ -0,0 +1,9 @@ +title: Pet +type: object +properties: + id: + type: integer + format: int64 + example: 10 + category: + $ref: './split-spec/split-spec.yaml#/components/schemas/Category' diff --git a/packages/@ama-sdk/schematics/testing/mocks/spec-chunk3/spec-chunk3.yaml b/packages/@ama-sdk/schematics/testing/mocks/spec-chunk3/spec-chunk3.yaml new file mode 100644 index 0000000000..950f434f21 --- /dev/null +++ b/packages/@ama-sdk/schematics/testing/mocks/spec-chunk3/spec-chunk3.yaml @@ -0,0 +1,9 @@ +title: Pet +type: object +properties: + id: + type: integer + format: int64 + example: 10 + category: + $ref: '../spec-chunk4/spec-chunk4.yaml#/components/schemas/Category' diff --git a/packages/@ama-sdk/schematics/testing/mocks/spec-chunk4/spec-chunk4.yaml b/packages/@ama-sdk/schematics/testing/mocks/spec-chunk4/spec-chunk4.yaml new file mode 100644 index 0000000000..c600013052 --- /dev/null +++ b/packages/@ama-sdk/schematics/testing/mocks/spec-chunk4/spec-chunk4.yaml @@ -0,0 +1,12 @@ +components: + schemas: + Category: + type: object + properties: + id: + type: integer + format: int64 + example: 10 + name: + type: string + example: "test" diff --git a/packages/@ama-sdk/schematics/testing/mocks/split-spec/spec-chunk1.yaml b/packages/@ama-sdk/schematics/testing/mocks/split-spec/spec-chunk1.yaml new file mode 100644 index 0000000000..7273682c85 --- /dev/null +++ b/packages/@ama-sdk/schematics/testing/mocks/split-spec/spec-chunk1.yaml @@ -0,0 +1,9 @@ +title: Pet +type: object +properties: + id: + type: integer + format: int64 + example: 10 + category: + $ref: './split-spec.yaml#/components/schemas/Category' diff --git a/packages/@ama-sdk/schematics/testing/mocks/split-spec/split-spec.yaml b/packages/@ama-sdk/schematics/testing/mocks/split-spec/split-spec.yaml new file mode 100644 index 0000000000..09bb4c7f42 --- /dev/null +++ b/packages/@ama-sdk/schematics/testing/mocks/split-spec/split-spec.yaml @@ -0,0 +1,45 @@ +openapi: 3.0.2 +info: + description: test + title: test + version: 0.0.0 +paths: + /test: + get: + responses: + '200': + description: test + content: + application/json: + schema: + $ref: './spec-chunk1.yaml' + /test2: + get: + responses: + '200': + description: test + content: + application/json: + schema: + $ref: '../spec-chunk2.yaml' + /test3: + get: + responses: + '200': + description: test + content: + application/json: + schema: + $ref: '../spec-chunk3/spec-chunk3.yaml' +components: + schemas: + Category: + type: object + properties: + id: + type: integer + format: int64 + example: 10 + name: + type: string + example: "test" From a0b37c2adeaf371651fbc90afedccc3083ed3631 Mon Sep 17 00:00:00 2001 From: mrednic Date: Tue, 20 Aug 2024 15:59:54 +0200 Subject: [PATCH 2/2] fix: local file path update in specs --- .../helpers/copy-referenced-files.spec.ts | 19 ++++++-- .../core/helpers/copy-referenced-files.ts | 13 +++--- .../schematics/typescript/core/index.ts | 1 + .../testing/mocks/split-spec/spec-chunk1.yaml | 2 + .../split-spec/spec-with-updated-paths.yaml | 45 +++++++++++++++++++ 5 files changed, 68 insertions(+), 12 deletions(-) create mode 100644 packages/@ama-sdk/schematics/testing/mocks/split-spec/spec-with-updated-paths.yaml diff --git a/packages/@ama-sdk/schematics/schematics/typescript/core/helpers/copy-referenced-files.spec.ts b/packages/@ama-sdk/schematics/schematics/typescript/core/helpers/copy-referenced-files.spec.ts index b9f91fe7a8..4aaf1a2bd2 100644 --- a/packages/@ama-sdk/schematics/schematics/typescript/core/helpers/copy-referenced-files.spec.ts +++ b/packages/@ama-sdk/schematics/schematics/typescript/core/helpers/copy-referenced-files.spec.ts @@ -2,11 +2,11 @@ import { cleanVirtualFileSystem, useVirtualFileSystem } from '@o3r/test-helpers' import { readFile } from 'node:fs/promises'; import { dirname, join } from 'node:path'; -describe('Copy Referenced Files', () => { +describe('Specs processing', () => { const virtualFileSystem = useVirtualFileSystem(); - const copyReferencedFiles = require('./copy-referenced-files').copyReferencedFiles; + const {copyReferencedFiles, updateLocalRelativeRefs} = require('./copy-referenced-files'); - const migrationScriptMocksPath = join(__dirname, '../../../../testing/mocks'); + const specsMocksPath = join(__dirname, '../../../../testing/mocks'); const specFilePath = '../models/split-spec/split-spec.yaml'; const outputDirectory = './local-references'; @@ -14,7 +14,7 @@ describe('Copy Referenced Files', () => { if (!virtualFileSystem.existsSync(dirname(virtualPath))) { await virtualFileSystem.promises.mkdir(dirname(virtualPath), {recursive: true}); } - await virtualFileSystem.promises.writeFile(virtualPath, await readFile(join(migrationScriptMocksPath, realPath), {encoding: 'utf8'})); + await virtualFileSystem.promises.writeFile(virtualPath, await readFile(join(specsMocksPath, realPath), {encoding: 'utf8'})); }; beforeAll(async () => { @@ -39,4 +39,15 @@ describe('Copy Referenced Files', () => { expect(virtualFileSystem.existsSync(join(outputDirectory, 'spec-chunk3/spec-chunk3.yaml'))).toBe(true); expect(virtualFileSystem.existsSync(join(outputDirectory, 'spec-chunk4/spec-chunk4.yaml'))).toBe(true); }); + + it('should update with new local basepath', async () => { + const specWitheRelativesFilePath = 'split-spec/split-spec.yaml'; + const expectedSpecWitheRelativesFilePath = 'split-spec/spec-with-updated-paths.yaml'; + const expectedContent = await readFile(join(specsMocksPath, expectedSpecWitheRelativesFilePath), {encoding: 'utf8'}); + const specContent = await readFile(join(specsMocksPath, specWitheRelativesFilePath), {encoding: 'utf8'}); + + const baseRelativePath = await copyReferencedFiles(specFilePath, './output-local-directory'); + const newSpecContent = await updateLocalRelativeRefs(specContent, baseRelativePath); + expect(newSpecContent).toBe(expectedContent); + }); }); diff --git a/packages/@ama-sdk/schematics/schematics/typescript/core/helpers/copy-referenced-files.ts b/packages/@ama-sdk/schematics/schematics/typescript/core/helpers/copy-referenced-files.ts index ac73243b3d..041c6771b4 100644 --- a/packages/@ama-sdk/schematics/schematics/typescript/core/helpers/copy-referenced-files.ts +++ b/packages/@ama-sdk/schematics/schematics/typescript/core/helpers/copy-referenced-files.ts @@ -1,6 +1,6 @@ import { existsSync } from 'node:fs'; import { copyFile, mkdir, readFile, rm } from 'node:fs/promises'; -import { dirname, join, normalize, posix, relative, resolve } from 'node:path'; +import { dirname, join, normalize, posix, relative, resolve, sep } from 'node:path'; const refMatcher = /\B['"]?[$]ref['"]?\s*:\s*([^#\n]+)/g; @@ -53,7 +53,7 @@ export function updateLocalRelativeRefs(specContent: string, newBaseRelativePath return specContent.replace(refMatcher, (match, ref: string) => { const refPath = ref.replace(/['"]/g, ''); return refPath.startsWith('.') ? - match.replace(refPath, formatPath(normalize(posix.join(newBaseRelativePath, refPath)))) + match.replace(refPath, formatPath(normalize(posix.join(newBaseRelativePath.replaceAll(sep, posix.sep), refPath)))) : match; }); } @@ -64,12 +64,9 @@ export function updateLocalRelativeRefs(specContent: string, newBaseRelativePath * @param outputDirectory */ export async function copyReferencedFiles(specFilePath: string, outputDirectory: string) { - const dedupe = (paths: string[]) => - paths.filter((refPath, index) => { - const actualPath = join(dirname(specFilePath), refPath); - return paths.findIndex((otherRefPath) => join(dirname(specFilePath), otherRefPath) === actualPath) === index; - }); - const refPaths = dedupe(await extractRefPathRecursive(specFilePath, specFilePath, new Set())); + const dedupe = (paths: string[]) => ([...new Set(paths)]); + const allRefPaths = await extractRefPathRecursive(specFilePath, specFilePath, new Set()); + const refPaths = dedupe(allRefPaths); if (refPaths.length) { if (existsSync(outputDirectory)) { await rm(outputDirectory, { recursive: true }); diff --git a/packages/@ama-sdk/schematics/schematics/typescript/core/index.ts b/packages/@ama-sdk/schematics/schematics/typescript/core/index.ts index da50794267..75b4c64604 100644 --- a/packages/@ama-sdk/schematics/schematics/typescript/core/index.ts +++ b/packages/@ama-sdk/schematics/schematics/typescript/core/index.ts @@ -162,6 +162,7 @@ function ngGenerateTypescriptSDKFn(options: NgGenerateTypescriptSDKCoreSchematic if (path.relative(process.cwd(), specPath).startsWith('..')) { // TODO would be better to create files on tree instead of FS + // https://github.com/AmadeusITGroup/otter/issues/2078 const newRelativePath = await copyReferencedFiles(specPath, './spec-local-references'); if (newRelativePath) { specContent = updateLocalRelativeRefs(specContent, newRelativePath); diff --git a/packages/@ama-sdk/schematics/testing/mocks/split-spec/spec-chunk1.yaml b/packages/@ama-sdk/schematics/testing/mocks/split-spec/spec-chunk1.yaml index 7273682c85..f05a4a1d9c 100644 --- a/packages/@ama-sdk/schematics/testing/mocks/split-spec/spec-chunk1.yaml +++ b/packages/@ama-sdk/schematics/testing/mocks/split-spec/spec-chunk1.yaml @@ -7,3 +7,5 @@ properties: example: 10 category: $ref: './split-spec.yaml#/components/schemas/Category' + category2: + $ref: '../spec-chunk4/spec-chunk4.yaml#/components/schemas/Category' diff --git a/packages/@ama-sdk/schematics/testing/mocks/split-spec/spec-with-updated-paths.yaml b/packages/@ama-sdk/schematics/testing/mocks/split-spec/spec-with-updated-paths.yaml new file mode 100644 index 0000000000..4f5fc58aed --- /dev/null +++ b/packages/@ama-sdk/schematics/testing/mocks/split-spec/spec-with-updated-paths.yaml @@ -0,0 +1,45 @@ +openapi: 3.0.2 +info: + description: test + title: test + version: 0.0.0 +paths: + /test: + get: + responses: + '200': + description: test + content: + application/json: + schema: + $ref: './output-local-directory/split-spec/spec-chunk1.yaml' + /test2: + get: + responses: + '200': + description: test + content: + application/json: + schema: + $ref: './output-local-directory/spec-chunk2.yaml' + /test3: + get: + responses: + '200': + description: test + content: + application/json: + schema: + $ref: './output-local-directory/spec-chunk3/spec-chunk3.yaml' +components: + schemas: + Category: + type: object + properties: + id: + type: integer + format: int64 + example: 10 + name: + type: string + example: "test"