From dd0fe7c2f7461d52adb9408173550b220670833f Mon Sep 17 00:00:00 2001 From: Thomas Boutell Date: Wed, 20 Nov 2024 12:55:24 -0500 Subject: [PATCH 1/9] wip --- lib/methods/export.js | 105 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 96 insertions(+), 9 deletions(-) diff --git a/lib/methods/export.js b/lib/methods/export.js index 54cd2e3..9ebba9c 100644 --- a/lib/methods/export.js +++ b/lib/methods/export.js @@ -210,6 +210,57 @@ module.exports = self => { mode = doc.aposMode || req.mode }) { recursion++; + if ((doc.type === '@apostrophecms/rich-text') && (type === 'relationship')) { + const linkedDocs = await self.apos.doc.db.find({ + aposDocId: { + $in: doc.permalinkIds + } + }).project({ + type: 1, + aposDocId: 1 + }).toArray(); + const linkedIdsByType = new Map(); + for (const linkedDoc of linkedDocs) { + // Normalization is a little different here because these + // are individual pages or pieces + const name = linkedDoc.type.startsWith('/') ? '@apostrophecms/any-page-type' : linkedDoc.type; + const linkedIds = linkedIdsByType.get(name) || new Set(); + linkedIds.add(linkedDoc.aposDocId); + if (linkedIds.size === 1) { + linkedIdsByType.set(name, linkedIds); + } + } + if (doc.imageIds?.length > 0) { + linkedIdsByType.set('@apostrophecms/image', new Set(doc.imageIds)); + } + const virtualDoc = { + type: '@apostrophecms/rich-text_related' + }; + const virtualSchema = []; + for (const [ linkedType, linkedIds ] of linkedIdsByType.entries()) { + console.log(`linked type: ${linkedType}`); + const baseName = self.apos.util.slugify(linkedType); + const fieldName = `_${baseName}`; + const idsStorage = `${baseName}Ids`; + virtualSchema.push({ + name: fieldName, + type: 'relationship', + withType: linkedType, + idsStorage + }); + const ids = [...linkedIds.values()]; + virtualDoc[idsStorage] = ids; + } + await self.getRelatedDocsFromSchema(req, { + doc: virtualDoc, + schema: virtualSchema, + relatedTypes, + storedData, + type: 'relationship', + recursion, + mode + }); + } for (const field of schema) { const fieldValue = doc[field.name]; const shouldRecurse = recursion <= MAX_RECURSION; @@ -217,10 +268,12 @@ module.exports = self => { if (!field.withType && !fieldValue) { continue; } - if (field.withType && relatedTypes && !relatedTypes.includes(field.withType)) { + if (field.withType && relatedTypes && !relatedTypesIncludes(field.withType)) { + console.log('*** skipping:', field.withType, field); continue; } if (field.withType && !self.canExport(req, field.withType)) { + console.log('*** cannot export:', field.withType); continue; } @@ -282,6 +335,11 @@ module.exports = self => { }); } } + + function relatedTypesIncludes(name) { + name = normalizeName(name); + return relatedTypes.includes(name); + } }, async handleRelatedField(req, { @@ -478,12 +536,12 @@ module.exports = self => { }, getRelatedTypes(req, schema = [], related = []) { - return findSchemaRelatedTypes(schema, related); - + findSchemaRelatedTypes(schema, related); + return related; function findSchemaRelatedTypes(schema, related, recursions = 0) { recursions++; if (recursions >= MAX_RECURSION) { - return related; + return; } for (const field of schema) { if ( @@ -491,22 +549,51 @@ module.exports = self => { self.canExport(req, field.withType) && !related.includes(field.withType) ) { - related.push(field.withType); + pushRelated(field.withType); const relatedManager = self.apos.doc.getManager(field.withType); findSchemaRelatedTypes(relatedManager.schema, related, recursions); } else if ([ 'array', 'object' ].includes(field.type)) { findSchemaRelatedTypes(field.schema, related, recursions); } else if (field.type === 'area') { const widgets = self.apos.area.getWidgets(field.options); - for (const widget of Object.keys(widgets)) { - const { schema = [] } = self.apos.modules[`${widget}-widget`]; + for (const [ widget, options ] of Object.entries(widgets)) { + const manager = self.apos.area.getWidgetManager(widget); + const schema = manager.schema || []; + if (widget === '@apostrophecms/rich-text') { + const rteOptions = { + ...manager.options.defaultOptions, + ...options + }; + if ((rteOptions.toolbar?.includes('image') || rteOptions.insert?.includes('image')) && !related.includes('@apostrophecms/image')) { + pushRelated('@apostrophecms/image'); + } + if (rteOptions.toolbar?.includes('link')) { + for (const name of manager.linkFields.linkTo.choices.map(choice => choice.value)) { + if (self.apos.doc.getManager(name) && !related.includes(name)) { + pushRelated(name); + } + } + } + } findSchemaRelatedTypes(schema, related, recursions); } } } - - return related; + function pushRelated(name) { + name = normalizeName(name); + if (!related.includes(name)) { + related.push(name); + } + } } } }; }; + +function normalizeName(name) { + if (name === '@apostrophecms/page') { + return '@apostrophecms/any-page-type'; + } else { + return name; + } +} From 8b0d4ab208a2a5a1e355a20fc5f7a681a0a6192b Mon Sep 17 00:00:00 2001 From: Thomas Boutell Date: Wed, 20 Nov 2024 13:43:30 -0500 Subject: [PATCH 2/9] working --- lib/methods/export.js | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/methods/export.js b/lib/methods/export.js index 9ebba9c..962340a 100644 --- a/lib/methods/export.js +++ b/lib/methods/export.js @@ -217,13 +217,14 @@ module.exports = self => { } }).project({ type: 1, - aposDocId: 1 + aposDocId: 1, + slug: 1 }).toArray(); const linkedIdsByType = new Map(); for (const linkedDoc of linkedDocs) { // Normalization is a little different here because these // are individual pages or pieces - const name = linkedDoc.type.startsWith('/') ? '@apostrophecms/any-page-type' : linkedDoc.type; + const name = linkedDoc.slug.startsWith('/') ? '@apostrophecms/any-page-type' : linkedDoc.type; const linkedIds = linkedIdsByType.get(name) || new Set(); linkedIds.add(linkedDoc.aposDocId); if (linkedIds.size === 1) { @@ -238,7 +239,6 @@ module.exports = self => { }; const virtualSchema = []; for (const [ linkedType, linkedIds ] of linkedIdsByType.entries()) { - console.log(`linked type: ${linkedType}`); const baseName = self.apos.util.slugify(linkedType); const fieldName = `_${baseName}`; const idsStorage = `${baseName}Ids`; @@ -250,6 +250,7 @@ module.exports = self => { }); const ids = [...linkedIds.values()]; virtualDoc[idsStorage] = ids; + virtualDoc[fieldName] = ids.map(id => ({ aposDocId: id })); } await self.getRelatedDocsFromSchema(req, { doc: virtualDoc, @@ -269,11 +270,9 @@ module.exports = self => { continue; } if (field.withType && relatedTypes && !relatedTypesIncludes(field.withType)) { - console.log('*** skipping:', field.withType, field); continue; } if (field.withType && !self.canExport(req, field.withType)) { - console.log('*** cannot export:', field.withType); continue; } From 4a274b93d99f6217d87a05b0efdebb67f4e2e402 Mon Sep 17 00:00:00 2001 From: Thomas Boutell Date: Wed, 20 Nov 2024 14:05:27 -0500 Subject: [PATCH 3/9] refactored --- lib/methods/export.js | 161 +++++++++++++++++++++++------------------- 1 file changed, 90 insertions(+), 71 deletions(-) diff --git a/lib/methods/export.js b/lib/methods/export.js index 962340a..83ab8d3 100644 --- a/lib/methods/export.js +++ b/lib/methods/export.js @@ -211,56 +211,7 @@ module.exports = self => { }) { recursion++; if ((doc.type === '@apostrophecms/rich-text') && (type === 'relationship')) { - const linkedDocs = await self.apos.doc.db.find({ - aposDocId: { - $in: doc.permalinkIds - } - }).project({ - type: 1, - aposDocId: 1, - slug: 1 - }).toArray(); - const linkedIdsByType = new Map(); - for (const linkedDoc of linkedDocs) { - // Normalization is a little different here because these - // are individual pages or pieces - const name = linkedDoc.slug.startsWith('/') ? '@apostrophecms/any-page-type' : linkedDoc.type; - const linkedIds = linkedIdsByType.get(name) || new Set(); - linkedIds.add(linkedDoc.aposDocId); - if (linkedIds.size === 1) { - linkedIdsByType.set(name, linkedIds); - } - } - if (doc.imageIds?.length > 0) { - linkedIdsByType.set('@apostrophecms/image', new Set(doc.imageIds)); - } - const virtualDoc = { - type: '@apostrophecms/rich-text_related' - }; - const virtualSchema = []; - for (const [ linkedType, linkedIds ] of linkedIdsByType.entries()) { - const baseName = self.apos.util.slugify(linkedType); - const fieldName = `_${baseName}`; - const idsStorage = `${baseName}Ids`; - virtualSchema.push({ - name: fieldName, - type: 'relationship', - withType: linkedType, - idsStorage - }); - const ids = [...linkedIds.values()]; - virtualDoc[idsStorage] = ids; - virtualDoc[fieldName] = ids.map(id => ({ aposDocId: id })); - } - await self.getRelatedDocsFromSchema(req, { - doc: virtualDoc, - schema: virtualSchema, - relatedTypes, - storedData, - type: 'relationship', - recursion, - mode - }); + await self.getRelatedDocsFromRichTextWidget(req, { doc, relatedTypes, storedData, recursion, mode }); } for (const field of schema) { const fieldValue = doc[field.name]; @@ -341,6 +292,65 @@ module.exports = self => { } }, + async getRelatedDocsFromRichTextWidget(req, { + doc, + relatedTypes, + storedData, + recursion, + mode + }) { + const linkedDocs = await self.apos.doc.db.find({ + aposDocId: { + $in: doc.permalinkIds + } + }).project({ + type: 1, + aposDocId: 1, + slug: 1 + }).toArray(); + const linkedIdsByType = new Map(); + for (const linkedDoc of linkedDocs) { + // Normalization is a little different here because these + // are individual pages or pieces + const name = linkedDoc.slug.startsWith('/') ? '@apostrophecms/any-page-type' : linkedDoc.type; + const linkedIds = linkedIdsByType.get(name) || new Set(); + linkedIds.add(linkedDoc.aposDocId); + if (linkedIds.size === 1) { + linkedIdsByType.set(name, linkedIds); + } + } + if (doc.imageIds?.length > 0) { + linkedIdsByType.set('@apostrophecms/image', new Set(doc.imageIds)); + } + const virtualDoc = { + type: '@apostrophecms/rich-text_related' + }; + const virtualSchema = []; + for (const [ linkedType, linkedIds ] of linkedIdsByType.entries()) { + const baseName = self.apos.util.slugify(linkedType); + const fieldName = `_${baseName}`; + const idsStorage = `${baseName}Ids`; + virtualSchema.push({ + name: fieldName, + type: 'relationship', + withType: linkedType, + idsStorage + }); + const ids = [...linkedIds.values()]; + virtualDoc[idsStorage] = ids; + virtualDoc[fieldName] = ids.map(id => ({ aposDocId: id })); + } + await self.getRelatedDocsFromSchema(req, { + doc: virtualDoc, + schema: virtualSchema, + relatedTypes, + storedData, + type: 'relationship', + recursion, + mode + }); + }, + async handleRelatedField(req, { doc, field, @@ -548,7 +558,7 @@ module.exports = self => { self.canExport(req, field.withType) && !related.includes(field.withType) ) { - pushRelated(field.withType); + pushRelated(related, field.withType); const relatedManager = self.apos.doc.getManager(field.withType); findSchemaRelatedTypes(relatedManager.schema, related, recursions); } else if ([ 'array', 'object' ].includes(field.type)) { @@ -556,32 +566,34 @@ module.exports = self => { } else if (field.type === 'area') { const widgets = self.apos.area.getWidgets(field.options); for (const [ widget, options ] of Object.entries(widgets)) { - const manager = self.apos.area.getWidgetManager(widget); - const schema = manager.schema || []; + const schema = self.apos.area.getWidgetManager(widget).schema; if (widget === '@apostrophecms/rich-text') { - const rteOptions = { - ...manager.options.defaultOptions, - ...options - }; - if ((rteOptions.toolbar?.includes('image') || rteOptions.insert?.includes('image')) && !related.includes('@apostrophecms/image')) { - pushRelated('@apostrophecms/image'); - } - if (rteOptions.toolbar?.includes('link')) { - for (const name of manager.linkFields.linkTo.choices.map(choice => choice.value)) { - if (self.apos.doc.getManager(name) && !related.includes(name)) { - pushRelated(name); - } - } - } + self.getRelatedTypesFromRichTextWidget(req, { options, related }); } findSchemaRelatedTypes(schema, related, recursions); } } } - function pushRelated(name) { - name = normalizeName(name); - if (!related.includes(name)) { - related.push(name); + } + }, + // Does not currently utilize req, but it could be relevant in overrides and is + // always the first argument by convention, so it is included in the signature + getRelatedTypesFromRichTextWidget(req, { + options, + related + }) { + const manager = self.apos.modules['@apostrophecms/rich-text-widget']; + const rteOptions = { + ...manager.options.defaultOptions, + ...options + }; + if ((rteOptions.toolbar?.includes('image') || rteOptions.insert?.includes('image')) && !related.includes('@apostrophecms/image')) { + pushRelated(related, '@apostrophecms/image'); + } + if (rteOptions.toolbar?.includes('link')) { + for (const name of manager.linkFields.linkTo.choices.map(choice => choice.value)) { + if (self.apos.doc.getManager(name) && !related.includes(name)) { + pushRelated(related, name); } } } @@ -596,3 +608,10 @@ function normalizeName(name) { return name; } } + +function pushRelated(related, name) { + name = normalizeName(name); + if (!related.includes(name)) { + related.push(name); + } +} From 62822a85b5adf6712c6baa860d53901064e9f815 Mon Sep 17 00:00:00 2001 From: Thomas Boutell Date: Mon, 25 Nov 2024 16:07:19 -0500 Subject: [PATCH 4/9] wip --- test/import-page.js | 64 ++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 60 insertions(+), 4 deletions(-) diff --git a/test/import-page.js b/test/import-page.js index 7e3a22f..3df08bd 100644 --- a/test/import-page.js +++ b/test/import-page.js @@ -42,7 +42,19 @@ const server = { } }, 'custom-page': { - extend: '@apostrophecms/page-type' + extend: '@apostrophecms/page-type', + fields: { + add: { + main: { + type: 'area', + widgets: { + '@apostrophecms/rich-text': { + insert: [ 'image' ] + } + } + } + } + } }, 'test-page': { extend: '@apostrophecms/page-type' @@ -109,6 +121,17 @@ describe('@apostrophecms/import-export:import-page', function () { slug: '/level-1-page-3' } ); + const attachment = apos.attachment.insert(req, { + name: 'test-image.jpg', + path: `${apos.rootDir}/public/test-image.jpg` + }); + + const inlineImage = await apos.image.insert(req, { + ...apos.image.newInstance(), + title: 'inline-image', + attachment + }); + const level2Page1 = await apos.page.insert( req, level1Page1._id, @@ -116,7 +139,23 @@ describe('@apostrophecms/import-export:import-page', function () { { title: 'Level 2 Page 1', type: 'test-page', - slug: '/level-1-page-1/level-2-page-1' + slug: '/level-1-page-1/level-2-page-1', + main: { + items: [ + { + type: '@apostrophecms/rich-text', + content: ` +

+ Test Link +

+
+ alt text +
+
+ ` + } + ] + } } ); const level3Page1 = await apos.page.insert( @@ -254,7 +293,10 @@ describe('@apostrophecms/import-export:import-page', function () { const homePublished = await apos.page.find(apos.task.getReq({ mode: 'published' }), { slug: '/' }).toObject(); const actual = { - docs: importedDocs + docs: importedDocs.map(doc => { + const { main, ...rest } = doc; + return rest; + }) }; const expected = { docs: [ @@ -418,6 +460,8 @@ describe('@apostrophecms/import-export:import-page', function () { }; assert.deepEqual(actual, expected); + console.log('-->', actual.docs.length); + console.log('-->', actual.docs.map(doc => doc.type)); }); it('should import pages from same tarball twice without issues', async function () { @@ -505,7 +549,10 @@ describe('@apostrophecms/import-export:import-page', function () { const homePublished = await apos.page.find(apos.task.getReq({ mode: 'published' }), { slug: '/' }).toObject(); const actual = { - docs: importedDocs + docs: importedDocs.map(doc => { + const { main, ...rest } = doc; + return rest; + }) }; const expected = { docs: [ @@ -669,6 +716,15 @@ describe('@apostrophecms/import-export:import-page', function () { }; assert.deepEqual(actual, expected); + const importedLevel2Page1 = importedDocs.find(({ title }) => title === 'Level 2 Page 1'); + console.log('>>>', importedLevel2Page1.main.items[0].content); + const importedLevel1Page3 = importedDocs.find(({ title }) => title === 'Level 1 Page 3'); + assert(importedLevel1Page3); + const inlineImage = importedDocs.find(({ title }) => title === 'inline-image'); + assert(inlineImage); + assert(importedLevel1Page3); + assert(importedLevel2Page1.main.items[0].content.includes(inlineImage.aposDocId)); + assert(importedLevel2Page1.main.items[0].content.includes(importedLevel1Page3.aposDocId)); }); it('should import pages with existing parkedId and children', async function () { From c3317385e8e73f1a1d30065c6c5476b7accb9238 Mon Sep 17 00:00:00 2001 From: Thomas Boutell Date: Mon, 2 Dec 2024 13:58:10 -0500 Subject: [PATCH 5/9] tests passing --- CHANGELOG.md | 7 ++ lib/methods/export.js | 1 - lib/methods/import.js | 1 - test/import-page.js | 178 ++++++++++++++++++++++++++++++++---------- 4 files changed, 143 insertions(+), 44 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a9108a..787b321 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## UNRELEASED + +### Adds + +* Pages linked a page via the "Internal Page" option in the rich text editor are now candidates to be exported as related documents. +* Images embedded inline in rich text widgets via the `insert: [ 'image' ]` option are now candidates to be exported as related documents. + ## 2.5.0 (2024-11-08) ### Adds diff --git a/lib/methods/export.js b/lib/methods/export.js index 83ab8d3..1bffe76 100644 --- a/lib/methods/export.js +++ b/lib/methods/export.js @@ -35,7 +35,6 @@ module.exports = self => { const docs = (await self.getDocs(req, ids, hasRelatedTypes, manager, reporting)) .map((doc) => self.apos.util.clonePermanent(doc)); - if (!hasRelatedTypes) { return self.exportFile( req, diff --git a/lib/methods/import.js b/lib/methods/import.js index ff99681..34b2e51 100644 --- a/lib/methods/import.js +++ b/lib/methods/import.js @@ -705,7 +705,6 @@ module.exports = self => { const type = doc.type; const docToInsert = doc; - await manager.convert(_req, doc, docToInsert, { presentFieldsOnly: true, fetchRelationships: false diff --git a/test/import-page.js b/test/import-page.js index 3df08bd..9c7c620 100644 --- a/test/import-page.js +++ b/test/import-page.js @@ -37,19 +37,28 @@ const server = { { name: 'custom-page', label: 'Custom Page' + }, + { + name: 'rich-text-page', + label: 'Rich Text Page' } ] } }, 'custom-page': { + extend: '@apostrophecms/page-type' + }, + 'rich-text-page': { extend: '@apostrophecms/page-type', fields: { add: { main: { type: 'area', - widgets: { - '@apostrophecms/rich-text': { - insert: [ 'image' ] + options: { + widgets: { + '@apostrophecms/rich-text': { + insert: [ 'image' ] + } } } } @@ -73,6 +82,15 @@ const server = { }, stop: async (apos) => { await t.destroy(apos); + }, + cleanup: async (apos) => { + await apos.doc.db.deleteMany({ type: '@apostrophecms/archive-page' }); + await apos.doc.db.deleteMany({ type: '@apostrophecms/home-page' }); + await apos.doc.db.deleteMany({ type: '@apostrophecms/image' }); + await apos.doc.db.deleteMany({ type: '@apostrophecms/image-tag' }); + await apos.doc.db.deleteMany({ type: 'custom-page' }); + await apos.doc.db.deleteMany({ type: 'test-page' }); + await apos.doc.db.deleteMany({ type: 'rich-text-page' }); } }; @@ -132,32 +150,54 @@ describe('@apostrophecms/import-export:import-page', function () { attachment }); - const level2Page1 = await apos.page.insert( + const richTextRaw = { + type: '@apostrophecms/rich-text', + content: ` +

+ Test Link +

+
+ alt text +
+
+ ` + }; + + const richTextWidget = await apos.modules['@apostrophecms/rich-text-widget'].sanitize(req, richTextRaw, { + insert: [ 'image' ] + }); + + const level1Page4 = await apos.page.insert( req, - level1Page1._id, + '_home', 'lastChild', { - title: 'Level 2 Page 1', - type: 'test-page', - slug: '/level-1-page-1/level-2-page-1', + title: 'Level 1 Page 4', + type: 'rich-text-page', + slug: '/level-1-page-4', main: { + metaType: 'area', items: [ - { - type: '@apostrophecms/rich-text', - content: ` -

- Test Link -

-
- alt text -
-
- ` - } + richTextWidget ] } } ); + + const level1Page4Found = await apos.page.find(req, { + title: 'Level 1 Page 4' + }).toObject(); + + const level2Page1 = await apos.page.insert( + req, + level1Page1._id, + 'lastChild', + { + title: 'Level 2 Page 1', + type: 'test-page', + slug: '/level-1-page-1/level-2-page-1' + } + ); const level3Page1 = await apos.page.insert( req, level2Page1._id, @@ -202,6 +242,7 @@ describe('@apostrophecms/import-export:import-page', function () { await apos.page.publish(req, level1Page1); await apos.page.publish(req, level1Page2); await apos.page.publish(req, level1Page3); + await apos.page.publish(req, level1Page4); await apos.page.publish(req, level2Page1); await apos.page.publish(req, level3Page1); await apos.page.publish(req, level4Page1); @@ -258,13 +299,7 @@ describe('@apostrophecms/import-export:import-page', function () { const importFilePath = path.join(tempPath, fileName); await fs.copyFile(exportFilePath, importFilePath); - // cleanup - await apos.doc.db.deleteMany({ type: '@apostrophecms/archive-page' }); - await apos.doc.db.deleteMany({ type: '@apostrophecms/home-page' }); - await apos.doc.db.deleteMany({ type: '@apostrophecms/image' }); - await apos.doc.db.deleteMany({ type: '@apostrophecms/image-tag' }); - await apos.doc.db.deleteMany({ type: 'custom-page' }); - await apos.doc.db.deleteMany({ type: 'test-page' }); + await server.cleanup(apos); await server.stop(apos); apos = await server.start(); @@ -289,6 +324,7 @@ describe('@apostrophecms/import-export:import-page', function () { aposMode: 1 }) .toArray(); + const homeDraft = await apos.page.find(apos.task.getReq({ mode: 'draft' }), { slug: '/' }).toObject(); const homePublished = await apos.page.find(apos.task.getReq({ mode: 'published' }), { slug: '/' }).toObject(); @@ -460,8 +496,6 @@ describe('@apostrophecms/import-export:import-page', function () { }; assert.deepEqual(actual, expected); - console.log('-->', actual.docs.length); - console.log('-->', actual.docs.map(doc => doc.type)); }); it('should import pages from same tarball twice without issues', async function () { @@ -504,13 +538,7 @@ describe('@apostrophecms/import-export:import-page', function () { await fs.copyFile(exportFilePath, importFilePath); await fs.copyFile(exportFilePath, importFilePathDuplicate); - // cleanup - await apos.doc.db.deleteMany({ type: '@apostrophecms/archive-page' }); - await apos.doc.db.deleteMany({ type: '@apostrophecms/home-page' }); - await apos.doc.db.deleteMany({ type: '@apostrophecms/image' }); - await apos.doc.db.deleteMany({ type: '@apostrophecms/image-tag' }); - await apos.doc.db.deleteMany({ type: 'custom-page' }); - await apos.doc.db.deleteMany({ type: 'test-page' }); + await server.cleanup(apos); await server.stop(apos); apos = await server.start(); @@ -545,6 +573,7 @@ describe('@apostrophecms/import-export:import-page', function () { aposMode: 1 }) .toArray(); + const homeDraft = await apos.page.find(apos.task.getReq({ mode: 'draft' }), { slug: '/' }).toObject(); const homePublished = await apos.page.find(apos.task.getReq({ mode: 'published' }), { slug: '/' }).toObject(); @@ -716,15 +745,80 @@ describe('@apostrophecms/import-export:import-page', function () { }; assert.deepEqual(actual, expected); - const importedLevel2Page1 = importedDocs.find(({ title }) => title === 'Level 2 Page 1'); - console.log('>>>', importedLevel2Page1.main.items[0].content); - const importedLevel1Page3 = importedDocs.find(({ title }) => title === 'Level 1 Page 3'); - assert(importedLevel1Page3); + }); + + it.only('should import related documents referenced by rich text', async function() { + const req = apos.task.getReq({ mode: 'draft' }); + + const manager = apos.page; + const ids = await manager + .find( + req, + { + title: 'Level 1 Page 4' + }, + { + project: { + _id: 1 + } + } + ) + .toArray(); + + // export + const exportReq = apos.task.getReq({ + body: { + _ids: ids.map(({ _id }) => _id), + extension: 'gzip', + // Because @apostrophecms/any-page-type is what is allowed by default by the rich text editor link widget, + // and will show up accordingly as a related type choice. An explicit page type here won't match + relatedTypes: [ '@apostrophecms/image', '@apostrophecms/any-page-type' ], + type: req.t('apostrophe:pages') + } + }); + const { url } = await apos.modules['@apostrophecms/import-export'].export(exportReq, manager); + const fileName = path.basename(url); + const exportFilePath = path.join(exportsPath, fileName); + const importFilePath = path.join(tempPath, fileName); + await fs.copyFile(exportFilePath, importFilePath); + + await server.cleanup(apos); + await server.stop(apos); + apos = await server.start(); + + // import + const mimeType = apos.modules['@apostrophecms/import-export'].formats.gzip.allowedTypes.at(0); + const importReq = apos.task.getReq({ + body: {}, + files: { + file: { + path: importFilePath, + type: mimeType + } + } + }); + await apos.modules['@apostrophecms/import-export'].import(importReq); + + const importedDocs = await apos.doc.db + .find({ type: /rich-text-page|@apostrophecms\/image|test-page/ }) + .sort({ + type: 1, + title: 1, + aposMode: 1 + }) + .toArray(); + const importedLevel1Page4 = importedDocs.find(({ title }) => title === 'Level 1 Page 4'); + assert(importedLevel1Page4); + assert.strictEqual(importedLevel1Page4.type, 'rich-text-page'); + const content = importedLevel1Page4.main?.items?.[0]?.content; + assert(content.includes(' title === 'inline-image'); assert(inlineImage); + assert(content.includes(inlineImage.aposDocId)); + const importedLevel1Page3 = importedDocs.find(({ title }) => title === 'Level 1 Page 3'); assert(importedLevel1Page3); - assert(importedLevel2Page1.main.items[0].content.includes(inlineImage.aposDocId)); - assert(importedLevel2Page1.main.items[0].content.includes(importedLevel1Page3.aposDocId)); + assert(content.includes(importedLevel1Page3.aposDocId)); }); it('should import pages with existing parkedId and children', async function () { From 524b5fec75ee004d15470632e9ce5ab34fb124c2 Mon Sep 17 00:00:00 2001 From: Thomas Boutell Date: Mon, 2 Dec 2024 14:01:45 -0500 Subject: [PATCH 6/9] test cleanup, lint --- lib/methods/export.js | 15 ++++++++++++--- test/import-page.js | 4 ---- test/index.js | 6 ++++-- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/lib/methods/export.js b/lib/methods/export.js index 1bffe76..80bed5c 100644 --- a/lib/methods/export.js +++ b/lib/methods/export.js @@ -210,7 +210,13 @@ module.exports = self => { }) { recursion++; if ((doc.type === '@apostrophecms/rich-text') && (type === 'relationship')) { - await self.getRelatedDocsFromRichTextWidget(req, { doc, relatedTypes, storedData, recursion, mode }); + await self.getRelatedDocsFromRichTextWidget(req, { + doc, + relatedTypes, + storedData, + recursion, + mode + }); } for (const field of schema) { const fieldValue = doc[field.name]; @@ -335,7 +341,7 @@ module.exports = self => { withType: linkedType, idsStorage }); - const ids = [...linkedIds.values()]; + const ids = [ ...linkedIds.values() ]; virtualDoc[idsStorage] = ids; virtualDoc[fieldName] = ids.map(id => ({ aposDocId: id })); } @@ -567,7 +573,10 @@ module.exports = self => { for (const [ widget, options ] of Object.entries(widgets)) { const schema = self.apos.area.getWidgetManager(widget).schema; if (widget === '@apostrophecms/rich-text') { - self.getRelatedTypesFromRichTextWidget(req, { options, related }); + self.getRelatedTypesFromRichTextWidget(req, { + options, + related + }); } findSchemaRelatedTypes(schema, related, recursions); } diff --git a/test/import-page.js b/test/import-page.js index 9c7c620..a164225 100644 --- a/test/import-page.js +++ b/test/import-page.js @@ -184,10 +184,6 @@ describe('@apostrophecms/import-export:import-page', function () { } ); - const level1Page4Found = await apos.page.find(req, { - title: 'Level 1 Page 4' - }).toObject(); - const level2Page1 = await apos.page.insert( req, level1Page1._id, diff --git a/test/index.js b/test/index.js index d486f7f..e91f18c 100644 --- a/test/index.js +++ b/test/index.js @@ -935,8 +935,10 @@ describe('@apostrophecms/import-export', function () { relatedTypesTopics }; const expected = { - relatedTypesArticles: [ 'topic', '@apostrophecms/image', '@apostrophecms/image-tag' ], - relatedTypesTopics: [ 'topic' ] + // @apostrophecms/any-page-type is a candidate because of rich text widgets, which are + // allowed to contain Internal Page links that are candidates to be related documents + relatedTypesArticles: [ 'topic', '@apostrophecms/any-page-type', '@apostrophecms/image', '@apostrophecms/image-tag' ], + relatedTypesTopics: [ '@apostrophecms/any-page-type', 'topic' ] }; assert.deepEqual(actual, expected); From 6973fb1e49c5ab739519bea9114a89dd8249cc2e Mon Sep 17 00:00:00 2001 From: Thomas Boutell Date: Mon, 9 Dec 2024 08:44:00 -0500 Subject: [PATCH 7/9] cleanup --- lib/methods/export.js | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/lib/methods/export.js b/lib/methods/export.js index 80bed5c..8a5c441 100644 --- a/lib/methods/export.js +++ b/lib/methods/export.js @@ -292,8 +292,8 @@ module.exports = self => { } function relatedTypesIncludes(name) { - name = normalizeName(name); - return relatedTypes.includes(name); + const normalizedName = normalizeName(name); + return relatedTypes.includes(normalizedName); } }, @@ -317,11 +317,12 @@ module.exports = self => { for (const linkedDoc of linkedDocs) { // Normalization is a little different here because these // are individual pages or pieces - const name = linkedDoc.slug.startsWith('/') ? '@apostrophecms/any-page-type' : linkedDoc.type; - const linkedIds = linkedIdsByType.get(name) || new Set(); + const docType = linkedDoc.slug.startsWith('/') ? '@apostrophecms/any-page-type' : linkedDoc.type; + const isTypeStored = linkedIdsByType.has(docType); + const linkedIds = isTypeStored ? linkedIdsByType.get(docType) : new Set(); linkedIds.add(linkedDoc.aposDocId); - if (linkedIds.size === 1) { - linkedIdsByType.set(name, linkedIds); + if (!isTypeStored) { + linkedIdsByType.set(docType, linkedIds); } } if (doc.imageIds?.length > 0) { @@ -595,7 +596,10 @@ module.exports = self => { ...manager.options.defaultOptions, ...options }; - if ((rteOptions.toolbar?.includes('image') || rteOptions.insert?.includes('image')) && !related.includes('@apostrophecms/image')) { + if ( + (rteOptions.toolbar?.includes('image') || rteOptions.insert?.includes('image')) && + !related.includes('@apostrophecms/image') + ) { pushRelated(related, '@apostrophecms/image'); } if (rteOptions.toolbar?.includes('link')) { From 7c43d5a552e3bf1132b94e0966fc0af37e60578e Mon Sep 17 00:00:00 2001 From: Thomas Boutell Date: Thu, 12 Dec 2024 14:47:17 -0500 Subject: [PATCH 8/9] a lot of changes around page types to satisfy tests & ensure functionality --- lib/methods/export.js | 108 +++++++++++++------------ test/import-page.js | 2 +- test/index.js | 29 ++++++- ui/apos/components/AposExportModal.vue | 7 +- 4 files changed, 87 insertions(+), 59 deletions(-) diff --git a/lib/methods/export.js b/lib/methods/export.js index 8a5c441..ed4e769 100644 --- a/lib/methods/export.js +++ b/lib/methods/export.js @@ -292,8 +292,13 @@ module.exports = self => { } function relatedTypesIncludes(name) { - const normalizedName = normalizeName(name); - return relatedTypes.includes(normalizedName); + if (name === '@apostrophecms/any-page-type') { + return relatedTypes.some(type => { + const module = self.apos.modules[type]; + return self.apos.instanceOf(module, '@apostrophecms/page-type'); + }); + } + return relatedTypes.includes(name); } }, @@ -304,7 +309,7 @@ module.exports = self => { recursion, mode }) { - const linkedDocs = await self.apos.doc.db.find({ + let linkedDocs = await self.apos.doc.db.find({ aposDocId: { $in: doc.permalinkIds } @@ -313,6 +318,10 @@ module.exports = self => { aposDocId: 1, slug: 1 }).toArray(); + // We're likely going to fetch them all with an @apostrophecms/any-page-type query, so we + // need to do our real related types check early or we'll allow all page types + // whenever we allow even one + linkedDocs = linkedDocs.filter(doc => relatedTypes.includes(doc.type)); const linkedIdsByType = new Map(); for (const linkedDoc of linkedDocs) { // Normalization is a little different here because these @@ -549,48 +558,56 @@ module.exports = self => { timeoutId }; }, - + // The entry point. Modifies `related`, also returns `related` because code elsewhere + // expects that getRelatedTypes(req, schema = [], related = []) { - findSchemaRelatedTypes(schema, related); + self.findSchemaRelatedTypes(req, schema, related, 0); return related; - function findSchemaRelatedTypes(schema, related, recursions = 0) { - recursions++; - if (recursions >= MAX_RECURSION) { - return; - } - for (const field of schema) { - if ( - field.type === 'relationship' && - self.canExport(req, field.withType) && - !related.includes(field.withType) - ) { - pushRelated(related, field.withType); - const relatedManager = self.apos.doc.getManager(field.withType); - findSchemaRelatedTypes(relatedManager.schema, related, recursions); - } else if ([ 'array', 'object' ].includes(field.type)) { - findSchemaRelatedTypes(field.schema, related, recursions); - } else if (field.type === 'area') { - const widgets = self.apos.area.getWidgets(field.options); - for (const [ widget, options ] of Object.entries(widgets)) { - const schema = self.apos.area.getWidgetManager(widget).schema; - if (widget === '@apostrophecms/rich-text') { - self.getRelatedTypesFromRichTextWidget(req, { - options, - related - }); - } - findSchemaRelatedTypes(schema, related, recursions); + }, + // Called recursively for you. Modifies `related`, has no useful return value + findSchemaRelatedTypes(req, schema, related, recursions) { + recursions++; + if (recursions >= MAX_RECURSION) { + return; + } + for (const field of schema) { + if ( + field.type === 'relationship' && + self.canExport(req, field.withType) && + !related.includes(field.withType) + ) { + self.pushRelatedType(req, related, field.withType, recursions); + } else if ([ 'array', 'object' ].includes(field.type)) { + self.findSchemaRelatedTypes(req, field.schema, related, recursions); + } else if (field.type === 'area') { + const widgets = self.apos.area.getWidgets(field.options); + for (const [ widget, options ] of Object.entries(widgets)) { + const schema = self.apos.area.getWidgetManager(widget).schema || []; + if (widget === '@apostrophecms/rich-text') { + self.getRelatedTypesFromRichTextWidget(req, options, related, recursions); } + self.findSchemaRelatedTypes(req, schema, related, recursions); } } } }, + pushRelatedType(req, related, type, recursions) { + if ((type === '@apostrophecms/page') || (type === '@apostrophecms/any-page-type')) { + const pageTypes = Object.entries(self.apos.doc.managers).filter(([ name, module ]) => self.apos.instanceOf(module, '@apostrophecms/page-type')).map(([ name, module ]) => name); + for (const type of pageTypes) { + self.pushRelatedType(req, related, type, recursions); + } + return; + } + if (!related.includes(type)) { + related.push(type); + const relatedManager = self.apos.doc.getManager(type); + self.findSchemaRelatedTypes(req, relatedManager.schema, related, recursions); + } + }, // Does not currently utilize req, but it could be relevant in overrides and is // always the first argument by convention, so it is included in the signature - getRelatedTypesFromRichTextWidget(req, { - options, - related - }) { + getRelatedTypesFromRichTextWidget(req, options, related, recursions) { const manager = self.apos.modules['@apostrophecms/rich-text-widget']; const rteOptions = { ...manager.options.defaultOptions, @@ -600,30 +617,15 @@ module.exports = self => { (rteOptions.toolbar?.includes('image') || rteOptions.insert?.includes('image')) && !related.includes('@apostrophecms/image') ) { - pushRelated(related, '@apostrophecms/image'); + self.pushRelatedType(req, related, '@apostrophecms/image', recursions); } if (rteOptions.toolbar?.includes('link')) { for (const name of manager.linkFields.linkTo.choices.map(choice => choice.value)) { if (self.apos.doc.getManager(name) && !related.includes(name)) { - pushRelated(related, name); + self.pushRelatedType(req, related, name, recursions); } } } } }; }; - -function normalizeName(name) { - if (name === '@apostrophecms/page') { - return '@apostrophecms/any-page-type'; - } else { - return name; - } -} - -function pushRelated(related, name) { - name = normalizeName(name); - if (!related.includes(name)) { - related.push(name); - } -} diff --git a/test/import-page.js b/test/import-page.js index a164225..d6dcf18 100644 --- a/test/import-page.js +++ b/test/import-page.js @@ -768,7 +768,7 @@ describe('@apostrophecms/import-export:import-page', function () { extension: 'gzip', // Because @apostrophecms/any-page-type is what is allowed by default by the rich text editor link widget, // and will show up accordingly as a related type choice. An explicit page type here won't match - relatedTypes: [ '@apostrophecms/image', '@apostrophecms/any-page-type' ], + relatedTypes: [ '@apostrophecms/image', 'test-page' ], type: req.t('apostrophe:pages') } }); diff --git a/test/index.js b/test/index.js index e91f18c..0321edf 100644 --- a/test/index.js +++ b/test/index.js @@ -935,10 +935,31 @@ describe('@apostrophecms/import-export', function () { relatedTypesTopics }; const expected = { - // @apostrophecms/any-page-type is a candidate because of rich text widgets, which are - // allowed to contain Internal Page links that are candidates to be related documents - relatedTypesArticles: [ 'topic', '@apostrophecms/any-page-type', '@apostrophecms/image', '@apostrophecms/image-tag' ], - relatedTypesTopics: [ '@apostrophecms/any-page-type', 'topic' ] + // All page types are in play because rich text internal page links are in play. + // Articles are in play because a page type has a relationship to them, so: see above + // (remember this is quite recursive) + relatedTypesArticles: [ + 'topic', + '@apostrophecms/home-page', + '@apostrophecms/archive-page', + '@apostrophecms/search', + 'home-page', + 'default-page', + '@apostrophecms/image', + '@apostrophecms/image-tag', + 'article' + ], + relatedTypesTopics: [ + '@apostrophecms/home-page', + '@apostrophecms/archive-page', + '@apostrophecms/search', + 'home-page', + 'default-page', + '@apostrophecms/image', + '@apostrophecms/image-tag', + 'article', + 'topic' + ] }; assert.deepEqual(actual, expected); diff --git a/ui/apos/components/AposExportModal.vue b/ui/apos/components/AposExportModal.vue index de0812a..ba6c834 100644 --- a/ui/apos/components/AposExportModal.vue +++ b/ui/apos/components/AposExportModal.vue @@ -287,7 +287,12 @@ export default { }, getRelatedTypeLabel(moduleName) { const moduleOptions = apos.modules[moduleName]; - return this.$t(moduleOptions.label); + if (moduleOptions.label) { + return this.$t(moduleOptions.label); + } else { + // Often not set for page types etc. + return moduleName; + } }, onFormatChange(formatName) { this.formatName = this.formats.find(format => format.name === formatName).name; From 6b9794f70792c1b8ca318376fefde30951cd5598 Mon Sep 17 00:00:00 2001 From: Thomas Boutell Date: Thu, 12 Dec 2024 16:26:01 -0500 Subject: [PATCH 9/9] allow both valid aliases for "any page type" --- lib/methods/export.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/methods/export.js b/lib/methods/export.js index ed4e769..f746f13 100644 --- a/lib/methods/export.js +++ b/lib/methods/export.js @@ -292,7 +292,7 @@ module.exports = self => { } function relatedTypesIncludes(name) { - if (name === '@apostrophecms/any-page-type') { + if ([ '@apostrophecms/any-page-type', '@apostrophecms/page' ].includes(name)) { return relatedTypes.some(type => { const module = self.apos.modules[type]; return self.apos.instanceOf(module, '@apostrophecms/page-type');