From d738b9041ffc9777220e47a069c939b74a82778e Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Wed, 25 Oct 2023 15:45:02 +0200 Subject: [PATCH 1/5] TASK: Remove legacy `unstructured` node type --- Neos.Neos/NodeTypes/Unstructured.yaml | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 Neos.Neos/NodeTypes/Unstructured.yaml diff --git a/Neos.Neos/NodeTypes/Unstructured.yaml b/Neos.Neos/NodeTypes/Unstructured.yaml deleted file mode 100644 index 91b17e7d48e..00000000000 --- a/Neos.Neos/NodeTypes/Unstructured.yaml +++ /dev/null @@ -1,5 +0,0 @@ -# legacy type for nodes without type -'unstructured': - constraints: - nodeTypes: - '*': true From 50e8ff69eaeb36d4b2bed42deb6a4d30b24ebccc Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Thu, 26 Oct 2023 08:29:25 +0200 Subject: [PATCH 2/5] TASK: Remove `'unstructured': {}` NodeType declaration from behat tests --- .../Tests/Behavior/Features/Assets.feature | 1 - .../Tests/Behavior/Features/Basic.feature | 1 - .../Tests/Behavior/Features/Errors.feature | 1 - .../Tests/Behavior/Features/References.feature | 1 - .../Tests/Behavior/Features/Variants.feature | 1 - 5 files changed, 5 deletions(-) diff --git a/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Features/Assets.feature b/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Features/Assets.feature index bc5cf9671ea..f4ca3e07455 100644 --- a/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Features/Assets.feature +++ b/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Features/Assets.feature @@ -7,7 +7,6 @@ Feature: Export of used Assets, Image Variants and Persistent Resources | language | en | en, de, ch | ch->de | And using the following node types: """yaml - 'unstructured': {} 'Neos.Neos:Site': {} 'Some.Package:Homepage': superTypes: diff --git a/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Features/Basic.feature b/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Features/Basic.feature index 3623e2fe61a..a1dea4af085 100644 --- a/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Features/Basic.feature +++ b/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Features/Basic.feature @@ -5,7 +5,6 @@ Feature: Simple migrations without content dimensions Given using no content dimensions And using the following node types: """yaml - 'unstructured': {} 'Neos.Neos:Site': {} 'Some.Package:Homepage': superTypes: diff --git a/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Features/Errors.feature b/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Features/Errors.feature index 4cf57e96a4c..33af167cd03 100644 --- a/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Features/Errors.feature +++ b/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Features/Errors.feature @@ -5,7 +5,6 @@ Feature: Exceptional cases during migrations Given using no content dimensions And using the following node types: """yaml - 'unstructured': {} 'Neos.Neos:Site': {} 'Some.Package:Homepage': superTypes: diff --git a/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Features/References.feature b/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Features/References.feature index 7c67812eb50..52e659f6017 100644 --- a/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Features/References.feature +++ b/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Features/References.feature @@ -5,7 +5,6 @@ Feature: Migrations that contain nodes with "reference" or "references propertie Given using no content dimensions And using the following node types: """yaml - 'unstructured': {} 'Neos.Neos:Site': {} 'Some.Package:Homepage': superTypes: diff --git a/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Features/Variants.feature b/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Features/Variants.feature index e6285a44323..a0e35e8fc92 100644 --- a/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Features/Variants.feature +++ b/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Features/Variants.feature @@ -7,7 +7,6 @@ Feature: Migrating nodes with content dimensions | language | en | en, de, ch | ch->de | And using the following node types: """yaml - 'unstructured': {} 'Neos.Neos:Site': {} 'Some.Package:Homepage': superTypes: From 9fce857f72e3ba60bf45732e3afd1637743c0434 Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Thu, 26 Oct 2023 13:47:35 +0200 Subject: [PATCH 3/5] TASK: Legacy Asset extractor, handle if "unstructured" nodetype is not available --- .../Classes/NodeDataToAssetsProcessor.php | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Neos.ContentRepository.LegacyNodeMigration/Classes/NodeDataToAssetsProcessor.php b/Neos.ContentRepository.LegacyNodeMigration/Classes/NodeDataToAssetsProcessor.php index a1495d6174e..6f67d5ea3bf 100644 --- a/Neos.ContentRepository.LegacyNodeMigration/Classes/NodeDataToAssetsProcessor.php +++ b/Neos.ContentRepository.LegacyNodeMigration/Classes/NodeDataToAssetsProcessor.php @@ -36,15 +36,15 @@ public function run(): ProcessorResult $numberOfErrors = 0; foreach ($this->nodeDataRows as $nodeDataRow) { $nodeTypeName = NodeTypeName::fromString($nodeDataRow['nodetype']); - try { + if ($this->nodeTypeManager->hasNodeType($nodeTypeName)) { $nodeType = $this->nodeTypeManager->getNodeType($nodeTypeName); - } catch (NodeTypeNotFoundException $exception) { - $numberOfErrors ++; - $this->dispatch(Severity::ERROR, '%s. Node: "%s"', $exception->getMessage(), $nodeDataRow['identifier']); - continue; + foreach ($nodeType->getProperties() as $nodeTypePropertyName => $nodeTypePropertyValue) { + $propertyTypes[$nodeTypePropertyName] = $nodeTypePropertyValue['type'] ?? null; + } + } else { + $this->dispatch(Severity::WARNING, 'The node type "%s" of node "%s" is not available. Falling back to "string" typed properties.', $nodeTypeName->value, $nodeDataRow['identifier']); + $propertyTypes = []; } - // HACK the following line is required in order to fully initialize the node type - $nodeType->getFullConfiguration(); try { $properties = json_decode($nodeDataRow['properties'], true, 512, JSON_THROW_ON_ERROR); } catch (\JsonException $exception) { @@ -53,7 +53,7 @@ public function run(): ProcessorResult continue; } foreach ($properties as $propertyName => $propertyValue) { - $propertyType = $nodeType->getPropertyType($propertyName); + $propertyType = $propertyTypes[$propertyName] ?? 'string'; // string is the fallback, in case the property is not defined or the node type does not exist. foreach ($this->extractAssetIdentifiers($propertyType, $propertyValue) as $assetId) { if (array_key_exists($assetId, $this->processedAssetIds)) { continue; From 24a6d99e03e12338e689d874045824e1d68df05d Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Wed, 1 Nov 2023 09:29:50 +0100 Subject: [PATCH 4/5] Revert "TASK: Legacy Asset extractor, handle if "unstructured" nodetype is not available" This reverts commit 9fce857f72e3ba60bf45732e3afd1637743c0434. --- .../Classes/NodeDataToAssetsProcessor.php | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Neos.ContentRepository.LegacyNodeMigration/Classes/NodeDataToAssetsProcessor.php b/Neos.ContentRepository.LegacyNodeMigration/Classes/NodeDataToAssetsProcessor.php index e47b522337c..0d60b591cee 100644 --- a/Neos.ContentRepository.LegacyNodeMigration/Classes/NodeDataToAssetsProcessor.php +++ b/Neos.ContentRepository.LegacyNodeMigration/Classes/NodeDataToAssetsProcessor.php @@ -45,15 +45,15 @@ public function run(): ProcessorResult $numberOfErrors = 0; foreach ($this->nodeDataRows as $nodeDataRow) { $nodeTypeName = NodeTypeName::fromString($nodeDataRow['nodetype']); - if ($this->nodeTypeManager->hasNodeType($nodeTypeName)) { + try { $nodeType = $this->nodeTypeManager->getNodeType($nodeTypeName); - foreach ($nodeType->getProperties() as $nodeTypePropertyName => $nodeTypePropertyValue) { - $propertyTypes[$nodeTypePropertyName] = $nodeTypePropertyValue['type'] ?? null; - } - } else { - $this->dispatch(Severity::WARNING, 'The node type "%s" of node "%s" is not available. Falling back to "string" typed properties.', $nodeTypeName->value, $nodeDataRow['identifier']); - $propertyTypes = []; + } catch (NodeTypeNotFoundException $exception) { + $numberOfErrors ++; + $this->dispatch(Severity::ERROR, '%s. Node: "%s"', $exception->getMessage(), $nodeDataRow['identifier']); + continue; } + // HACK the following line is required in order to fully initialize the node type + $nodeType->getFullConfiguration(); try { $properties = json_decode($nodeDataRow['properties'], true, 512, JSON_THROW_ON_ERROR); } catch (\JsonException $exception) { @@ -62,7 +62,7 @@ public function run(): ProcessorResult continue; } foreach ($properties as $propertyName => $propertyValue) { - $propertyType = $propertyTypes[$propertyName] ?? 'string'; // string is the fallback, in case the property is not defined or the node type does not exist. + $propertyType = $nodeType->getPropertyType($propertyName); foreach ($this->extractAssetIdentifiers($propertyType, $propertyValue) as $assetId) { if (array_key_exists($assetId, $this->processedAssetIds)) { continue; From 7af08e48d3ee66231b139fd1cc93555d2e2ce65f Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Wed, 1 Nov 2023 10:15:39 +0100 Subject: [PATCH 5/5] TASK: Fix unstructured node handling in legacy migration --- .../Classes/NodeDataToAssetsProcessor.php | 14 +++++++------- .../Classes/NodeDataToEventsProcessor.php | 14 +++++++++++--- .../Behavior/Bootstrap/FeatureContext.php | 2 +- .../Tests/Behavior/Features/Errors.feature | 4 ++-- .../Tests/Behavior/Features/Variants.feature | 19 ++++++++++--------- 5 files changed, 31 insertions(+), 22 deletions(-) diff --git a/Neos.ContentRepository.LegacyNodeMigration/Classes/NodeDataToAssetsProcessor.php b/Neos.ContentRepository.LegacyNodeMigration/Classes/NodeDataToAssetsProcessor.php index 0d60b591cee..f96db3b58cc 100644 --- a/Neos.ContentRepository.LegacyNodeMigration/Classes/NodeDataToAssetsProcessor.php +++ b/Neos.ContentRepository.LegacyNodeMigration/Classes/NodeDataToAssetsProcessor.php @@ -44,16 +44,16 @@ public function run(): ProcessorResult { $numberOfErrors = 0; foreach ($this->nodeDataRows as $nodeDataRow) { + if ($nodeDataRow['path'] === '/sites') { + // the sites node has no properties and is unstructured + continue; + } $nodeTypeName = NodeTypeName::fromString($nodeDataRow['nodetype']); - try { - $nodeType = $this->nodeTypeManager->getNodeType($nodeTypeName); - } catch (NodeTypeNotFoundException $exception) { - $numberOfErrors ++; - $this->dispatch(Severity::ERROR, '%s. Node: "%s"', $exception->getMessage(), $nodeDataRow['identifier']); + if (!$this->nodeTypeManager->hasNodeType($nodeTypeName)) { + $this->dispatch(Severity::ERROR, 'The node type "%s" is not available. Node: "%s"', $nodeTypeName->value, $nodeDataRow['identifier']); continue; } - // HACK the following line is required in order to fully initialize the node type - $nodeType->getFullConfiguration(); + $nodeType = $this->nodeTypeManager->getNodeType($nodeTypeName); try { $properties = json_decode($nodeDataRow['properties'], true, 512, JSON_THROW_ON_ERROR); } catch (\JsonException $exception) { diff --git a/Neos.ContentRepository.LegacyNodeMigration/Classes/NodeDataToEventsProcessor.php b/Neos.ContentRepository.LegacyNodeMigration/Classes/NodeDataToEventsProcessor.php index 4f8bb63a3e9..cc3ad13609b 100644 --- a/Neos.ContentRepository.LegacyNodeMigration/Classes/NodeDataToEventsProcessor.php +++ b/Neos.ContentRepository.LegacyNodeMigration/Classes/NodeDataToEventsProcessor.php @@ -256,16 +256,24 @@ public function processNodeDataWithoutFallbackToEmptyDimension(NodeAggregateId $ $nodeName = end($pathParts); assert($nodeName !== false); $nodeTypeName = NodeTypeName::fromString($nodeDataRow['nodetype']); - $nodeType = $this->nodeTypeManager->getNodeType($nodeTypeName); - $serializedPropertyValuesAndReferences = $this->extractPropertyValuesAndReferences($nodeDataRow, $nodeType); + + $nodeType = $this->nodeTypeManager->hasNodeType($nodeTypeName) ? $this->nodeTypeManager->getNodeType($nodeTypeName) : null; $isSiteNode = $nodeDataRow['parentpath'] === '/sites'; - if ($isSiteNode && !$nodeType->isOfType(NodeTypeNameFactory::NAME_SITE)) { + if ($isSiteNode && !$nodeType?->isOfType(NodeTypeNameFactory::NAME_SITE)) { throw new MigrationException(sprintf( 'The site node "%s" (type: "%s") must be of type "%s"', $nodeDataRow['identifier'], $nodeTypeName->value, NodeTypeNameFactory::NAME_SITE ), 1695801620); } + if (!$nodeType) { + $this->dispatch(Severity::ERROR, 'The node type "%s" is not available. Node: "%s"', $nodeTypeName->value, $nodeDataRow['identifier']); + return; + } + + $nodeType = $this->nodeTypeManager->getNodeType($nodeTypeName); + $serializedPropertyValuesAndReferences = $this->extractPropertyValuesAndReferences($nodeDataRow, $nodeType); + if ($this->isAutoCreatedChildNode($parentNodeAggregate->nodeTypeName, $nodeName) && !$this->visitedNodes->containsNodeAggregate($nodeAggregateId)) { // Create tethered node if the node was not found before. // If the node was already visited, we want to create a node variant (and keep the tethering status) diff --git a/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Bootstrap/FeatureContext.php b/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Bootstrap/FeatureContext.php index 53b87836684..32b714bbbb3 100644 --- a/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Bootstrap/FeatureContext.php +++ b/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Bootstrap/FeatureContext.php @@ -193,7 +193,7 @@ public function iExpectTheFollowingEventsToBeExported(TableNode $table): void */ public function iExpectTheFollwingErrorsToBeLogged(TableNode $table): void { - Assert::assertSame($this->loggedErrors, $table->getColumn(0), 'Expected logged errors do not match'); + Assert::assertSame($table->getColumn(0), $this->loggedErrors, 'Expected logged errors do not match'); $this->loggedErrors = []; } diff --git a/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Features/Errors.feature b/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Features/Errors.feature index 33af167cd03..5058e8674d6 100644 --- a/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Features/Errors.feature +++ b/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Features/Errors.feature @@ -100,11 +100,11 @@ Feature: Exceptional cases during migrations When I have the following node data rows: | Identifier | Path | Properties | Node Type | | sites | /sites | | | - | a | /sites/a | not json | Some.Package:SomeNodeType | + | a | /sites/a | not json | Some.Package:Homepage | And I run the event migration Then I expect a MigrationError with the message """ - Failed to decode properties "not json" of node "a" (type: "Some.Package:SomeNodeType"): Could not convert database value "not json" to Doctrine Type flow_json_array + Failed to decode properties "not json" of node "a" (type: "Some.Package:Homepage"): Could not convert database value "not json" to Doctrine Type flow_json_array """ Scenario: Node variants with the same dimension diff --git a/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Features/Variants.feature b/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Features/Variants.feature index a0e35e8fc92..40ff8fca40a 100644 --- a/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Features/Variants.feature +++ b/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Features/Variants.feature @@ -11,6 +11,7 @@ Feature: Migrating nodes with content dimensions 'Some.Package:Homepage': superTypes: 'Neos.Neos:Site': true + 'Some.Package:Thing': {} """ And using identifier "default", I define a content repository And I am in content repository "default" @@ -66,10 +67,10 @@ Feature: Migrating nodes with content dimensions | Identifier | Path | Node Type | Dimension Values | | sites | /sites | unstructured | | | site | /sites/site | Some.Package:Homepage | {"language": ["de"]} | - | a | /sites/site/a | unstructured | {"language": ["de"]} | - | a1 | /sites/site/a/a1 | unstructured | {"language": ["de"]} | - | b | /sites/site/b | unstructured | {"language": ["de"]} | - | a1 | /sites/site/b/a1 | unstructured | {"language": ["ch"]} | + | a | /sites/site/a | Some.Package:Thing | {"language": ["de"]} | + | a1 | /sites/site/a/a1 | Some.Package:Thing | {"language": ["de"]} | + | b | /sites/site/b | Some.Package:Thing | {"language": ["de"]} | + | a1 | /sites/site/b/a1 | Some.Package:Thing | {"language": ["ch"]} | And I run the event migration Then I expect the following events to be exported | Type | Payload | @@ -87,11 +88,11 @@ Feature: Migrating nodes with content dimensions | Identifier | Path | Node Type | Dimension Values | | sites | /sites | unstructured | | | site | /sites/site | Some.Package:Homepage | {"language": ["de"]} | - | a | /sites/site/a | unstructured | {"language": ["de"]} | - | a1 | /sites/site/a/a1 | unstructured | {"language": ["de"]} | - | b | /sites/site/b | unstructured | {"language": ["de"]} | - | a | /sites/site/b/a | unstructured | {"language": ["ch"]} | - | a1 | /sites/site/b/a/a1 | unstructured | {"language": ["ch"]} | + | a | /sites/site/a | Some.Package:Thing | {"language": ["de"]} | + | a1 | /sites/site/a/a1 | Some.Package:Thing | {"language": ["de"]} | + | b | /sites/site/b | Some.Package:Thing | {"language": ["de"]} | + | a | /sites/site/b/a | Some.Package:Thing | {"language": ["ch"]} | + | a1 | /sites/site/b/a/a1 | Some.Package:Thing | {"language": ["ch"]} | And I run the event migration Then I expect the following events to be exported | Type | Payload |