TASK: Deterministic projection schema with sensible defaults

Projections should be created with deterministic charset and collation and columns containing the same data should behave the same to have similar conditions between installations, therefore the configured charset/collation should be applied when creating projection schemas. This includes some additional optimizations for column definitions limiting the possible contents based on the currently existing limitations in the code. Fixes: #4729
neos · Nov 16, 2023 · 460bd04 · 460bd04
1 parent f622bde
commit 460bd04
Show file tree

Hide file tree

Showing 10 changed files with 203 additions and 168 deletions.
diff --git a/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphProjection.php b/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphProjection.php
@@ -122,7 +122,7 @@ private function setupTables(): SetupResult
             throw new \RuntimeException('Failed to retrieve Schema Manager', 1625653914);
         }
 
-        $schema = (new DoctrineDbalContentGraphSchemaBuilder($this->tableNamePrefix))->buildSchema();
+        $schema = (new DoctrineDbalContentGraphSchemaBuilder($this->tableNamePrefix))->buildSchema($schemaManager);
 
         $schemaDiff = (new Comparator())->compare($schemaManager->createSchema(), $schema);
         foreach ($schemaDiff->toSaveSql($connection->getDatabasePlatform()) as $statement) {
@@ -807,7 +807,7 @@ function (NodeRecord $node) use ($eventEnvelope) {
                         'nodeanchorpoint' => $nodeAnchorPoint?->value,
                         'destinationnodeaggregateid' => $reference->targetNodeAggregateId->value,
                         'properties' => $reference->properties
-                            ? \json_encode($reference->properties, JSON_THROW_ON_ERROR)
+                            ? \json_encode($reference->properties, JSON_THROW_ON_ERROR & JSON_FORCE_OBJECT)
                             : null
                     ]);
                     $position++;

diff --git a/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphSchemaBuilder.php b/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphSchemaBuilder.php
@@ -2,22 +2,26 @@
 
 namespace Neos\ContentGraph\DoctrineDbalAdapter;
 
+use Doctrine\DBAL\Schema\AbstractSchemaManager;
 use Doctrine\DBAL\Schema\Schema;
 use Doctrine\DBAL\Types\Types;
+use Neos\ContentRepository\Core\Infrastructure\DbalSchemaFactory;
 
 /**
  * @internal
  */
 class DoctrineDbalContentGraphSchemaBuilder
 {
+    private const DEFAULT_TEXT_COLLATION = 'utf8mb4_unicode_520_ci';
+
     public function __construct(
-        private readonly string $tableNamePrefix,
+        private readonly string $tableNamePrefix
     ) {
     }
 
-    public function buildSchema(): Schema
+    public function buildSchema(AbstractSchemaManager $schemaManager): Schema
     {
-        $schema = new Schema();
+        $schema = DbalSchemaFactory::createEmptySchema($schemaManager);
 
         $this->createNodeTable($schema);
         $this->createHierarchyRelationTable($schema);
@@ -29,27 +33,19 @@ public function buildSchema(): Schema
 
     private function createNodeTable(Schema $schema): void
     {
-
         $table = $schema->createTable($this->tableNamePrefix . '_node');
-        $table->addColumn('relationanchorpoint', Types::STRING)
-            ->setLength(255)
-            ->setNotnull(true);
-        $table->addColumn('nodeaggregateid', Types::STRING)
-            ->setLength(64)
-            ->setNotnull(false);
-        $table->addColumn('origindimensionspacepoint', Types::TEXT)
-            ->setNotnull(false);
-        $table->addColumn('origindimensionspacepointhash', Types::STRING)
-            ->setLength(255)
-            ->setNotnull(false);
-        $table->addColumn('nodetypename', Types::STRING)
-            ->setLength(255)
-            ->setNotnull(true);
-        $table->addColumn('properties', Types::TEXT) // TODO longtext?
-        ->setNotnull(true);
+        $table = DbalSchemaFactory::addColumnForNodeAnchorPoint($table, 'relationanchorpoint');
+        $table = DbalSchemaFactory::addColumnForNodeAggregateId($table, 'nodeaggregateid', false);
+        $table = DbalSchemaFactory::addColumnForDimensionSpacePoint($table, 'origindimensionspacepoint', false);
+        $table = DbalSchemaFactory::addColumnForDimensionSpacePointHash($table, 'origindimensionspacepointhash', false);
+        $table = DbalSchemaFactory::addColumnForNodeTypeName($table, 'nodetypename');
+        $table->addColumn('properties', Types::TEXT)
+            ->setNotnull(true)
+            ->setCustomSchemaOption('collation', self::DEFAULT_TEXT_COLLATION);
         $table->addColumn('classification', Types::STRING)
-            ->setLength(255)
-            ->setNotnull(true);
+            ->setLength(20)
+            ->setNotnull(true)
+            ->setCustomSchemaOption('charset', 'binary');
         $table->addColumn('created', Types::DATETIME_IMMUTABLE)
             ->setDefault('CURRENT_TIMESTAMP')
             ->setNotnull(true);
@@ -73,27 +69,20 @@ private function createHierarchyRelationTable(Schema $schema): void
         $table = $schema->createTable($this->tableNamePrefix . '_hierarchyrelation');
         $table->addColumn('name', Types::STRING)
             ->setLength(255)
-            ->setNotnull(false);
+            ->setNotnull(false)
+            ->setCustomSchemaOption('collation', self::DEFAULT_TEXT_COLLATION);
         $table->addColumn('position', Types::INTEGER)
             ->setNotnull(true);
-        $table->addColumn('contentstreamid', Types::STRING)
-            ->setLength(40)
-            ->setNotnull(true);
-        $table->addColumn('dimensionspacepoint', Types::TEXT)
-            ->setNotnull(true);
-        $table->addColumn('dimensionspacepointhash', Types::STRING)
-            ->setLength(255)
-            ->setNotnull(true);
-        $table->addColumn('parentnodeanchor', Types::STRING)
-            ->setLength(255)
-            ->setNotnull(true);
-        $table->addColumn('childnodeanchor', Types::STRING)
-            ->setLength(255)
-            ->setNotnull(true);
+        $table = DbalSchemaFactory::addColumnForContentStreamId($table, 'contentstreamid', true);
+        $table = DbalSchemaFactory::addColumnForDimensionSpacePoint($table, 'dimensionspacepoint', true);
+        $table = DbalSchemaFactory::addColumnForDimensionSpacePointHash($table, 'dimensionspacepointhash', true);
+        $table = DbalSchemaFactory::addColumnForNodeAnchorPoint($table, 'parentnodeanchor');
+        $table = DbalSchemaFactory::addColumnForNodeAnchorPoint($table, 'childnodeanchor');
         $table
             ->addIndex(['childnodeanchor'])
             ->addIndex(['contentstreamid'])
             ->addIndex(['parentnodeanchor'])
+            ->addIndex(['contentstreamid', 'childnodeanchor', 'dimensionspacepointhash'])
             ->addIndex(['contentstreamid', 'dimensionspacepointhash']);
     }
 
@@ -102,17 +91,16 @@ private function createReferenceRelationTable(Schema $schema): void
         $table = $schema->createTable($this->tableNamePrefix . '_referencerelation');
         $table->addColumn('name', Types::STRING)
             ->setLength(255)
-            ->setNotnull(true);
+            ->setNotnull(true)
+            ->setCustomSchemaOption('charset', 'ascii')
+            ->setCustomSchemaOption('collation', 'ascii_general_ci');
         $table->addColumn('position', Types::INTEGER)
             ->setNotnull(true);
-        $table->addColumn('nodeanchorpoint', Types::STRING)
-            ->setLength(255)
-            ->setNotnull(true);
+        $table = DbalSchemaFactory::addColumnForNodeAnchorPoint($table, 'nodeanchorpoint');
         $table->addColumn('properties', Types::TEXT)
-            ->setNotnull(false);
-        $table->addColumn('destinationnodeaggregateid', Types::STRING)
-            ->setLength(64)
-            ->setNotnull(true);
+            ->setNotnull(false)
+            ->setCustomSchemaOption('collation', self::DEFAULT_TEXT_COLLATION);
+        $table = DbalSchemaFactory::addColumnForNodeAggregateId($table, 'destinationnodeaggregateid', true);
 
         $table
             ->setPrimaryKey(['name', 'position', 'nodeanchorpoint']);
@@ -121,18 +109,10 @@ private function createReferenceRelationTable(Schema $schema): void
     private function createRestrictionRelationTable(Schema $schema): void
     {
         $table = $schema->createTable($this->tableNamePrefix . '_restrictionrelation');
-        $table->addColumn('contentstreamid', Types::STRING)
-            ->setLength(40)
-            ->setNotnull(true);
-        $table->addColumn('dimensionspacepointhash', Types::STRING)
-            ->setLength(255)
-            ->setNotnull(true);
-        $table->addColumn('originnodeaggregateid', Types::STRING)
-            ->setLength(64)
-            ->setNotnull(true);
-        $table->addColumn('affectednodeaggregateid', Types::STRING)
-            ->setLength(64)
-            ->setNotnull(true);
+        $table = DbalSchemaFactory::addColumnForContentStreamId($table, 'contentstreamid', true);
+        $table = DbalSchemaFactory::addColumnForDimensionSpacePointHash($table, 'dimensionspacepointhash', true);
+        $table = DbalSchemaFactory::addColumnForNodeAggregateId($table, 'originnodeaggregateid', true);
+        $table = DbalSchemaFactory::addColumnForNodeAggregateId($table, 'affectednodeaggregateid', true);
 
         $table
             ->setPrimaryKey([

diff --git a/Neos.ContentRepository.Core/Classes/Infrastructure/DbalSchemaFactory.php b/Neos.ContentRepository.Core/Classes/Infrastructure/DbalSchemaFactory.php
@@ -0,0 +1,87 @@
+<?php
+namespace Neos\ContentRepository\Core\Infrastructure;
+
+use Doctrine\DBAL\Schema\AbstractSchemaManager;
+use Doctrine\DBAL\Schema\Schema;
+use Doctrine\DBAL\Schema\Table;
+use Doctrine\DBAL\Types\Types;
+
+/**
+ * Provide doctrine DBAL column schema definitions for common types in the content repository to
+ * produce consistent columns across projections.
+ *
+ * @internal Because we might need to check for platform later on and generally change the input and output format of functions within.
+ */
+final class DbalSchemaFactory
+{
+    public static function addColumnForNodeAggregateId(Table $table, string $columnName, bool $notNull): Table
+    {
+        $table->addColumn($columnName, Types::STRING)
+                ->setLength(64)
+                ->setNotnull($notNull)
+                ->setCustomSchemaOption('charset', 'ascii')
+                ->setCustomSchemaOption('collation', 'ascii_general_ci');
+
+        return $table;
+    }
+
+    public static function addColumnForContentStreamId(Table $table, string $columnName, bool $notNull): Table
+    {
+        $table->addColumn($columnName, Types::STRING)
+            ->setLength(36)
+            ->setNotnull($notNull)
+            ->setCustomSchemaOption('charset', 'binary');
+
+        return $table;
+    }
+
+    public static function addColumnForNodeAnchorPoint(Table $table, string $columnName): Table
+    {
+        $table->addColumn($columnName, Types::BINARY)
+            ->setLength(36)
+            ->setNotnull(true);
+
+        return $table;
+    }
+
+    public static function addColumnForDimensionSpacePoint(Table $table, string $columnName, bool $notNull): Table
+    {
+        $table->addColumn($columnName, Types::TEXT)
+            ->setNotnull($notNull)
+            ->setDefault('{}')
+            ->setCustomSchemaOption('collation', 'utf8mb4_unicode_520_ci');
+
+        return $table;
+    }
+
+    public static function addColumnForDimensionSpacePointHash(Table $table, string $columnName, bool $notNull): Table
+    {
+        $table->addColumn($columnName, Types::BINARY)
+            ->setLength(32)
+            ->setDefault('')
+            ->setNotnull($notNull);
+
+        return $table;
+    }
+
+    public static function addColumnForNodeTypeName(Table $table, string $columnName): Table
+    {
+        $table->addColumn($columnName, Types::STRING)
+            ->setLength(255)
+            ->setNotnull(true)
+            ->setCustomSchemaOption('charset', 'ascii')
+            ->setCustomSchemaOption('collation', 'ascii_general_ci');
+
+        return $table;
+    }
+
+    public static function createEmptySchema(AbstractSchemaManager $schemaManager): Schema
+    {
+        $schemaConfig = $schemaManager->createSchemaConfig();
+        $schemaConfig->setDefaultTableOptions([
+            'charset' => 'utf8mb4'
+        ]);
+
+        return new Schema([], [], $schemaConfig);
+    }
+}
diff --git a/Neos.ContentRepository.Core/Classes/Projection/ContentStream/ContentStreamProjection.php b/Neos.ContentRepository.Core/Classes/Projection/ContentStream/ContentStreamProjection.php
@@ -18,6 +18,7 @@
 use Doctrine\DBAL\Schema\AbstractSchemaManager;
 use Doctrine\DBAL\Schema\Comparator;
 use Doctrine\DBAL\Schema\Schema;
+use Doctrine\DBAL\Schema\SchemaConfig;
 use Doctrine\DBAL\Types\Types;
 use Neos\ContentRepository\Core\EventStore\EventInterface;
 use Neos\ContentRepository\Core\Feature\Common\EmbedsContentStreamAndNodeAggregateId;
@@ -34,6 +35,7 @@
 use Neos\ContentRepository\Core\Feature\WorkspaceRebase\Event\WorkspaceRebaseFailed;
 use Neos\ContentRepository\Core\Feature\WorkspaceRebase\Event\WorkspaceWasRebased;
 use Neos\ContentRepository\Core\Infrastructure\DbalClientInterface;
+use Neos\ContentRepository\Core\Infrastructure\DbalSchemaFactory;
 use Neos\ContentRepository\Core\Projection\ProjectionInterface;
 use Neos\ContentRepository\Core\Projection\ProjectionStateInterface;
 use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamId;
@@ -89,17 +91,14 @@ private function setupTables(): void
             $connection->executeStatement(sprintf("UPDATE %s SET state='FORKED' WHERE state='REBASING'; ", $this->tableName));
         }
 
-        $schema = new Schema();
+        $schema = DbalSchemaFactory::createEmptySchema($schemaManager);
         $contentStreamTable = $schema->createTable($this->tableName);
-        $contentStreamTable->addColumn('contentStreamId', Types::STRING)
-            ->setLength(40)
-            ->setNotnull(true);
+        $contentStreamTable = DbalSchemaFactory::addColumnForContentStreamId($contentStreamTable, 'contentStreamId', true);
         $contentStreamTable->addColumn('version', Types::INTEGER)
             ->setNotnull(true);
-        $contentStreamTable->addColumn('sourceContentStreamId', Types::STRING)
-            ->setLength(40)
-            ->setNotnull(false);
-        $contentStreamTable->addColumn('state', Types::STRING)
+        $contentStreamTable = DbalSchemaFactory::addColumnForContentStreamId($contentStreamTable, 'sourceContentStreamId', false);
+        // Should become a DB ENUM or int (latter needs adaption to code)
+        $contentStreamTable->addColumn('state', Types::BINARY)
             ->setLength(20)
             ->setNotnull(true);
         $contentStreamTable->addColumn('removed', Types::BOOLEAN)

diff --git a/Neos.ContentRepository.Core/Classes/Projection/NodeHiddenState/NodeHiddenStateProjection.php b/Neos.ContentRepository.Core/Classes/Projection/NodeHiddenState/NodeHiddenStateProjection.php
@@ -17,14 +17,14 @@
 use Doctrine\DBAL\Connection;
 use Doctrine\DBAL\Schema\AbstractSchemaManager;
 use Doctrine\DBAL\Schema\Comparator;
-use Doctrine\DBAL\Schema\Schema;
 use Doctrine\DBAL\Types\Types;
 use Neos\ContentRepository\Core\EventStore\EventInterface;
 use Neos\ContentRepository\Core\Feature\ContentStreamForking\Event\ContentStreamWasForked;
 use Neos\ContentRepository\Core\Feature\DimensionSpaceAdjustment\Event\DimensionSpacePointWasMoved;
 use Neos\ContentRepository\Core\Feature\NodeDisabling\Event\NodeAggregateWasDisabled;
 use Neos\ContentRepository\Core\Feature\NodeDisabling\Event\NodeAggregateWasEnabled;
 use Neos\ContentRepository\Core\Infrastructure\DbalClientInterface;
+use Neos\ContentRepository\Core\Infrastructure\DbalSchemaFactory;
 use Neos\ContentRepository\Core\Projection\ProjectionInterface;
 use Neos\EventStore\CatchUp\CheckpointStorageInterface;
 use Neos\EventStore\DoctrineAdapter\DoctrineCheckpointStorage;
@@ -65,19 +65,12 @@ private function setupTables(): void
         if (!$schemaManager instanceof AbstractSchemaManager) {
             throw new \RuntimeException('Failed to retrieve Schema Manager', 1625653914);
         }
-        $schema = new Schema();
+        $schema = DbalSchemaFactory::createEmptySchema($schemaManager);
         $contentStreamTable = $schema->createTable($this->tableName);
-        $contentStreamTable->addColumn('contentstreamid', Types::STRING)
-            ->setLength(40)
-            ->setNotnull(true);
-        $contentStreamTable->addColumn('nodeaggregateid', Types::STRING)
-            ->setLength(64)
-            ->setNotnull(true);
-        $contentStreamTable->addColumn('dimensionspacepointhash', Types::STRING)
-            ->setLength(255)
-            ->setNotnull(true);
-        $contentStreamTable->addColumn('dimensionspacepoint', Types::TEXT)
-            ->setNotnull(false);
+        $contentStreamTable = DbalSchemaFactory::addColumnForContentStreamId($contentStreamTable, 'contentstreamid', true);
+        $contentStreamTable = DbalSchemaFactory::addColumnForNodeAggregateId($contentStreamTable, 'nodeaggregateid', false);
+        $contentStreamTable = DbalSchemaFactory::addColumnForDimensionSpacePointHash($contentStreamTable, 'dimensionspacepointhash', false);
+        $contentStreamTable = DbalSchemaFactory::addColumnForDimensionSpacePoint($contentStreamTable, 'dimensionspacepoint', false);
         $contentStreamTable->addColumn('hidden', Types::BOOLEAN)
             ->setDefault(false)
             ->setNotnull(false);