FEATURE: Configure table content to be excluded from dump

Define tables of which only the schema should be cloned, without the table content. This can be used to exclude large tables or tables containing confident content
sitegeist · Feb 23, 2021 · d79be9a · d79be9a
1 parent 6e08442
commit d79be9a
Show file tree

Hide file tree

Showing 5 changed files with 102 additions and 25 deletions.
diff --git a/Classes/Command/CloneCommandController.php b/Classes/Command/CloneCommandController.php
@@ -7,6 +7,8 @@
  *                                                                        */
 
 use Neos\Flow\Annotations as Flow;
+use Neos\Flow\Cli\Exception\StopCommandException;
+use Neos\Flow\Mvc\Exception\StopActionException;
 use Neos\Utility\Arrays;
 use Neos\Flow\Core\Bootstrap;
 use Sitegeist\MagicWand\DBAL\SimpleDBAL;
@@ -65,7 +67,7 @@ public function listCommand()
      * @param boolean $yes confirm execution without further input
      * @param boolean $keepDb skip dropping of database during sync
      */
-    public function defaultCommand(bool $yes = false, bool $keepDb = false) : void
+    public function defaultCommand(bool $yes = false, bool $keepDb = false): void
     {
         if ($this->defaultPreset === null || $this->defaultPreset === '') {
             $this->renderLine('There is no default preset configured!');
@@ -97,17 +99,12 @@ public function presetCommand($presetName, $yes = false, $keepDb = false)
                     $configuration['port'],
                     $configuration['path'],
                     $configuration['context'],
-                    (isset($configuration['postClone']) ?
-                        $configuration['postClone'] : null
-                    ),
+                    $configuration['clone'] ?? null,
+                    $configuration['postClone'] ?? null,
                     $yes,
                     $keepDb,
-                    (isset($configuration['flowCommand']) ?
-                        $configuration['flowCommand'] : null
-                    ),
-                    (isset($configuration['sshOptions']) ?
-                        $configuration['sshOptions'] : ''
-                    )
+                    $configuration['flowCommand'] ?? null,
+                    $configuration['sshOptions'] ?? ''
                 );
             } else {
                 $this->renderLine('The preset ' . $presetName . ' was not found!');
@@ -127,18 +124,22 @@ public function presetCommand($presetName, $yes = false, $keepDb = false)
      * @param string $port ssh port
      * @param string $path path on the remote server
      * @param string $context flow_context on the remote server
-     * @param mixded $postClone command or array of commands to be executed after cloning
+     * @param null $clone
+     * @param null $postClone command or array of commands to be executed after cloning
      * @param boolean $yes confirm execution without further input
      * @param boolean $keepDb skip dropping of database during sync
-     * @param string $remoteFlowCommand the flow command to execute on the remote system
+     * @param null $remoteFlowCommand the flow command to execute on the remote system
      * @param string $sshOptions additional options for the ssh command
+     * @throws StopCommandException
+     * @throws StopActionException
      */
     protected function cloneRemoteHost(
         $host,
         $user,
         $port,
         $path,
         $context = 'Production',
+        $clone = null,
         $postClone = null,
         $yes = false,
         $keepDb = false,
@@ -176,7 +177,7 @@ protected function cloneRemoteHost(
         );
 
         if ($remotePersistenceConfigurationYaml) {
-            $remotePersistenceConfiguration = \Symfony\Component\Yaml\Yaml::parse($remotePersistenceConfigurationYaml);
+            $remotePersistenceConfiguration = Yaml::parse($remotePersistenceConfigurationYaml);
         }
         $remoteDataPersistentPath = $path . '/Data/Persistent';
 
@@ -265,6 +266,8 @@ protected function cloneRemoteHost(
         #  Transfer Database #
         ######################
 
+        $tableContentToSkip = $clone['database']['excludeTableContent'] ?? [];
+
         $this->renderHeadLine('Transfer Database');
         $this->executeLocalShellCommand(
             'ssh -p %s %s %s@%s -- %s | %s',
@@ -273,13 +276,14 @@ protected function cloneRemoteHost(
                 $sshOptions,
                 $user,
                 $host,
-                $this->dbal->buildDumpCmd(
+                $this->dbal->buildDataDumpCmd(
                     $remotePersistenceConfiguration['driver'],
                     $remotePersistenceConfiguration['host'],
                     (int)$remotePersistenceConfiguration['port'],
                     $remotePersistenceConfiguration['user'],
                     escapeshellcmd($remotePersistenceConfiguration['password']),
-                    $remotePersistenceConfiguration['dbname']
+                    $remotePersistenceConfiguration['dbname'],
+                    $tableContentToSkip
                 ),
                 $this->dbal->buildCmd(
                     $this->databaseConfiguration['driver'],
@@ -292,6 +296,35 @@ protected function cloneRemoteHost(
             ]
         );
 
+        if (count($tableContentToSkip) > 0) {
+            $this->executeLocalShellCommand(
+                'ssh -p %s %s %s@%s -- %s | %s',
+                [
+                    $port,
+                    $sshOptions,
+                    $user,
+                    $host,
+                    $this->dbal->buildSchemaDumpCmd(
+                        $remotePersistenceConfiguration['driver'],
+                        $remotePersistenceConfiguration['host'],
+                        (int)$remotePersistenceConfiguration['port'],
+                        $remotePersistenceConfiguration['user'],
+                        escapeshellcmd($remotePersistenceConfiguration['password']),
+                        $remotePersistenceConfiguration['dbname'],
+                        $tableContentToSkip
+                    ),
+                    $this->dbal->buildCmd(
+                        $this->databaseConfiguration['driver'],
+                        $this->databaseConfiguration['host'],
+                        (int)$this->databaseConfiguration['port'],
+                        $this->databaseConfiguration['user'],
+                        $this->databaseConfiguration['password'],
+                        $this->databaseConfiguration['dbname']
+                    )
+                ]
+            );
+        }
+
         ##################
         # Transfer Files #
         ##################
@@ -372,7 +405,7 @@ protected function cloneRemoteHost(
         ##################
         # Set DB charset #
         ##################
-        if ($this->databaseConfiguration['driver'] == 'pdo_mysql' && $remotePersistenceConfiguration['charset'] != 'utf8mb4' ) {
+        if ($this->databaseConfiguration['driver'] == 'pdo_mysql' && $remotePersistenceConfiguration['charset'] != 'utf8mb4') {
             $this->renderHeadLine('Set DB charset');
             $this->executeLocalFlowCommand('database:setcharset');
         }
@@ -420,9 +453,9 @@ protected function cloneRemoteHost(
     /**
      * @param $remotePersistenceConfiguration
      * @param $this ->databaseConfiguration
-     * @throws \Neos\Flow\Mvc\Exception\StopActionException
+     * @throws StopCommandException
      */
-    protected function checkConfiguration($remotePersistenceConfiguration)
+    protected function checkConfiguration($remotePersistenceConfiguration): void
     {
         $this->renderHeadLine('Check Configuration');
         if (!$this->dbal->driverIsSupported($remotePersistenceConfiguration['driver'])

diff --git a/Classes/Command/StashCommandController.php b/Classes/Command/StashCommandController.php
@@ -90,7 +90,7 @@ public function createCommand($name)
         $this->renderHeadLine('Backup Database');
 
         $this->executeLocalShellCommand(
-            $this->dbal->buildDumpCmd(
+            $this->dbal->buildDataDumpCmd(
                 $this->databaseConfiguration['driver'],
                 $this->databaseConfiguration['host'],
                 (int)$this->databaseConfiguration['port'],

diff --git a/Classes/DBAL/SimpleDBAL.php b/Classes/DBAL/SimpleDBAL.php
@@ -9,10 +9,11 @@
 use Neos\Utility\Arrays;
 use Neos\Flow\Core\Bootstrap;
 
-class SimpleDBAL {
+class SimpleDBAL
+{
     /**
      * @param string $driver
-     * @param string $host
+     * @param string|null $host
      * @param int $port
      * @param string $username
      * @param string $password
@@ -30,22 +31,55 @@ public function buildCmd(string $driver, ?string $host, int $port, string $usern
 
     /**
      * @param string $driver
-     * @param string $host
+     * @param string|null $host
      * @param int $port
      * @param string $username
      * @param string $password
      * @param string $database
+     * @param array $excludeTables
      * @return string
      */
-    public function buildDumpCmd(string $driver, ?string $host, int $port, string $username, string $password, string $database): string
+    public function buildDataDumpCmd(string $driver, ?string $host, int $port, string $username, string $password, string $database, array $excludeTables = []): string
     {
+        $buildExcludeTableParameters = static function (string $parameterName) use ($excludeTables, $database) {
+            return implode(' ', array_map(static function (string $excludeTable) use ($parameterName, $database) {
+                return sprintf('%s %s.%s', $parameterName, $database, $excludeTable);
+            }, $excludeTables));
+        };
+
         if ($driver === 'pdo_mysql') {
-            return sprintf('mysqldump --single-transaction --add-drop-table --no-tablespaces --host=%s --port=%s --user=%s --password=%s %s', escapeshellarg($host), escapeshellarg($port), escapeshellarg($username), escapeshellarg($password), escapeshellarg($database));
+            return sprintf('mysqldump --single-transaction --add-drop-table --no-tablespaces --host=%s --port=%s --user=%s --password=%s %s %s', escapeshellarg($host), escapeshellarg($port), escapeshellarg($username), escapeshellarg($password), $buildExcludeTableParameters('--ignore-table'), escapeshellarg($database));
         } else if ($driver === 'pdo_pgsql') {
-            return sprintf('PGPASSWORD=%s pg_dump --host=%s --port=%s --username=%s --dbname=%s --schema=public --no-owner --no-privileges', escapeshellarg($password), escapeshellarg($host), escapeshellarg($port), escapeshellarg($username), escapeshellarg($database));
+            return sprintf('PGPASSWORD=%s pg_dump --host=%s --port=%s --username=%s  %s --dbname=%s --schema=public --no-owner --no-privileges', escapeshellarg($password), escapeshellarg($host), escapeshellarg($port), escapeshellarg($username), $buildExcludeTableParameters('--exclude-table'), escapeshellarg($database));
         }
     }
 
+    /**
+     * @param string $driver
+     * @param string|null $host
+     * @param int $port
+     * @param string $username
+     * @param string $password
+     * @param string $database
+     * @param array $tables
+     * @return string
+     */
+    public function buildSchemaDumpCmd(string $driver, ?string $host, int $port, string $username, string $password, string $database, array $tables = []): string
+    {
+        $buildOnlyTableParameters = static function (string $parameterName = '') use ($tables) {
+            return implode(' ', array_map(static function (string $table) use ($parameterName) {
+                return trim($parameterName . ' ' . $table);
+            }, $tables));
+        };
+
+        if ($driver === 'pdo_mysql') {
+            return sprintf('mysqldump --single-transaction --add-drop-table --no-tablespaces --no-data --host=%s --port=%s --user=%s --password=%s %s %s', escapeshellarg($host), escapeshellarg($port), escapeshellarg($username), escapeshellarg($password), escapeshellarg($database), $buildOnlyTableParameters());
+        } else if ($driver === 'pdo_pgsql') {
+            return sprintf('PGPASSWORD=%s pg_dump --host=%s --port=%s --username=%s --dbname=%s --schema=public --no-owner --no-privileges --schema-only %s', escapeshellarg($password), escapeshellarg($host), escapeshellarg($port), escapeshellarg($username), escapeshellarg($database), $buildOnlyTableParameters('-t'));
+        }
+    }
+
+
     /**
      * @param string $driver
      * @param string $database

diff --git a/Configuration/Settings.yaml b/Configuration/Settings.yaml
@@ -31,6 +31,13 @@ Sitegeist:
 #        # default is the main flowCommand-Setting
 #        flowCommand: ~
 #
+#        # options to adjust the clone process
+#        clone:
+#          database:
+#            # List tables of which only the schema should be cloned without data. This can be used to exclude
+#            # large tables or tables containing confident conten
+#            excludeTableContent: []
+#
 #        # commands to execute after cloning
 #        postClone:
 #         - './flow help'

diff --git a/README.md b/README.md
@@ -64,6 +64,9 @@ Sitegeist:
         # default is the main flowCommand-Setting
         flowCommand: ~
 
+        # options to adjust the clone process
+        clone: []
+
         # commands to execute after cloning like ./flow user:create ...
         postClone: []