From d79be9ac72252e9f9ce45dab39018a2c09a9b427 Mon Sep 17 00:00:00 2001 From: Daniel Lienert Date: Thu, 18 Feb 2021 17:05:54 +0100 Subject: [PATCH] FEATURE: Configure table content to be excluded from dump Define tables of which only the schema should be cloned, without the table content. This can be used to exclude large tables or tables containing confident content --- Classes/Command/CloneCommandController.php | 69 ++++++++++++++++------ Classes/Command/StashCommandController.php | 2 +- Classes/DBAL/SimpleDBAL.php | 46 +++++++++++++-- Configuration/Settings.yaml | 7 +++ README.md | 3 + 5 files changed, 102 insertions(+), 25 deletions(-) diff --git a/Classes/Command/CloneCommandController.php b/Classes/Command/CloneCommandController.php index 169126e..bf42e7e 100644 --- a/Classes/Command/CloneCommandController.php +++ b/Classes/Command/CloneCommandController.php @@ -7,6 +7,8 @@ * */ use Neos\Flow\Annotations as Flow; +use Neos\Flow\Cli\Exception\StopCommandException; +use Neos\Flow\Mvc\Exception\StopActionException; use Neos\Utility\Arrays; use Neos\Flow\Core\Bootstrap; use Sitegeist\MagicWand\DBAL\SimpleDBAL; @@ -65,7 +67,7 @@ public function listCommand() * @param boolean $yes confirm execution without further input * @param boolean $keepDb skip dropping of database during sync */ - public function defaultCommand(bool $yes = false, bool $keepDb = false) : void + public function defaultCommand(bool $yes = false, bool $keepDb = false): void { if ($this->defaultPreset === null || $this->defaultPreset === '') { $this->renderLine('There is no default preset configured!'); @@ -97,17 +99,12 @@ public function presetCommand($presetName, $yes = false, $keepDb = false) $configuration['port'], $configuration['path'], $configuration['context'], - (isset($configuration['postClone']) ? - $configuration['postClone'] : null - ), + $configuration['clone'] ?? null, + $configuration['postClone'] ?? null, $yes, $keepDb, - (isset($configuration['flowCommand']) ? - $configuration['flowCommand'] : null - ), - (isset($configuration['sshOptions']) ? - $configuration['sshOptions'] : '' - ) + $configuration['flowCommand'] ?? null, + $configuration['sshOptions'] ?? '' ); } else { $this->renderLine('The preset ' . $presetName . ' was not found!'); @@ -127,11 +124,14 @@ public function presetCommand($presetName, $yes = false, $keepDb = false) * @param string $port ssh port * @param string $path path on the remote server * @param string $context flow_context on the remote server - * @param mixded $postClone command or array of commands to be executed after cloning + * @param null $clone + * @param null $postClone command or array of commands to be executed after cloning * @param boolean $yes confirm execution without further input * @param boolean $keepDb skip dropping of database during sync - * @param string $remoteFlowCommand the flow command to execute on the remote system + * @param null $remoteFlowCommand the flow command to execute on the remote system * @param string $sshOptions additional options for the ssh command + * @throws StopCommandException + * @throws StopActionException */ protected function cloneRemoteHost( $host, @@ -139,6 +139,7 @@ protected function cloneRemoteHost( $port, $path, $context = 'Production', + $clone = null, $postClone = null, $yes = false, $keepDb = false, @@ -176,7 +177,7 @@ protected function cloneRemoteHost( ); if ($remotePersistenceConfigurationYaml) { - $remotePersistenceConfiguration = \Symfony\Component\Yaml\Yaml::parse($remotePersistenceConfigurationYaml); + $remotePersistenceConfiguration = Yaml::parse($remotePersistenceConfigurationYaml); } $remoteDataPersistentPath = $path . '/Data/Persistent'; @@ -265,6 +266,8 @@ protected function cloneRemoteHost( # Transfer Database # ###################### + $tableContentToSkip = $clone['database']['excludeTableContent'] ?? []; + $this->renderHeadLine('Transfer Database'); $this->executeLocalShellCommand( 'ssh -p %s %s %s@%s -- %s | %s', @@ -273,13 +276,14 @@ protected function cloneRemoteHost( $sshOptions, $user, $host, - $this->dbal->buildDumpCmd( + $this->dbal->buildDataDumpCmd( $remotePersistenceConfiguration['driver'], $remotePersistenceConfiguration['host'], (int)$remotePersistenceConfiguration['port'], $remotePersistenceConfiguration['user'], escapeshellcmd($remotePersistenceConfiguration['password']), - $remotePersistenceConfiguration['dbname'] + $remotePersistenceConfiguration['dbname'], + $tableContentToSkip ), $this->dbal->buildCmd( $this->databaseConfiguration['driver'], @@ -292,6 +296,35 @@ protected function cloneRemoteHost( ] ); + if (count($tableContentToSkip) > 0) { + $this->executeLocalShellCommand( + 'ssh -p %s %s %s@%s -- %s | %s', + [ + $port, + $sshOptions, + $user, + $host, + $this->dbal->buildSchemaDumpCmd( + $remotePersistenceConfiguration['driver'], + $remotePersistenceConfiguration['host'], + (int)$remotePersistenceConfiguration['port'], + $remotePersistenceConfiguration['user'], + escapeshellcmd($remotePersistenceConfiguration['password']), + $remotePersistenceConfiguration['dbname'], + $tableContentToSkip + ), + $this->dbal->buildCmd( + $this->databaseConfiguration['driver'], + $this->databaseConfiguration['host'], + (int)$this->databaseConfiguration['port'], + $this->databaseConfiguration['user'], + $this->databaseConfiguration['password'], + $this->databaseConfiguration['dbname'] + ) + ] + ); + } + ################## # Transfer Files # ################## @@ -372,7 +405,7 @@ protected function cloneRemoteHost( ################## # Set DB charset # ################## - if ($this->databaseConfiguration['driver'] == 'pdo_mysql' && $remotePersistenceConfiguration['charset'] != 'utf8mb4' ) { + if ($this->databaseConfiguration['driver'] == 'pdo_mysql' && $remotePersistenceConfiguration['charset'] != 'utf8mb4') { $this->renderHeadLine('Set DB charset'); $this->executeLocalFlowCommand('database:setcharset'); } @@ -420,9 +453,9 @@ protected function cloneRemoteHost( /** * @param $remotePersistenceConfiguration * @param $this ->databaseConfiguration - * @throws \Neos\Flow\Mvc\Exception\StopActionException + * @throws StopCommandException */ - protected function checkConfiguration($remotePersistenceConfiguration) + protected function checkConfiguration($remotePersistenceConfiguration): void { $this->renderHeadLine('Check Configuration'); if (!$this->dbal->driverIsSupported($remotePersistenceConfiguration['driver']) diff --git a/Classes/Command/StashCommandController.php b/Classes/Command/StashCommandController.php index 75b30d7..90a5993 100644 --- a/Classes/Command/StashCommandController.php +++ b/Classes/Command/StashCommandController.php @@ -90,7 +90,7 @@ public function createCommand($name) $this->renderHeadLine('Backup Database'); $this->executeLocalShellCommand( - $this->dbal->buildDumpCmd( + $this->dbal->buildDataDumpCmd( $this->databaseConfiguration['driver'], $this->databaseConfiguration['host'], (int)$this->databaseConfiguration['port'], diff --git a/Classes/DBAL/SimpleDBAL.php b/Classes/DBAL/SimpleDBAL.php index e8492b4..5840aa3 100644 --- a/Classes/DBAL/SimpleDBAL.php +++ b/Classes/DBAL/SimpleDBAL.php @@ -9,10 +9,11 @@ use Neos\Utility\Arrays; use Neos\Flow\Core\Bootstrap; -class SimpleDBAL { +class SimpleDBAL +{ /** * @param string $driver - * @param string $host + * @param string|null $host * @param int $port * @param string $username * @param string $password @@ -30,22 +31,55 @@ public function buildCmd(string $driver, ?string $host, int $port, string $usern /** * @param string $driver - * @param string $host + * @param string|null $host * @param int $port * @param string $username * @param string $password * @param string $database + * @param array $excludeTables * @return string */ - public function buildDumpCmd(string $driver, ?string $host, int $port, string $username, string $password, string $database): string + public function buildDataDumpCmd(string $driver, ?string $host, int $port, string $username, string $password, string $database, array $excludeTables = []): string { + $buildExcludeTableParameters = static function (string $parameterName) use ($excludeTables, $database) { + return implode(' ', array_map(static function (string $excludeTable) use ($parameterName, $database) { + return sprintf('%s %s.%s', $parameterName, $database, $excludeTable); + }, $excludeTables)); + }; + if ($driver === 'pdo_mysql') { - return sprintf('mysqldump --single-transaction --add-drop-table --no-tablespaces --host=%s --port=%s --user=%s --password=%s %s', escapeshellarg($host), escapeshellarg($port), escapeshellarg($username), escapeshellarg($password), escapeshellarg($database)); + return sprintf('mysqldump --single-transaction --add-drop-table --no-tablespaces --host=%s --port=%s --user=%s --password=%s %s %s', escapeshellarg($host), escapeshellarg($port), escapeshellarg($username), escapeshellarg($password), $buildExcludeTableParameters('--ignore-table'), escapeshellarg($database)); } else if ($driver === 'pdo_pgsql') { - return sprintf('PGPASSWORD=%s pg_dump --host=%s --port=%s --username=%s --dbname=%s --schema=public --no-owner --no-privileges', escapeshellarg($password), escapeshellarg($host), escapeshellarg($port), escapeshellarg($username), escapeshellarg($database)); + return sprintf('PGPASSWORD=%s pg_dump --host=%s --port=%s --username=%s %s --dbname=%s --schema=public --no-owner --no-privileges', escapeshellarg($password), escapeshellarg($host), escapeshellarg($port), escapeshellarg($username), $buildExcludeTableParameters('--exclude-table'), escapeshellarg($database)); } } + /** + * @param string $driver + * @param string|null $host + * @param int $port + * @param string $username + * @param string $password + * @param string $database + * @param array $tables + * @return string + */ + public function buildSchemaDumpCmd(string $driver, ?string $host, int $port, string $username, string $password, string $database, array $tables = []): string + { + $buildOnlyTableParameters = static function (string $parameterName = '') use ($tables) { + return implode(' ', array_map(static function (string $table) use ($parameterName) { + return trim($parameterName . ' ' . $table); + }, $tables)); + }; + + if ($driver === 'pdo_mysql') { + return sprintf('mysqldump --single-transaction --add-drop-table --no-tablespaces --no-data --host=%s --port=%s --user=%s --password=%s %s %s', escapeshellarg($host), escapeshellarg($port), escapeshellarg($username), escapeshellarg($password), escapeshellarg($database), $buildOnlyTableParameters()); + } else if ($driver === 'pdo_pgsql') { + return sprintf('PGPASSWORD=%s pg_dump --host=%s --port=%s --username=%s --dbname=%s --schema=public --no-owner --no-privileges --schema-only %s', escapeshellarg($password), escapeshellarg($host), escapeshellarg($port), escapeshellarg($username), escapeshellarg($database), $buildOnlyTableParameters('-t')); + } + } + + /** * @param string $driver * @param string $database diff --git a/Configuration/Settings.yaml b/Configuration/Settings.yaml index 60cb385..5fdf748 100644 --- a/Configuration/Settings.yaml +++ b/Configuration/Settings.yaml @@ -31,6 +31,13 @@ Sitegeist: # # default is the main flowCommand-Setting # flowCommand: ~ # +# # options to adjust the clone process +# clone: +# database: +# # List tables of which only the schema should be cloned without data. This can be used to exclude +# # large tables or tables containing confident conten +# excludeTableContent: [] +# # # commands to execute after cloning # postClone: # - './flow help' diff --git a/README.md b/README.md index 6c6d7ec..1b37645 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,9 @@ Sitegeist: # default is the main flowCommand-Setting flowCommand: ~ + # options to adjust the clone process + clone: [] + # commands to execute after cloning like ./flow user:create ... postClone: []