Mydumper integration

assets/dev-env.lando.template.yml.ejs

@@ -271,6 +271,14 @@ tooling:
     cmd:
       - wp
 
+tooling:
+  db-myloader:
+    service: php
+    description: "Run WP-CLI command"
+    user: www-data
+    cmd:
+      - myloader -h database -u wordpress -p wordpress --database wordpress
+
   db:
     service: php
     description: "Connect to the DB using mysql client (e.g. allow to run imports)"

src/commands/dev-env-import-sql.ts

@@ -1,16 +1,18 @@
 import chalk from 'chalk';
 import fs from 'fs';
+import os from 'os';
 
 import * as exit from '../lib/cli/exit';
 import { getFileMeta, unzipFile } from '../lib/client-file-uploader';
+import { getSqlDumpDetails, SqlDumpDetails, SqlDumpType } from '../lib/database';
 import {
 	processBooleanOption,
 	validateDependencies,
 } from '../lib/dev-environment/dev-environment-cli';
 import {
+	exec,
 	getEnvironmentPath,
 	resolveImportPath,
-	exec,
 } from '../lib/dev-environment/dev-environment-core';
 import { bootstrapLando, isEnvUp } from '../lib/dev-environment/dev-environment-lando';
 import UserError from '../lib/user-error';
@@ -38,6 +40,9 @@ export class DevEnvImportSQLCommand {
 
 		validateImportFileExtension( this.fileName );
 
+		const dumpDetails = await getSqlDumpDetails( this.fileName );
+		const isMyDumper = dumpDetails.type === SqlDumpType.MYDUMPER;
+
 		// Check if file is compressed and if so, extract the
 		const fileMeta = await getFileMeta( this.fileName );
 		if ( fileMeta.isCompressed ) {
@@ -66,7 +71,7 @@ export class DevEnvImportSQLCommand {
 		const resolvedPath = await resolveImportPath(
 			this.slug,
 			this.fileName,
-			searchReplace,
+			isMyDumper ? [] : searchReplace,
 			inPlace
 		);
 
@@ -84,9 +89,8 @@ export class DevEnvImportSQLCommand {
 		}
 
 		const fd = await fs.promises.open( resolvedPath, 'r' );
-		const importArg = [ 'db', '--disable-auto-rehash' ].concat(
-			this.options.quiet ? '--silent' : []
-		);
+		const importArg = this.getImportArgs( dumpDetails );
+
 		const origIsTTY = process.stdin.isTTY;
 
 		try {
@@ -111,6 +115,23 @@ export class DevEnvImportSQLCommand {
 			fs.unlinkSync( resolvedPath );
 		}
 
+		if ( isMyDumper && searchReplace?.length ) {
+			for ( const pair of searchReplace ) {
+				const [ from, to ] = pair.split( ',' ).map( item => item.trim() );
+				// TODO: Investigate if it's worth it to refactor everything to only use wp's version of wp search-replace
+				// eslint-disable-next-line no-await-in-loop
+				await exec( lando, this.slug, [
+					'wp',
+					'search-replace',
+					'--all-tables',
+					`//${ from }`,
+					`//${ to }`,
+					'--skip-plugins',
+					'--skip-themes',
+				] );
+			}
+		}
+
 		const cacheArg = [ 'wp', 'cache', 'flush', '--skip-plugins', '--skip-themes' ].concat(
 			this.options.quiet ? '--quiet' : []
 		);
@@ -145,4 +166,29 @@ export class DevEnvImportSQLCommand {
 		].concat( this.options.quiet ? '--quiet' : [] );
 		await exec( lando, this.slug, addUserArg );
 	}
+
+	public getImportArgs( dumpDetails: SqlDumpDetails ) {
+		let importArg = [ 'db', '--disable-auto-rehash' ].concat(
+			this.options.quiet ? '--silent' : []
+		);
+		const threadCount = Math.max( os.cpus().length - 2, 1 );
+		if ( dumpDetails.type === SqlDumpType.MYDUMPER ) {
+			importArg = [
+				'db-myloader',
+				'--overwrite-tables',
+				`--threads=${ threadCount }`,
+				'--max-threads-for-schema-creation=10',
+				'--max-threads-for-index-creation=10',
+				'--skip-triggers',
+				'--skip-post',
+				'--checksum="SKIP"',
+				'--metadata-refresh-interval=2000000',
+				'--stream',
+				'--source-db',
+				dumpDetails.sourceDb,
+			].concat( this.options.quiet ? [ '--verbose=0' ] : [ '--verbose=3' ] );
+		}
+
+		return importArg;
+	}
 }

src/commands/dev-env-sync-sql.ts

@@ -13,6 +13,8 @@ import { TrackFunction } from '../lib/analytics/clients/tracks';
 import { BackupStorageAvailability } from '../lib/backup-storage-availability/backup-storage-availability';
 import * as exit from '../lib/cli/exit';
 import { unzipFile } from '../lib/client-file-uploader';
+import { getSqlDumpDetails, SqlDumpType } from '../lib/database';
+import { exec } from '../lib/dev-environment/dev-environment-core';
 import { makeTempDir } from '../lib/utils';
 import { getReadInterface } from '../lib/validations/line-by-line';
 
@@ -62,6 +64,7 @@ export class DevEnvSyncSQLCommand {
 	public siteUrls: string[] = [];
 	public searchReplaceMap: Record< string, string > = {};
 	public track: TrackFunction;
+	private _sqlDumpType?: SqlDumpType;
 
 	/**
 	 * Creates a new instance of the command
@@ -95,6 +98,19 @@ export class DevEnvSyncSQLCommand {
 		return `${ this.tmpDir }/sql-export.sql.gz`;
 	}
 
+	private getSqlDumpType(): SqlDumpType {
+		if ( ! this._sqlDumpType ) {
+			throw new Error( 'SQL Dump type not initialized' );
+		}
+
+		return this._sqlDumpType;
+	}
+
+	private async initSqlDumpType(): Promise< void > {
+		const dumpDetails = await getSqlDumpDetails( this.sqlFile );
+		this._sqlDumpType = dumpDetails.type;
+	}
+
 	private async confirmEnoughStorage( job: Job ) {
 		const storageAvailability = BackupStorageAvailability.createFromDbCopyJob( job );
 		return await storageAvailability.validateAndPromptDiskSpaceWarningForDevEnvBackupImport();
@@ -138,6 +154,23 @@ export class DevEnvSyncSQLCommand {
 		} );
 	}
 
+	public async runWPSearchReplace(): Promise< void > {
+		const replacements = this.searchReplaceMap;
+		for ( const url in replacements ) {
+			const replacement = replacements[ url ];
+			// eslint-disable-next-line no-await-in-loop
+			await exec( this.lando, this.slug, [
+				'wp',
+				'search-replace',
+				'--all-tables',
+				`//${ url }`,
+				`//${ replacement }`,
+				'--skip-plugins',
+				'--skip-themes',
+			] );
+		}
+	}
+
 	public generateSearchReplaceMap(): void {
 		this.searchReplaceMap = {};
 
@@ -212,6 +245,7 @@ export class DevEnvSyncSQLCommand {
 		try {
 			console.log( `Extracting the exported file ${ this.gzFile }...` );
 			await unzipFile( this.gzFile, this.sqlFile );
+			await this.initSqlDumpType();
 			console.log( `${ chalk.green( '✓' ) } Extracted to ${ this.sqlFile }` );
 		} catch ( err ) {
 			const error = err as Error;
@@ -239,29 +273,30 @@ export class DevEnvSyncSQLCommand {
 		console.log( 'Generating search-replace configuration...' );
 		this.generateSearchReplaceMap();
 
-		try {
-			console.log( 'Running the following search-replace operations on the SQL file:' );
-			for ( const [ domain, landoDomain ] of Object.entries( this.searchReplaceMap ) ) {
-				console.log( `  ${ domain } -> ${ landoDomain }` );
+		if ( this.getSqlDumpType() === SqlDumpType.MYSQLDUMP ) {
+			try {
+				console.log( 'Running the following search-replace operations on the SQL file:' );
+				for ( const [ domain, landoDomain ] of Object.entries( this.searchReplaceMap ) ) {
+					console.log( `  ${ domain } -> ${ landoDomain }` );
+				}
+
+				await this.runSearchReplace();
+				console.log( `${ chalk.green( '✓' ) } Search-replace operation is complete` );
+			} catch ( err ) {
+				const error = err as Error;
+				await this.track( 'error', {
+					error_type: 'search_replace',
+					error_message: error.message,
+					stack: error.stack,
+				} );
+				exit.withError( `Error replacing domains: ${ error.message }` );
 			}
-
-			await this.runSearchReplace();
-			console.log( `${ chalk.green( '✓' ) } Search-replace operation is complete` );
-		} catch ( err ) {
-			const error = err as Error;
-			await this.track( 'error', {
-				error_type: 'search_replace',
-				error_message: error.message,
-				stack: error.stack,
-			} );
-			exit.withError( `Error replacing domains: ${ error.message }` );
 		}
 
 		try {
 			console.log( 'Importing the SQL file...' );
 			await this.runImport();
 			console.log( `${ chalk.green( '✓' ) } SQL file imported` );
-			return true;
 		} catch ( err ) {
 			const error = err as Error;
 			await this.track( 'error', {
@@ -271,5 +306,27 @@ export class DevEnvSyncSQLCommand {
 			} );
 			exit.withError( `Error importing SQL file: ${ error.message }` );
 		}
+
+		if ( this.getSqlDumpType() === SqlDumpType.MYDUMPER ) {
+			try {
+				console.log( 'Running the following search-replace operations on the database:' );
+				for ( const [ domain, landoDomain ] of Object.entries( this.searchReplaceMap ) ) {
+					console.log( `  ${ domain } -> ${ landoDomain }` );
+				}
+
+				await this.runWPSearchReplace();
+				console.log( `${ chalk.green( '✓' ) } Search-replace operation is complete` );
+			} catch ( err ) {
+				const error = err as Error;
+				await this.track( 'error', {
+					error_type: 'search_replace',
+					error_message: error.message,
+					stack: error.stack,
+				} );
+				exit.withError( `Error replacing domains: ${ error.message }` );
+			}
+		}
+
+		return true;
 	}
 }