diff --git a/__fixtures__/dev-env-e2e/mydumper-detection.expected.sql b/__fixtures__/dev-env-e2e/mydumper-detection.expected.sql new file mode 100644 index 000000000..746290981 --- /dev/null +++ b/__fixtures__/dev-env-e2e/mydumper-detection.expected.sql @@ -0,0 +1,16 @@ + +-- metadata.header -1 +# Started dump at: 2024-07-26 03:00:36 +[config] +quote_character = BACKTICK + +[myloader_session_variables] +SQL_MODE='NO_AUTO_VALUE_ON_ZERO,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION' /*!40101 + + +-- some_db-schema-create.sql -1 +/*!40101 SET NAMES utf8mb4*/; +/*!40014 SET FOREIGN_KEY_CHECKS=0*/; +/*!40101 SET SQL_MODE='NO_AUTO_VALUE_ON_ZERO,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION'*/; +/*!40103 SET TIME_ZONE='+00:00' */; +CREATE DATABASE /*!32312 IF NOT EXISTS*/ `some_db` /*!40100 DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci */ /*!80016 DEFAULT ENCRYPTION='N' */; diff --git a/__fixtures__/dev-env-e2e/mydumper-detection.sql b/__fixtures__/dev-env-e2e/mydumper-detection.sql new file mode 100644 index 000000000..b0187f4d3 --- /dev/null +++ b/__fixtures__/dev-env-e2e/mydumper-detection.sql @@ -0,0 +1,16 @@ + +-- metadata.header 198 +# Started dump at: 2024-07-26 03:00:36 +[config] +quote_character = BACKTICK + +[myloader_session_variables] +SQL_MODE='NO_AUTO_VALUE_ON_ZERO,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION' /*!40101 + + +-- some_db-schema-create.sql 370 +/*!40101 SET NAMES utf8mb4*/; +/*!40014 SET FOREIGN_KEY_CHECKS=0*/; +/*!40101 SET SQL_MODE='NO_AUTO_VALUE_ON_ZERO,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION'*/; +/*!40103 SET TIME_ZONE='+00:00' */; +CREATE DATABASE /*!32312 IF NOT EXISTS*/ `some_db` /*!40100 DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci */ /*!80016 DEFAULT ENCRYPTION='N' */; diff --git a/__fixtures__/dev-env-e2e/mydumper-detection.sql.gz b/__fixtures__/dev-env-e2e/mydumper-detection.sql.gz new file mode 100644 index 000000000..c599e7274 Binary files /dev/null and b/__fixtures__/dev-env-e2e/mydumper-detection.sql.gz differ diff --git a/__fixtures__/dev-env-e2e/mysqldump-detection.sql b/__fixtures__/dev-env-e2e/mysqldump-detection.sql new file mode 100644 index 000000000..b1f1e879b --- /dev/null +++ b/__fixtures__/dev-env-e2e/mysqldump-detection.sql @@ -0,0 +1,16 @@ +-- MySQL dump 10.13 Distrib 8.0.28, for Linux (x86_64) +-- +-- Host: localhost Database: some_db +-- ------------------------------------------------------ +-- Server version 8.0.28 + +/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; +/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; +/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; +/*!50503 SET NAMES utf8mb4 */; +/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; +/*!40103 SET TIME_ZONE='+00:00' */; +/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; +/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; +/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; +/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; diff --git a/__fixtures__/dev-env-e2e/mysqldump-detection.sql.gz b/__fixtures__/dev-env-e2e/mysqldump-detection.sql.gz new file mode 100644 index 000000000..746a9fcd7 Binary files /dev/null and b/__fixtures__/dev-env-e2e/mysqldump-detection.sql.gz differ diff --git a/__tests__/commands/dev-env-sync-sql.ts b/__tests__/commands/dev-env-sync-sql.ts index 6dac6401e..a0392447d 100644 --- a/__tests__/commands/dev-env-sync-sql.ts +++ b/__tests__/commands/dev-env-sync-sql.ts @@ -1,74 +1,24 @@ import { replace } from '@automattic/vip-search-replace'; -import fs, { ReadStream } from 'fs'; +import fs from 'fs'; import Lando from 'lando'; -import { WriteStream } from 'node:fs'; -import { Interface } from 'node:readline'; -import { PassThrough } from 'stream'; +import path from 'path'; import { DevEnvImportSQLCommand } from '../../src/commands/dev-env-import-sql'; import { DevEnvSyncSQLCommand } from '../../src/commands/dev-env-sync-sql'; import { ExportSQLCommand } from '../../src/commands/export-sql'; -import { unzipFile } from '../../src/lib/client-file-uploader'; -import { getReadInterface } from '../../src/lib/validations/line-by-line'; - -/** - * - * @param {Array<{name, data}>} eventArgs Event arguments - * @param {timeout} timeout - * - * @return {Stream} A passthrough stream - */ -function getMockStream( eventArgs: { name: string; data?: string }[], timeout = 10 ) { - const mockStream = new PassThrough(); - - if ( ! eventArgs ) { - eventArgs = [ { name: 'finish' } ]; - } - - // Leave 10ms of room for the listeners to setup - setTimeout( () => { - eventArgs.forEach( ( { name, data } ) => { - mockStream.emit( name, data ); - } ); - }, timeout ); - - return mockStream; -} - -const mockReadStream: ReadStream = getMockStream( - [ { name: 'finish' }, { name: 'data', data: 'data' } ], - 10 -) as unknown as ReadStream; -const mockWriteStream: WriteStream = getMockStream( - [ { name: 'finish' } ], - 20 -) as unknown as WriteStream; - -jest.spyOn( fs, 'createReadStream' ).mockReturnValue( mockReadStream ); -jest.spyOn( fs, 'createWriteStream' ).mockReturnValue( mockWriteStream ); -jest.spyOn( fs, 'renameSync' ).mockImplementation( () => {} ); -jest.mock( '@automattic/vip-search-replace', () => { - return { - replace: jest.fn(), - }; -} ); -jest.mock( '../../src/lib/client-file-uploader', () => { - return { - unzipFile: jest.fn(), - }; -} ); +import * as clientFileUploader from '../../src/lib/client-file-uploader'; -jest.mock( '../../src/lib/validations/line-by-line', () => { +jest.mock( '@automattic/vip-search-replace', () => { + // eslint-disable-next-line @typescript-eslint/no-var-requires + const { PassThrough } = require( 'node:stream' ) as typeof import('node:stream'); return { - getReadInterface: jest.fn(), + replace: jest.fn( ( ...args ) => { + return Promise.resolve( new PassThrough().pipe( args[ 0 ] ) ); + } ), }; } ); -jest.mocked( replace ).mockResolvedValue( mockReadStream ); -jest.mocked( unzipFile ).mockResolvedValue(); -jest - .mocked( getReadInterface ) - .mockResolvedValue( getMockStream( [ { name: 'close' } ], 100 ) as unknown as Interface ); +jest.spyOn( clientFileUploader, 'unzipFile' ); jest.spyOn( console, 'log' ).mockImplementation( () => {} ); @@ -126,17 +76,55 @@ describe( 'commands/DevEnvSyncSQLCommand', () => { } ); describe( '.runSearchReplace', () => { - it( 'should run search-replace operation on the SQL file', async () => { + it( 'should run search-replace operation on the mysqldump file', async () => { const cmd = new DevEnvSyncSQLCommand( app, env, 'test-slug', lando ); + fs.copyFileSync( + path.join( __dirname, '../../__fixtures__/dev-env-e2e/mysqldump-detection.sql.gz' ), + cmd.gzFile + ); + fs.copyFileSync( + path.join( __dirname, '../../__fixtures__/dev-env-e2e/mysqldump-detection.sql' ), + cmd.sqlFile + ); + await cmd.initSqlDumpType(); cmd.searchReplaceMap = { 'test.go-vip.com': 'test-slug.vipdev.lndo.site' }; cmd.slug = 'test-slug'; await cmd.runSearchReplace(); - expect( replace ).toHaveBeenCalledWith( mockReadStream, [ + expect( replace ).toHaveBeenCalledWith( expect.any( Object ), [ 'test.go-vip.com', 'test-slug.vipdev.lndo.site', ] ); } ); + + it( 'should run search-replace operation on the mydumper file', async () => { + const cmd = new DevEnvSyncSQLCommand( app, env, 'test-slug', lando ); + fs.copyFileSync( + path.join( __dirname, '../../__fixtures__/dev-env-e2e/mydumper-detection.sql.gz' ), + cmd.gzFile + ); + fs.copyFileSync( + path.join( __dirname, '../../__fixtures__/dev-env-e2e/mydumper-detection.sql' ), + cmd.sqlFile + ); + await cmd.initSqlDumpType(); + cmd.searchReplaceMap = { 'test.go-vip.com': 'test-slug.vipdev.lndo.site' }; + cmd.slug = 'test-slug'; + + await cmd.runSearchReplace(); + expect( replace ).toHaveBeenCalledWith( expect.any( Object ), [ + 'test.go-vip.com', + 'test-slug.vipdev.lndo.site', + ] ); + + const fileContentExpected = fs.readFileSync( + path.join( __dirname, '../../__fixtures__/dev-env-e2e/mydumper-detection.expected.sql' ), + 'utf8' + ); + const fileContent = fs.readFileSync( cmd.sqlFile, 'utf8' ); + + expect( fileContent ).toBe( fileContentExpected ); + } ); } ); describe( '.runImport', () => { @@ -159,6 +147,14 @@ describe( 'commands/DevEnvSyncSQLCommand', () => { const importSpy = jest.spyOn( syncCommand, 'runImport' ); beforeAll( () => { + fs.copyFileSync( + path.join( __dirname, '../../__fixtures__/dev-env-e2e/mysqldump-detection.sql.gz' ), + syncCommand.gzFile + ); + fs.copyFileSync( + path.join( __dirname, '../../__fixtures__/dev-env-e2e/mysqldump-detection.sql' ), + syncCommand.sqlFile + ); exportSpy.mockResolvedValue(); searchReplaceSpy.mockResolvedValue(); importSpy.mockResolvedValue(); @@ -174,7 +170,7 @@ describe( 'commands/DevEnvSyncSQLCommand', () => { await syncCommand.run(); expect( exportSpy ).toHaveBeenCalled(); - expect( unzipFile ).toHaveBeenCalled(); + expect( clientFileUploader.unzipFile ).toHaveBeenCalled(); expect( generateSearchReplaceMapSpy ).toHaveBeenCalled(); expect( searchReplaceSpy ).toHaveBeenCalled(); expect( importSpy ).toHaveBeenCalled(); diff --git a/assets/dev-env.lando.template.yml.ejs b/assets/dev-env.lando.template.yml.ejs index 450cccbd2..10118f715 100644 --- a/assets/dev-env.lando.template.yml.ejs +++ b/assets/dev-env.lando.template.yml.ejs @@ -271,6 +271,13 @@ tooling: cmd: - wp + db-myloader: + service: php + description: "Run mydumper's myloader to import database dumps generated by mydumper" + user: root + cmd: + - myloader -h database -u wordpress -p wordpress --database wordpress + db: service: php description: "Connect to the DB using mysql client (e.g. allow to run imports)" diff --git a/src/bin/vip-dev-env-import-sql.js b/src/bin/vip-dev-env-import-sql.js index 175ee8c56..66b3b047a 100755 --- a/src/bin/vip-dev-env-import-sql.js +++ b/src/bin/vip-dev-env-import-sql.js @@ -2,6 +2,7 @@ import { DevEnvImportSQLCommand } from '../commands/dev-env-import-sql'; import command from '../lib/cli/command'; +import { getSqlDumpDetails } from '../lib/database'; import { getEnvTrackingInfo, handleCLIException, @@ -66,9 +67,16 @@ command( { .argv( process.argv, async ( unmatchedArgs, opt ) => { const [ fileName ] = unmatchedArgs; const slug = await getEnvironmentName( opt ); + if ( opt.searchReplace && ! Array.isArray( opt.searchReplace ) ) { + opt.searchReplace = [ opt.searchReplace ]; + } const cmd = new DevEnvImportSQLCommand( fileName, opt, slug ); + const dumpDetails = await getSqlDumpDetails( fileName ); const trackingInfo = getEnvTrackingInfo( cmd.slug ); - const trackerFn = makeCommandTracker( 'dev_env_import_sql', trackingInfo ); + const trackerFn = makeCommandTracker( 'dev_env_import_sql', { + ...trackingInfo, + sqldump_type: dumpDetails.type, + } ); await trackerFn( 'execute' ); try { diff --git a/src/commands/dev-env-import-sql.ts b/src/commands/dev-env-import-sql.ts index cf51d14e8..6c1512012 100644 --- a/src/commands/dev-env-import-sql.ts +++ b/src/commands/dev-env-import-sql.ts @@ -1,8 +1,10 @@ import chalk from 'chalk'; import fs from 'fs'; +import os from 'os'; import * as exit from '../lib/cli/exit'; import { getFileMeta, unzipFile } from '../lib/client-file-uploader'; +import { getSqlDumpDetails, SqlDumpDetails, SqlDumpType } from '../lib/database'; import { processBooleanOption, validateDependencies, @@ -43,6 +45,9 @@ export class DevEnvImportSQLCommand { validateImportFileExtension( this.fileName ); + const dumpDetails = await getSqlDumpDetails( this.fileName ); + const isMyDumper = dumpDetails.type === SqlDumpType.MYDUMPER; + // Check if file is compressed and if so, extract the const fileMeta = await getFileMeta( this.fileName ); if ( fileMeta.isCompressed ) { @@ -83,13 +88,13 @@ export class DevEnvImportSQLCommand { const expectedDomain = `${ this.slug }.${ lando.config.domain }`; await validateSQL( resolvedPath, { isImport: false, - skipChecks: [], + skipChecks: isMyDumper ? [ 'dropTable', 'dropDB' ] : [], extraCheckParams: { siteHomeUrlLando: expectedDomain }, } ); } const fd = await fs.promises.open( resolvedPath, 'r' ); - const importArg = this.getImportArgs(); + const importArg = this.getImportArgs( dumpDetails ); const origIsTTY = process.stdin.isTTY; @@ -131,10 +136,27 @@ export class DevEnvImportSQLCommand { await addAdminUser( lando, this.slug ); } - public getImportArgs() { - const importArg = [ 'db', '--disable-auto-rehash' ].concat( + public getImportArgs( dumpDetails: SqlDumpDetails ) { + let importArg = [ 'db', '--disable-auto-rehash' ].concat( this.options.quiet ? '--silent' : [] ); + const threadCount = Math.max( os.cpus().length - 2, 1 ); + if ( dumpDetails.type === SqlDumpType.MYDUMPER ) { + importArg = [ + 'db-myloader', + '--overwrite-tables', + `--source-db=${ dumpDetails.sourceDb }`, + `--threads=${ threadCount }`, + '--max-threads-for-schema-creation=10', + '--max-threads-for-index-creation=10', + '--skip-triggers', + '--skip-post', + '--innodb-optimize-keys', + '--checksum=SKIP', + '--metadata-refresh-interval=2000000', + '--stream', + ].concat( this.options.quiet ? [ '--verbose=0' ] : [ '--verbose=3' ] ); + } return importArg; } diff --git a/src/commands/dev-env-sync-sql.ts b/src/commands/dev-env-sync-sql.ts index 12e85f525..4de7c557b 100644 --- a/src/commands/dev-env-sync-sql.ts +++ b/src/commands/dev-env-sync-sql.ts @@ -4,15 +4,17 @@ import { replace } from '@automattic/vip-search-replace'; import chalk from 'chalk'; import fs from 'fs'; import Lando from 'lando'; +import { pipeline } from 'node:stream/promises'; import urlLib from 'url'; -import { DevEnvImportSQLCommand } from './dev-env-import-sql'; +import { DevEnvImportSQLCommand, DevEnvImportSQLOptions } from './dev-env-import-sql'; import { ExportSQLCommand } from './export-sql'; import { App, AppEnvironment, Job } from '../graphqlTypes'; import { TrackFunction } from '../lib/analytics/clients/tracks'; import { BackupStorageAvailability } from '../lib/backup-storage-availability/backup-storage-availability'; import * as exit from '../lib/cli/exit'; import { unzipFile } from '../lib/client-file-uploader'; +import { fixMyDumperTransform, getSqlDumpDetails, SqlDumpType } from '../lib/database'; import { makeTempDir } from '../lib/utils'; import { getReadInterface } from '../lib/validations/line-by-line'; @@ -23,7 +25,7 @@ import { getReadInterface } from '../lib/validations/line-by-line'; * @return Site home url. null if not found */ function findSiteHomeUrl( sql: string ): string | null { - const regex = "'(siteurl|home)',\\s?'(.*?)'"; + const regex = `['"](siteurl|home)['"],\\s?['"](.*?)['"]`; const url = sql.match( regex )?.[ 2 ] || ''; return urlLib.parse( url ).hostname || null; @@ -61,7 +63,8 @@ export class DevEnvSyncSQLCommand { public tmpDir: string; public siteUrls: string[] = []; public searchReplaceMap: Record< string, string > = {}; - public track: TrackFunction; + public _track: TrackFunction; + private _sqlDumpType?: SqlDumpType; /** * Creates a new instance of the command @@ -79,22 +82,42 @@ export class DevEnvSyncSQLCommand { public lando: Lando, trackerFn: TrackFunction = () => {} ) { - this.track = trackerFn; + this._track = trackerFn; this.tmpDir = makeTempDir(); } + public track( name: string, eventProps: Record< string, unknown > ) { + return this._track( name, { + ...eventProps, + sqldump_type: this._sqlDumpType, + } ); + } + private get landoDomain(): string { return `${ this.slug }.${ this.lando.config.domain }`; } - private get sqlFile(): string { + public get sqlFile(): string { return `${ this.tmpDir }/sql-export.sql`; } - private get gzFile(): string { + public get gzFile(): string { return `${ this.tmpDir }/sql-export.sql.gz`; } + private getSqlDumpType(): SqlDumpType { + if ( ! this._sqlDumpType ) { + throw new Error( 'SQL Dump type not initialized' ); + } + + return this._sqlDumpType; + } + + public async initSqlDumpType(): Promise< void > { + const dumpDetails = await getSqlDumpDetails( this.sqlFile ); + this._sqlDumpType = dumpDetails.type; + } + private async confirmEnoughStorage( job: Job ) { const storageAvailability = BackupStorageAvailability.createFromDbCopyJob( job ); return await storageAvailability.validateAndPromptDiskSpaceWarningForDevEnvBackupImport(); @@ -109,7 +132,7 @@ export class DevEnvSyncSQLCommand { this.app, this.env, { outputFile: this.gzFile, confirmEnoughStorageHook: this.confirmEnoughStorage.bind( this ) }, - this.track + this.track.bind( this ) ); await exportCommand.run(); } @@ -127,15 +150,18 @@ export class DevEnvSyncSQLCommand { const replacedStream = await replace( readStream, replacements ); const outputFile = `${ this.tmpDir }/sql-export-sr.sql`; - replacedStream.pipe( fs.createWriteStream( outputFile ) ); + const streams: ( NodeJS.ReadableStream | NodeJS.WritableStream | NodeJS.ReadWriteStream )[] = [ + replacedStream, + ]; + if ( this.getSqlDumpType() === SqlDumpType.MYDUMPER ) { + streams.push( fixMyDumperTransform() ); + } - return new Promise( ( resolve, reject ) => { - replacedStream.on( 'finish', () => { - fs.renameSync( outputFile, this.sqlFile ); - resolve(); - } ); - replacedStream.on( 'error', reject ); - } ); + streams.push( fs.createWriteStream( outputFile ) ); + + await pipeline( streams ); + + fs.renameSync( outputFile, this.sqlFile ); } public generateSearchReplaceMap(): void { @@ -178,7 +204,7 @@ export class DevEnvSyncSQLCommand { * @throws {Error} If there is an error importing the file */ public async runImport(): Promise< void > { - const importOptions = { + const importOptions: DevEnvImportSQLOptions = { inPlace: true, skipValidate: true, quiet: true, @@ -212,6 +238,7 @@ export class DevEnvSyncSQLCommand { try { console.log( `Extracting the exported file ${ this.gzFile }...` ); await unzipFile( this.gzFile, this.sqlFile ); + await this.initSqlDumpType(); console.log( `${ chalk.green( '✓' ) } Extracted to ${ this.sqlFile }` ); } catch ( err ) { const error = err as Error; @@ -261,7 +288,6 @@ export class DevEnvSyncSQLCommand { console.log( 'Importing the SQL file...' ); await this.runImport(); console.log( `${ chalk.green( '✓' ) } SQL file imported` ); - return true; } catch ( err ) { const error = err as Error; await this.track( 'error', { @@ -271,5 +297,7 @@ export class DevEnvSyncSQLCommand { } ); exit.withError( `Error importing SQL file: ${ error.message }` ); } + + return true; } } diff --git a/src/lib/database.ts b/src/lib/database.ts new file mode 100644 index 000000000..6a878045f --- /dev/null +++ b/src/lib/database.ts @@ -0,0 +1,131 @@ +import fs from 'node:fs'; +import readline from 'node:readline'; +import { Transform, TransformCallback } from 'node:stream'; +import zlib from 'node:zlib'; + +import { createExternalizedPromise } from './promise'; + +export enum SqlDumpType { + MYDUMPER = 'MYDUMPER', + MYSQLDUMP = 'MYSQLDUMP', +} + +export interface SqlDumpDetails { + type: SqlDumpType; + sourceDb: string; +} + +export const getSqlDumpDetails = async ( filePath: string ): Promise< SqlDumpDetails > => { + const isCompressed = filePath.endsWith( '.gz' ); + let fileStream: fs.ReadStream | zlib.Gunzip; + // eslint-disable-next-line @typescript-eslint/no-invalid-void-type + const fileStreamExternalPromise = createExternalizedPromise< void >(); + + if ( isCompressed ) { + fileStream = await getSqlFileStreamFromCompressedFile( filePath ); + } else { + fileStream = fs.createReadStream( filePath ); + } + + const readLine = readline.createInterface( { + input: fileStream, + crlfDelay: Infinity, + } ); + + let isMyDumper = false; + let sourceDB = ''; + let currentLineNumber = 0; + + for await ( const line of readLine ) { + if ( line === '' ) { + continue; + } + + const metadataMatch = line.match( /^-- metadata.header / ); + + const sourceDBMatch = line.match( /^-- (.*)-schema-create.sql/ ) ?? []; + const sourceDBName = sourceDBMatch[ 1 ]; + + if ( metadataMatch && ! isMyDumper ) { + isMyDumper = true; + } + + if ( sourceDBMatch && ! sourceDB ) { + sourceDB = sourceDBName; + } + + if ( sourceDB && isMyDumper ) { + // all fields found? end the search early. + break; + } + + if ( currentLineNumber > 100 ) { + // we'll assume that this isn't the correct file if we still haven't found `-- metadata.header` even at the 100th line. + break; + } + + currentLineNumber++; + } + + if ( fileStream instanceof fs.ReadStream ) { + fileStream.on( 'close', () => { + fileStreamExternalPromise.resolve(); + } ); + } else { + fileStreamExternalPromise.resolve(); + } + + readLine.close(); + fileStream.close(); + await fileStreamExternalPromise.promise; + + return { + type: isMyDumper ? SqlDumpType.MYDUMPER : SqlDumpType.MYSQLDUMP, + sourceDb: sourceDB, + }; +}; + +const verifyFileExists = async ( filePath: string ) => { + try { + await fs.promises.access( filePath, fs.constants.F_OK ); + } catch { + throw new Error( 'File not accessible. Does file exist?' ); + } +}; + +const getSqlFileStreamFromGz = async ( filePath: string ): Promise< zlib.Gunzip > => { + await verifyFileExists( filePath ); + return fs.createReadStream( filePath ).pipe( zlib.createGunzip() ); +}; + +const getSqlFileStreamFromCompressedFile = async ( filePath: string ): Promise< zlib.Gunzip > => { + if ( filePath.endsWith( '.gz' ) ) { + return await getSqlFileStreamFromGz( filePath ); + } + + throw new Error( 'Not a supported compressed file' ); +}; + +export const fixMyDumperTransform = () => { + return new Transform( { + transform( chunk: string, _encoding: BufferEncoding, callback: TransformCallback ) { + const chunkString = chunk.toString(); + const lineEnding = chunkString.includes( '\r\n' ) ? '\r\n' : '\n'; + const regex = /^-- ([^ ]+) [0-9]+$/; + const lines = chunk + .toString() + .split( lineEnding ) + .map( line => { + const match = line.match( regex ); + + if ( ! match ) { + return line; + } + + const tablePart = match[ 1 ]; + return `-- ${ tablePart } -1`; + } ); + callback( null, lines.join( lineEnding ) ); + }, + } ); +}; diff --git a/src/lib/promise.ts b/src/lib/promise.ts new file mode 100644 index 000000000..ceaca7ac8 --- /dev/null +++ b/src/lib/promise.ts @@ -0,0 +1,22 @@ +export const createExternalizedPromise = < T >(): { + promise: Promise< T >; + resolve: ( value: T ) => void; + reject: ( reason?: Error ) => void; +} => { + let externalResolve: ( ( value: T ) => void ) | null = null; + let externalReject: ( ( reason?: Error ) => void ) | null = null; + const externalizedPromise = new Promise< T >( ( resolve, reject ) => { + externalResolve = resolve; + externalReject = reject; + } ); + + if ( ! externalReject || ! externalResolve ) { + throw new Error( "Somehow, externalReject or externalResolve didn't get set." ); + } + + return { + promise: externalizedPromise, + resolve: externalResolve, + reject: externalReject, + }; +}; diff --git a/src/lib/search-and-replace.ts b/src/lib/search-and-replace.ts index d9e708e52..6ad55946d 100644 --- a/src/lib/search-and-replace.ts +++ b/src/lib/search-and-replace.ts @@ -2,17 +2,17 @@ import { replace } from '@automattic/vip-search-replace'; import { red } from 'chalk'; import debugLib from 'debug'; import fs from 'fs'; +import { Readable, Writable } from 'node:stream'; import { pipeline } from 'node:stream/promises'; import path from 'path'; +import { fixMyDumperTransform, getSqlDumpDetails, SqlDumpType } from './database'; import { makeTempDir } from './utils'; import * as exit from '../lib/cli/exit'; import { confirm } from '../lib/cli/prompt'; import { getFileSize } from '../lib/client-file-uploader'; import { trackEvent } from '../lib/tracker'; -import type { Readable, Writable } from 'node:stream'; - const debug = debugLib( '@automattic/vip:lib:search-and-replace' ); export interface GetReadAndWriteStreamsOptions { @@ -116,7 +116,14 @@ export const searchAndReplace = async ( { isImport = true, inPlace = false, output = process.stdout }: SearchReplaceOptions, binary: string | null = null ): Promise< SearchReplaceOutput > => { - await trackEvent( 'searchreplace_started', { is_import: isImport, in_place: inPlace } ); + const dumpDetails = await getSqlDumpDetails( fileName ); + const isMyDumper = dumpDetails.type === SqlDumpType.MYDUMPER; + + await trackEvent( 'searchreplace_started', { + is_import: isImport, + in_place: inPlace, + sqldump_type: dumpDetails.type, + } ); const startTime = process.hrtime(); const fileSize = getFileSize( fileName ); @@ -146,6 +153,7 @@ export const searchAndReplace = async ( await trackEvent( 'search_replace_in_place_cancelled', { is_import: isImport, in_place: inPlace, + sqldump_type: dumpDetails.type, } ); process.exit(); } @@ -157,7 +165,7 @@ export const searchAndReplace = async ( output, } ); - let replacedStream; + let replacedStream: NodeJS.ReadableStream; try { replacedStream = await replace( readStream, replacements, binary ); } catch ( replaceError ) { @@ -165,8 +173,18 @@ export const searchAndReplace = async ( exit.withError( replaceError as string | Error ); } + const streams: ( NodeJS.ReadableStream | NodeJS.ReadWriteStream | NodeJS.WritableStream )[] = [ + replacedStream, + ]; + + if ( isMyDumper ) { + streams.push( fixMyDumperTransform() ); + } + + streams.push( writeStream ); + try { - await pipeline( replacedStream, writeStream ); + await pipeline( streams ); } catch ( error ) { console.log( red( @@ -179,7 +197,11 @@ export const searchAndReplace = async ( const endTime = process.hrtime( startTime ); const end = endTime[ 1 ] / 1000000; // time in ms - await trackEvent( 'searchreplace_completed', { time_to_run: end, file_size: fileSize } ); + await trackEvent( 'searchreplace_completed', { + time_to_run: end, + file_size: fileSize, + sqldump_type: dumpDetails.type, + } ); return { inputFileName: fileName, diff --git a/src/lib/types.ts b/src/lib/types.ts new file mode 100644 index 000000000..bab8ffd05 --- /dev/null +++ b/src/lib/types.ts @@ -0,0 +1,9 @@ +/** + * https://github.com/sindresorhus/type-fest/blob/f361912c779dfb81c10cd5fcf860f60a80358058/source/omit-index-signature.d.ts#L103C1-L107C3 + */ +export type OmitIndexSignature< ObjectType > = { + // eslint-disable-next-line @typescript-eslint/ban-types + [ KeyType in keyof ObjectType as {} extends Record< KeyType, unknown > + ? never + : KeyType ]: ObjectType[ KeyType ]; +}; diff --git a/src/lib/validations/sql.ts b/src/lib/validations/sql.ts index c8eda8690..486d3d617 100644 --- a/src/lib/validations/sql.ts +++ b/src/lib/validations/sql.ts @@ -9,6 +9,7 @@ import { type PostLineExecutionProcessingParams, getReadInterface, } from '../../lib/validations/line-by-line'; +import { OmitIndexSignature } from '../types'; let problemsFound = 0; let lineNum = 1; @@ -61,9 +62,14 @@ export interface Checks { [ key: string ]: CheckType; } +export type CheckName = keyof OmitIndexSignature< Checks >; + interface ValidationOptions { isImport: boolean; - skipChecks: string[]; + // ignoring eslint for the following - this info is still useful for code completion + // at least until we can refactor to this to be stricter. + // eslint-disable-next-line @typescript-eslint/no-redundant-type-constituents + skipChecks: ( string | CheckName )[]; extraCheckParams: Record< string, string >; } @@ -326,7 +332,7 @@ const checks: Checks = { recommendation: "Disabling 'UNIQUE_CHECKS' is not allowed. These lines should be removed", }, siteHomeUrl: { - matcher: "'(siteurl|home)',\\s?'(.*?)'", + matcher: `['"](siteurl|home)['"],\\s?['"](.*?)['"]`, matchHandler: ( lineNumber, results ) => ( { text: results[ 1 ] + ' ' + results[ 2 ] } ), outputFormatter: infoCheckFormatter, results: [], @@ -335,7 +341,7 @@ const checks: Checks = { recommendation: '', }, siteHomeUrlLando: { - matcher: "'(siteurl|home)',\\s?'([^']+)'", + matcher: `['"](siteurl|home)['"],\\s?['"]([^'"]+)['"]`, matchHandler: ( lineNumber, results, expectedDomain ) => { let foundDomain = results[ 2 ]; if ( ! /^https?:\/\//i.test( foundDomain ) ) {