FORNO-1704: SQL Import - Improve handling of compressed files

__tests__/bin/vip-import-sql.js

@@ -1,21 +1,23 @@
 /**
  * External dependencies
  */
+import path from 'path';
 
 /**
  * Internal dependencies
  */
-import { validateAndGetTableNames } from '../../src/bin/vip-import-sql';
+import { validateAndGetTableNames, gates } from '../../src/bin/vip-import-sql';
 import * as exit from '../../src/lib/cli/exit';
 
 jest.mock( '../../src/lib/tracker' );
 jest.mock( '../../src/lib/validations/site-type' );
 jest.mock( '../../src/lib/validations/is-multi-site' );
 jest.mock( '../../src/lib/api/feature-flags' );
 jest.spyOn( process, 'exit' ).mockImplementation( () => {} );
-jest.spyOn( exit, 'withError' );
 jest.spyOn( console, 'log' ).mockImplementation( () => {} );
 
+const mockExit = jest.spyOn( exit, 'withError' );
+
 describe( 'vip-import-sql', () => {
 	describe( 'validateAndGetTableNames', () => {
 		it( 'returns an empty array when skipValidate is true', async () => {
@@ -53,4 +55,53 @@ describe( 'vip-import-sql', () => {
 			expect( result ).toEqual( expected );
 		} );
 	} );
+
+	describe( 'gates', () => {
+		const opts = {
+			app: {
+				id: 1,
+				typeId: 2,
+				organization: {
+					id: 2,
+				},
+			},
+			env: {
+				id: 1,
+				type: 'develop',
+				importStatus: {
+					dbOperationInProgress: false,
+					importInProgress: false,
+				},
+			},
+		};
+
+		beforeEach( async () => {
+			mockExit.mockClear();
+		} );
+
+		it( 'fails if the import file has an invalid extension', async () => {
+			const invalidFilePath = path.join(
+				process.cwd(),
+				'__fixtures__',
+				'validations',
+				'empty.zip'
+			);
+
+			const fileMeta = { fileName: invalidFilePath, basename: 'empty.zip' };
+			await gates( opts.app, opts.env, fileMeta );
+			expect( mockExit ).toHaveBeenCalledWith(
+				'Invalid file extension. Please provide a .sql or .gz file.'
+			);
+		} );
+
+		it.each( [ 'bad-sql-dump.sql.gz', 'bad-sql-dump.sql' ] )(
+			'passes if the import file has a valid extension',
+			async basename => {
+				const validFilePath = path.join( process.cwd(), '__fixtures__', 'validations', basename );
+				const fileMeta = { fileName: validFilePath, basename };
+				await gates( opts.app, opts.env, fileMeta );
+				expect( mockExit ).not.toHaveBeenCalled();
+			}
+		);
+	} );
 } );

__tests__/devenv-e2e/010-import-sql.spec.js

@@ -86,47 +86,52 @@ describe( 'vip dev-env import sql', () => {
 			}
 		} );
 
-		it( 'should fail if the file fails validation', async () => {
-			const file = path.join( __dirname, '../../__fixtures__/dev-env-e2e/fail-validation.sql' );
-			const result = await cliTest.spawn(
-				[ process.argv[ 0 ], vipDevEnvImportSQL, '--slug', slug, file ],
-				{ env }
-			);
-			expect( result.rc ).toBeGreaterThan( 0 );
-			expect( result.stderr ).toContain( 'SQL Error: DROP TABLE was not found' );
-			expect( result.stderr ).toContain( 'SQL Error: CREATE TABLE was not found' );
-		} );
-
-		it( 'should allow to bypass validation', async () => {
-			const file = path.join( __dirname, '../../__fixtures__/dev-env-e2e/fail-validation.sql' );
-			let result = await cliTest.spawn(
-				[ process.argv[ 0 ], vipDevEnvImportSQL, '--slug', slug, file, '--skip-validate' ],
-				{ env },
-				true
-			);
-			expect( result.rc ).toBe( 0 );
-			expect( result.stdout ).toContain( 'Success: Database imported' );
-			expect( result.stdout ).toContain( 'Success: The cache was flushed' );
-
-			result = await cliTest.spawn(
-				[
-					process.argv[ 0 ],
-					vipDevEnvExec,
-					'--slug',
-					slug,
-					'--quiet',
-					'--',
-					'wp',
-					'option',
-					'get',
-					'e2etest',
-				],
-				{ env },
-				true
-			);
-			expect( result.rc ).toBe( 0 );
-			expect( result.stdout.trim() ).toBe( '200' );
-		} );
+		it.each( [ 'fail-validation.sql.gz', 'fail-validation.sql' ] )(
+			'should fail if the file fails validation',
+			async baseName => {
+				const file = path.join( __dirname, `../../__fixtures__/dev-env-e2e/${ baseName }` );
+				const result = await cliTest.spawn(
+					[ process.argv[ 0 ], vipDevEnvImportSQL, '--slug', slug, file ],
+					{ env }
+				);
+				expect( result.rc ).toBeGreaterThan( 0 );
+				expect( result.stderr ).toContain( 'SQL Error: DROP TABLE was not found' );
+				expect( result.stderr ).toContain( 'SQL Error: CREATE TABLE was not found' );
+			}
+		);
+
+		it.each( [ 'fail-validation.sql.gz', 'fail-validation.sql' ] )(
+			'should allow to bypass validation',
+			async baseName => {
+				const file = path.join( __dirname, `../../__fixtures__/dev-env-e2e/${ baseName }` );
+				let result = await cliTest.spawn(
+					[ process.argv[ 0 ], vipDevEnvImportSQL, '--slug', slug, file, '--skip-validate' ],
+					{ env },
+					true
+				);
+				expect( result.rc ).toBe( 0 );
+				expect( result.stdout ).toContain( 'Success: Database imported' );
+				expect( result.stdout ).toContain( 'Success: The cache was flushed' );
+				result = await cliTest.spawn(
+					[
+						process.argv[ 0 ],
+						vipDevEnvExec,
+						'--slug',
+						slug,
+						'--quiet',
+						'--',
+						'wp',
+						'option',
+						'get',
+						'e2etest',
+					],
+					{ env },
+					true
+				);
+				expect( result.rc ).toBe( 0 );
+				expect( result.stdout.trim() ).toBe( '200' );
+			}
+		);
 
 		it( 'should correctly perform replace', async () => {
 			let result = await cliTest.spawn(

__tests__/lib/validations/sql.js

@@ -134,4 +134,18 @@ describe( 'lib/validations/sql', () => {
 			);
 		} );
 	} );
+
+	describe( 'it fails when the import file is compressed', () => {
+		it( '.zip', async () => {
+			const compressedFilePath = path.join(
+				process.cwd(),
+				'__fixtures__',
+				'validations',
+				'empty.zip'
+			);
+
+			await validate( compressedFilePath );
+			expect( mockExit ).toHaveBeenCalledWith( ERROR_CODE );
+		} );
+	} );
 } );

src/bin/vip-import-sql.js

@@ -28,7 +28,12 @@ import {
 	uploadImportSqlFileToS3,
 } from '../lib/client-file-uploader';
 import { trackEventWithEnv } from '../lib/tracker';
-import { staticSqlValidations, getTableNames } from '../lib/validations/sql';
+import {
+	staticSqlValidations,
+	getTableNames,
+	validateImportFileExtension,
+	validateFilename,
+} from '../lib/validations/sql';
 import { siteTypeValidations } from '../lib/validations/site-type';
 import { searchAndReplace } from '../lib/search-and-replace';
 import API from '../lib/api';
@@ -90,11 +95,27 @@ const SQL_IMPORT_PREFLIGHT_PROGRESS_STEPS = [
 /**
  * @param {AppForImport} app
  * @param {EnvForImport} env
- * @param {string} fileName
+ * @param {FileMeta} fileMeta
  */
-export async function gates( app, env, fileName ) {
+export async function gates( app, env, fileMeta ) {
 	const { id: envId, appId } = env;
 	const track = trackEventWithEnv.bind( null, appId, envId );
+	const { fileName, basename } = fileMeta;
+
+	try {
+		// Extract base file name and exit if it contains unsafe character
+		validateFilename( basename );
+	} catch ( error ) {
+		await track( 'import_sql_command_error', { error_type: 'invalid-filename' } );
+		exit.withError( error );
+	}
+
+	try {
+		validateImportFileExtension( fileName );
+	} catch ( error ) {
+		await track( 'import_sql_command_error', { error_type: 'invalid-extension' } );
+		exit.withError( error );
+	}
 
 	if ( ! currentUserCanImportForApp( app ) ) {
 		await track( 'import_sql_command_error', { error_type: 'unauthorized' } );
@@ -264,17 +285,6 @@ If you are confident the file does not contain unsupported statements, you can r
 	return getTableNames();
 }
 
-function validateFilename( filename ) {
-	const re = /^[a-z0-9\-_.]+$/i;
-
-	// Exits if filename contains anything outside a-z A-Z - _ .
-	if ( ! re.test( filename ) ) {
-		exit.withError(
-			'Error: The characters used in the name of a file for import are limited to [0-9,a-z,A-Z,-,_,.]'
-		);
-	}
-}
-
 const displayPlaybook = ( {
 	launched,
 	tableNames,
@@ -418,16 +428,13 @@ void command( {
 		await track( 'import_sql_command_execute' );
 
 		// // halt operation of the import based on some rules
-		await gates( app, env, fileName );
+		await gates( app, env, fileMeta );
 
 		// Log summary of import details
 		const domain = env?.primaryDomain?.name ? env.primaryDomain.name : `#${ env.id }`;
 		const formattedEnvironment = formatEnvironment( opts.env.type );
 		const launched = opts.env.launched;
 
-		// Extract base file name and exit if it contains unsafe character
-		validateFilename( fileMeta.basename );
-
 		let fileNameToUpload = fileName;
 
 		// SQL file validations

src/commands/dev-env-import-sql.js

@@ -16,7 +16,10 @@ import {
 } from '../lib/dev-environment/dev-environment-core';
 import { bootstrapLando, isEnvUp } from '../lib/dev-environment/dev-environment-lando';
 import UserError from '../lib/user-error';
-import { validate as validateSQL } from '../lib/validations/sql';
+import { validate as validateSQL, validateImportFileExtension } from '../lib/validations/sql';
+import { getFileMeta, unzipFile } from '../lib/client-file-uploader';
+import { makeTempDir } from '../lib/utils';
+import * as exit from '../lib/cli/exit';
 
 export class DevEnvImportSQLCommand {
 	fileName;
@@ -34,6 +37,24 @@ export class DevEnvImportSQLCommand {
 		const lando = await bootstrapLando();
 		await validateDependencies( lando, this.slug, silent );
 
+		validateImportFileExtension( this.fileName );
+
+		// Check if file is compressed and if so, extract the
+		const fileMeta = await getFileMeta( this.fileName );
+		if ( fileMeta.isCompressed ) {
+			const tmpDir = makeTempDir();
+			const sqlFile = `${ tmpDir }/sql-import.sql`;
+
+			try {
+				console.log( `Extracting the compressed file ${ this.fileName }...` );
+				await unzipFile( this.fileName, sqlFile );
+				console.log( `${ chalk.green( '✓' ) } Extracted to ${ sqlFile }` );
+				this.fileName = sqlFile;
+			} catch ( err ) {
+				exit.withError( `Error extracting the SQL file: ${ err.message }` );
+			}
+		}
+
 		const { searchReplace, inPlace } = this.options;
 		const resolvedPath = await resolveImportPath(
 			this.slug,

src/lib/client-file-uploader.ts

@@ -9,7 +9,7 @@ import os from 'os';
 import path from 'path';
 import fetch, { HeaderInit, RequestInfo, RequestInit, Response } from 'node-fetch';
 import chalk from 'chalk';
-import { createGunzip, createGzip } from 'zlib';
+import { createGunzip, createGzip, Gunzip, ZlibOptions } from 'zlib';
 import { createHash } from 'crypto';
 import { pipeline } from 'node:stream/promises';
 import { PassThrough } from 'stream';
@@ -53,7 +53,7 @@ interface WithId {
 	id: number;
 }
 
-interface FileMeta {
+export interface FileMeta {
 	basename: string;
 	fileContent?: string | Buffer | ReadStream;
 	fileName: string;
@@ -123,9 +123,27 @@ export const unzipFile = async (
 	inputFilename: string,
 	outputFilename: string
 ): Promise< void > => {
+	const mimeType = await detectCompressedMimeType( inputFilename );
+
+	const extractFunctions: Record<
+		string,
+		{ extractor: ( options?: ZlibOptions | undefined ) => Gunzip }
+	> = {
+		'application/gzip': {
+			extractor: createGunzip,
+		},
+	};
+
+	const extractionInfo = extractFunctions[ mimeType ];
+
+	// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
+	if ( ! extractionInfo ) {
+		throw new Error( `unsupported file format: ${ mimeType }` );
+	}
+
 	const source = createReadStream( inputFilename );
 	const destination = createWriteStream( outputFilename );
-	await pipeline( source, createGunzip(), destination );
+	await pipeline( source, extractionInfo.extractor(), destination );
 };
 
 export async function getFileMeta( fileName: string ): Promise< FileMeta > {