Skip to content

Commit

Permalink
Merge branch 'trunk' into update/nodejs_versions_and_format
Browse files Browse the repository at this point in the history 
* trunk:
  Revert "New package release: v3.9.0"
  chore: bump package version to v3.9.0
  Add comment explaining the reason why the limit default value is set manually
  Remove setImmediate
  Validate logs limit argument
  Update socket error handler
  build(deps): bump uuid from 11.0.2 to 11.0.3
  build(deps-dev): bump nock from 13.5.5 to 13.5.6
  fix(dev-env): detection of URLs to replace for multisites (#2081)
  • Loading branch information
@luiztiago
luiztiago committed Nov 13, 2024
2 parents fed73f8 + 9a00cdf commit daad20e
Show file tree
Hide file tree
Showing 8 changed files with 128 additions and 41 deletions.
11 changes: 8 additions & 3 deletions __tests__/commands/dev-env-sync-sql.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,10 @@ describe( 'commands/DevEnvSyncSQLCommand', () => {
blogId: 2,
homeUrl: 'https://subsite.com',
},
{
blogId: 3,
homeUrl: 'https://another.com/path',
},
],
},
};
Expand All @@ -56,7 +60,7 @@ describe( 'commands/DevEnvSyncSQLCommand', () => {
it( 'should return a map of search-replace values', () => {
const cmd = new DevEnvSyncSQLCommand( app, env, 'test-slug', lando );
cmd.slug = 'test-slug';
cmd.siteUrls = [ 'test.go-vip.com' ];
cmd.siteUrls = [ 'http://test.go-vip.com' ];
cmd.generateSearchReplaceMap();

expect( cmd.searchReplaceMap ).toEqual( { 'test.go-vip.com': 'test-slug.vipdev.lndo.site' } );
Expand All @@ -65,12 +69,13 @@ describe( 'commands/DevEnvSyncSQLCommand', () => {
it( 'should return a map of search-replace values for multisite', () => {
const cmd = new DevEnvSyncSQLCommand( app, msEnv, 'test-slug', lando );
cmd.slug = 'test-slug';
cmd.siteUrls = [ 'test.go-vip.com', 'subsite.com' ];
cmd.siteUrls = [ 'https://test.go-vip.com', 'http://subsite.com', 'http://another.com/path' ];
cmd.generateSearchReplaceMap();

expect( cmd.searchReplaceMap ).toEqual( {
'test.go-vip.com': 'test-slug.vipdev.lndo.site',
'subsite.com': 'subsite-com-2.test-slug.vipdev.lndo.site',
'subsite.com': 'subsite-com.test-slug.vipdev.lndo.site',
'another.com/path': 'another-com.test-slug.vipdev.lndo.site/path',
} );
} );
} );
Expand Down
28 changes: 14 additions & 14 deletions npm-shrinkwrap.json
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,7 @@
"dockerode": "^4.0.0",
"eslint": "^8.35.0",
"jest": "^29.7.0",
"nock": "13.5.5",
"nock": "13.5.6",
"prettier": "npm:[email protected]",
"rimraf": "6.0.1",
"typescript": "^5.2.2"
Expand Down Expand Up @@ -175,7 +175,7 @@
"socks-proxy-agent": "^5.0.1",
"tar": "^7.4.0",
"update-notifier": "7.3.1",
"uuid": "11.0.2",
"uuid": "11.0.3",
"xdg-basedir": "^4.0.0",
"xml2js": "^0.5.0"
},
Expand Down
8 changes: 7 additions & 1 deletion src/bin/vip-logs.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import { trackEvent } from '../lib/tracker';

const LIMIT_MIN = 1;
const LIMIT_MAX = 5000;
const LIMIT_DEFAULT = 500;
const ALLOWED_TYPES = [ 'app', 'batch' ];
const ALLOWED_FORMATS = [ 'csv', 'json', 'table' ];
const DEFAULT_POLLING_DELAY_IN_SECONDS = 30;
Expand Down Expand Up @@ -161,6 +162,10 @@ function printLogs( logs, format ) {
* @param {string} format
*/
export function validateInputs( type, limit, format ) {
if ( limit === undefined ) {
limit = LIMIT_DEFAULT;
}

if ( ! ALLOWED_TYPES.includes( type ) ) {
exit.withError(
`Invalid type: ${ type }. The supported types are: ${ ALLOWED_TYPES.join( ', ' ) }.`
Expand Down Expand Up @@ -202,7 +207,8 @@ command( {
module: 'logs',
} )
.option( 'type', 'The type of logs to be returned: "app" or "batch"', 'app' )
.option( 'limit', 'The maximum number of log lines', 500 )
// The default limit is set manually in the validateInputs function to address validation issues, avoiding incorrect replacement of the default value.
.option( 'limit', `The maximum number of log lines (defaults to ${ LIMIT_DEFAULT })` )
.option( 'follow', 'Keep fetching new logs as they are generated' )
.option( 'format', 'Output the log lines in CSV or JSON format', 'table' )
.examples( [
Expand Down
20 changes: 17 additions & 3 deletions src/bin/vip-wp.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,11 +53,25 @@ const unpipeStreamsFromProcess = ( { stdin, stdout: outStream } ) => {
};

const bindStreamEvents = ( { subShellRl, commonTrackingParams, isSubShell, stdoutStream } ) => {
const criticalErrors = [
'ECONNRESET',
'ETIMEDOUT',
'EHOSTUNREACH',
'ENOSPC',
'EACCES',
'EMFILE',
'ENOMEM',
];

stdoutStream.on( 'error', err => {
commandRunning = false;

// TODO handle this better
console.error( 'Error: ' + err.message );
if ( criticalErrors.includes( err.code ) ) {
console.error( `Error: ${ err.message }` );
} else {
// TODO handle this better
debug( 'Error: ' + err.message );
}
} );

stdoutStream.on( 'end', async () => {
Expand Down Expand Up @@ -231,7 +245,7 @@ const bindReconnectEvents = ( {
} );

currentJob.socket.io.on( 'reconnect_attempt', attempt => {
console.error( 'There was an error connecting to the server. Retrying...' );
debug( 'There was an error connecting to the server. Retrying...' );

if ( attempt > 1 ) {
return;
Expand Down
2 changes: 1 addition & 1 deletion src/commands/dev-env-import-sql.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ export class DevEnvImportSQLCommand {
* Therefore, for the things to work, we have to pretend that stdin is not a TTY :-)
*/
process.stdin.isTTY = false;
await exec( lando, this.slug, importArg, { stdio: [ fd, 'pipe', 'pipe' ] } );
await exec( lando, this.slug, importArg, { stdio: [ fd.fd, 'pipe', 'pipe' ] } );

if ( ! this.options.quiet ) {
console.log( `${ chalk.green.bold( 'Success:' ) } Database imported.` );
Expand Down
91 changes: 76 additions & 15 deletions src/commands/dev-env-sync-sql.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ import chalk from 'chalk';
import fs from 'fs';
import Lando from 'lando';
import { pipeline } from 'node:stream/promises';
import urlLib from 'url';

import { DevEnvImportSQLCommand, DevEnvImportSQLOptions } from './dev-env-import-sql';
import { ExportSQLCommand } from './export-sql';
Expand All @@ -18,6 +17,27 @@ import { fixMyDumperTransform, getSqlDumpDetails, SqlDumpType } from '../lib/dat
import { makeTempDir } from '../lib/utils';
import { getReadInterface } from '../lib/validations/line-by-line';

/**
* Replaces the domain in the given URL
*
* @param string str The URL to replace the domain in.
* @param string domain The new domain
* @return The URL with the new domain
*/
const replaceDomain = ( str: string, domain: string ): string =>
str.replace( /^([^:]+:\/\/)([^:/]+)/, `$1${ domain }` );

/**
* Strips the protocol from the URL
*
* @param string url The URL to strip the protocol from
* @return The URL without the protocol
*/
function stripProtocol( url: string ): string {
const parts = url.split( '//', 2 );
return parts.length > 1 ? parts[ 1 ] : parts[ 0 ];
}

/**
* Finds the site home url from the SQL line
*
Expand All @@ -27,8 +47,12 @@ import { getReadInterface } from '../lib/validations/line-by-line';
function findSiteHomeUrl( sql: string ): string | null {
const regex = `['"](siteurl|home)['"],\\s?['"](.*?)['"]`;
const url = sql.match( regex )?.[ 2 ] || '';

return urlLib.parse( url ).hostname || null;
try {
new URL( url );
return url;
} catch {
return null;
}
}

/**
Expand All @@ -42,17 +66,17 @@ async function extractSiteUrls( sqlFile: string ): Promise< string[] > {
const readInterface = await getReadInterface( sqlFile );

return new Promise( ( resolve, reject ) => {
const domains: Set< string > = new Set();
const urls: Set< string > = new Set();
readInterface.on( 'line', line => {
const domain = findSiteHomeUrl( line );
if ( domain ) {
domains.add( domain );
const url = findSiteHomeUrl( line );
if ( url ) {
urls.add( url );
}
} );

readInterface.on( 'close', () => {
// Soring by length so that longest domains are replaced first
resolve( Array.from( domains ).sort( ( dom1, dom2 ) => dom2.length - dom1.length ) );
// Soring by length so that longest URLs are replaced first
resolve( Array.from( urls ).sort( ( url1, url2 ) => url2.length - url1.length ) );
} );

readInterface.on( 'error', reject );
Expand Down Expand Up @@ -168,7 +192,9 @@ export class DevEnvSyncSQLCommand {
this.searchReplaceMap = {};

for ( const url of this.siteUrls ) {
this.searchReplaceMap[ url ] = this.landoDomain;
this.searchReplaceMap[ stripProtocol( url ) ] = stripProtocol(
replaceDomain( url, this.landoDomain )
);
}

const networkSites = this.env.wpSitesSDS?.nodes;
Expand All @@ -177,12 +203,18 @@ export class DevEnvSyncSQLCommand {
for ( const site of networkSites ) {
if ( ! site?.blogId || site.blogId === 1 ) continue;

const url = site?.homeUrl?.replace( /https?:\/\//, '' );
if ( ! url || ! this.searchReplaceMap[ url ] ) continue;
const url = site?.homeUrl;
if ( ! url ) continue;

const strippedUrl = stripProtocol( url );
if ( ! this.searchReplaceMap[ strippedUrl ] ) continue;

const domain = new URL( url ).hostname;
const newDomain = `${ this.slugifyDomain( domain ) }.${ this.landoDomain }`;

this.searchReplaceMap[ url ] = `${ this.slugifyDomain( url ) }-${ site.blogId }.${
this.landoDomain
}`;
this.searchReplaceMap[ stripProtocol( url ) ] = stripProtocol(
replaceDomain( url, newDomain )
);
}
}

Expand Down Expand Up @@ -213,6 +245,34 @@ export class DevEnvSyncSQLCommand {
await importCommand.run();
}

public async fixBlogsTable(): Promise< void > {
const networkSites = this.env.wpSitesSDS?.nodes;
if ( ! networkSites ) {
return;
}

const queries: string[] = [];
for ( const site of networkSites ) {
if ( ! site?.blogId || ! site?.homeUrl ) {
continue;
}

const oldDomain = new URL( site.homeUrl ).hostname;
const newDomain =
site.blogId !== 1
? `${ this.slugifyDomain( oldDomain ) }.${ this.landoDomain }`
: this.landoDomain;

queries.push(
`UPDATE wp_blogs SET domain = '${ newDomain }' WHERE blog_id = ${ Number( site.blogId ) };`
);
}

if ( queries.length ) {
await fs.promises.appendFile( this.sqlFile, queries.join( '\n' ) );
}
}

/**
* Sequentially runs the commands to export, search-replace, and import the SQL file
* to the local environment
Expand Down Expand Up @@ -273,6 +333,7 @@ export class DevEnvSyncSQLCommand {
}

await this.runSearchReplace();
await this.fixBlogsTable();
console.log( `${ chalk.green( '✓' ) } Search-replace operation is complete` );
} catch ( err ) {
const error = err as Error;
Expand Down
5 changes: 3 additions & 2 deletions src/lib/dev-environment/dev-environment-lando.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import Lando, { type LandoConfig } from 'lando/lib/lando';
import landoUtils, { type AppInfo } from 'lando/plugins/lando-core/lib/utils';
import landoBuildTask from 'lando/plugins/lando-tooling/lib/build';
import { lookup } from 'node:dns/promises';
import { FileHandle, mkdir, rename } from 'node:fs/promises';
import { mkdir, rename } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import path, { dirname } from 'node:path';
import { satisfies } from 'semver';
Expand All @@ -24,9 +24,10 @@ import UserError from '../user-error';

import type { NetworkInspectInfo } from 'dockerode';
import type Landerode from 'lando/lib/docker';
import type { StdioOptions } from 'node:child_process';

export interface LandoExecOptions {
stdio?: string | [ FileHandle, string, string ];
stdio?: StdioOptions;
}

/**
Expand Down

0 comments on commit daad20e

Please sign in to comment.