Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add download failure details from url provided by import api to media import command #1820

Merged
Merged
3 changes: 3 additions & 0 deletions src/bin/vip-import-media.js
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ import { MediaImportProgressTracker } from '../lib/media-import/progress';
import { mediaImportCheckStatus } from '../lib/media-import/status';
import { trackEventWithEnv } from '../lib/tracker';

const API_VERSION = 'v2';

const appQuery = `
id,
name,
Expand Down Expand Up @@ -142,6 +144,7 @@ Importing Media into your App...
archiveUrl: url,
overwriteExistingFiles,
importIntermediateImages,
apiVersion: API_VERSION,
},
},
} );
Expand Down
4 changes: 4 additions & 0 deletions src/graphqlTypes.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -784,6 +784,8 @@ export type AppEnvironmentMediaImportStatus = {
__typename?: 'AppEnvironmentMediaImportStatus';
/** Media Import failure details */
failureDetails?: Maybe< AppEnvironmentMediaImportStatusFailureDetails >;
/** URL to download the media import error log */
failureDetailsUrl?: Maybe< Scalars[ 'String' ][ 'output' ] >;
/** Total number of media files that were imported */
filesProcessed?: Maybe< Scalars[ 'Int' ][ 'output' ] >;
/** Total number of media files that are to be import */
Expand Down Expand Up @@ -945,6 +947,8 @@ export type AppEnvironmentStartDbBackupCopyPayload = {

/** Mutation request input to start a Media Import */
export type AppEnvironmentStartMediaImportInput = {
/** API version to be used for the media import */
apiVersion?: InputMaybe< Scalars[ 'String' ][ 'input' ] >;
/** The unique ID of the Application */
applicationId: Scalars[ 'Int' ][ 'input' ];
/** Publicly accessible URL that contains an archive of the media files to be imported */
Expand Down
1 change: 1 addition & 0 deletions src/lib/media-import/status.generated.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ export type AppQuery = {
status?: string | null;
filesTotal?: number | null;
filesProcessed?: number | null;
failureDetailsUrl?: string | null;
failureDetails?: {
__typename?: 'AppEnvironmentMediaImportStatusFailureDetails';
previousStatus?: string | null;
Expand Down
132 changes: 105 additions & 27 deletions src/lib/media-import/status.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { ApolloClient, NormalizedCacheObject } from '@apollo/client';
import chalk from 'chalk';
import { prompt } from 'enquirer';
import gql from 'graphql-tag';
import { writeFile } from 'node:fs/promises';
import { resolve } from 'node:path';
Expand All @@ -13,7 +14,7 @@ import {
Maybe,
} from '../../graphqlTypes';
import API from '../../lib/api';
import { capitalize, formatEnvironment, formatData, RunningSprite } from '../../lib/cli/format';
import { capitalize, formatData, formatEnvironment, RunningSprite } from '../../lib/cli/format';
import {
AppForMediaImport,
currentUserCanImportForApp,
Expand All @@ -38,6 +39,7 @@ const IMPORT_MEDIA_PROGRESS_QUERY = gql`
status
filesTotal
filesProcessed
failureDetailsUrl
failureDetails {
previousStatus
globalErrors
Expand Down Expand Up @@ -271,44 +273,120 @@ ${ maybeExitPrompt }
void checkStatus( IMPORT_MEDIA_PROGRESS_POLL_INTERVAL );
} );

async function exportFailureDetails(
fileErrors: Maybe< AppEnvironmentMediaImportStatusFailureDetailsFileErrors >[]
) {
const formattedData = buildFileErrors( fileErrors, exportFileErrorsToJson );
const errorsFile = `media-import-${ app.name ?? '' }-${ Date.now() }${
exportFileErrorsToJson ? '.json' : '.txt'
}`;
try {
await writeFile( errorsFile, formattedData );
progressTracker.suffix += `${ chalk.yellow(
`⚠️ All errors have been exported to ${ chalk.bold( resolve( errorsFile ) ) }`
) }`;
} catch ( writeFileErr ) {
progressTracker.suffix += `${ chalk.red(
`Could not export errors to file\n${ ( writeFileErr as Error ).message }`
) }`;
}
}

async function fetchFailureDetails( failureDetailsUrl: string ) {
progressTracker.suffix += `
=============================================================
Downloading errors details from ${ failureDetailsUrl }...
\n`;
try {
const response = await fetch( failureDetailsUrl );
return ( await response.json() ) as AppEnvironmentMediaImportStatusFailureDetailsFileErrors[];
} catch ( err ) {
progressTracker.suffix += `${ chalk.red(
`Could not download file import errors report\n${ ( err as Error ).message }`
) }`;
throw err;
}
}

async function promptFailureDetailsDownload( failureDetailsUrl: string ) {
progressTracker.suffix += `${ chalk.yellow(
`⚠️ Error details can be found on ${ chalk.bold(
failureDetailsUrl
) }\n${ chalk.italic.yellow(
'(This link will be valid for the next 15 minutes. The report is retained for 7 days from the completion of the import.)'
) }. `
) }\n`;
progressTracker.print( { clearAfter: true } );

const failureDetails = await prompt( {
type: 'confirm',
name: 'download',
message: 'Download file import errors report now?',
} );

if ( ! failureDetails.download ) {
return;
}

const failureDetailsErrors = await fetchFailureDetails( failureDetailsUrl );
await exportFailureDetails( failureDetailsErrors );
}

function printFileErrorsReportLinkExpiredError( results: AppEnvironmentMediaImportStatus ) {
Fixed Show fixed Hide fixed
if (
results.filesTotal &&
results.filesProcessed &&
results.filesTotal !== results.filesProcessed
) {
const errorsFound = results.filesTotal - results.filesProcessed;
progressTracker.suffix += `${ chalk.yellow(
`⚠️ ${ errorsFound } error(s) were found. File import errors report link expired.`
) }`;
}
}

async function printFailureDetails(
Fixed Show fixed Hide fixed
fileErrors: Maybe< AppEnvironmentMediaImportStatusFailureDetailsFileErrors >[],
results: AppEnvironmentMediaImportStatus
) {
progressTracker.suffix += `${ chalk.yellow(
`⚠️ ${ fileErrors.length } file import error(s) were found`
) }`;

if ( ( results.filesTotal ?? 0 ) - ( results.filesProcessed ?? 0 ) !== fileErrors.length ) {
progressTracker.suffix += `. ${ chalk.italic.yellow(
'File import errors report size threshold reached.'
) }`;
}
await exportFailureDetails( fileErrors );
}

try {
const results = await getResults();
const results: AppEnvironmentMediaImportStatus = await getResults();
overallStatus = results.status ?? 'unknown';

progressTracker.stopPrinting();

setProgressTrackerSuffix();
progressTracker.print();

const fileErrors = results.failureDetails?.fileErrors ?? [];
if ( fileErrors.length > 0 ) {
progressTracker.suffix += `${ chalk.yellow(
`⚠️ ${ fileErrors.length } file error(s) have been extracted`
) }`;
if ( ( results.filesTotal ?? 0 ) - ( results.filesProcessed ?? 0 ) !== fileErrors.length ) {
progressTracker.suffix += `. ${ chalk.italic.yellow(
'File-errors report size threshold reached.'
) }`;
}
const formattedData = buildFileErrors( fileErrors, exportFileErrorsToJson );
const errorsFile = `media-import-${ app.name ?? '' }-${ Date.now() }${
exportFileErrorsToJson ? '.json' : '.txt'
}`;
try {
await writeFile( errorsFile, formattedData );
progressTracker.suffix += `\n\n${ chalk.yellow(
`All errors have been exported to ${ chalk.bold( resolve( errorsFile ) ) }`
) }\n\n`;
} catch ( writeFileErr ) {
progressTracker.suffix += `\n\n${ chalk.red(
`Could not export errors to file\n${ ( writeFileErr as Error ).message }`
) }\n\n`;
if ( results.failureDetailsUrl ) {
await promptFailureDetailsDownload( results.failureDetailsUrl as unknown as string );
} else {
const fileErrors = results.failureDetails?.fileErrors ?? [];

if ( fileErrors.length > 0 ) {
// Errors were observed and are present in the dto
// Fall back to exporting errors to local file
await printFailureDetails( fileErrors, results );
} else {
// Errors are not present in the dto
// And file error details report link is not available
printFileErrorsReportLinkExpiredError( results );
}
}

// Print one final time
progressTracker.print( { clearAfter: true } );

process.exit( 0 );
} catch ( importFailed ) {
progressTracker.stopPrinting();
Expand Down