From 7360f2c2fbe2b45e61eee5be4d343109c64a8241 Mon Sep 17 00:00:00 2001 From: Wexx Date: Sat, 6 Jan 2024 18:56:39 -0800 Subject: [PATCH 1/5] util: new xivapi cli tool --- package.json | 1 + util/console_logger.ts | 4 + util/index.ts | 2 + util/query_xivapi.ts | 535 +++++++++++++++++++++++++++++++++++++++++ util/xivapi.ts | 23 +- 5 files changed, 557 insertions(+), 8 deletions(-) create mode 100644 util/query_xivapi.ts diff --git a/package.json b/package.json index aa2d443afa..241ef0057c 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,7 @@ "find-translations": "npm run util -- findTranslations", "translate-timeline": "npm run util -- translateTimeline", "generate": "npm run util -- generate", + "query": "npm run util -- query", "process-triggers": "node --loader=ts-node/esm util/process_triggers_folder.ts", "generate-log-guide": "node --loader=ts-node/esm util/gen_log_guide.ts", "validate-versions": "node --loader=ts-node/esm util/validate_versions.ts", diff --git a/util/console_logger.ts b/util/console_logger.ts index 94160b9125..7dcb021332 100644 --- a/util/console_logger.ts +++ b/util/console_logger.ts @@ -68,4 +68,8 @@ export class ConsoleLogger { console.log(`ERROR: ${msg} Exiting...`); process.exit(1); } + + printNoHeader(msg: string): void { + console.log(`${msg}`); + } } diff --git a/util/index.ts b/util/index.ts index 7d663ab258..7a8ad32ba6 100644 --- a/util/index.ts +++ b/util/index.ts @@ -4,6 +4,7 @@ import inquirerFuzzyPath, { FuzzyPathQuestionOptions } from 'inquirer-fuzzy-path import { registerFindMissingTranslations } from './find_missing_translations_action'; import { registerGenerateDataFiles } from './generate_data_files'; +import { registerQueryXivApi } from './query_xivapi'; import { registerTranslateTimeline } from './translate_timeline'; declare module 'inquirer' { @@ -34,6 +35,7 @@ const subparsers = argumentParser.addSubparsers({ registerTranslateTimeline(actionChoices, subparsers); registerGenerateDataFiles(actionChoices, subparsers); registerFindMissingTranslations(actionChoices, subparsers); +registerQueryXivApi(actionChoices, subparsers); inquirer.registerPrompt('fuzzypath', inquirerFuzzyPath); diff --git a/util/query_xivapi.ts b/util/query_xivapi.ts new file mode 100644 index 0000000000..07340da1ed --- /dev/null +++ b/util/query_xivapi.ts @@ -0,0 +1,535 @@ +// A basic CLI tool to query XIVAPI and return Typescript-formatted JSON to the console. +// Accepts -e ENDPOINT -c COLUMNS -f FILTER/S, or they can be specified through inquirer. +// Use `npm run query -- -h` for help on wildcard usage and how to format the FILTER param. + +import { Namespace, RawTextHelpFormatter, SubParser } from 'argparse'; +import inquirer from 'inquirer'; + +import { UnreachableCode } from '../resources/not_reached'; + +import { ConsoleLogger, LogLevelKey } from './console_logger'; +import { XivApi } from './xivapi'; + +import { ActionChoiceType } from '.'; + +const _LOGLEVEL: LogLevelKey = 'info'; + +// XIVAPI does not require a ?columns=xx param in the URL string, +// but without one, on a primary endpoint (e.g. Status), it will +// simply return a list of sub-nodes available (e.g. Status/1, Status/2) without meaningful data +// But specifying columns=* actually returns full data for each record in the underlying table. +const _COLUMNS_DEFAULT = '*'; + +type XivApiColumnValue = string | number | Record | null; +type XivApiQueryRecord = { + [column: string]: XivApiColumnValue; +}; +type XivApiQueryResult = XivApiQueryRecord[]; + +type QueryXivApiNamespaceInterface = { + 'endpoint': string | null; + 'columns': string | null; + 'filters': string | null; +}; + +class QueryXivApiNamespace extends Namespace implements QueryXivApiNamespaceInterface { + 'endpoint': string | null; + 'columns': string | null; + 'filters': string | null; +} + +type QueryXivApiInquirerType = { + [name in keyof QueryXivApiNamespaceInterface]: QueryXivApiNamespaceInterface[name]; +}; + +const includeOps = ['=', '~', '>', '<'] as const; +const excludeOps = ['!=', '!~'] as const; +const allOps = [...includeOps, ...excludeOps] as const; +const arrayValueOps = ['~', '!~'] as const; +const arithmeticOps = ['>', '<'] as const; + +type AllOp = typeof allOps[number]; +type IncludeOp = typeof includeOps[number]; +type ExcludeOp = typeof excludeOps[number]; +type ArrayValueOp = typeof arrayValueOps[number]; +type ArithmeticOp = typeof arithmeticOps[number]; +type IncludeStringsOp = Exclude; + +type IncludeFilter = + | { + column: string; + op: IncludeOp & ArrayValueOp; // ~ + value: string[]; + } + | { + column: string; + op: Exclude; // = + value: string; + } + | { + column: string; + op: ArithmeticOp; // <, > + value: number; + }; + +type ExcludeFilter = + | { + column: string; + op: ExcludeOp & ArrayValueOp; // !~ + value: string[]; + } + | { + column: string; + op: Exclude; // != + value: string; + }; + +type FilterSet = { + includes: IncludeFilter[]; + excludes: ExcludeFilter[]; +}; + +const isOp = (op: string): op is AllOp => { + return (allOps as readonly string[]).includes(op); +}; + +const isIncludeOp = (op: string): op is IncludeOp => { + return (includeOps as readonly string[]).includes(op); +}; + +const isArrayValueOp = (op: string): op is ArrayValueOp => { + return (arrayValueOps as readonly string[]).includes(op); +}; + +const isArithmeticOp = (op: string): op is ArithmeticOp => { + return (arithmeticOps as readonly string[]).includes(op); +}; + +const isIncludeStringsOp = (op: string): op is IncludeStringsOp => { + return (includeOps.filter((o) => !isArithmeticOp(o)) as readonly string[]).includes(op); +}; + +const stringToNum = (val: string | number): number | undefined => { + const dataNum = typeof val === 'number' ? val : parseFloat(val); + return typeof dataNum === 'number' ? dataNum : undefined; +}; + +const log = new ConsoleLogger(); +log.setLogLevel(_LOGLEVEL); + +// called by inquirer +const queryApiFunc = async (args: Namespace): Promise => { + if (!(args instanceof QueryXivApiNamespace)) + throw new UnreachableCode(); + const questions = [ + { + type: 'input', + name: 'endpoint', + message: 'Specify XIVAPI endpoint to query: ', + when: () => args.endpoint === null || args.endpoint === undefined, + }, + { + type: 'input', + name: 'columns', + message: 'Specify columns/parameters to fetch: ', + default: _COLUMNS_DEFAULT, + when: () => args.columns === null || args.columns === undefined, + }, + { + type: 'input', + name: 'filters', + message: 'Specify filter(s) to apply on resulting data: ', + }, + ] as const; + const bottomBar = new inquirer.ui.BottomBar(); + bottomBar.log.write('\nRe-run this script with -h for help with these options.'); + return inquirer.prompt(questions) + .then((answers) => { + const myEndpoint = answers.endpoint ?? args.endpoint?.toString() ?? ''; + const myColumns = answers.columns ?? args.columns?.toString() ?? _COLUMNS_DEFAULT; + const myFilters = answers.filters ?? args.filters?.toString() ?? ''; + return queryApi(myEndpoint, myColumns, myFilters); + }).catch(console.error); +}; + +// import for the util master tool +export const registerQueryXivApi = ( + actionChoices: ActionChoiceType, + subparsers: SubParser, +): void => { + actionChoices.query = { + name: 'Query XIVAPI', + callback: queryApiFunc, + namespace: QueryXivApiNamespace, + }; + + const queryParser = subparsers.addParser('query', { + description: actionChoices.query.name, + formatterClass: RawTextHelpFormatter, + epilog: ` + ENDPOINT: Specify a single XIVAPI endpoint, e.g. Pet or Status/968. + + COLUMNS: Specify a comma-separated list of columns. Whitespace is ignored. Use * for all (default). + + FILTERS: If the --filter option is used, it should be in the format [column] [operator] [value]. + Multiple filters must be separated by commas. * in a [value] acts as a wildcard. + Any [column] used for filtering must be included in the columns returned from the API. + Valid [operators] are: =, !=, >, <, ~ (in), !~ (not in). + For ~ or !~, give a comma-separated list of values inside parentheses, e.g. ~ (123, 456). + Whitespace is ignored unless it's part of an element in an array-like list. + + Example: ID > 30, Patch != 88, Name ~ (Topaz Titan, Ruby Carb*) + `, + }); + + queryParser.addArgument(['-e', '--endpoint'], { + nargs: 1, + type: 'string', + help: 'The name of the XIVAPI endpoint to query', + }); + + queryParser.addArgument(['-c', '--columns'], { + nargs: 1, + type: 'string', + help: 'Columns/parameters to obtain from the endpoint', + }); + + queryParser.addArgument(['-f', '--filter'], { + nargs: '+', + type: 'string', + help: 'Filter(s) to apply on returned data', + }); +}; + +const queryApi = async ( + endpoint: string, + columns: string, + filters: string, +): Promise => { + const outputToConsole = (result: XivApiQueryResult): void => { + log.printNoHeader(JSON.stringify(result, null, 2).replace(/"/g, '\'')); + }; + + const _ENDPOINT = endpoint.trim(); + if (endpoint.length < 1) { + log.fatalError('No endpoint specified.'); + return; + } + + const _COLUMNS = columns.replace(/\s+/g, '').split(',').filter((c) => c !== ''); + if (_COLUMNS.length < 1 || _COLUMNS[0] === undefined) { + log.fatalError('No columns specified.'); + return; + } + + const api = new XivApi(null, log); + + const apiData = await api.queryApi( + _ENDPOINT, + _COLUMNS, + ) as XivApiQueryResult; + + if (filters === '') { + log.debug('No filters were specified.'); + if (apiData.length === 0) + log.printNoHeader('No API data matched your query.'); + else + outputToConsole(apiData); + } else { + log.debug('Applying specified filters to data...'); + const filteredData = applyFilters(apiData, parseFilters(filters)); + if (filteredData.length === 0) + log.printNoHeader('No API data matched your query & filter conditions.'); + else + outputToConsole(filteredData); + } +}; + +const parseFilters = (filters: string): FilterSet => { + const parsedFilters: FilterSet = { + includes: [], + excludes: [], + }; + + // filters are comma-separated, but don't count commas inside (); that's an array. + const regexFilterSeparator = /(? f !== ''); + + const regexColumn = /^\s*(\w+)\s*/; + const regexOp = new RegExp(`(${allOps.join('|')})`); + const regexValueArr = /\s*[([][\w\s*]+,[\w\s*]+[,\w\s*]*[)\]]\s*/; + const regexValueStr = /[\w\s*]+/; + const regexValueEither = new RegExp(`(${regexValueStr.source}$|${regexValueArr.source}$)`); + const regexFilterParts = new RegExp( + `${regexColumn.source}${regexOp.source}${regexValueEither.source}`, + ); + + rawFilters.forEach((rf) => { + const filterParts = rf.match(regexFilterParts); + if (filterParts !== null && filterParts.length === 4) { + const [/* full match */, col, op, val] = filterParts; + if (col === undefined || op === undefined || val === undefined) { + log.fatalError(`Could not parse filter: ${rf}.`); + return; + } + if (col === '') { + log.fatalError(`Invalid column name (${col}) in filter (${rf}).`); + return; + } + if (!isOp(op)) { + log.fatalError(`Invalid operator (${op}) in filter (${rf}).`); + return; + } + if (val.trim() === '') { + log.fatalError(`Invalid value (${val}) in filter (${rf}).`); + return; + } + + let valArr; + if (isArrayValueOp(op)) { // ~, !~ + if (val.match(new RegExp(`^${regexValueArr.source}$`)) === null) { + log.fatalError( + `Invalid filter type (${rf}) specified for operator (${op}). Must be array-like string list (x,y).`, + ); + return; + } + valArr = val.replace(/[()]/g, '').trim().split(',').map((e) => e.trim().toString()); + if (valArr.length < 2 || valArr[0] === '') { + log.fatalError(`Could not extract array from array-like string filter (${rf}).`); + return; + } + if (isIncludeOp(op)) + parsedFilters.includes.push({ + column: col, + op: op, + value: valArr, + }); + else + parsedFilters.excludes.push({ + column: col, + op: op, + value: valArr, + }); + } else if (isArithmeticOp(op)) { // <, > + const trimNumVal = stringToNum(val.trim()); + if (trimNumVal === undefined) { + log.fatalError(`Cannot use numerical operator ('${op}') with text filter (${val}).`); + return; + } + parsedFilters.includes.push({ + column: col, + op: op, + value: trimNumVal, + }); + } else { // =, != + const trimVal = val.trim().toString(); + if (trimVal.match(new RegExp(`^${regexValueArr.source}$`)) !== null) { + log.fatalError(`Cannot use array-like filter (${rf}) with operator (${op}).`); + return; + } + if (isIncludeOp(op)) + parsedFilters.includes.push({ + column: col, + op: op, + value: trimVal, + }); + else + parsedFilters.excludes.push({ + column: col, + op: op, + value: trimVal, + }); + } + } else { + log.fatalError(`Could not parse filter (${rf}).`); + return; + } + }); + return parsedFilters; +}; + +const applyFilters = ( + data: XivApiQueryResult, + filters: FilterSet, +): XivApiQueryResult => { + // Type definitions already ensure that the filter value is appropriate for the operator type. + // But TypeScript doesn't seem to respect op<->value type limits when typeguarding just the op. + // So, separately typeguard the value based on the given op. + const isStringsFilter = ( + str: string | string[] | number, + op: AllOp, + ): str is string | string[] => { + return (allOps.filter((op) => !isArithmeticOp(op)) as readonly string[]).includes(op); + }; + + const isValidRecordData = (data?: XivApiColumnValue): data is string | number | null => { + // return false if undefined or a non-null pure object (null object is okay) + return data !== undefined && + ( + data === null || + ( + typeof data !== 'object' && + Object.getPrototypeOf(data) !== Object.prototype + ) + ); + }; + + const matchesWildcardRegex = (filterVal: string, recordData: string) => { + const valToRegex = (val: string) => val.replace(/([.+?^=!:${}|\-\[\]\/\\])/g, '\\$1'); + return new RegExp(`^${filterVal.split('*').map(valToRegex).join('.*')}$`).test(recordData); + }; + + const textValueMatch = (filters: string | string[], recordLoose: string | number): boolean => { + const record = recordLoose.toString(); + if (Array.isArray(filters)) { + if (filters.includes(record)) + return true; + for (const f of filters) { + if (f.includes('*') && matchesWildcardRegex(f, record)) + return true; + } + } else if (filters === record || matchesWildcardRegex(filters, record)) + return true; + return false; + }; + + const numValueMatch = (op: ArithmeticOp, filter: number, record: number): boolean => { + return (op === '>' && record > filter) || (op === '<' && record < filter); + }; + + const filteredData: XivApiQueryResult = []; + // Process include filters first, then exclude filters. + // If a record matches both types, it is excluded. + for (const record of data) { + // Start by assuming we'll include the record (if no include filters, we move on to excludes). + let includeRecord = true; + for (const iFilter of filters.includes) { + // at least 1 include filter, so loop and each time assume no match until there is one. + includeRecord = false; + const column = iFilter.column; + const op = iFilter.op; + const filterVal = iFilter.value; + log.debug(`Checking against include filter: ${column} ${op} ${filterVal.toString()}`); + + // since we don't have a schema for the user's query, recordDataToMatch could be: + // - a string or number + // - an object (the column may be a link to another endpoint/table, + // in which case XIVAPI will reproduce the associated record as a nested object) + // - null, if there's no data for that column + // (unless that's the filter value, that's OK for excludes, but not OK for includes) + // - undefined, if the user forgot to include the column in their query + const recordDataToMatch = record[column]; + if (!isValidRecordData(recordDataToMatch)) { // filter out objects & undefined + log.fatalError( + `Cannot apply filter(s) on column ${column}: column not in query set or does not contain a filterable value.`, + ); + break; // Not necessary, but TypeScript doesn't know that + } + + // Special case for 'null' - if it's a filter value, check the column for a match + // otherwise, if not a filter value and the column is null, the include filter fails + if (recordDataToMatch === null) { + // treat the record data as a string ('null') for matching, + // as that's how it would be stored in filters{}. + if ( + isIncludeStringsOp(op) && + isStringsFilter(filterVal, op) && + textValueMatch(filterVal, 'null') + ) { + log.debug(`Matched include filter for ${column}: Filter: null Record: null`); + includeRecord = true; + continue; + } else { + includeRecord = false; + break; + } + } + + if ( + isIncludeStringsOp(op) && + isStringsFilter(filterVal, op) && + textValueMatch(filterVal, recordDataToMatch) + ) { + log.debug( + `Matched include filter for ${column}: (Filter: ${op} ${filterVal.toString()} | Record: ${recordDataToMatch})`, + ); + includeRecord = true; + } else if (isArithmeticOp(op) && !isStringsFilter(filterVal, op)) { + const recordDataToMatchAsNum = stringToNum(recordDataToMatch); + if (recordDataToMatchAsNum === undefined) { + log.fatalError( + `Column value (${column}) is non-numerical, and cannot be used with ${op} operator.`, + ); + break; + } + if (numValueMatch(op, filterVal, recordDataToMatchAsNum)) { + log.debug( + `Matched include filter for ${column}: (Filter: ${op} ${filterVal.toString()} | Record: ${recordDataToMatch})`, + ); + includeRecord = true; + } + } else + log.debug( + `No match: (Filter: ${op} ${filterVal.toString()} | Record: ${recordDataToMatch}).`, + ); + + // if we haven't passed this filter, no need to continue cheking include filters + if (includeRecord === false) { + log.debug('Did not match include filter.'); + break; + } + } + + // if we failed somewhere during include filtering, skip this record + if (includeRecord === false) { + log.debug('Failed to match all include filters. Record excluded.'); + continue; + } + + for (const eFilter of filters.excludes) { + // conversely to includes, assume we match unless specifically excluded + includeRecord = true; + const column = eFilter.column; + const op = eFilter.op; + const filterVal = eFilter.value; + log.debug(`Checking against exclude filter: ${column} ${op} ${filterVal.toString()}`); + + const recordDataToMatch = record[column]; + if (!isValidRecordData(recordDataToMatch)) { + log.fatalError( + `Cannot apply filter(s) on column ${column}: column not in query set or does not contain a filterable value.`, + ); + break; // Not necessary, but TypeScript doesn't know that + } + + // Special case for 'null' - if it's a filter value, check the column; + // otherwise, if the column is null, exclude filter passes. + if (recordDataToMatch === null) { + // treat the record data as a string ('null') for matching, + // as that's how it would be stored in filters{}. + if (textValueMatch(filterVal, 'null')) { + log.debug(`Matched exlcude filter for ${column}. Filter: null Record: null.`); + includeRecord = false; + break; + } else { + includeRecord = true; + continue; + } + } + + if (textValueMatch(filterVal, recordDataToMatch)) { + log.debug( + `Matched exclude filter for ${column}: (Filter: ${filterVal.toString()} | Record: ${recordDataToMatch})`, + ); + includeRecord = false; + break; + } + } + + if (includeRecord) { + log.debug(`Adding record to dataset.`); + filteredData.push(record); + } + } + return filteredData; +}; diff --git a/util/xivapi.ts b/util/xivapi.ts index 1726f7e24e..64e36a4190 100644 --- a/util/xivapi.ts +++ b/util/xivapi.ts @@ -81,6 +81,7 @@ export class XivApi { let currentPage = 0; let maxPage = 1; const output: XivApiOutput = []; + const specificNodeRequested = endpoint.includes('/'); while (currentPage < maxPage) { currentPage++; let url = `${_XIVAPI_URL}${endpoint}?limit=${_XIVAPI_RESULTS_LIMIT}&columns=${ @@ -94,10 +95,11 @@ export class XivApi { let jsonResult; try { const response = await fetch(url); - if (!response.ok) - throw new Error(`Network error occurred fetching API results.`); jsonResult = (await response.json()) as XivApiResult; - const pageNum = jsonResult.Pagination.Page; + if (!response.ok) + throw new Error(`Error occurred fetching API results.`); + // If hitting a specific endpoint node (e.g. Status/968), no Pagination object is returned. + const pageNum = specificNodeRequested ? 1 : jsonResult.Pagination.Page; if (pageNum === null || pageNum === undefined) throw new Error(`Invalid data returned from API query.`); } catch (e) { @@ -114,13 +116,18 @@ export class XivApi { } if (currentPage === 1) { - maxPage = typeof jsonResult.Pagination.PageTotal === 'string' - ? parseInt(jsonResult.Pagination.PageTotal) - : jsonResult.Pagination.PageTotal; + // If hitting a specific endpoint node (e.g. Status/968), only one page is returned. + maxPage = specificNodeRequested ? 1 : ( + typeof jsonResult.Pagination.PageTotal === 'string' + ? parseInt(jsonResult.Pagination.PageTotal) + : jsonResult.Pagination.PageTotal + ); this.log.debug(`API endpoint ${endpoint} has ${maxPage} page(s).`); } - - output.push(...Object.values(jsonResult.Results)); + if (specificNodeRequested) + output.push(jsonResult); + else + output.push(...Object.values(jsonResult.Results)); } this.log.info(`API query successful for endpoint: ${endpoint}`); From c6af3c96f5f6dd4c3f745e1d3e00c4cf14414c4d Mon Sep 17 00:00:00 2001 From: Wexx Date: Sat, 6 Jan 2024 19:08:18 -0800 Subject: [PATCH 2/5] add loglevel arg --- util/query_xivapi.ts | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/util/query_xivapi.ts b/util/query_xivapi.ts index 07340da1ed..d51ab79dc9 100644 --- a/util/query_xivapi.ts +++ b/util/query_xivapi.ts @@ -7,12 +7,12 @@ import inquirer from 'inquirer'; import { UnreachableCode } from '../resources/not_reached'; -import { ConsoleLogger, LogLevelKey } from './console_logger'; +import { ConsoleLogger, LogLevelKey, logLevels } from './console_logger'; import { XivApi } from './xivapi'; import { ActionChoiceType } from '.'; -const _LOGLEVEL: LogLevelKey = 'info'; +const _LOGLEVEL_DEFAULT: LogLevelKey = 'info'; // XIVAPI does not require a ?columns=xx param in the URL string, // but without one, on a primary endpoint (e.g. Status), it will @@ -30,12 +30,14 @@ type QueryXivApiNamespaceInterface = { 'endpoint': string | null; 'columns': string | null; 'filters': string | null; + 'loglevel': LogLevelKey | null; }; class QueryXivApiNamespace extends Namespace implements QueryXivApiNamespaceInterface { 'endpoint': string | null; 'columns': string | null; 'filters': string | null; + 'loglevel': LogLevelKey | null; } type QueryXivApiInquirerType = { @@ -115,7 +117,7 @@ const stringToNum = (val: string | number): number | undefined => { }; const log = new ConsoleLogger(); -log.setLogLevel(_LOGLEVEL); +log.setLogLevel(_LOGLEVEL_DEFAULT); // called by inquirer const queryApiFunc = async (args: Namespace): Promise => { @@ -148,7 +150,11 @@ const queryApiFunc = async (args: Namespace): Promise => { const myEndpoint = answers.endpoint ?? args.endpoint?.toString() ?? ''; const myColumns = answers.columns ?? args.columns?.toString() ?? _COLUMNS_DEFAULT; const myFilters = answers.filters ?? args.filters?.toString() ?? ''; - return queryApi(myEndpoint, myColumns, myFilters); + // args.loglevel returns as an object, rather than a string primitive. + // so force it here to a string and then re-apply the type with an assertion. + // parsearg already limits user-input values to the respective type literal. + const myLogLevel: LogLevelKey = args.loglevel ?? _LOGLEVEL_DEFAULT; + return queryApi(myEndpoint, myColumns, myFilters, myLogLevel.toString() as LogLevelKey); }).catch(console.error); }; @@ -199,17 +205,27 @@ export const registerQueryXivApi = ( type: 'string', help: 'Filter(s) to apply on returned data', }); + + queryParser.addArgument(['-ll', '--loglevel'], { + nargs: 1, + type: 'string', + choices: logLevels.map((ll) => ll[0]), + help: `The level of console output you want to see (default: ${_LOGLEVEL_DEFAULT})`, + }); }; const queryApi = async ( endpoint: string, columns: string, filters: string, + loglevel: LogLevelKey, ): Promise => { const outputToConsole = (result: XivApiQueryResult): void => { log.printNoHeader(JSON.stringify(result, null, 2).replace(/"/g, '\'')); }; + log.setLogLevel(loglevel); + const _ENDPOINT = endpoint.trim(); if (endpoint.length < 1) { log.fatalError('No endpoint specified.'); @@ -228,7 +244,7 @@ const queryApi = async ( _ENDPOINT, _COLUMNS, ) as XivApiQueryResult; - + if (filters === '') { log.debug('No filters were specified.'); if (apiData.length === 0) From e30bbf1995a4b78b883f747ad57f1e0daf304fed Mon Sep 17 00:00:00 2001 From: Wexx Date: Sat, 6 Jan 2024 19:13:22 -0800 Subject: [PATCH 3/5] add note to PatchUpdateChecklist --- docs/PatchUpdateChecklist.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/PatchUpdateChecklist.md b/docs/PatchUpdateChecklist.md index b9e9e08032..2ea3b5924c 100644 --- a/docs/PatchUpdateChecklist.md +++ b/docs/PatchUpdateChecklist.md @@ -32,6 +32,11 @@ in the console as the scripts are run. It is recommended that you choose at least the 'Alert' level to ensure you are notified of any problems that require manual intervetion before merging the file changes. +If you run into collisions or other data issues that require resolution before merge, +you can use the XIVAPI CLI helper utility to request and filter XIVAPI data -- +and see it as JSON console output -- by running `npm run query`. +Use `npm run query -- -h` for info on how to use the utility. + #### effect_id data As new status effects are added to the game, those names may conflict with existing names. From 3940cb40e3d226724646fa09bbfffa695778fad7 Mon Sep 17 00:00:00 2001 From: Wexx Date: Sun, 7 Jan 2024 14:21:21 -0800 Subject: [PATCH 4/5] fix help text --- util/query_xivapi.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/query_xivapi.ts b/util/query_xivapi.ts index d51ab79dc9..2b20892291 100644 --- a/util/query_xivapi.ts +++ b/util/query_xivapi.ts @@ -177,7 +177,7 @@ export const registerQueryXivApi = ( COLUMNS: Specify a comma-separated list of columns. Whitespace is ignored. Use * for all (default). - FILTERS: If the --filter option is used, it should be in the format [column] [operator] [value]. + FILTER: If the --filter option is used, it should be in the format [column] [operator] [value]. Multiple filters must be separated by commas. * in a [value] acts as a wildcard. Any [column] used for filtering must be included in the columns returned from the API. Valid [operators] are: =, !=, >, <, ~ (in), !~ (not in). From c3c83dd44fcdc807f93726baf77a01163f6bab8a Mon Sep 17 00:00:00 2001 From: Wexx Date: Wed, 10 Jan 2024 17:13:42 -0800 Subject: [PATCH 5/5] add comments & regex cleanup --- util/query_xivapi.ts | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/util/query_xivapi.ts b/util/query_xivapi.ts index 2b20892291..b40c5141ec 100644 --- a/util/query_xivapi.ts +++ b/util/query_xivapi.ts @@ -267,13 +267,21 @@ const parseFilters = (filters: string): FilterSet => { excludes: [], }; - // filters are comma-separated, but don't count commas inside (); that's an array. - const regexFilterSeparator = /(? 30, ID < 40, Name ~ (Ruby Carbuncle, Obsidian Carbuncle) + // Whitespace is generally ignored unless inside an array-like element (example above). + + // use commas to seperate different filters, ignoring commas inside () or [] + // match: ID>30,ID<40 -- don't match: ID~(30,40) + const regexFilterSeparator = /(? f !== ''); const regexColumn = /^\s*(\w+)\s*/; const regexOp = new RegExp(`(${allOps.join('|')})`); - const regexValueArr = /\s*[([][\w\s*]+,[\w\s*]+[,\w\s*]*[)\]]\s*/; + // for regexValueArr, match all alphanumeric values/whitespace/literal-* inside () or [] + // requires 2 comma-separated elements, optionally accepts more + // match: (30,40) or [Topaz, Obs*, *uby] -- don't match: [10,] or (Emerald) + const regexValueArr = /\s*[\(\[][\w\s*]+,[\w\s*]+[,\w\s*]*[\)\]]\s*/; const regexValueStr = /[\w\s*]+/; const regexValueEither = new RegExp(`(${regexValueStr.source}$|${regexValueArr.source}$)`); const regexFilterParts = new RegExp( @@ -309,7 +317,7 @@ const parseFilters = (filters: string): FilterSet => { ); return; } - valArr = val.replace(/[()]/g, '').trim().split(',').map((e) => e.trim().toString()); + valArr = val.replace(/[\(\)\[\]]/g, '').trim().split(',').map((e) => e.trim().toString()); if (valArr.length < 2 || valArr[0] === '') { log.fatalError(`Could not extract array from array-like string filter (${rf}).`); return;