util: add logging & err handling, docs updates

util/README.md

@@ -1,17 +1,10 @@
-# How to run python utils
+# Utils
 
-Download and install [python](https://www.python.org/).
+This directory contains a set of utiliy scripts
+and libraries to assist in cactbot development.
 
-Download [SaintCoinach.Cmd-master-\*-\*.zip](https://github.com/ufx/SaintCoinach/releases) and extract on C Drive root or D Drive root.
+Many are already referenced or documented
+in the various project [docs](https://github.com/OverlayPlugin/cactbot/tree/main/docs).
 
-Saint coinach is only accepted in 2 directories by default, 'C:\\SaintCoinach\\' and 'D:\\SaintCoinach\\'  
-These MUST include the one of those must include the SaintCoinach.Cmd.exe (with all other needed files for it to run).  
-If you use a different path, you can add it to the coinach.py _DEFAULT_COINACH_PATHS variable
-
-## Troubleshooting with SaintCoinach
-
-### SaintCoinach FFXIV client version mismatch
-
-When you run SaintCoinach manually, does it shows you need to update? This means that definitions are not updated to the latest patch. For minor patches, SaintCoinach does not need to update definitions, so you need to do is just change the version data to latest version.
-
-In the SainCoinach dir open the \Definitions\game.ver file and change the version number to latest version which showed when you launch SaintCoinach manually.
+You can run the most common scripts by running `npm run util`
+from inside your cactbot root directory.

util/gen_effect_id.ts

@@ -1,6 +1,7 @@
 import path from 'path';
 
 import { cleanName } from './csv_util';
+import { ConsoleLogger, LogLevelKey } from './generate_data_files';
 import { OutputFileAttributes, XivApi } from './xivapi';
 
 const _EFFECT_ID: OutputFileAttributes = {
@@ -34,6 +35,9 @@ type OutputEffectId = {
   [name: string]: string; // the id is converted to hex, so use string
 };
 
+const _SCRIPT_NAME = path.basename(import.meta.url);
+let log: ConsoleLogger;
+
 // TODO: add renaming?
 // Almagest: 563
 
@@ -90,16 +94,12 @@ const customMapping: Readonly<MappingTable> = {
   'EmboldenSelf': 1239,
 };
 
-const printError = (
-  header: string,
-  what: string,
-) => console.error(`${header} ${what}`);
-
 const assembleData = (apiData: XivApiStatus): OutputEffectId => {
   const formattedData: OutputEffectId = {};
   const foundNames = new Set();
   const map = new Map<string, number>();
 
+  log.debug('Processing & assembling data...');
   for (const effect of apiData) {
     const id = effect.ID;
     const rawName = effect.Name;
@@ -110,50 +110,63 @@ const assembleData = (apiData: XivApiStatus): OutputEffectId => {
     if (!name)
       continue;
 
-    // TODO: The below printError() calls generate a ton of noise.  That's to be expected,
-    // but we might want to add a flag to suppress these entirely, or filter out
-    // existing/known conflicts so we can just see what's changing each patch.
+    // See comment above specifically about known mappings.
+    // Conflicts here are only logged at a 'debug' level because of the noise generated.
+    // If a future patch makes job changes resulting in a new status ID,
+    // we have to trust someone will notice the jobs module is no longer tracking,
+    // and then update the known mapping manually.
     if (rawName in knownMapping) {
       if (id !== knownMapping[rawName]) {
-        printError('skipping', rawName);
+        log.debug(`Conflict with known/static mapping: ${name} (ID: ${id})`);
         continue;
       }
     }
 
     if (map.has(name)) {
-      printError('collision', name);
-      printError('collision', name);
+      log.info(
+        `Collision detected: ${name} (IDs: ${id}, ${map.get(name) ?? ''}).  Skipping...`,
+      );
       map.delete(name);
       continue;
     }
     if (foundNames.has(name)) {
-      printError('collision', name);
+      log.debug(`Additional collision: ${name} (new ID: ${id}). Skipping...`);
       continue;
     }
 
     foundNames.add(name);
     map.set(name, id);
+    log.debug(`Adding ${name} (ID: ${id}) to data output.`);
   }
+  log.debug('Completed initial pass. Starting post-processing...');
 
   // Make sure everything specified in known_mapping was found in the above loop.
   for (const rawName of Object.keys(knownMapping)) {
     const name = cleanName(rawName);
     if (name && !foundNames.has(name))
-      printError('missing known name', rawName);
+      log.alert(`Known name missing from data: ${rawName}.  Please investigate.`);
   }
+  log.debug('Known name mapping check complete.');
 
   // Add custom effect name for necessary duplicates.
-  for (const [name, id] of Object.entries(customMapping))
+  for (const [name, id] of Object.entries(customMapping)) {
     map.set(name, id);
+    log.debug(`Added custom mapping: ${name} (ID: ${id})`);
+  }
+  log.debug('Custom name mappings added.');
 
   // Store ids as hex.
   map.forEach((id, name) => formattedData[name] = id.toString(16).toUpperCase());
-
+  log.debug('Data assembly/formatting complete.');
   return formattedData;
 };
 
-export default async (): Promise<void> => {
-  const api = new XivApi(null, true);
+export default async (logLevel: LogLevelKey): Promise<void> => {
+  log = new ConsoleLogger();
+  log.setLogLevel(logLevel);
+  log.info(`Starting processing for ${_SCRIPT_NAME}`);
+
+  const api = new XivApi(null, log);
 
   const apiData = await api.queryApi(
     _ENDPOINT,
@@ -163,8 +176,10 @@ export default async (): Promise<void> => {
   const outputData = assembleData(apiData);
 
   await api.writeFile(
-    path.basename(import.meta.url),
+    _SCRIPT_NAME,
     _EFFECT_ID,
     outputData,
   );
+
+  log.successDone(`Completed processing for ${_SCRIPT_NAME}`);
 };

util/gen_hunt_data.ts

@@ -3,6 +3,7 @@ import path from 'path';
 import { LocaleObject } from '../types/trigger';
 
 import { getCnTable, getKoTable } from './csv_util';
+import { ConsoleLogger, LogLevelKey } from './generate_data_files';
 import { OutputFileAttributes, XivApi } from './xivapi';
 
 const _HUNT: OutputFileAttributes = {
@@ -43,6 +44,11 @@ const _COLUMNS = [
   'BNpcName.Name_ja',
 ];
 
+type LocaleOutputColumns = [key: string, ...indices: string[]];
+const _LOCALE_TABLE = 'BNpcName';
+const _LOCALE_INPUT_COLS = ['#', 'Singular'];
+const _LOCALE_OUTPUT_COLS: LocaleOutputColumns = ['BNpcNameId', 'LocaleName'];
+
 // SS- (minions) and SS+ (boss) mobs are rank 1 & 3 respectively
 // so we can only differentiate them with known BNpcBaseIds
 // This requires manual additions for future expansions.
@@ -88,13 +94,16 @@ type OutputHuntMap = {
   };
 };
 
+const _SCRIPT_NAME = path.basename(import.meta.url);
+let log: ConsoleLogger;
+
 const deLocaleSubstitutions = (replaceString: string): string | string[] => {
   const substitutionMap: { [param: string]: string[] } = {
     '[t]': ['der', 'die', 'das'],
     '[a]': ['e', 'er', 'es'],
     '[A]': ['e', 'er', 'es'],
   };
-
+  log.debug(`Doing 'de' locale substitutions on: ${replaceString}`);
   replaceString = replaceString.replace('[p]', '');
   let results: string[] = [replaceString];
 
@@ -115,16 +124,23 @@ const deLocaleSubstitutions = (replaceString: string): string | string[] => {
 };
 
 const fetchLocaleCsvTables = async () => {
-  const cnBNpcNames = await getCnTable('BNpcName', ['#', 'Singular'], ['BNpcNameId', 'LocaleName']);
-  const koBNpcNames = await getKoTable('BNpcName', ['#', 'Singular'], ['BNpcNameId', 'LocaleName']);
+  log.debug(
+    `Table: ${_LOCALE_TABLE} | Query columns: [${_LOCALE_INPUT_COLS.toString()}] | Output: [${_LOCALE_OUTPUT_COLS.toString()}]`,
+  );
+  log.debug('Fetching \'cn\' table...');
+  const cnBNpcNames = await getCnTable(_LOCALE_TABLE, _LOCALE_INPUT_COLS, _LOCALE_OUTPUT_COLS);
+  log.debug('Fetching \'ko\' table...');
+  const koBNpcNames = await getKoTable(_LOCALE_TABLE, _LOCALE_INPUT_COLS, _LOCALE_OUTPUT_COLS);
   return {
     cn: cnBNpcNames,
     ko: koBNpcNames,
   };
 };
 
 const assembleData = async (apiData: XivApiNotoriousMonster): Promise<OutputHuntMap> => {
+  log.debug('Processing & assembling data...');
   const formattedData: OutputHuntMap = {};
+  log.info('Fetching locale CSV tables...');
   const localeCsvTables = await fetchLocaleCsvTables();
 
   for (const record of apiData) {
@@ -186,27 +202,36 @@ const assembleData = async (apiData: XivApiNotoriousMonster): Promise<OutputHunt
     if (typeof koLocaleName === 'string' && koLocaleName !== '')
       localeNames['ko'] = koLocaleName;
 
+    log.debug(`Collected hunt data for ${record.BNpcName.Name_en} (ID: ${nameId})`);
     formattedData[name] = {
       id: nameId,
       name: localeNames,
       rank: rank,
     };
   }
-
+  log.debug('Data assembly/formatting complete.');
   return formattedData;
 };
 
-const api = new XivApi(null, true);
+export default async (logLevel: LogLevelKey): Promise<void> => {
+  log = new ConsoleLogger();
+  log.setLogLevel(logLevel);
+  log.info(`Starting processing for ${_SCRIPT_NAME}`);
 
-const apiData = await api.queryApi(
-  _ENDPOINT,
-  _COLUMNS,
-) as XivApiNotoriousMonster;
+  const api = new XivApi(null, log);
 
-const outputData = await assembleData(apiData);
+  const apiData = await api.queryApi(
+    _ENDPOINT,
+    _COLUMNS,
+  ) as XivApiNotoriousMonster;
 
-await api.writeFile(
-  path.basename(import.meta.url),
-  _HUNT,
-  outputData,
-);
+  const outputData = await assembleData(apiData);
+
+  await api.writeFile(
+    _SCRIPT_NAME,
+    _HUNT,
+    outputData,
+  );
+
+  log.successDone(`Completed processing for ${_SCRIPT_NAME}`);
+};

util/gen_pet_names.ts

@@ -1,6 +1,7 @@
 import path from 'path';
 
 import { getCnTable, getKoTable } from './csv_util';
+import { ConsoleLogger, LogLevelKey } from './generate_data_files';
 import { OutputFileAttributes, XivApi } from './xivapi';
 
 const _PET_NAMES: OutputFileAttributes = {
@@ -26,10 +27,10 @@ const _COLUMNS = [
 ];
 
 const _LOCALE_TABLE = 'Pet';
-
 const _LOCALE_COLUMNS = ['Name'];
 
 type ResultPet = {
+  ID: number;
   Name_de: string | null;
   Name_en: string | null;
   Name_fr: string | null;
@@ -47,8 +48,14 @@ type OutputPetNames = {
   ko: string[];
 };
 
+const _SCRIPT_NAME = path.basename(import.meta.url);
+let log: ConsoleLogger;
+
 const fetchLocaleCsvTables = async () => {
+  log.debug(`Table: ${_LOCALE_TABLE} | Query columns: [${_LOCALE_COLUMNS.toString()}]`);
+  log.debug('Fetching \'cn\' table...');
   const cnPet = await getCnTable(_LOCALE_TABLE, _LOCALE_COLUMNS);
+  log.debug('Fetching \'ko\' table...');
   const koPet = await getKoTable(_LOCALE_TABLE, _LOCALE_COLUMNS);
   return {
     cn: cnPet,
@@ -57,6 +64,7 @@ const fetchLocaleCsvTables = async () => {
 };
 
 const assembleData = async (apiData: XivApiPet): Promise<OutputPetNames> => {
+  log.debug('Processing & assembling data...');
   // This isn't really a locale object, and ordering is alpha in the current file, so:
   // eslint-disable-next-line rulesdir/cactbot-locale-order
   const formattedData: OutputPetNames = {
@@ -80,21 +88,29 @@ const assembleData = async (apiData: XivApiPet): Promise<OutputPetNames> => {
       formattedData.fr.push(pet.Name_fr);
     if (pet.Name_ja !== null)
       formattedData.ja.push(pet.Name_ja);
+    log.debug(`Collected base pet data for ${pet.Name_en} (ID: ${pet.ID})`);
   }
 
+  log.info('Fetching locale CSV tables...');
   const localeCsvTables = await fetchLocaleCsvTables();
   for (const name of Object.keys(localeCsvTables.cn).filter((k) => k !== '')) {
     formattedData.cn.push(name);
+    log.debug(`Collected 'cn' pet data for ${name}`);
   }
   for (const name of Object.keys(localeCsvTables.ko).filter((k) => k !== '')) {
     formattedData.ko.push(name);
+    log.debug(`Collected 'ko' pet data for ${name}`);
   }
-
+  log.debug('Data assembly/formatting complete.');
   return formattedData;
 };
 
-export default async (): Promise<void> => {
-  const api = new XivApi(null, true);
+export default async (logLevel: LogLevelKey): Promise<void> => {
+  log = new ConsoleLogger();
+  log.setLogLevel(logLevel);
+  log.info(`Starting processing for ${_SCRIPT_NAME}`);
+
+  const api = new XivApi(null, log);
 
   const apiData = await api.queryApi(
     _ENDPOINT,
@@ -104,8 +120,10 @@ export default async (): Promise<void> => {
   const outputData = await assembleData(apiData);
 
   await api.writeFile(
-    path.basename(import.meta.url),
+    _SCRIPT_NAME,
     _PET_NAMES,
     outputData,
   );
+
+  log.successDone(`Completed processing for ${_SCRIPT_NAME}`);
 };