From b414366471741aa731ae8089c0117f451aa4ca5d Mon Sep 17 00:00:00 2001 From: "Christian Landgren (aider)" Date: Wed, 4 Dec 2024 15:08:27 +0100 Subject: [PATCH 01/82] refactor: Divide saveToAPI worker into specialized workers for emissions, economy, goals, and initiatives --- src/lib/saveUtils.ts | 31 +++++ src/workers/saveEconomy.ts | 55 +++++++++ src/workers/saveEmissions.ts | 113 ++++++++++++++++++ src/workers/saveGoals.ts | 39 +++++++ src/workers/saveInitiatives.ts | 39 +++++++ src/workers/saveToAPI.ts | 207 +++++---------------------------- 6 files changed, 308 insertions(+), 176 deletions(-) create mode 100644 src/lib/saveUtils.ts create mode 100644 src/workers/saveEconomy.ts create mode 100644 src/workers/saveEmissions.ts create mode 100644 src/workers/saveGoals.ts create mode 100644 src/workers/saveInitiatives.ts diff --git a/src/lib/saveUtils.ts b/src/lib/saveUtils.ts new file mode 100644 index 00000000..103b3a31 --- /dev/null +++ b/src/lib/saveUtils.ts @@ -0,0 +1,31 @@ +import { getReportingPeriodDates } from './reportingPeriodDates' +import { apiFetch } from './api' + +export function formatAsReportingPeriods( + entries: { year: number }[], + fiscalYear: { startMonth: number; endMonth: number }, + category: 'emissions' | 'economy' +) { + return entries.map(({ year, ...data }) => { + const [startDate, endDate] = getReportingPeriodDates( + year, + fiscalYear.startMonth, + fiscalYear.endMonth + ) + return { + startDate, + endDate, + [category]: + category === 'economy' + ? (data as { economy: any }).economy + : { + ...data, + }, + } + }) +} + +export const defaultMetadata = (url: string) => ({ + source: url, + comment: 'Parsed by Garbo AI', +}) diff --git a/src/workers/saveEconomy.ts b/src/workers/saveEconomy.ts new file mode 100644 index 00000000..7ce9fd75 --- /dev/null +++ b/src/workers/saveEconomy.ts @@ -0,0 +1,55 @@ +import { DiscordJob, DiscordWorker } from '../lib/DiscordWorker' +import { apiFetch } from '../lib/api' +import { defaultMetadata, formatAsReportingPeriods } from '../lib/saveUtils' +import redis from '../config/redis' +import { getReportingPeriodDates } from '../lib/reportingPeriodDates' + +export class JobData extends DiscordJob { + declare data: DiscordJob['data'] & { + companyName: string + wikidata: any + fiscalYear: any + economy?: any[] + } +} + +const saveEconomy = new DiscordWorker( + 'saveEconomy', + async (job) => { + const { url, fiscalYear, wikidata, economy = [] } = job.data + const wikidataId = wikidata.node + const metadata = defaultMetadata(url) + + if (economy?.length) { + job.editMessage(`🤖 Sparar ekonomidata...`) + return Promise.all( + economy.map(async ({ year, economy }) => { + const [startDate, endDate] = getReportingPeriodDates( + year, + fiscalYear.startMonth, + fiscalYear.endMonth + ) + job.log(`Saving economy for ${startDate}-${endDate}`) + job.sendMessage(`🤖 Sparar ekonomidata för ${year}...`) + const body = { + startDate, + endDate, + economy, + metadata, + } + + return await apiFetch(`/companies/${wikidataId}/${year}/economy`, { + body, + }) + }) + ) + } + + return null + }, + { + connection: redis, + } +) + +export default saveEconomy diff --git a/src/workers/saveEmissions.ts b/src/workers/saveEmissions.ts new file mode 100644 index 00000000..d7a970e1 --- /dev/null +++ b/src/workers/saveEmissions.ts @@ -0,0 +1,113 @@ +import { DiscordJob, DiscordWorker } from '../lib/DiscordWorker' +import { apiFetch } from '../lib/api' +import { defaultMetadata, formatAsReportingPeriods } from '../lib/saveUtils' +import redis from '../config/redis' + +export class JobData extends DiscordJob { + declare data: DiscordJob['data'] & { + companyName: string + wikidata: any + fiscalYear: any + scope12?: any[] + scope3?: any[] + biogenic?: any[] + } +} + +const saveEmissions = new DiscordWorker( + 'saveEmissions', + async (job) => { + const { url, fiscalYear, wikidata, scope12 = [], scope3 = [], biogenic = [] } = job.data + const wikidataId = wikidata.node + const metadata = defaultMetadata(url) + + if (scope12?.length || scope3?.length || biogenic?.length) { + job.editMessage(`🤖 Sparar utsläppsdata...`) + + return Promise.all([ + ...(await scope12.reduce(async (lastPromise, { year, scope1, scope2 }) => { + const arr = await lastPromise + const [startDate, endDate] = getReportingPeriodDates( + year, + fiscalYear.startMonth, + fiscalYear.endMonth + ) + job.log(`Saving scope1 and scope2 for ${startDate}-${endDate}`) + job.sendMessage(`🤖 Sparar utsläppsdata scope 1+2 för ${year}...`) + const body = { + startDate, + endDate, + emissions: { + scope1, + scope2, + }, + metadata, + } + return [ + ...arr, + await apiFetch(`/companies/${wikidataId}/${year}/emissions`, { + body, + }), + ] + }, Promise.resolve([]))), + + ...(await scope3.reduce(async (lastPromise, { year, scope3 }) => { + const arr = await lastPromise + const [startDate, endDate] = getReportingPeriodDates( + year, + fiscalYear.startMonth, + fiscalYear.endMonth + ) + job.sendMessage(`🤖 Sparar utsläppsdata scope 3 för ${year}...`) + job.log(`Saving scope3 for ${year}`) + const body = { + startDate, + endDate, + emissions: { + scope3, + }, + metadata, + } + return [ + ...arr, + await apiFetch(`/companies/${wikidataId}/${year}/emissions`, { + body, + }), + ] + }, Promise.resolve([]))), + + ...(await biogenic.reduce(async (lastPromise, { year, biogenic }) => { + const arr = await lastPromise + const [startDate, endDate] = getReportingPeriodDates( + year, + fiscalYear.startMonth, + fiscalYear.endMonth + ) + job.sendMessage(`🤖 Sparar utsläppsdata biogenic för ${year}...`) + job.log(`Saving biogenic for ${year}`) + const body = { + startDate, + endDate, + emissions: { + biogenic, + }, + metadata, + } + return [ + ...arr, + await apiFetch(`/companies/${wikidataId}/${year}/emissions`, { + body, + }), + ] + }, Promise.resolve([]))), + ]) + } + + return null + }, + { + connection: redis, + } +) + +export default saveEmissions diff --git a/src/workers/saveGoals.ts b/src/workers/saveGoals.ts new file mode 100644 index 00000000..be976a0b --- /dev/null +++ b/src/workers/saveGoals.ts @@ -0,0 +1,39 @@ +import { DiscordJob, DiscordWorker } from '../lib/DiscordWorker' +import { apiFetch } from '../lib/api' +import { defaultMetadata } from '../lib/saveUtils' +import redis from '../config/redis' + +export class JobData extends DiscordJob { + declare data: DiscordJob['data'] & { + companyName: string + wikidata: any + goals?: any + } +} + +const saveGoals = new DiscordWorker( + 'saveGoals', + async (job) => { + const { url, wikidata, goals } = job.data + const wikidataId = wikidata.node + const metadata = defaultMetadata(url) + + if (goals) { + job.editMessage(`🤖 Sparar mål...`) + return await apiFetch(`/companies/${wikidataId}/goals`, { + body: { + goals, + metadata, + }, + method: 'POST', + }) + } + + return null + }, + { + connection: redis, + } +) + +export default saveGoals diff --git a/src/workers/saveInitiatives.ts b/src/workers/saveInitiatives.ts new file mode 100644 index 00000000..117d046d --- /dev/null +++ b/src/workers/saveInitiatives.ts @@ -0,0 +1,39 @@ +import { DiscordJob, DiscordWorker } from '../lib/DiscordWorker' +import { apiFetch } from '../lib/api' +import { defaultMetadata } from '../lib/saveUtils' +import redis from '../config/redis' + +export class JobData extends DiscordJob { + declare data: DiscordJob['data'] & { + companyName: string + wikidata: any + initiatives?: any + } +} + +const saveInitiatives = new DiscordWorker( + 'saveInitiatives', + async (job) => { + const { url, wikidata, initiatives } = job.data + const wikidataId = wikidata.node + const metadata = defaultMetadata(url) + + if (initiatives) { + job.editMessage(`🤖 Sparar initiativ...`) + return await apiFetch(`/companies/${wikidataId}/initiatives`, { + body: { + initiatives, + metadata, + }, + method: 'POST', + }) + } + + return null + }, + { + connection: redis, + } +) + +export default saveInitiatives diff --git a/src/workers/saveToAPI.ts b/src/workers/saveToAPI.ts index 51c80767..9a7cd56a 100644 --- a/src/workers/saveToAPI.ts +++ b/src/workers/saveToAPI.ts @@ -1,9 +1,9 @@ -import { askPrompt } from '../lib/openai' import { DiscordJob, DiscordWorker } from '../lib/DiscordWorker' import { apiFetch } from '../lib/api' -import { getReportingPeriodDates } from '../lib/reportingPeriodDates' import discord from '../discord' import redis from '../config/redis' +import { askPrompt } from '../lib/openai' +import { formatAsReportingPeriods } from '../lib/saveUtils' export class JobData extends DiscordJob { declare data: DiscordJob['data'] & { @@ -24,30 +24,6 @@ export class JobData extends DiscordJob { const ONE_DAY = 1000 * 60 * 60 * 24 -function formatAsReportingPeriods( - entries: { year: number }[], - fiscalYear: { startMonth: number; endMonth: number }, - category: 'emissions' | 'economy' -) { - return entries.map(({ year, ...data }) => { - const [startDate, endDate] = getReportingPeriodDates( - year, - fiscalYear.startMonth, - fiscalYear.endMonth - ) - return { - startDate, - endDate, - [category]: - category === 'economy' - ? (data as { economy: any }).economy - : { - ...data, - }, - } - }) -} - const askDiff = async ( existingCompany, { @@ -81,16 +57,11 @@ const askDiff = async ( initiatives, } - /** - * Normalise company data to allow comparing with the same structure - */ const getCompanyBeforeAfter = () => Object.keys(updated).reduce( ([before, after], key) => { - // Only keep the updated data, and only if there are meaningful changes in array fields if (Array.isArray(updated[key]) ? updated[key].length : updated[key]) { if (key === 'economy') { - // only keep relevant properties for each reportingPeriod before['reportingPeriods'] = ( existingCompany.reportingPeriods ?? [] ).map(({ startDate, endDate, economy }) => ({ @@ -104,7 +75,6 @@ const askDiff = async ( 'economy' ) } else if (key === 'scope12') { - // only keep relevant properties for each reportingPeriod before['reportingPeriods'] = ( existingCompany.reportingPeriods ?? [] ).map(({ startDate, endDate, emissions }) => ({ @@ -123,7 +93,6 @@ const askDiff = async ( 'emissions' ) } else if (key === 'scope3') { - // only keep relevant properties for each reportingPeriod before['reportingPeriods'] = ( existingCompany.reportingPeriods ?? [] ).map(({ startDate, endDate, emissions }) => ({ @@ -141,7 +110,6 @@ const askDiff = async ( 'emissions' ) } else if (key === 'biogenic') { - // only keep relevant properties for each reportingPeriod before['reportingPeriods'] = ( existingCompany.reportingPeriods ?? [] ).map(({ startDate, endDate, emissions }) => ({ @@ -170,7 +138,6 @@ const askDiff = async ( const [before, after] = getCompanyBeforeAfter() - // IDEA: Use a diff helper to compare objects and generate markdown diff const diff = await askPrompt( `What is changed between these two json values? Please respond in clear text with markdown formatting. The purpose is to let an editor approve the changes or suggest changes in Discord. @@ -212,10 +179,6 @@ const saveToAPI = new DiscordWorker( () => null ) - const metadata = { - source: url, - comment: 'Parsed by Garbo AI', - } const diff = !approved ? await askDiff(existingCompany, { scope12, @@ -247,149 +210,41 @@ const saveToAPI = new DiscordWorker( }) return await job.moveToDelayed(Date.now() + ONE_DAY) - } else { - if (scope12?.length || scope3?.length || biogenic?.length) { - job.editMessage(`🤖 Sparar utsläppsdata...`) - return Promise.all([ - ...(await scope12.reduce( - async (lastPromise, { year, scope1, scope2 }) => { - const arr = await lastPromise - const [startDate, endDate] = getReportingPeriodDates( - year, - fiscalYear.startMonth, - fiscalYear.endMonth - ) - job.log(`Saving scope1 and scope2 for ${startDate}-${endDate}`) - job.sendMessage(`🤖 Sparar utsläppsdata scope 1+2 för ${year}...`) - const body = { - startDate, - endDate, - emissions: { - scope1, - scope2, - }, - metadata, - } - return [ - ...arr, - await apiFetch(`/companies/${wikidataId}/${year}/emissions`, { - body, - }), - ] - }, - Promise.resolve([]) - )), - ...(await scope3.reduce(async (lastPromise, { year, scope3 }) => { - const arr = await lastPromise - const [startDate, endDate] = getReportingPeriodDates( - year, - fiscalYear.startMonth, - fiscalYear.endMonth - ) - job.sendMessage(`🤖 Sparar utsläppsdata scope 3 för ${year}...`) - job.log(`Saving scope3 for ${year}`) - const body = { - startDate, - endDate, - emissions: { - scope3, - }, - metadata, - } - return [ - ...arr, - await apiFetch(`/companies/${wikidataId}/${year}/emissions`, { - body, - }), - ] - }, Promise.resolve([]))), - ...(await biogenic.reduce(async (lastPromise, { year, biogenic }) => { - const arr = await lastPromise - const [startDate, endDate] = getReportingPeriodDates( - year, - fiscalYear.startMonth, - fiscalYear.endMonth - ) - job.sendMessage(`🤖 Sparar utsläppsdata biogenic för ${year}...`) - job.log(`Saving biogenic for ${year}`) - const body = { - startDate, - endDate, - emissions: { - biogenic, - }, - metadata, - } - return [ - ...arr, - await apiFetch(`/companies/${wikidataId}/${year}/emissions`, { - body, - }), - ] - }, Promise.resolve([]))), - ]) - } - - if (industry) { - job.editMessage(`🤖 Sparar GICS industri...`) - return await apiFetch(`/companies/${wikidataId}/industry`, { - body: { - industry, - metadata, - }, - method: 'PUT', - }) - } + } - if (goals) { - job.editMessage(`🤖 Sparar mål...`) - return await apiFetch(`/companies/${wikidataId}/goals`, { - body: { - goals, - metadata, - }, - method: 'POST', - }) - } + // Queue the appropriate specialized worker based on the data + if (scope12?.length || scope3?.length || biogenic?.length) { + return await job.queueChild('saveEmissions', { + scope12, + scope3, + biogenic, + }) + } - if (initiatives) { - job.editMessage(`🤖 Sparar initiativ...`) - return await apiFetch(`/companies/${wikidataId}/initiatives`, { - body: { - initiatives, - metadata, - }, - method: 'POST', - }) - } + if (industry) { + job.editMessage(`🤖 Sparar GICS industri...`) + return await apiFetch(`/companies/${wikidataId}/industry`, { + body: { + industry, + metadata: defaultMetadata(url), + }, + method: 'PUT', + }) + } - if (economy?.length) { - job.editMessage(`🤖 Sparar ekonomidata...`) - return Promise.all([ - ...economy.map(async ({ year, economy }) => { - const [startDate, endDate] = getReportingPeriodDates( - year, - fiscalYear.startMonth, - fiscalYear.endMonth - ) - job.log(`Saving economy for ${startDate}-${endDate}`) - job.sendMessage(`🤖 Sparar ekonomidata för ${year}...`) - const body = { - startDate, - endDate, - economy, - metadata, - } + if (goals) { + return await job.queueChild('saveGoals', { goals }) + } - return await apiFetch(`/companies/${wikidataId}/${year}/economy`, { - body, - }) - }), - ]) - } + if (initiatives) { + return await job.queueChild('saveInitiatives', { initiatives }) + } - throw new Error('No data to save') + if (economy?.length) { + return await job.queueChild('saveEconomy', { economy }) } + + throw new Error('No data to save') }, { concurrency: 10, From 26db2491160829ff4d98fe0d998f717356985b96 Mon Sep 17 00:00:00 2001 From: "Christian Landgren (aider)" Date: Wed, 4 Dec 2024 15:09:42 +0100 Subject: [PATCH 02/82] refactor: Move diff comparison logic to specialized workers --- src/workers/saveEmissions.ts | 58 +++++++++++++ src/workers/saveToAPI.ts | 161 ----------------------------------- 2 files changed, 58 insertions(+), 161 deletions(-) diff --git a/src/workers/saveEmissions.ts b/src/workers/saveEmissions.ts index d7a970e1..2f81becf 100644 --- a/src/workers/saveEmissions.ts +++ b/src/workers/saveEmissions.ts @@ -2,6 +2,10 @@ import { DiscordJob, DiscordWorker } from '../lib/DiscordWorker' import { apiFetch } from '../lib/api' import { defaultMetadata, formatAsReportingPeriods } from '../lib/saveUtils' import redis from '../config/redis' +import { askPrompt } from '../lib/openai' +import discord from '../discord' + +const ONE_DAY = 1000 * 60 * 60 * 24 export class JobData extends DiscordJob { declare data: DiscordJob['data'] & { @@ -11,12 +15,66 @@ export class JobData extends DiscordJob { scope12?: any[] scope3?: any[] biogenic?: any[] + approved?: boolean } } +const askDiff = async (existingCompany, { scope12, scope3, biogenic, fiscalYear }) => { + if (!existingCompany.reportingPeriods?.length) return '' + + const before = { + reportingPeriods: existingCompany.reportingPeriods.map(({ startDate, endDate, emissions }) => ({ + startDate, + endDate, + emissions, + })) + } + + const after = { + reportingPeriods: [ + ...(scope12 ? formatAsReportingPeriods(scope12, fiscalYear, 'emissions') : []), + ...(scope3 ? formatAsReportingPeriods(scope3, fiscalYear, 'emissions') : []), + ...(biogenic ? formatAsReportingPeriods(biogenic, fiscalYear, 'emissions') : []) + ] + } + + return await askPrompt( + `What is changed between these two json values? Please respond in clear text with markdown formatting. +The purpose is to let an editor approve the changes or suggest changes in Discord. +Be as brief as possible. Never be technical - meaning no comments about structure changes, fields renames etc. +Focus only on the actual values that have changed. +When handling years and ambiguous dates, always use the last year in the period (e.g. startDate: 2020 - endDate: 2021 should be referred to as 2021). +NEVER REPEAT UNCHANGED VALUES OR UNCHANGED YEARS! If nothing important has changed, just write "NO_CHANGES".`, + JSON.stringify({ before, after }) + ) +} + const saveEmissions = new DiscordWorker( 'saveEmissions', async (job) => { + const { approved = false } = job.data + const existingCompany = await apiFetch(`/companies/${job.data.wikidata.node}`).catch(() => null) + + if (!approved) { + const diff = await askDiff(existingCompany, job.data) + + if (diff) { + if (diff.includes('NO_CHANGES')) { + await job.sendMessage({ + content: `# ${job.data.companyName}: emissions\n${diff}`.slice(0, 2000), + }) + return diff + } + + const buttonRow = discord.createButtonRow(job.id!) + await job.sendMessage({ + content: `# ${job.data.companyName}: emissions\n${diff}`.slice(0, 2000), + components: [buttonRow], + }) + + return await job.moveToDelayed(Date.now() + ONE_DAY) + } + } const { url, fiscalYear, wikidata, scope12 = [], scope3 = [], biogenic = [] } = job.data const wikidataId = wikidata.node const metadata = defaultMetadata(url) diff --git a/src/workers/saveToAPI.ts b/src/workers/saveToAPI.ts index 9a7cd56a..9de5f866 100644 --- a/src/workers/saveToAPI.ts +++ b/src/workers/saveToAPI.ts @@ -24,135 +24,6 @@ export class JobData extends DiscordJob { const ONE_DAY = 1000 * 60 * 60 * 24 -const askDiff = async ( - existingCompany, - { - scope12, - scope3, - biogenic, - industry, - economy, - goals, - initiatives, - fiscalYear, - } -) => { - if ( - (scope12 || scope3 || biogenic) && - !existingCompany.reportingPeriods?.length - ) - return '' - if (economy && !existingCompany.reportingPeriods.length) return '' - if (goals && !existingCompany.goals) return '' - if (initiatives && !existingCompany.initiatives) return '' - if (industry && !existingCompany.industry) return '' - - const updated = { - scope12, - scope3, - biogenic, - industry, - economy, - goals, - initiatives, - } - - const getCompanyBeforeAfter = () => - Object.keys(updated).reduce( - ([before, after], key) => { - if (Array.isArray(updated[key]) ? updated[key].length : updated[key]) { - if (key === 'economy') { - before['reportingPeriods'] = ( - existingCompany.reportingPeriods ?? [] - ).map(({ startDate, endDate, economy }) => ({ - startDate, - endDate, - economy, - })) - after['reportingPeriods'] = formatAsReportingPeriods( - updated.economy, - fiscalYear, - 'economy' - ) - } else if (key === 'scope12') { - before['reportingPeriods'] = ( - existingCompany.reportingPeriods ?? [] - ).map(({ startDate, endDate, emissions }) => ({ - startDate, - endDate, - emissions: emissions - ? { - scope1: emissions.scope1, - scope2: emissions.scope2, - } - : null, - })) - after['reportingPeriods'] = formatAsReportingPeriods( - updated.scope12, - fiscalYear, - 'emissions' - ) - } else if (key === 'scope3') { - before['reportingPeriods'] = ( - existingCompany.reportingPeriods ?? [] - ).map(({ startDate, endDate, emissions }) => ({ - startDate, - endDate, - emissions: emissions - ? { - scope3: emissions.scope3, - } - : null, - })) - after['reportingPeriods'] = formatAsReportingPeriods( - updated.scope3, - fiscalYear, - 'emissions' - ) - } else if (key === 'biogenic') { - before['reportingPeriods'] = ( - existingCompany.reportingPeriods ?? [] - ).map(({ startDate, endDate, emissions }) => ({ - startDate, - endDate, - emissions: emissions - ? { - biogenic: emissions.biogenic, - } - : null, - })) - after['reportingPeriods'] = formatAsReportingPeriods( - updated.biogenic, - fiscalYear, - 'emissions' - ) - } else if (['initiatives', 'goals', 'industry'].includes(key)) { - before[key] = existingCompany[key] - after[key] = updated[key] - } - } - return [before, after] - }, - [{}, {}] - ) - - const [before, after] = getCompanyBeforeAfter() - - const diff = await askPrompt( - `What is changed between these two json values? Please respond in clear text with markdown formatting. -The purpose is to let an editor approve the changes or suggest changes in Discord. -Be as breif as possible. Never be technical - meaning no comments about structure changes, fields renames etc. -Focus only on the actual values that have changed. -When handling years and ambigous dates, always use the last year in the period (e.g. startDate: 2020 - endDate: 2021 should be referred to as 2021). -NEVER REPEAT UNCHANGED VALUES OR UNCHANGED YEARS! If nothing important has changed, just write "NO_CHANGES".`, - JSON.stringify({ - before, - after, - }) - ) - - return diff -} const saveToAPI = new DiscordWorker( 'saveToAPI', @@ -179,38 +50,6 @@ const saveToAPI = new DiscordWorker( () => null ) - const diff = !approved - ? await askDiff(existingCompany, { - scope12, - scope3, - biogenic, - industry, - goals, - initiatives, - economy, - fiscalYear, - }) - : '' - - if (diff) { - if (diff.includes('NO_CHANGES')) { - await job.sendMessage({ - content: `# ${companyName}: \`${apiSubEndpoint}\` - ${diff}`.slice(0, 2000), - }) - - return diff - } - - const buttonRow = discord.createButtonRow(job.id!) - await job.sendMessage({ - content: `# ${companyName}: \`${apiSubEndpoint}\` - ${diff}`.slice(0, 2000), - components: [buttonRow], - }) - - return await job.moveToDelayed(Date.now() + ONE_DAY) - } // Queue the appropriate specialized worker based on the data if (scope12?.length || scope3?.length || biogenic?.length) { From 4f8a7a73c832bf4d9436ccf143a282b5ab78f302 Mon Sep 17 00:00:00 2001 From: "Christian Landgren (aider)" Date: Wed, 4 Dec 2024 15:13:47 +0100 Subject: [PATCH 03/82] refactor: Migrate saveToAPI logic to flow-based approach in checkDB --- src/workers/checkDB.ts | 86 ++++++++++++++++++++++++------------------ 1 file changed, 49 insertions(+), 37 deletions(-) diff --git a/src/workers/checkDB.ts b/src/workers/checkDB.ts index de53a456..e359d333 100644 --- a/src/workers/checkDB.ts +++ b/src/workers/checkDB.ts @@ -51,71 +51,83 @@ const checkDB = new DiscordWorker('checkDB', async (job: JobData) => { await apiFetch(`/companies`, { body }) } - const { scope12, scope3, biogenic, industry, economy, goals, initiatives } = - childrenValues + const { scope12, scope3, biogenic, industry, economy, goals, initiatives } = childrenValues + const base = { - companyName, - url, - fiscalYear, - wikidata, - threadId, - channelId, + name: companyName, + data: { + companyName, + url, + fiscalYear, + wikidata, + threadId, + channelId, + }, + opts: { + attempts: 3, + } } - // TODO convert to flow - // Send to done, which is a simple worker to post a message to discord - // Include the link to the company webpage, and the link to the JSON data for the company - // Link to localhost or the production website - // We want to know when all steps have been completed to print a message to discord + await job.editMessage(`🤖 Sparar data...`) if (scope12 || scope3 || biogenic) { - await job.editMessage(`🤖 Skapar jobb för att spara utsläppsdata...`) - saveToAPI.queue.add(companyName + ' emissions', { + await flow.add({ ...base, - apiSubEndpoint: 'emissions', - scope12, - scope3, - biogenic, + queueName: 'saveEmissions', + data: { + ...base.data, + scope12, + scope3, + biogenic + } }) } if (industry) { - await job.editMessage(`🤖 Skapar jobb för att spara branschdata...`) - saveToAPI.queue.add(companyName + ' industry', { + await flow.add({ ...base, - apiSubEndpoint: 'industry', - industry, + queueName: 'saveIndustry', + data: { + ...base.data, + industry + } }) } if (economy) { - await job.editMessage(`🤖 Skapar jobb för att spara ekonomidata...`) - saveToAPI.queue.add(companyName + ' economy', { + await flow.add({ ...base, - apiSubEndpoint: 'economy', - economy, + queueName: 'saveEconomy', + data: { + ...base.data, + economy + } }) } if (goals) { - await job.editMessage(`🤖 Skapar jobb för att spara mål...`) - saveToAPI.queue.add(companyName + ' goals', { + await flow.add({ ...base, - apiSubEndpoint: 'goals', - goals, + queueName: 'saveGoals', + data: { + ...base.data, + goals + } }) } if (initiatives) { - await job.editMessage(`🤖 Skapar jobb för att spara initiativ...`) - saveToAPI.queue.add(companyName + ' initiatives', { - ...base, - apiSubEndpoint: 'initiatives', - initiatives, + await flow.add({ + ...base, + queueName: 'saveInitiatives', + data: { + ...base.data, + initiatives + } }) } - return JSON.stringify(childrenValues, null, 2) + return JSON.stringify({ saved: true }, null, 2) }) export default checkDB From ea64383b677ae426aa81f56e4486962439aea030 Mon Sep 17 00:00:00 2001 From: "Christian Landgren (aider)" Date: Wed, 4 Dec 2024 15:14:59 +0100 Subject: [PATCH 04/82] feat: Add worker to send company link with green checkmark to Discord --- src/workers/checkDB.ts | 9 +++++++++ src/workers/sendCompanyLink.ts | 37 ++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 src/workers/sendCompanyLink.ts diff --git a/src/workers/checkDB.ts b/src/workers/checkDB.ts index e359d333..094b86f0 100644 --- a/src/workers/checkDB.ts +++ b/src/workers/checkDB.ts @@ -127,6 +127,15 @@ const checkDB = new DiscordWorker('checkDB', async (job: JobData) => { }) } + // Add final step to send company link + await flow.add({ + ...base, + queueName: 'sendCompanyLink', + data: { + ...base.data, + } + }) + return JSON.stringify({ saved: true }, null, 2) }) diff --git a/src/workers/sendCompanyLink.ts b/src/workers/sendCompanyLink.ts new file mode 100644 index 00000000..c95d9b71 --- /dev/null +++ b/src/workers/sendCompanyLink.ts @@ -0,0 +1,37 @@ +import { DiscordJob, DiscordWorker } from '../lib/DiscordWorker' +import redis from '../config/redis' + +export class JobData extends DiscordJob { + declare data: DiscordJob['data'] & { + companyName: string + wikidata: any + } +} + +const sendCompanyLink = new DiscordWorker( + 'sendCompanyLink', + async (job) => { + const { companyName, wikidata } = job.data + const wikidataId = wikidata.node + + // Create URL-safe company name + const urlSafeCompanyName = companyName + .toLowerCase() + .replace(/[åä]/g, 'a') + .replace(/[ö]/g, 'o') + .replace(/[^a-z0-9]/g, '-') + .replace(/-+/g, '-') + .replace(/^-|-$/g, '') + + const url = `http://beta.klimatkollen.se/companies/${wikidataId}-${urlSafeCompanyName}` + + await job.sendMessage(`✅ Företaget har sparats! Se resultatet här: ${url}`) + + return { url } + }, + { + connection: redis, + } +) + +export default sendCompanyLink From 5538a033c17ce2867673b565d33f6ae39ce80fa9 Mon Sep 17 00:00:00 2001 From: "Christian Landgren (aider)" Date: Wed, 4 Dec 2024 15:19:11 +0100 Subject: [PATCH 05/82] refactor: Restructure checkDB flow with sendCompanyLink as main job --- src/workers/checkDB.ts | 104 ++++++++++++++++++----------------------- 1 file changed, 45 insertions(+), 59 deletions(-) diff --git a/src/workers/checkDB.ts b/src/workers/checkDB.ts index 094b86f0..eb965ceb 100644 --- a/src/workers/checkDB.ts +++ b/src/workers/checkDB.ts @@ -70,70 +70,56 @@ const checkDB = new DiscordWorker('checkDB', async (job: JobData) => { await job.editMessage(`🤖 Sparar data...`) - if (scope12 || scope3 || biogenic) { - await flow.add({ - ...base, - queueName: 'saveEmissions', - data: { - ...base.data, - scope12, - scope3, - biogenic - } - }) - } - - if (industry) { - await flow.add({ - ...base, - queueName: 'saveIndustry', - data: { - ...base.data, - industry - } - }) - } - - if (economy) { - await flow.add({ - ...base, - queueName: 'saveEconomy', - data: { - ...base.data, - economy - } - }) - } - - if (goals) { - await flow.add({ - ...base, - queueName: 'saveGoals', - data: { - ...base.data, - goals - } - }) - } - - if (initiatives) { - await flow.add({ - ...base, - queueName: 'saveInitiatives', - data: { - ...base.data, - initiatives - } - }) - } - - // Add final step to send company link await flow.add({ ...base, queueName: 'sendCompanyLink', data: { ...base.data, - } + }, + children: [ + scope12 || scope3 || biogenic ? { + ...base, + queueName: 'saveEmissions', + data: { + ...base.data, + scope12, + scope3, + biogenic + } + } : null, + industry ? { + ...base, + queueName: 'saveIndustry', + data: { + ...base.data, + industry + } + } : null, + economy ? { + ...base, + queueName: 'saveEconomy', + data: { + ...base.data, + economy + } + } : null, + goals ? { + ...base, + queueName: 'saveGoals', + data: { + ...base.data, + goals + } + } : null, + initiatives ? { + ...base, + queueName: 'saveInitiatives', + data: { + ...base.data, + initiatives + } + } : null, + ].filter(Boolean) }) return JSON.stringify({ saved: true }, null, 2) From 1955d3d22b2b235043aaa3fb781f161598c7552a Mon Sep 17 00:00:00 2001 From: "Christian Landgren (aider)" Date: Wed, 4 Dec 2024 15:19:38 +0100 Subject: [PATCH 06/82] feat: Initialize flow producer with redis connection in checkDB worker --- src/workers/checkDB.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/workers/checkDB.ts b/src/workers/checkDB.ts index eb965ceb..1439b83b 100644 --- a/src/workers/checkDB.ts +++ b/src/workers/checkDB.ts @@ -1,6 +1,7 @@ +import { FlowProducer } from 'bullmq' import { DiscordJob, DiscordWorker } from '../lib/DiscordWorker' import { apiFetch } from '../lib/api' -import saveToAPI from './saveToAPI' +import redis from '../config/redis' export class JobData extends DiscordJob { declare data: DiscordJob['data'] & { @@ -13,6 +14,8 @@ export class JobData extends DiscordJob { } } +const flow = new FlowProducer({ connection: redis }) + const checkDB = new DiscordWorker('checkDB', async (job: JobData) => { const { companyName, From 3f8023b77f98fcba876bbc3b6da99c861a41da73 Mon Sep 17 00:00:00 2001 From: "Christian Landgren (aider)" Date: Wed, 4 Dec 2024 15:26:19 +0100 Subject: [PATCH 07/82] feat: Add saveIndustry worker for saving company industry data --- src/workers/saveIndustry.ts | 39 +++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 src/workers/saveIndustry.ts diff --git a/src/workers/saveIndustry.ts b/src/workers/saveIndustry.ts new file mode 100644 index 00000000..f62da007 --- /dev/null +++ b/src/workers/saveIndustry.ts @@ -0,0 +1,39 @@ +import { DiscordJob, DiscordWorker } from '../lib/DiscordWorker' +import { apiFetch } from '../lib/api' +import { defaultMetadata } from '../lib/saveUtils' +import redis from '../config/redis' + +export class JobData extends DiscordJob { + declare data: DiscordJob['data'] & { + companyName: string + wikidata: any + industry?: any + } +} + +const saveIndustry = new DiscordWorker( + 'saveIndustry', + async (job) => { + const { url, wikidata, industry } = job.data + const wikidataId = wikidata.node + const metadata = defaultMetadata(url) + + if (industry) { + job.editMessage(`🤖 Sparar branschdata...`) + return await apiFetch(`/companies/${wikidataId}/industry`, { + body: { + industry, + metadata, + }, + method: 'POST', + }) + } + + return null + }, + { + connection: redis, + } +) + +export default saveIndustry From 9e00165599bb379b138a16122b4687253b37f30e Mon Sep 17 00:00:00 2001 From: "Christian Landgren (aider)" Date: Wed, 4 Dec 2024 15:30:21 +0100 Subject: [PATCH 08/82] refactor: Divide saveEmissions into separate workers for scope12, scope3, and biogenic --- src/workers/extractEmissions.ts | 54 ++++++++++++++++++--------------- src/workers/saveBiogenic.ts | 52 +++++++++++++++++++++++++++++++ src/workers/saveScope12.ts | 53 ++++++++++++++++++++++++++++++++ src/workers/saveScope3.ts | 52 +++++++++++++++++++++++++++++++ 4 files changed, 187 insertions(+), 24 deletions(-) create mode 100644 src/workers/saveBiogenic.ts create mode 100644 src/workers/saveScope12.ts create mode 100644 src/workers/saveScope3.ts diff --git a/src/workers/extractEmissions.ts b/src/workers/extractEmissions.ts index 6a427453..4a7f4b59 100644 --- a/src/workers/extractEmissions.ts +++ b/src/workers/extractEmissions.ts @@ -44,30 +44,36 @@ const extractEmissions = new DiscordWorker( type: JobType.IndustryGics, }, }, - { - ...base, - name: 'scope1+2 ' + companyName, - data: { - ...base.data, - type: JobType.Scope12, - }, - }, - { - ...base, - name: 'scope3 ' + companyName, - data: { - ...base.data, - type: JobType.Scope3, - }, - }, - { - ...base, - name: 'biogenic ' + companyName, - data: { - ...base.data, - type: JobType.Biogenic, - }, - }, + scope12 + ? { + name: 'saveScope12 ' + companyName, + queueName: 'saveScope12', + data: { + ...base.data, + scope12, + }, + } + : null, + scope3 + ? { + name: 'saveScope3 ' + companyName, + queueName: 'saveScope3', + data: { + ...base.data, + scope3, + }, + } + : null, + biogenic + ? { + name: 'saveBiogenic ' + companyName, + queueName: 'saveBiogenic', + data: { + ...base.data, + biogenic, + }, + } + : null, { ...base, name: 'economy ' + companyName, diff --git a/src/workers/saveBiogenic.ts b/src/workers/saveBiogenic.ts new file mode 100644 index 00000000..f48ce58c --- /dev/null +++ b/src/workers/saveBiogenic.ts @@ -0,0 +1,52 @@ +import { DiscordJob, DiscordWorker } from '../lib/DiscordWorker' +import { apiFetch } from '../lib/api' +import { defaultMetadata } from '../lib/saveUtils' +import redis from '../config/redis' +import { getReportingPeriodDates } from '../lib/reportingPeriodDates' + +export class JobData extends DiscordJob { + declare data: DiscordJob['data'] & { + companyName: string + wikidata: any + fiscalYear: any + biogenic?: any[] + } +} + +const saveBiogenic = new DiscordWorker( + 'saveBiogenic', + async (job) => { + const { url, fiscalYear, wikidata, biogenic = [] } = job.data + const wikidataId = wikidata.node + const metadata = defaultMetadata(url) + + if (biogenic?.length) { + job.editMessage(`🤖 Sparar biogeniska utsläpp...`) + return Promise.all( + biogenic.map(async ({ year, biogenic }) => { + const [startDate, endDate] = getReportingPeriodDates( + year, + fiscalYear.startMonth, + fiscalYear.endMonth + ) + job.sendMessage(`🤖 Sparar utsläppsdata biogenic för ${year}...`) + job.log(`Saving biogenic for ${year}`) + const body = { + startDate, + endDate, + emissions: { + biogenic, + }, + metadata, + } + return await apiFetch(`/companies/${wikidataId}/${year}/emissions`, { + body, + }) + }) + ) + } + return null + } +) + +export default saveBiogenic diff --git a/src/workers/saveScope12.ts b/src/workers/saveScope12.ts new file mode 100644 index 00000000..5dc2c992 --- /dev/null +++ b/src/workers/saveScope12.ts @@ -0,0 +1,53 @@ +import { DiscordJob, DiscordWorker } from '../lib/DiscordWorker' +import { apiFetch } from '../lib/api' +import { defaultMetadata } from '../lib/saveUtils' +import redis from '../config/redis' +import { getReportingPeriodDates } from '../lib/reportingPeriodDates' + +export class JobData extends DiscordJob { + declare data: DiscordJob['data'] & { + companyName: string + wikidata: any + fiscalYear: any + scope12?: any[] + } +} + +const saveScope12 = new DiscordWorker( + 'saveScope12', + async (job) => { + const { url, fiscalYear, wikidata, scope12 = [] } = job.data + const wikidataId = wikidata.node + const metadata = defaultMetadata(url) + + if (scope12?.length) { + job.editMessage(`🤖 Sparar utsläppsdata scope 1+2...`) + return Promise.all( + scope12.map(async ({ year, scope1, scope2 }) => { + const [startDate, endDate] = getReportingPeriodDates( + year, + fiscalYear.startMonth, + fiscalYear.endMonth + ) + job.log(`Saving scope1 and scope2 for ${startDate}-${endDate}`) + job.sendMessage(`🤖 Sparar utsläppsdata scope 1+2 för ${year}...`) + const body = { + startDate, + endDate, + emissions: { + scope1, + scope2, + }, + metadata, + } + return await apiFetch(`/companies/${wikidataId}/${year}/emissions`, { + body, + }) + }) + ) + } + return null + } +) + +export default saveScope12 diff --git a/src/workers/saveScope3.ts b/src/workers/saveScope3.ts new file mode 100644 index 00000000..7dcbc777 --- /dev/null +++ b/src/workers/saveScope3.ts @@ -0,0 +1,52 @@ +import { DiscordJob, DiscordWorker } from '../lib/DiscordWorker' +import { apiFetch } from '../lib/api' +import { defaultMetadata } from '../lib/saveUtils' +import redis from '../config/redis' +import { getReportingPeriodDates } from '../lib/reportingPeriodDates' + +export class JobData extends DiscordJob { + declare data: DiscordJob['data'] & { + companyName: string + wikidata: any + fiscalYear: any + scope3?: any[] + } +} + +const saveScope3 = new DiscordWorker( + 'saveScope3', + async (job) => { + const { url, fiscalYear, wikidata, scope3 = [] } = job.data + const wikidataId = wikidata.node + const metadata = defaultMetadata(url) + + if (scope3?.length) { + job.editMessage(`🤖 Sparar utsläppsdata scope 3...`) + return Promise.all( + scope3.map(async ({ year, scope3 }) => { + const [startDate, endDate] = getReportingPeriodDates( + year, + fiscalYear.startMonth, + fiscalYear.endMonth + ) + job.sendMessage(`🤖 Sparar utsläppsdata scope 3 för ${year}...`) + job.log(`Saving scope3 for ${year}`) + const body = { + startDate, + endDate, + emissions: { + scope3, + }, + metadata, + } + return await apiFetch(`/companies/${wikidataId}/${year}/emissions`, { + body, + }) + }) + ) + } + return null + } +) + +export default saveScope3 From 5e776b3a3b7c838e68ba02d5d3d86991f58c263c Mon Sep 17 00:00:00 2001 From: "Christian Landgren (aider)" Date: Wed, 4 Dec 2024 15:33:55 +0100 Subject: [PATCH 09/82] feat: Add askDiff worker to analyze emissions changes between years --- src/workers/askDiff.ts | 57 +++++++++++++++++++++++++++++++++ src/workers/extractEmissions.ts | 7 ++++ 2 files changed, 64 insertions(+) create mode 100644 src/workers/askDiff.ts diff --git a/src/workers/askDiff.ts b/src/workers/askDiff.ts new file mode 100644 index 00000000..4e0cef4d --- /dev/null +++ b/src/workers/askDiff.ts @@ -0,0 +1,57 @@ +import { DiscordJob, DiscordWorker } from '../lib/DiscordWorker' +import { ask } from '../lib/openai' +import { vectorDB } from '../lib/vectordb' + +class JobData extends DiscordJob { + declare data: DiscordJob['data'] & { + companyName: string + wikidata: any + url: string + } +} + +const askDiff = new DiscordWorker( + 'askDiff', + async (job) => { + const { url } = job.data + + const markdown = await vectorDB.getRelevantMarkdown(url, [ + 'change', + 'changed', + 'difference', + 'compared', + 'previous', + 'year', + 'förändring', + 'skillnad', + 'jämfört', + 'föregående', + 'år', + ]) + + const response = await ask( + [ + { + role: 'system', + content: + 'Du är en expert på hållbarhetsrapportering som ska analysera förändringar i utsläpp mellan åren.', + }, + { + role: 'user', + content: `Analysera följande text och beskriv kortfattat de viktigaste förändringarna i utsläpp mellan åren. Fokusera på scope 1, 2 och 3. Svara på svenska. + +${markdown}`, + }, + ], + { + temperature: 0, + } + ) + + await job.sendMessage(`📊 Analys av förändringar:\n${response}`) + + return response + } +) + +export default askDiff diff --git a/src/workers/extractEmissions.ts b/src/workers/extractEmissions.ts index 4a7f4b59..a2d40624 100644 --- a/src/workers/extractEmissions.ts +++ b/src/workers/extractEmissions.ts @@ -99,6 +99,13 @@ const extractEmissions = new DiscordWorker( type: JobType.Initiatives, }, }, + { + name: 'askDiff ' + companyName, + queueName: 'askDiff', + data: { + ...base.data, + }, + }, ], opts: { attempts: 3, From d8bf098e3c6c771b341d323b8b28ccf5d91d3944 Mon Sep 17 00:00:00 2001 From: Christian Landgren Date: Wed, 4 Dec 2024 15:37:28 +0100 Subject: [PATCH 10/82] fix: typescript id --- src/workers/askDiff.ts | 57 ---------------------- src/workers/checkDB.ts | 105 +++++++++++++++++++++++------------------ 2 files changed, 58 insertions(+), 104 deletions(-) delete mode 100644 src/workers/askDiff.ts diff --git a/src/workers/askDiff.ts b/src/workers/askDiff.ts deleted file mode 100644 index 4e0cef4d..00000000 --- a/src/workers/askDiff.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { DiscordJob, DiscordWorker } from '../lib/DiscordWorker' -import { ask } from '../lib/openai' -import { vectorDB } from '../lib/vectordb' - -class JobData extends DiscordJob { - declare data: DiscordJob['data'] & { - companyName: string - wikidata: any - url: string - } -} - -const askDiff = new DiscordWorker( - 'askDiff', - async (job) => { - const { url } = job.data - - const markdown = await vectorDB.getRelevantMarkdown(url, [ - 'change', - 'changed', - 'difference', - 'compared', - 'previous', - 'year', - 'förändring', - 'skillnad', - 'jämfört', - 'föregående', - 'år', - ]) - - const response = await ask( - [ - { - role: 'system', - content: - 'Du är en expert på hållbarhetsrapportering som ska analysera förändringar i utsläpp mellan åren.', - }, - { - role: 'user', - content: `Analysera följande text och beskriv kortfattat de viktigaste förändringarna i utsläpp mellan åren. Fokusera på scope 1, 2 och 3. Svara på svenska. - -${markdown}`, - }, - ], - { - temperature: 0, - } - ) - - await job.sendMessage(`📊 Analys av förändringar:\n${response}`) - - return response - } -) - -export default askDiff diff --git a/src/workers/checkDB.ts b/src/workers/checkDB.ts index 1439b83b..1c15efd4 100644 --- a/src/workers/checkDB.ts +++ b/src/workers/checkDB.ts @@ -5,7 +5,7 @@ import redis from '../config/redis' export class JobData extends DiscordJob { declare data: DiscordJob['data'] & { - companyName?: string + companyName: string description?: string wikidata: any fiscalYear: any @@ -54,8 +54,9 @@ const checkDB = new DiscordWorker('checkDB', async (job: JobData) => { await apiFetch(`/companies`, { body }) } - const { scope12, scope3, biogenic, industry, economy, goals, initiatives } = childrenValues - + const { scope12, scope3, biogenic, industry, economy, goals, initiatives } = + childrenValues + const base = { name: companyName, data: { @@ -68,7 +69,7 @@ const checkDB = new DiscordWorker('checkDB', async (job: JobData) => { }, opts: { attempts: 3, - } + }, } await job.editMessage(`🤖 Sparar data...`) @@ -80,49 +81,59 @@ const checkDB = new DiscordWorker('checkDB', async (job: JobData) => { ...base.data, }, children: [ - scope12 || scope3 || biogenic ? { - ...base, - queueName: 'saveEmissions', - data: { - ...base.data, - scope12, - scope3, - biogenic - } - } : null, - industry ? { - ...base, - queueName: 'saveIndustry', - data: { - ...base.data, - industry - } - } : null, - economy ? { - ...base, - queueName: 'saveEconomy', - data: { - ...base.data, - economy - } - } : null, - goals ? { - ...base, - queueName: 'saveGoals', - data: { - ...base.data, - goals - } - } : null, - initiatives ? { - ...base, - queueName: 'saveInitiatives', - data: { - ...base.data, - initiatives - } - } : null, - ].filter(Boolean) + scope12 || scope3 || biogenic + ? { + ...base, + queueName: 'saveEmissions', + data: { + ...base.data, + scope12, + scope3, + biogenic, + }, + } + : null, + industry + ? { + ...base, + queueName: 'saveIndustry', + data: { + ...base.data, + industry, + }, + } + : null, + economy + ? { + ...base, + queueName: 'saveEconomy', + data: { + ...base.data, + economy, + }, + } + : null, + goals + ? { + ...base, + queueName: 'saveGoals', + data: { + ...base.data, + goals, + }, + } + : null, + initiatives + ? { + ...base, + queueName: 'saveInitiatives', + data: { + ...base.data, + initiatives, + }, + } + : null, + ].filter((e) => e !== null), }) return JSON.stringify({ saved: true }, null, 2) From 85b92e8c0943d2cc0e5208aa46287f8ad6a16042 Mon Sep 17 00:00:00 2001 From: "Christian Landgren (aider)" Date: Wed, 4 Dec 2024 15:41:58 +0100 Subject: [PATCH 11/82] refactor: Move askDiff to saveUtils and update emission workers --- src/lib/saveUtils.ts | 32 ++++++++++++++++++++++++++++++++ src/workers/saveBiogenic.ts | 12 ++++++++++++ src/workers/saveScope12.ts | 12 ++++++++++++ src/workers/saveScope3.ts | 12 ++++++++++++ 4 files changed, 68 insertions(+) diff --git a/src/lib/saveUtils.ts b/src/lib/saveUtils.ts index 103b3a31..5b5cc952 100644 --- a/src/lib/saveUtils.ts +++ b/src/lib/saveUtils.ts @@ -25,7 +25,39 @@ export function formatAsReportingPeriods( }) } +import { askPrompt } from './openai' + export const defaultMetadata = (url: string) => ({ source: url, comment: 'Parsed by Garbo AI', }) + +export const askDiff = async (existingCompany: any, { scope12, scope3, biogenic, fiscalYear }: any) => { + if (!existingCompany.reportingPeriods?.length) return '' + + const before = { + reportingPeriods: existingCompany.reportingPeriods.map(({ startDate, endDate, emissions }) => ({ + startDate, + endDate, + emissions, + })) + } + + const after = { + reportingPeriods: [ + ...(scope12 ? formatAsReportingPeriods(scope12, fiscalYear, 'emissions') : []), + ...(scope3 ? formatAsReportingPeriods(scope3, fiscalYear, 'emissions') : []), + ...(biogenic ? formatAsReportingPeriods(biogenic, fiscalYear, 'emissions') : []) + ] + } + + return await askPrompt( + `What is changed between these two json values? Please respond in clear text with markdown formatting. +The purpose is to let an editor approve the changes or suggest changes in Discord. +Be as brief as possible. Never be technical - meaning no comments about structure changes, fields renames etc. +Focus only on the actual values that have changed. +When handling years and ambiguous dates, always use the last year in the period (e.g. startDate: 2020 - endDate: 2021 should be referred to as 2021). +NEVER REPEAT UNCHANGED VALUES OR UNCHANGED YEARS! If nothing important has changed, just write "NO_CHANGES".`, + JSON.stringify({ before, after }) + ) +} diff --git a/src/workers/saveBiogenic.ts b/src/workers/saveBiogenic.ts index f48ce58c..145e2613 100644 --- a/src/workers/saveBiogenic.ts +++ b/src/workers/saveBiogenic.ts @@ -21,6 +21,18 @@ const saveBiogenic = new DiscordWorker( const metadata = defaultMetadata(url) if (biogenic?.length) { + const existingCompany = await apiFetch(`/companies/${wikidataId}`).catch(() => null) + const diff = await askDiff(existingCompany, { biogenic, fiscalYear }) + + if (diff && !diff.includes('NO_CHANGES')) { + const buttonRow = discord.createButtonRow(job.id!) + await job.sendMessage({ + content: `# ${job.data.companyName}: biogenic emissions\n${diff}`.slice(0, 2000), + components: [buttonRow], + }) + return await job.moveToDelayed(Date.now() + ONE_DAY) + } + job.editMessage(`🤖 Sparar biogeniska utsläpp...`) return Promise.all( biogenic.map(async ({ year, biogenic }) => { diff --git a/src/workers/saveScope12.ts b/src/workers/saveScope12.ts index 5dc2c992..dc1b48e8 100644 --- a/src/workers/saveScope12.ts +++ b/src/workers/saveScope12.ts @@ -21,6 +21,18 @@ const saveScope12 = new DiscordWorker( const metadata = defaultMetadata(url) if (scope12?.length) { + const existingCompany = await apiFetch(`/companies/${wikidataId}`).catch(() => null) + const diff = await askDiff(existingCompany, { scope12, fiscalYear }) + + if (diff && !diff.includes('NO_CHANGES')) { + const buttonRow = discord.createButtonRow(job.id!) + await job.sendMessage({ + content: `# ${job.data.companyName}: scope 1+2 emissions\n${diff}`.slice(0, 2000), + components: [buttonRow], + }) + return await job.moveToDelayed(Date.now() + ONE_DAY) + } + job.editMessage(`🤖 Sparar utsläppsdata scope 1+2...`) return Promise.all( scope12.map(async ({ year, scope1, scope2 }) => { diff --git a/src/workers/saveScope3.ts b/src/workers/saveScope3.ts index 7dcbc777..079bb9dc 100644 --- a/src/workers/saveScope3.ts +++ b/src/workers/saveScope3.ts @@ -21,6 +21,18 @@ const saveScope3 = new DiscordWorker( const metadata = defaultMetadata(url) if (scope3?.length) { + const existingCompany = await apiFetch(`/companies/${wikidataId}`).catch(() => null) + const diff = await askDiff(existingCompany, { scope3, fiscalYear }) + + if (diff && !diff.includes('NO_CHANGES')) { + const buttonRow = discord.createButtonRow(job.id!) + await job.sendMessage({ + content: `# ${job.data.companyName}: scope 3 emissions\n${diff}`.slice(0, 2000), + components: [buttonRow], + }) + return await job.moveToDelayed(Date.now() + ONE_DAY) + } + job.editMessage(`🤖 Sparar utsläppsdata scope 3...`) return Promise.all( scope3.map(async ({ year, scope3 }) => { From a2dc765804c8fdba861a22061f81d82e5f75efd5 Mon Sep 17 00:00:00 2001 From: "Christian Landgren (aider)" Date: Wed, 4 Dec 2024 15:44:21 +0100 Subject: [PATCH 12/82] refactor: Move reporting period dates and diff logic to scope-specific workers --- src/workers/saveBiogenic.ts | 5 ++++- src/workers/saveScope12.ts | 5 ++++- src/workers/saveScope3.ts | 5 ++++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/workers/saveBiogenic.ts b/src/workers/saveBiogenic.ts index 145e2613..4be77136 100644 --- a/src/workers/saveBiogenic.ts +++ b/src/workers/saveBiogenic.ts @@ -1,6 +1,9 @@ import { DiscordJob, DiscordWorker } from '../lib/DiscordWorker' import { apiFetch } from '../lib/api' -import { defaultMetadata } from '../lib/saveUtils' +import { defaultMetadata, askDiff } from '../lib/saveUtils' +import discord from '../discord' + +const ONE_DAY = 1000 * 60 * 60 * 24 import redis from '../config/redis' import { getReportingPeriodDates } from '../lib/reportingPeriodDates' diff --git a/src/workers/saveScope12.ts b/src/workers/saveScope12.ts index dc1b48e8..1f04b39f 100644 --- a/src/workers/saveScope12.ts +++ b/src/workers/saveScope12.ts @@ -1,6 +1,9 @@ import { DiscordJob, DiscordWorker } from '../lib/DiscordWorker' import { apiFetch } from '../lib/api' -import { defaultMetadata } from '../lib/saveUtils' +import { defaultMetadata, askDiff } from '../lib/saveUtils' +import discord from '../discord' + +const ONE_DAY = 1000 * 60 * 60 * 24 import redis from '../config/redis' import { getReportingPeriodDates } from '../lib/reportingPeriodDates' diff --git a/src/workers/saveScope3.ts b/src/workers/saveScope3.ts index 079bb9dc..2b144676 100644 --- a/src/workers/saveScope3.ts +++ b/src/workers/saveScope3.ts @@ -1,6 +1,9 @@ import { DiscordJob, DiscordWorker } from '../lib/DiscordWorker' import { apiFetch } from '../lib/api' -import { defaultMetadata } from '../lib/saveUtils' +import { defaultMetadata, askDiff } from '../lib/saveUtils' +import discord from '../discord' + +const ONE_DAY = 1000 * 60 * 60 * 24 import redis from '../config/redis' import { getReportingPeriodDates } from '../lib/reportingPeriodDates' From 0607fc50fb4b0be45230a2d0cd19a6d05e5526a3 Mon Sep 17 00:00:00 2001 From: Christian Landgren Date: Wed, 4 Dec 2024 15:48:26 +0100 Subject: [PATCH 13/82] fix: move askDiff to separate util function --- src/lib/saveUtils.ts | 32 ++++--- src/workers/saveEmissions.ts | 171 ----------------------------------- 2 files changed, 21 insertions(+), 182 deletions(-) delete mode 100644 src/workers/saveEmissions.ts diff --git a/src/lib/saveUtils.ts b/src/lib/saveUtils.ts index 5b5cc952..36a56932 100644 --- a/src/lib/saveUtils.ts +++ b/src/lib/saveUtils.ts @@ -1,5 +1,4 @@ import { getReportingPeriodDates } from './reportingPeriodDates' -import { apiFetch } from './api' export function formatAsReportingPeriods( entries: { year: number }[], @@ -32,23 +31,34 @@ export const defaultMetadata = (url: string) => ({ comment: 'Parsed by Garbo AI', }) -export const askDiff = async (existingCompany: any, { scope12, scope3, biogenic, fiscalYear }: any) => { +export const askDiff = async ( + existingCompany: any, + { scope12, scope3, biogenic, fiscalYear }: any +) => { if (!existingCompany.reportingPeriods?.length) return '' const before = { - reportingPeriods: existingCompany.reportingPeriods.map(({ startDate, endDate, emissions }) => ({ - startDate, - endDate, - emissions, - })) + reportingPeriods: existingCompany.reportingPeriods.map( + ({ startDate, endDate, emissions }) => ({ + startDate, + endDate, + emissions, + }) + ), } const after = { reportingPeriods: [ - ...(scope12 ? formatAsReportingPeriods(scope12, fiscalYear, 'emissions') : []), - ...(scope3 ? formatAsReportingPeriods(scope3, fiscalYear, 'emissions') : []), - ...(biogenic ? formatAsReportingPeriods(biogenic, fiscalYear, 'emissions') : []) - ] + ...(scope12 + ? formatAsReportingPeriods(scope12, fiscalYear, 'emissions') + : []), + ...(scope3 + ? formatAsReportingPeriods(scope3, fiscalYear, 'emissions') + : []), + ...(biogenic + ? formatAsReportingPeriods(biogenic, fiscalYear, 'emissions') + : []), + ], } return await askPrompt( diff --git a/src/workers/saveEmissions.ts b/src/workers/saveEmissions.ts deleted file mode 100644 index 2f81becf..00000000 --- a/src/workers/saveEmissions.ts +++ /dev/null @@ -1,171 +0,0 @@ -import { DiscordJob, DiscordWorker } from '../lib/DiscordWorker' -import { apiFetch } from '../lib/api' -import { defaultMetadata, formatAsReportingPeriods } from '../lib/saveUtils' -import redis from '../config/redis' -import { askPrompt } from '../lib/openai' -import discord from '../discord' - -const ONE_DAY = 1000 * 60 * 60 * 24 - -export class JobData extends DiscordJob { - declare data: DiscordJob['data'] & { - companyName: string - wikidata: any - fiscalYear: any - scope12?: any[] - scope3?: any[] - biogenic?: any[] - approved?: boolean - } -} - -const askDiff = async (existingCompany, { scope12, scope3, biogenic, fiscalYear }) => { - if (!existingCompany.reportingPeriods?.length) return '' - - const before = { - reportingPeriods: existingCompany.reportingPeriods.map(({ startDate, endDate, emissions }) => ({ - startDate, - endDate, - emissions, - })) - } - - const after = { - reportingPeriods: [ - ...(scope12 ? formatAsReportingPeriods(scope12, fiscalYear, 'emissions') : []), - ...(scope3 ? formatAsReportingPeriods(scope3, fiscalYear, 'emissions') : []), - ...(biogenic ? formatAsReportingPeriods(biogenic, fiscalYear, 'emissions') : []) - ] - } - - return await askPrompt( - `What is changed between these two json values? Please respond in clear text with markdown formatting. -The purpose is to let an editor approve the changes or suggest changes in Discord. -Be as brief as possible. Never be technical - meaning no comments about structure changes, fields renames etc. -Focus only on the actual values that have changed. -When handling years and ambiguous dates, always use the last year in the period (e.g. startDate: 2020 - endDate: 2021 should be referred to as 2021). -NEVER REPEAT UNCHANGED VALUES OR UNCHANGED YEARS! If nothing important has changed, just write "NO_CHANGES".`, - JSON.stringify({ before, after }) - ) -} - -const saveEmissions = new DiscordWorker( - 'saveEmissions', - async (job) => { - const { approved = false } = job.data - const existingCompany = await apiFetch(`/companies/${job.data.wikidata.node}`).catch(() => null) - - if (!approved) { - const diff = await askDiff(existingCompany, job.data) - - if (diff) { - if (diff.includes('NO_CHANGES')) { - await job.sendMessage({ - content: `# ${job.data.companyName}: emissions\n${diff}`.slice(0, 2000), - }) - return diff - } - - const buttonRow = discord.createButtonRow(job.id!) - await job.sendMessage({ - content: `# ${job.data.companyName}: emissions\n${diff}`.slice(0, 2000), - components: [buttonRow], - }) - - return await job.moveToDelayed(Date.now() + ONE_DAY) - } - } - const { url, fiscalYear, wikidata, scope12 = [], scope3 = [], biogenic = [] } = job.data - const wikidataId = wikidata.node - const metadata = defaultMetadata(url) - - if (scope12?.length || scope3?.length || biogenic?.length) { - job.editMessage(`🤖 Sparar utsläppsdata...`) - - return Promise.all([ - ...(await scope12.reduce(async (lastPromise, { year, scope1, scope2 }) => { - const arr = await lastPromise - const [startDate, endDate] = getReportingPeriodDates( - year, - fiscalYear.startMonth, - fiscalYear.endMonth - ) - job.log(`Saving scope1 and scope2 for ${startDate}-${endDate}`) - job.sendMessage(`🤖 Sparar utsläppsdata scope 1+2 för ${year}...`) - const body = { - startDate, - endDate, - emissions: { - scope1, - scope2, - }, - metadata, - } - return [ - ...arr, - await apiFetch(`/companies/${wikidataId}/${year}/emissions`, { - body, - }), - ] - }, Promise.resolve([]))), - - ...(await scope3.reduce(async (lastPromise, { year, scope3 }) => { - const arr = await lastPromise - const [startDate, endDate] = getReportingPeriodDates( - year, - fiscalYear.startMonth, - fiscalYear.endMonth - ) - job.sendMessage(`🤖 Sparar utsläppsdata scope 3 för ${year}...`) - job.log(`Saving scope3 for ${year}`) - const body = { - startDate, - endDate, - emissions: { - scope3, - }, - metadata, - } - return [ - ...arr, - await apiFetch(`/companies/${wikidataId}/${year}/emissions`, { - body, - }), - ] - }, Promise.resolve([]))), - - ...(await biogenic.reduce(async (lastPromise, { year, biogenic }) => { - const arr = await lastPromise - const [startDate, endDate] = getReportingPeriodDates( - year, - fiscalYear.startMonth, - fiscalYear.endMonth - ) - job.sendMessage(`🤖 Sparar utsläppsdata biogenic för ${year}...`) - job.log(`Saving biogenic for ${year}`) - const body = { - startDate, - endDate, - emissions: { - biogenic, - }, - metadata, - } - return [ - ...arr, - await apiFetch(`/companies/${wikidataId}/${year}/emissions`, { - body, - }), - ] - }, Promise.resolve([]))), - ]) - } - - return null - }, - { - connection: redis, - } -) - -export default saveEmissions From ad72765fd65fd0762df90fe5a4b2640d265c0a01 Mon Sep 17 00:00:00 2001 From: Christian Landgren Date: Wed, 4 Dec 2024 16:00:10 +0100 Subject: [PATCH 14/82] fix: prettier --- src/workers/saveBiogenic.ts | 88 ++++++++++++++++---------------- src/workers/saveEconomy.ts | 2 +- src/workers/saveGoals.ts | 2 +- src/workers/saveIndustry.ts | 2 +- src/workers/saveInitiatives.ts | 2 +- src/workers/saveScope12.ts | 91 ++++++++++++++++++---------------- src/workers/saveScope3.ts | 88 ++++++++++++++++---------------- src/workers/saveToAPI.ts | 2 - src/workers/sendCompanyLink.ts | 6 +-- 9 files changed, 144 insertions(+), 139 deletions(-) diff --git a/src/workers/saveBiogenic.ts b/src/workers/saveBiogenic.ts index 4be77136..d935bfba 100644 --- a/src/workers/saveBiogenic.ts +++ b/src/workers/saveBiogenic.ts @@ -16,52 +16,54 @@ export class JobData extends DiscordJob { } } -const saveBiogenic = new DiscordWorker( - 'saveBiogenic', - async (job) => { - const { url, fiscalYear, wikidata, biogenic = [] } = job.data - const wikidataId = wikidata.node - const metadata = defaultMetadata(url) +const saveBiogenic = new DiscordWorker('saveBiogenic', async (job) => { + const { url, fiscalYear, wikidata, biogenic = [] } = job.data + const wikidataId = wikidata.node + const metadata = defaultMetadata(url) - if (biogenic?.length) { - const existingCompany = await apiFetch(`/companies/${wikidataId}`).catch(() => null) - const diff = await askDiff(existingCompany, { biogenic, fiscalYear }) - - if (diff && !diff.includes('NO_CHANGES')) { - const buttonRow = discord.createButtonRow(job.id!) - await job.sendMessage({ - content: `# ${job.data.companyName}: biogenic emissions\n${diff}`.slice(0, 2000), - components: [buttonRow], - }) - return await job.moveToDelayed(Date.now() + ONE_DAY) - } + if (biogenic?.length) { + const existingCompany = await apiFetch(`/companies/${wikidataId}`).catch( + () => null + ) + const diff = await askDiff(existingCompany, { biogenic, fiscalYear }) - job.editMessage(`🤖 Sparar biogeniska utsläpp...`) - return Promise.all( - biogenic.map(async ({ year, biogenic }) => { - const [startDate, endDate] = getReportingPeriodDates( - year, - fiscalYear.startMonth, - fiscalYear.endMonth - ) - job.sendMessage(`🤖 Sparar utsläppsdata biogenic för ${year}...`) - job.log(`Saving biogenic for ${year}`) - const body = { - startDate, - endDate, - emissions: { - biogenic, - }, - metadata, - } - return await apiFetch(`/companies/${wikidataId}/${year}/emissions`, { - body, - }) - }) - ) + if (diff && !diff.includes('NO_CHANGES')) { + const buttonRow = discord.createButtonRow(job.id!) + await job.sendMessage({ + content: `# ${job.data.companyName}: biogenic emissions\n${diff}`.slice( + 0, + 2000 + ), + components: [buttonRow], + }) + return await job.moveToDelayed(Date.now() + ONE_DAY) } - return null + + job.editMessage(`🤖 Sparar biogeniska utsläpp...`) + return Promise.all( + biogenic.map(async ({ year, biogenic }) => { + const [startDate, endDate] = getReportingPeriodDates( + year, + fiscalYear.startMonth, + fiscalYear.endMonth + ) + job.sendMessage(`🤖 Sparar utsläppsdata biogenic för ${year}...`) + job.log(`Saving biogenic for ${year}`) + const body = { + startDate, + endDate, + emissions: { + biogenic, + }, + metadata, + } + return await apiFetch(`/companies/${wikidataId}/${year}/emissions`, { + body, + }) + }) + ) } -) + return null +}) export default saveBiogenic diff --git a/src/workers/saveEconomy.ts b/src/workers/saveEconomy.ts index 7ce9fd75..1adde075 100644 --- a/src/workers/saveEconomy.ts +++ b/src/workers/saveEconomy.ts @@ -44,7 +44,7 @@ const saveEconomy = new DiscordWorker( }) ) } - + return null }, { diff --git a/src/workers/saveGoals.ts b/src/workers/saveGoals.ts index be976a0b..bb940ace 100644 --- a/src/workers/saveGoals.ts +++ b/src/workers/saveGoals.ts @@ -28,7 +28,7 @@ const saveGoals = new DiscordWorker( method: 'POST', }) } - + return null }, { diff --git a/src/workers/saveIndustry.ts b/src/workers/saveIndustry.ts index f62da007..0156822d 100644 --- a/src/workers/saveIndustry.ts +++ b/src/workers/saveIndustry.ts @@ -28,7 +28,7 @@ const saveIndustry = new DiscordWorker( method: 'POST', }) } - + return null }, { diff --git a/src/workers/saveInitiatives.ts b/src/workers/saveInitiatives.ts index 117d046d..b933199b 100644 --- a/src/workers/saveInitiatives.ts +++ b/src/workers/saveInitiatives.ts @@ -28,7 +28,7 @@ const saveInitiatives = new DiscordWorker( method: 'POST', }) } - + return null }, { diff --git a/src/workers/saveScope12.ts b/src/workers/saveScope12.ts index 1f04b39f..5dda738d 100644 --- a/src/workers/saveScope12.ts +++ b/src/workers/saveScope12.ts @@ -16,53 +16,56 @@ export class JobData extends DiscordJob { } } -const saveScope12 = new DiscordWorker( - 'saveScope12', - async (job) => { - const { url, fiscalYear, wikidata, scope12 = [] } = job.data - const wikidataId = wikidata.node - const metadata = defaultMetadata(url) +const saveScope12 = new DiscordWorker('saveScope12', async (job) => { + const { url, fiscalYear, wikidata, scope12 = [] } = job.data + const wikidataId = wikidata.node + const metadata = defaultMetadata(url) - if (scope12?.length) { - const existingCompany = await apiFetch(`/companies/${wikidataId}`).catch(() => null) - const diff = await askDiff(existingCompany, { scope12, fiscalYear }) - - if (diff && !diff.includes('NO_CHANGES')) { - const buttonRow = discord.createButtonRow(job.id!) - await job.sendMessage({ - content: `# ${job.data.companyName}: scope 1+2 emissions\n${diff}`.slice(0, 2000), - components: [buttonRow], - }) - return await job.moveToDelayed(Date.now() + ONE_DAY) - } + if (scope12?.length) { + const existingCompany = await apiFetch(`/companies/${wikidataId}`).catch( + () => null + ) + const diff = await askDiff(existingCompany, { scope12, fiscalYear }) - job.editMessage(`🤖 Sparar utsläppsdata scope 1+2...`) - return Promise.all( - scope12.map(async ({ year, scope1, scope2 }) => { - const [startDate, endDate] = getReportingPeriodDates( - year, - fiscalYear.startMonth, - fiscalYear.endMonth - ) - job.log(`Saving scope1 and scope2 for ${startDate}-${endDate}`) - job.sendMessage(`🤖 Sparar utsläppsdata scope 1+2 för ${year}...`) - const body = { - startDate, - endDate, - emissions: { - scope1, - scope2, - }, - metadata, - } - return await apiFetch(`/companies/${wikidataId}/${year}/emissions`, { - body, - }) - }) - ) + if (diff && !diff.includes('NO_CHANGES')) { + const buttonRow = discord.createButtonRow(job.id!) + await job.sendMessage({ + content: + `# ${job.data.companyName}: scope 1+2 emissions\n${diff}`.slice( + 0, + 2000 + ), + components: [buttonRow], + }) + return await job.moveToDelayed(Date.now() + ONE_DAY) } - return null + + job.editMessage(`🤖 Sparar utsläppsdata scope 1+2...`) + return Promise.all( + scope12.map(async ({ year, scope1, scope2 }) => { + const [startDate, endDate] = getReportingPeriodDates( + year, + fiscalYear.startMonth, + fiscalYear.endMonth + ) + job.log(`Saving scope1 and scope2 for ${startDate}-${endDate}`) + job.sendMessage(`🤖 Sparar utsläppsdata scope 1+2 för ${year}...`) + const body = { + startDate, + endDate, + emissions: { + scope1, + scope2, + }, + metadata, + } + return await apiFetch(`/companies/${wikidataId}/${year}/emissions`, { + body, + }) + }) + ) } -) + return null +}) export default saveScope12 diff --git a/src/workers/saveScope3.ts b/src/workers/saveScope3.ts index 2b144676..506f4b32 100644 --- a/src/workers/saveScope3.ts +++ b/src/workers/saveScope3.ts @@ -16,52 +16,54 @@ export class JobData extends DiscordJob { } } -const saveScope3 = new DiscordWorker( - 'saveScope3', - async (job) => { - const { url, fiscalYear, wikidata, scope3 = [] } = job.data - const wikidataId = wikidata.node - const metadata = defaultMetadata(url) +const saveScope3 = new DiscordWorker('saveScope3', async (job) => { + const { url, fiscalYear, wikidata, scope3 = [] } = job.data + const wikidataId = wikidata.node + const metadata = defaultMetadata(url) - if (scope3?.length) { - const existingCompany = await apiFetch(`/companies/${wikidataId}`).catch(() => null) - const diff = await askDiff(existingCompany, { scope3, fiscalYear }) - - if (diff && !diff.includes('NO_CHANGES')) { - const buttonRow = discord.createButtonRow(job.id!) - await job.sendMessage({ - content: `# ${job.data.companyName}: scope 3 emissions\n${diff}`.slice(0, 2000), - components: [buttonRow], - }) - return await job.moveToDelayed(Date.now() + ONE_DAY) - } + if (scope3?.length) { + const existingCompany = await apiFetch(`/companies/${wikidataId}`).catch( + () => null + ) + const diff = await askDiff(existingCompany, { scope3, fiscalYear }) - job.editMessage(`🤖 Sparar utsläppsdata scope 3...`) - return Promise.all( - scope3.map(async ({ year, scope3 }) => { - const [startDate, endDate] = getReportingPeriodDates( - year, - fiscalYear.startMonth, - fiscalYear.endMonth - ) - job.sendMessage(`🤖 Sparar utsläppsdata scope 3 för ${year}...`) - job.log(`Saving scope3 for ${year}`) - const body = { - startDate, - endDate, - emissions: { - scope3, - }, - metadata, - } - return await apiFetch(`/companies/${wikidataId}/${year}/emissions`, { - body, - }) - }) - ) + if (diff && !diff.includes('NO_CHANGES')) { + const buttonRow = discord.createButtonRow(job.id!) + await job.sendMessage({ + content: `# ${job.data.companyName}: scope 3 emissions\n${diff}`.slice( + 0, + 2000 + ), + components: [buttonRow], + }) + return await job.moveToDelayed(Date.now() + ONE_DAY) } - return null + + job.editMessage(`🤖 Sparar utsläppsdata scope 3...`) + return Promise.all( + scope3.map(async ({ year, scope3 }) => { + const [startDate, endDate] = getReportingPeriodDates( + year, + fiscalYear.startMonth, + fiscalYear.endMonth + ) + job.sendMessage(`🤖 Sparar utsläppsdata scope 3 för ${year}...`) + job.log(`Saving scope3 for ${year}`) + const body = { + startDate, + endDate, + emissions: { + scope3, + }, + metadata, + } + return await apiFetch(`/companies/${wikidataId}/${year}/emissions`, { + body, + }) + }) + ) } -) + return null +}) export default saveScope3 diff --git a/src/workers/saveToAPI.ts b/src/workers/saveToAPI.ts index 9de5f866..72b8d443 100644 --- a/src/workers/saveToAPI.ts +++ b/src/workers/saveToAPI.ts @@ -24,7 +24,6 @@ export class JobData extends DiscordJob { const ONE_DAY = 1000 * 60 * 60 * 24 - const saveToAPI = new DiscordWorker( 'saveToAPI', async (job) => { @@ -50,7 +49,6 @@ const saveToAPI = new DiscordWorker( () => null ) - // Queue the appropriate specialized worker based on the data if (scope12?.length || scope3?.length || biogenic?.length) { return await job.queueChild('saveEmissions', { diff --git a/src/workers/sendCompanyLink.ts b/src/workers/sendCompanyLink.ts index c95d9b71..7440cb45 100644 --- a/src/workers/sendCompanyLink.ts +++ b/src/workers/sendCompanyLink.ts @@ -13,7 +13,7 @@ const sendCompanyLink = new DiscordWorker( async (job) => { const { companyName, wikidata } = job.data const wikidataId = wikidata.node - + // Create URL-safe company name const urlSafeCompanyName = companyName .toLowerCase() @@ -24,9 +24,9 @@ const sendCompanyLink = new DiscordWorker( .replace(/^-|-$/g, '') const url = `http://beta.klimatkollen.se/companies/${wikidataId}-${urlSafeCompanyName}` - + await job.sendMessage(`✅ Företaget har sparats! Se resultatet här: ${url}`) - + return { url } }, { From 025062ce3d392f83577fa45b0762f511edfaf355 Mon Sep 17 00:00:00 2001 From: Samuel Plumppu <6125097+Greenheart@users.noreply.github.com> Date: Wed, 4 Dec 2024 16:01:55 +0100 Subject: [PATCH 15/82] Update src/workers/sendCompanyLink.ts --- src/workers/sendCompanyLink.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/workers/sendCompanyLink.ts b/src/workers/sendCompanyLink.ts index 7440cb45..86c5aa83 100644 --- a/src/workers/sendCompanyLink.ts +++ b/src/workers/sendCompanyLink.ts @@ -23,7 +23,7 @@ const sendCompanyLink = new DiscordWorker( .replace(/-+/g, '-') .replace(/^-|-$/g, '') - const url = `http://beta.klimatkollen.se/companies/${wikidataId}-${urlSafeCompanyName}` + const url = `http://beta.klimatkollen.se/companies/${urlSafeCompanyName}-${wikidataId}` await job.sendMessage(`✅ Företaget har sparats! Se resultatet här: ${url}`) From 72689daf25b59d2f82d33a20d4e278c0da3aac3c Mon Sep 17 00:00:00 2001 From: Samuel Plumppu <6125097+Greenheart@users.noreply.github.com> Date: Wed, 4 Dec 2024 16:03:00 +0100 Subject: [PATCH 16/82] Update src/workers/sendCompanyLink.ts --- src/workers/sendCompanyLink.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/workers/sendCompanyLink.ts b/src/workers/sendCompanyLink.ts index 86c5aa83..3452f885 100644 --- a/src/workers/sendCompanyLink.ts +++ b/src/workers/sendCompanyLink.ts @@ -29,9 +29,6 @@ const sendCompanyLink = new DiscordWorker( return { url } }, - { - connection: redis, - } ) export default sendCompanyLink From 3b638aeea97ce9dbb6233e2c82aa2ab938cee9a3 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 3 Dec 2024 13:00:59 +0000 Subject: [PATCH 17/82] 3.4.10 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 95cf054d..77a29538 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "garbo", - "version": "3.4.9", + "version": "3.4.10", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "garbo", - "version": "3.4.9", + "version": "3.4.10", "license": "MIT License", "dependencies": { "@bull-board/api": "^6.5.3", diff --git a/package.json b/package.json index 55016901..ae526ab5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "garbo", - "version": "3.4.9", + "version": "3.4.10", "description": "", "type": "module", "engines": { From e6524ad628b78eae6ae42f8f42f02380e573ba0a Mon Sep 17 00:00:00 2001 From: fluxcdbot Date: Tue, 3 Dec 2024 13:03:49 +0000 Subject: [PATCH 18/82] ghcr.io/klimatbyran/garbo:3.4.10 --- k8s/api.yaml | 2 +- k8s/pre-deploy/db-migrate.yaml | 2 +- k8s/worker.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/k8s/api.yaml b/k8s/api.yaml index a5567838..b001f53e 100644 --- a/k8s/api.yaml +++ b/k8s/api.yaml @@ -14,7 +14,7 @@ spec: app: garbo spec: containers: - - image: ghcr.io/klimatbyran/garbo:3.4.9 # {"$imagepolicy": "flux-system:garbo"} + - image: ghcr.io/klimatbyran/garbo:3.4.10 # {"$imagepolicy": "flux-system:garbo"} resources: {} name: garbo ports: diff --git a/k8s/pre-deploy/db-migrate.yaml b/k8s/pre-deploy/db-migrate.yaml index 319de155..aa3a8167 100644 --- a/k8s/pre-deploy/db-migrate.yaml +++ b/k8s/pre-deploy/db-migrate.yaml @@ -9,7 +9,7 @@ spec: restartPolicy: Never containers: - name: migration - image: ghcr.io/klimatbyran/garbo:3.4.9 # {"$imagepolicy": "flux-system:garbo"} + image: ghcr.io/klimatbyran/garbo:3.4.10 # {"$imagepolicy": "flux-system:garbo"} command: ['npm', 'run', 'migrate'] env: - name: POSTGRES_PASSWORD diff --git a/k8s/worker.yaml b/k8s/worker.yaml index a6f896a6..29d690d6 100644 --- a/k8s/worker.yaml +++ b/k8s/worker.yaml @@ -14,7 +14,7 @@ spec: app: worker spec: containers: - - image: ghcr.io/klimatbyran/garbo:3.4.9 # {"$imagepolicy": "flux-system:garbo"} + - image: ghcr.io/klimatbyran/garbo:3.4.10 # {"$imagepolicy": "flux-system:garbo"} resources: {} command: ['npm', 'run', 'workers'] name: worker From 479e8a8aeaee9ed304f875d52df177212735a640 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 3 Dec 2024 13:42:08 +0000 Subject: [PATCH 19/82] 3.4.11 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 77a29538..a628ad1a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "garbo", - "version": "3.4.10", + "version": "3.4.11", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "garbo", - "version": "3.4.10", + "version": "3.4.11", "license": "MIT License", "dependencies": { "@bull-board/api": "^6.5.3", diff --git a/package.json b/package.json index ae526ab5..b34ed12d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "garbo", - "version": "3.4.10", + "version": "3.4.11", "description": "", "type": "module", "engines": { From 9ea0e69726e3705a6d8c89c41605cc22a492e52e Mon Sep 17 00:00:00 2001 From: fluxcdbot Date: Tue, 3 Dec 2024 13:44:56 +0000 Subject: [PATCH 20/82] ghcr.io/klimatbyran/garbo:3.4.11 --- k8s/api.yaml | 2 +- k8s/pre-deploy/db-migrate.yaml | 2 +- k8s/worker.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/k8s/api.yaml b/k8s/api.yaml index b001f53e..f86f4b39 100644 --- a/k8s/api.yaml +++ b/k8s/api.yaml @@ -14,7 +14,7 @@ spec: app: garbo spec: containers: - - image: ghcr.io/klimatbyran/garbo:3.4.10 # {"$imagepolicy": "flux-system:garbo"} + - image: ghcr.io/klimatbyran/garbo:3.4.11 # {"$imagepolicy": "flux-system:garbo"} resources: {} name: garbo ports: diff --git a/k8s/pre-deploy/db-migrate.yaml b/k8s/pre-deploy/db-migrate.yaml index aa3a8167..dd3cae45 100644 --- a/k8s/pre-deploy/db-migrate.yaml +++ b/k8s/pre-deploy/db-migrate.yaml @@ -9,7 +9,7 @@ spec: restartPolicy: Never containers: - name: migration - image: ghcr.io/klimatbyran/garbo:3.4.10 # {"$imagepolicy": "flux-system:garbo"} + image: ghcr.io/klimatbyran/garbo:3.4.11 # {"$imagepolicy": "flux-system:garbo"} command: ['npm', 'run', 'migrate'] env: - name: POSTGRES_PASSWORD diff --git a/k8s/worker.yaml b/k8s/worker.yaml index 29d690d6..28c0bd35 100644 --- a/k8s/worker.yaml +++ b/k8s/worker.yaml @@ -14,7 +14,7 @@ spec: app: worker spec: containers: - - image: ghcr.io/klimatbyran/garbo:3.4.10 # {"$imagepolicy": "flux-system:garbo"} + - image: ghcr.io/klimatbyran/garbo:3.4.11 # {"$imagepolicy": "flux-system:garbo"} resources: {} command: ['npm', 'run', 'workers'] name: worker From 9651dcff43536e555b05edfa1a18603c37fb6b3b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 4 Dec 2024 10:22:28 +0000 Subject: [PATCH 21/82] 3.4.12 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index a628ad1a..ec0306bc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "garbo", - "version": "3.4.11", + "version": "3.4.12", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "garbo", - "version": "3.4.11", + "version": "3.4.12", "license": "MIT License", "dependencies": { "@bull-board/api": "^6.5.3", diff --git a/package.json b/package.json index b34ed12d..fb828018 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "garbo", - "version": "3.4.11", + "version": "3.4.12", "description": "", "type": "module", "engines": { From 2df9ed47c70d12a9f5f7b0a1c91209bb7787cfc2 Mon Sep 17 00:00:00 2001 From: fluxcdbot Date: Wed, 4 Dec 2024 10:25:08 +0000 Subject: [PATCH 22/82] ghcr.io/klimatbyran/garbo:3.4.12 --- k8s/api.yaml | 2 +- k8s/pre-deploy/db-migrate.yaml | 2 +- k8s/worker.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/k8s/api.yaml b/k8s/api.yaml index f86f4b39..2663d82f 100644 --- a/k8s/api.yaml +++ b/k8s/api.yaml @@ -14,7 +14,7 @@ spec: app: garbo spec: containers: - - image: ghcr.io/klimatbyran/garbo:3.4.11 # {"$imagepolicy": "flux-system:garbo"} + - image: ghcr.io/klimatbyran/garbo:3.4.12 # {"$imagepolicy": "flux-system:garbo"} resources: {} name: garbo ports: diff --git a/k8s/pre-deploy/db-migrate.yaml b/k8s/pre-deploy/db-migrate.yaml index dd3cae45..ed3a7ef8 100644 --- a/k8s/pre-deploy/db-migrate.yaml +++ b/k8s/pre-deploy/db-migrate.yaml @@ -9,7 +9,7 @@ spec: restartPolicy: Never containers: - name: migration - image: ghcr.io/klimatbyran/garbo:3.4.11 # {"$imagepolicy": "flux-system:garbo"} + image: ghcr.io/klimatbyran/garbo:3.4.12 # {"$imagepolicy": "flux-system:garbo"} command: ['npm', 'run', 'migrate'] env: - name: POSTGRES_PASSWORD diff --git a/k8s/worker.yaml b/k8s/worker.yaml index 28c0bd35..b0c575cd 100644 --- a/k8s/worker.yaml +++ b/k8s/worker.yaml @@ -14,7 +14,7 @@ spec: app: worker spec: containers: - - image: ghcr.io/klimatbyran/garbo:3.4.11 # {"$imagepolicy": "flux-system:garbo"} + - image: ghcr.io/klimatbyran/garbo:3.4.12 # {"$imagepolicy": "flux-system:garbo"} resources: {} command: ['npm', 'run', 'workers'] name: worker From 8bd0a2849eb5d07528dfca45041a98d6d432932f Mon Sep 17 00:00:00 2001 From: Christian Landgren Date: Wed, 4 Dec 2024 12:09:31 +0100 Subject: [PATCH 23/82] feat: scale nlm to more workers --- k8s/nlm-ingestor.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/k8s/nlm-ingestor.yaml b/k8s/nlm-ingestor.yaml index 72566144..fd77840a 100644 --- a/k8s/nlm-ingestor.yaml +++ b/k8s/nlm-ingestor.yaml @@ -3,7 +3,7 @@ kind: Deployment metadata: name: nlm spec: - replicas: 1 + replicas: 5 selector: matchLabels: app: nlm @@ -16,7 +16,7 @@ spec: - image: ghcr.io/nlmatics/nlm-ingestor resources: limits: - cpu: 3000m + cpu: 1100m memory: 2Gi name: nlm ports: From b1d2076ac59b323ad098b51d987d52e3e099f775 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Bj=C3=B6rk?= Date: Wed, 4 Dec 2024 13:28:14 +0100 Subject: [PATCH 24/82] Post Process Paragraphs to Improve Chunking (#385) * post processed paragraphs to check for single line paragraphs and merge them * add clarifying commetns --- src/lib/vectordb.ts | 22 +++++++++++++++++++++- src/prompts/followUp/scope12.ts | 2 +- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/src/lib/vectordb.ts b/src/lib/vectordb.ts index de452eb8..aff4b494 100644 --- a/src/lib/vectordb.ts +++ b/src/lib/vectordb.ts @@ -23,9 +23,29 @@ async function addReport(url: string, markdown: string) { .map((p) => p.trim()) .filter((p) => p.length > 0) + let prefix = '' + const mergedParagraphs: string[] = [] + + // Combine standalone headers (titles without body) with the next paragraph that has a body. + for (let i = 0; i < paragraphs.length; i++) { + const current = paragraphs[i] + const hasBody = current.split('\n').length > 1 + + if (!hasBody) { + prefix += (prefix ? '\n' : '') + current + } else { + mergedParagraphs.push((prefix ? prefix + '\n' : '') + current) + prefix = '' + } + } + + if (prefix) { + mergedParagraphs.push(prefix) + } + const documentChunks: { chunk: string; paragraph: string }[] = [] - paragraphs.forEach((paragraph) => { + mergedParagraphs.forEach((paragraph) => { for (let i = 0; i < paragraph.length; i += CHUNK_SIZE - overlapSize) { const chunk = paragraph.slice(i, i + CHUNK_SIZE).trim() if (chunk.length > 0) { diff --git a/src/prompts/followUp/scope12.ts b/src/prompts/followUp/scope12.ts index 4795bdcf..02c02163 100644 --- a/src/prompts/followUp/scope12.ts +++ b/src/prompts/followUp/scope12.ts @@ -43,7 +43,7 @@ NEVER CALCULATE ANY EMISSIONS. ONLY REPORT THE DATA AS IT IS IN THE PDF. If you Example - feel free to add more fields and relevant data: { "scope12": [{ - "year": 2021, + "year": 2023, "scope1": { "total": 12.3 }, From ebce0dee2eedeb4bce9cf9327433c8e143b9595a Mon Sep 17 00:00:00 2001 From: Christian Landgren Date: Wed, 4 Dec 2024 13:33:41 +0100 Subject: [PATCH 25/82] fix: delay when parsing pdf with nlm-ingestor --- src/discord/commands/pdfs.ts | 18 ++++++++++++++---- src/workers/nlmParsePDF.ts | 3 +++ 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/discord/commands/pdfs.ts b/src/discord/commands/pdfs.ts index 79cd3097..d04e327f 100644 --- a/src/discord/commands/pdfs.ts +++ b/src/discord/commands/pdfs.ts @@ -54,10 +54,20 @@ export default { }) thread.send(`PDF i kö: ${url}`) - nlmParsePDF.queue.add('download ' + url.slice(-20), { - url, - threadId: thread.id, - }) + nlmParsePDF.queue.add( + 'download ' + url.slice(-20), + { + url, + threadId: thread.id, + }, + { + backoff: { + type: 'fixed', + delay: 60_000, + }, + attempts: 10, + } + ) }) } catch (error) { console.error('Pdfs: error', error) diff --git a/src/workers/nlmParsePDF.ts b/src/workers/nlmParsePDF.ts index ceb0ae18..0bac508a 100644 --- a/src/workers/nlmParsePDF.ts +++ b/src/workers/nlmParsePDF.ts @@ -44,6 +44,9 @@ const nlmParsePDF = new DiscordWorker( let json try { json = await extractJsonFromPdf(pdf) + } catch (err) { + job.editMessage(`❌ Fel vid tolkning av PDF: ${err.message}`) + throw new Error('Failed to parse PDF, retrying in one minute...') } finally { clearInterval(interval) } From 1d8ca50271b936388a064e41618f21948095c5d1 Mon Sep 17 00:00:00 2001 From: Christian Landgren Date: Wed, 4 Dec 2024 13:41:14 +0100 Subject: [PATCH 26/82] chore: better error messages --- src/workers/nlmParsePDF.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/workers/nlmParsePDF.ts b/src/workers/nlmParsePDF.ts index 0bac508a..a174d3ab 100644 --- a/src/workers/nlmParsePDF.ts +++ b/src/workers/nlmParsePDF.ts @@ -45,7 +45,15 @@ const nlmParsePDF = new DiscordWorker( try { json = await extractJsonFromPdf(pdf) } catch (err) { - job.editMessage(`❌ Fel vid tolkning av PDF: ${err.message}`) + if (job.attemptsMade < (job.opts?.attempts || 10)) { + job.editMessage( + `❌ Fel vid tolkning av PDF: ${err.message}. Försöker igen om en stund...` + ) + } else { + job.editMessage( + `❌ Fel vid tolkning av PDF: ${err.message}. Ger upp...` + ) + } throw new Error('Failed to parse PDF, retrying in one minute...') } finally { clearInterval(interval) From fa175f81ae8cfc2d143cdb307fc2cd3d188c1b87 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 4 Dec 2024 12:51:49 +0000 Subject: [PATCH 27/82] 3.4.13 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index ec0306bc..0f6889c3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "garbo", - "version": "3.4.12", + "version": "3.4.13", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "garbo", - "version": "3.4.12", + "version": "3.4.13", "license": "MIT License", "dependencies": { "@bull-board/api": "^6.5.3", diff --git a/package.json b/package.json index fb828018..c47ab34a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "garbo", - "version": "3.4.12", + "version": "3.4.13", "description": "", "type": "module", "engines": { From 7831ff045ba2c82779e34aa56164cdce4e1e0afb Mon Sep 17 00:00:00 2001 From: fluxcdbot Date: Wed, 4 Dec 2024 12:54:23 +0000 Subject: [PATCH 28/82] ghcr.io/klimatbyran/garbo:3.4.13 --- k8s/api.yaml | 2 +- k8s/pre-deploy/db-migrate.yaml | 2 +- k8s/worker.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/k8s/api.yaml b/k8s/api.yaml index 2663d82f..705acd3b 100644 --- a/k8s/api.yaml +++ b/k8s/api.yaml @@ -14,7 +14,7 @@ spec: app: garbo spec: containers: - - image: ghcr.io/klimatbyran/garbo:3.4.12 # {"$imagepolicy": "flux-system:garbo"} + - image: ghcr.io/klimatbyran/garbo:3.4.13 # {"$imagepolicy": "flux-system:garbo"} resources: {} name: garbo ports: diff --git a/k8s/pre-deploy/db-migrate.yaml b/k8s/pre-deploy/db-migrate.yaml index ed3a7ef8..9e4205e2 100644 --- a/k8s/pre-deploy/db-migrate.yaml +++ b/k8s/pre-deploy/db-migrate.yaml @@ -9,7 +9,7 @@ spec: restartPolicy: Never containers: - name: migration - image: ghcr.io/klimatbyran/garbo:3.4.12 # {"$imagepolicy": "flux-system:garbo"} + image: ghcr.io/klimatbyran/garbo:3.4.13 # {"$imagepolicy": "flux-system:garbo"} command: ['npm', 'run', 'migrate'] env: - name: POSTGRES_PASSWORD diff --git a/k8s/worker.yaml b/k8s/worker.yaml index b0c575cd..d57cfc73 100644 --- a/k8s/worker.yaml +++ b/k8s/worker.yaml @@ -14,7 +14,7 @@ spec: app: worker spec: containers: - - image: ghcr.io/klimatbyran/garbo:3.4.12 # {"$imagepolicy": "flux-system:garbo"} + - image: ghcr.io/klimatbyran/garbo:3.4.13 # {"$imagepolicy": "flux-system:garbo"} resources: {} command: ['npm', 'run', 'workers'] name: worker From 39581a7e878be0d84c8192d52bf9d798cae75fd9 Mon Sep 17 00:00:00 2001 From: Christian Landgren Date: Wed, 4 Dec 2024 16:02:32 +0100 Subject: [PATCH 29/82] fix: remove old job --- src/workers/saveToAPI.ts | 92 ---------------------------------------- 1 file changed, 92 deletions(-) delete mode 100644 src/workers/saveToAPI.ts diff --git a/src/workers/saveToAPI.ts b/src/workers/saveToAPI.ts deleted file mode 100644 index 72b8d443..00000000 --- a/src/workers/saveToAPI.ts +++ /dev/null @@ -1,92 +0,0 @@ -import { DiscordJob, DiscordWorker } from '../lib/DiscordWorker' -import { apiFetch } from '../lib/api' -import discord from '../discord' -import redis from '../config/redis' -import { askPrompt } from '../lib/openai' -import { formatAsReportingPeriods } from '../lib/saveUtils' - -export class JobData extends DiscordJob { - declare data: DiscordJob['data'] & { - apiSubEndpoint: string - companyName?: string - wikidata: any - fiscalYear: any - scope12?: any - scope3?: any - biogenic?: any - industry?: any - economy?: any - goals?: any - initiatives?: any - approved?: boolean - } -} - -const ONE_DAY = 1000 * 60 * 60 * 24 - -const saveToAPI = new DiscordWorker( - 'saveToAPI', - async (job) => { - const { - apiSubEndpoint = 'general', - companyName, - url, - fiscalYear, - wikidata, - scope12 = [], - scope3 = [], - biogenic = [], - economy = [], - goals, - initiatives, - industry, - approved = false, - } = job.data - - job.sendMessage(`🤖 sparar ${companyName}.${apiSubEndpoint} till API...`) - const wikidataId = wikidata.node - const existingCompany = await apiFetch(`/companies/${wikidataId}`).catch( - () => null - ) - - // Queue the appropriate specialized worker based on the data - if (scope12?.length || scope3?.length || biogenic?.length) { - return await job.queueChild('saveEmissions', { - scope12, - scope3, - biogenic, - }) - } - - if (industry) { - job.editMessage(`🤖 Sparar GICS industri...`) - return await apiFetch(`/companies/${wikidataId}/industry`, { - body: { - industry, - metadata: defaultMetadata(url), - }, - method: 'PUT', - }) - } - - if (goals) { - return await job.queueChild('saveGoals', { goals }) - } - - if (initiatives) { - return await job.queueChild('saveInitiatives', { initiatives }) - } - - if (economy?.length) { - return await job.queueChild('saveEconomy', { economy }) - } - - throw new Error('No data to save') - }, - { - concurrency: 10, - connection: redis, - } -) - -export default saveToAPI From d8e17a5747cfc0558acf7dd644462d167e118064 Mon Sep 17 00:00:00 2001 From: Samuel Plumppu <6125097+Greenheart@users.noreply.github.com> Date: Wed, 4 Dec 2024 16:25:35 +0100 Subject: [PATCH 30/82] Update src/workers/checkDB.ts --- src/workers/checkDB.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/workers/checkDB.ts b/src/workers/checkDB.ts index 1c15efd4..29a424d2 100644 --- a/src/workers/checkDB.ts +++ b/src/workers/checkDB.ts @@ -1,7 +1,6 @@ import { FlowProducer } from 'bullmq' import { DiscordJob, DiscordWorker } from '../lib/DiscordWorker' import { apiFetch } from '../lib/api' -import redis from '../config/redis' export class JobData extends DiscordJob { declare data: DiscordJob['data'] & { From 24935cc558c2a9acca9aec4fe2f9fff5b5a67add Mon Sep 17 00:00:00 2001 From: Samuel Plumppu <6125097+Greenheart@users.noreply.github.com> Date: Wed, 4 Dec 2024 16:25:45 +0100 Subject: [PATCH 31/82] Update src/workers/saveBiogenic.ts --- src/workers/saveBiogenic.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/workers/saveBiogenic.ts b/src/workers/saveBiogenic.ts index d935bfba..a5356d9e 100644 --- a/src/workers/saveBiogenic.ts +++ b/src/workers/saveBiogenic.ts @@ -4,7 +4,6 @@ import { defaultMetadata, askDiff } from '../lib/saveUtils' import discord from '../discord' const ONE_DAY = 1000 * 60 * 60 * 24 -import redis from '../config/redis' import { getReportingPeriodDates } from '../lib/reportingPeriodDates' export class JobData extends DiscordJob { From 0eaf7240380372c747896489fb77eab8797ce8bd Mon Sep 17 00:00:00 2001 From: Samuel Plumppu <6125097+Greenheart@users.noreply.github.com> Date: Wed, 4 Dec 2024 16:25:54 +0100 Subject: [PATCH 32/82] Update src/workers/saveEconomy.ts --- src/workers/saveEconomy.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/workers/saveEconomy.ts b/src/workers/saveEconomy.ts index 1adde075..f16b33e6 100644 --- a/src/workers/saveEconomy.ts +++ b/src/workers/saveEconomy.ts @@ -1,7 +1,6 @@ import { DiscordJob, DiscordWorker } from '../lib/DiscordWorker' import { apiFetch } from '../lib/api' import { defaultMetadata, formatAsReportingPeriods } from '../lib/saveUtils' -import redis from '../config/redis' import { getReportingPeriodDates } from '../lib/reportingPeriodDates' export class JobData extends DiscordJob { From 8dbead181de121589e0169fa1e4342eead2e5b1a Mon Sep 17 00:00:00 2001 From: Samuel Plumppu <6125097+Greenheart@users.noreply.github.com> Date: Wed, 4 Dec 2024 16:26:01 +0100 Subject: [PATCH 33/82] Update src/workers/saveInitiatives.ts --- src/workers/saveInitiatives.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/workers/saveInitiatives.ts b/src/workers/saveInitiatives.ts index b933199b..ccc74682 100644 --- a/src/workers/saveInitiatives.ts +++ b/src/workers/saveInitiatives.ts @@ -1,7 +1,6 @@ import { DiscordJob, DiscordWorker } from '../lib/DiscordWorker' import { apiFetch } from '../lib/api' import { defaultMetadata } from '../lib/saveUtils' -import redis from '../config/redis' export class JobData extends DiscordJob { declare data: DiscordJob['data'] & { From 724e296f0adee3dc47d0065659861316c0563e97 Mon Sep 17 00:00:00 2001 From: Samuel Plumppu <6125097+Greenheart@users.noreply.github.com> Date: Wed, 4 Dec 2024 16:26:08 +0100 Subject: [PATCH 34/82] Update src/workers/saveIndustry.ts --- src/workers/saveIndustry.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/workers/saveIndustry.ts b/src/workers/saveIndustry.ts index 0156822d..54a366a0 100644 --- a/src/workers/saveIndustry.ts +++ b/src/workers/saveIndustry.ts @@ -31,9 +31,6 @@ const saveIndustry = new DiscordWorker( return null }, - { - connection: redis, - } ) export default saveIndustry From 744cb155097608c06fdb5f71057148a7f1fc6180 Mon Sep 17 00:00:00 2001 From: Samuel Plumppu <6125097+Greenheart@users.noreply.github.com> Date: Wed, 4 Dec 2024 16:26:13 +0100 Subject: [PATCH 35/82] Update src/workers/sendCompanyLink.ts --- src/workers/sendCompanyLink.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/workers/sendCompanyLink.ts b/src/workers/sendCompanyLink.ts index 3452f885..4e521ce0 100644 --- a/src/workers/sendCompanyLink.ts +++ b/src/workers/sendCompanyLink.ts @@ -1,5 +1,4 @@ import { DiscordJob, DiscordWorker } from '../lib/DiscordWorker' -import redis from '../config/redis' export class JobData extends DiscordJob { declare data: DiscordJob['data'] & { From baac35bf6258c68caf1c6049d919615361bd2226 Mon Sep 17 00:00:00 2001 From: Samuel Plumppu <6125097+Greenheart@users.noreply.github.com> Date: Wed, 4 Dec 2024 16:26:20 +0100 Subject: [PATCH 36/82] Update src/workers/saveInitiatives.ts --- src/workers/saveInitiatives.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/workers/saveInitiatives.ts b/src/workers/saveInitiatives.ts index ccc74682..f7a8cc93 100644 --- a/src/workers/saveInitiatives.ts +++ b/src/workers/saveInitiatives.ts @@ -30,9 +30,6 @@ const saveInitiatives = new DiscordWorker( return null }, - { - connection: redis, - } ) export default saveInitiatives From 1274808fc7819f8d6f508410ad8a70e7adf105ae Mon Sep 17 00:00:00 2001 From: Samuel Plumppu <6125097+Greenheart@users.noreply.github.com> Date: Wed, 4 Dec 2024 16:26:29 +0100 Subject: [PATCH 37/82] Update src/workers/saveEconomy.ts --- src/workers/saveEconomy.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/workers/saveEconomy.ts b/src/workers/saveEconomy.ts index f16b33e6..286cf870 100644 --- a/src/workers/saveEconomy.ts +++ b/src/workers/saveEconomy.ts @@ -46,9 +46,6 @@ const saveEconomy = new DiscordWorker( return null }, - { - connection: redis, - } ) export default saveEconomy From d594dbf202bf247cada883da4360578db46ae499 Mon Sep 17 00:00:00 2001 From: Samuel Plumppu <6125097+Greenheart@users.noreply.github.com> Date: Wed, 4 Dec 2024 16:26:35 +0100 Subject: [PATCH 38/82] Update src/workers/saveScope3.ts --- src/workers/saveScope3.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/workers/saveScope3.ts b/src/workers/saveScope3.ts index 506f4b32..c3608fc0 100644 --- a/src/workers/saveScope3.ts +++ b/src/workers/saveScope3.ts @@ -1,7 +1,6 @@ import { DiscordJob, DiscordWorker } from '../lib/DiscordWorker' import { apiFetch } from '../lib/api' import { defaultMetadata, askDiff } from '../lib/saveUtils' -import discord from '../discord' const ONE_DAY = 1000 * 60 * 60 * 24 import redis from '../config/redis' From 6170c67463d5b53ec30957103b4484b8fa86d727 Mon Sep 17 00:00:00 2001 From: Samuel Plumppu <6125097+Greenheart@users.noreply.github.com> Date: Wed, 4 Dec 2024 16:26:44 +0100 Subject: [PATCH 39/82] Update src/workers/saveGoals.ts --- src/workers/saveGoals.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/workers/saveGoals.ts b/src/workers/saveGoals.ts index bb940ace..75fd9875 100644 --- a/src/workers/saveGoals.ts +++ b/src/workers/saveGoals.ts @@ -1,7 +1,6 @@ import { DiscordJob, DiscordWorker } from '../lib/DiscordWorker' import { apiFetch } from '../lib/api' import { defaultMetadata } from '../lib/saveUtils' -import redis from '../config/redis' export class JobData extends DiscordJob { declare data: DiscordJob['data'] & { From 3b9bf0254e8d70992e399b62ee52a4173c971b4b Mon Sep 17 00:00:00 2001 From: Samuel Plumppu <6125097+Greenheart@users.noreply.github.com> Date: Wed, 4 Dec 2024 16:26:55 +0100 Subject: [PATCH 40/82] Update src/workers/saveGoals.ts --- src/workers/saveGoals.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/workers/saveGoals.ts b/src/workers/saveGoals.ts index 75fd9875..07db9f75 100644 --- a/src/workers/saveGoals.ts +++ b/src/workers/saveGoals.ts @@ -30,9 +30,6 @@ const saveGoals = new DiscordWorker( return null }, - { - connection: redis, - } ) export default saveGoals From 0c3f35ca977490978e126f32e81dcbb448c84e34 Mon Sep 17 00:00:00 2001 From: Samuel Plumppu <6125097+Greenheart@users.noreply.github.com> Date: Wed, 4 Dec 2024 16:27:04 +0100 Subject: [PATCH 41/82] Update src/workers/saveIndustry.ts --- src/workers/saveIndustry.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/workers/saveIndustry.ts b/src/workers/saveIndustry.ts index 54a366a0..9f397687 100644 --- a/src/workers/saveIndustry.ts +++ b/src/workers/saveIndustry.ts @@ -1,7 +1,6 @@ import { DiscordJob, DiscordWorker } from '../lib/DiscordWorker' import { apiFetch } from '../lib/api' import { defaultMetadata } from '../lib/saveUtils' -import redis from '../config/redis' export class JobData extends DiscordJob { declare data: DiscordJob['data'] & { From ef16ce7b06d73d7a4e963963bdaee3a9a7d2303a Mon Sep 17 00:00:00 2001 From: "Christian Landgren (aider)" Date: Wed, 4 Dec 2024 16:26:13 +0100 Subject: [PATCH 42/82] feat: Extract API save logic to dedicated worker for Discord approvals --- src/discord/interactions/approve.ts | 21 +++++++++++++--- src/workers/apiSaveWorker.ts | 37 +++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 3 deletions(-) create mode 100644 src/workers/apiSaveWorker.ts diff --git a/src/discord/interactions/approve.ts b/src/discord/interactions/approve.ts index d8fc3c4f..8ac58c12 100644 --- a/src/discord/interactions/approve.ts +++ b/src/discord/interactions/approve.ts @@ -1,14 +1,29 @@ import { ButtonInteraction } from 'discord.js' import { DiscordJob } from '../../lib/DiscordWorker' +import { Queue } from 'bullmq' + +const apiSaveQueue = new Queue('api-save', { + connection: { + host: process.env.REDIS_HOST || 'localhost', + port: parseInt(process.env.REDIS_PORT || '6379'), + }, +}) export default { async execute(interaction: ButtonInteraction, job: DiscordJob) { await job.updateData({ ...job.data, approved: true }) - await job.promote() + + // Add to API save queue + await apiSaveQueue.add('save-approved', { + wikidataId: job.data.wikidataId, + approved: true + }) + job.log(`Approving company edit: ${job.data.wikidataId}`) - interaction.reply({ + await interaction.reply({ content: `Tack för din granskning, ${interaction?.user?.username}!`, }) - //discord.lockThread(threadId) + + await job.promote() }, } diff --git a/src/workers/apiSaveWorker.ts b/src/workers/apiSaveWorker.ts new file mode 100644 index 00000000..01ecfca1 --- /dev/null +++ b/src/workers/apiSaveWorker.ts @@ -0,0 +1,37 @@ +import { Job, Worker } from 'bullmq' +import { DiscordJob } from '../lib/DiscordWorker' + +export interface ApiSaveJob extends Job { + data: { + wikidataId: string + approved: boolean + } +} + +export const apiSaveWorker = new Worker( + 'api-save', + async (job: ApiSaveJob) => { + try { + const { wikidataId, approved } = job.data + if (!approved) { + throw new Error('Cannot save unapproved data') + } + + // TODO: Implement actual API save logic here + console.log(`Saving approved data for ${wikidataId} to API`) + + return { success: true } + } catch (error) { + console.error('API Save error:', error) + throw error + } + }, + { + connection: { + host: process.env.REDIS_HOST || 'localhost', + port: parseInt(process.env.REDIS_PORT || '6379'), + }, + } +) + +export default apiSaveWorker From 592f37539ccb15218813110cc02bfb61f26173aa Mon Sep 17 00:00:00 2001 From: "Christian Landgren (aider)" Date: Wed, 4 Dec 2024 16:26:37 +0100 Subject: [PATCH 43/82] refactor: Standardize job types and data flow across Discord and API save worker --- src/discord.ts | 14 +++++++++++--- src/discord/interactions/approve.ts | 2 +- src/workers/apiSaveWorker.ts | 4 ++++ 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/src/discord.ts b/src/discord.ts index 65f93e65..a6909d16 100644 --- a/src/discord.ts +++ b/src/discord.ts @@ -15,9 +15,17 @@ import commands from './discord/commands' import config from './config/discord' import approve from './discord/interactions/approve' import reject from './discord/interactions/reject' -import saveToAPI, { JobData as SaveToApiJob } from './workers/saveToAPI' +import { Queue } from 'bullmq' +import { ApiSaveJob } from './workers/apiSaveWorker' -const getJob = (jobId: string) => saveToAPI.queue.getJob(jobId) +const apiSaveQueue = new Queue('api-save', { + connection: { + host: process.env.REDIS_HOST || 'localhost', + port: parseInt(process.env.REDIS_PORT || '6379'), + }, +}) + +const getJob = (jobId: string) => apiSaveQueue.getJob(jobId) export class Discord { client: Client @@ -75,7 +83,7 @@ export class Discord { try { switch (action) { case 'approve': { - const job = (await getJob(jobId)) as SaveToApiJob + const job = (await getJob(jobId)) as ApiSaveJob if (!job) await interaction.reply('Job not found') else await approve.execute(interaction, job) break diff --git a/src/discord/interactions/approve.ts b/src/discord/interactions/approve.ts index 8ac58c12..8fd453c2 100644 --- a/src/discord/interactions/approve.ts +++ b/src/discord/interactions/approve.ts @@ -15,7 +15,7 @@ export default { // Add to API save queue await apiSaveQueue.add('save-approved', { - wikidataId: job.data.wikidataId, + ...job.data, approved: true }) diff --git a/src/workers/apiSaveWorker.ts b/src/workers/apiSaveWorker.ts index 01ecfca1..0210a71d 100644 --- a/src/workers/apiSaveWorker.ts +++ b/src/workers/apiSaveWorker.ts @@ -5,6 +5,10 @@ export interface ApiSaveJob extends Job { data: { wikidataId: string approved: boolean + url?: string + threadId?: string + channelId?: string + messageId?: string } } From c44fbff3a03a2ffde2bbcc41d01f2c632394e50d Mon Sep 17 00:00:00 2001 From: "Christian Landgren (aider)" Date: Wed, 4 Dec 2024 16:27:08 +0100 Subject: [PATCH 44/82] refactor: Rename apiSaveWorker to saveToApi --- src/discord.ts | 4 +-- src/discord/interactions/approve.ts | 2 +- src/workers/saveToApi.ts | 41 +++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 3 deletions(-) create mode 100644 src/workers/saveToApi.ts diff --git a/src/discord.ts b/src/discord.ts index a6909d16..63eebfc6 100644 --- a/src/discord.ts +++ b/src/discord.ts @@ -16,7 +16,7 @@ import config from './config/discord' import approve from './discord/interactions/approve' import reject from './discord/interactions/reject' import { Queue } from 'bullmq' -import { ApiSaveJob } from './workers/apiSaveWorker' +import { SaveToApiJob } from './workers/saveToApi' const apiSaveQueue = new Queue('api-save', { connection: { @@ -83,7 +83,7 @@ export class Discord { try { switch (action) { case 'approve': { - const job = (await getJob(jobId)) as ApiSaveJob + const job = (await getJob(jobId)) as SaveToApiJob if (!job) await interaction.reply('Job not found') else await approve.execute(interaction, job) break diff --git a/src/discord/interactions/approve.ts b/src/discord/interactions/approve.ts index 8fd453c2..713ddca4 100644 --- a/src/discord/interactions/approve.ts +++ b/src/discord/interactions/approve.ts @@ -2,7 +2,7 @@ import { ButtonInteraction } from 'discord.js' import { DiscordJob } from '../../lib/DiscordWorker' import { Queue } from 'bullmq' -const apiSaveQueue = new Queue('api-save', { +const saveToApiQueue = new Queue('api-save', { connection: { host: process.env.REDIS_HOST || 'localhost', port: parseInt(process.env.REDIS_PORT || '6379'), diff --git a/src/workers/saveToApi.ts b/src/workers/saveToApi.ts new file mode 100644 index 00000000..fda9e0b7 --- /dev/null +++ b/src/workers/saveToApi.ts @@ -0,0 +1,41 @@ +import { Job, Worker } from 'bullmq' +import { DiscordJob } from '../lib/DiscordWorker' + +export interface SaveToApiJob extends Job { + data: { + wikidataId: string + approved: boolean + url?: string + threadId?: string + channelId?: string + messageId?: string + } +} + +export const saveToApiWorker = new Worker( + 'api-save', + async (job: SaveToApiJob) => { + try { + const { wikidataId, approved } = job.data + if (!approved) { + throw new Error('Cannot save unapproved data') + } + + // TODO: Implement actual API save logic here + console.log(`Saving approved data for ${wikidataId} to API`) + + return { success: true } + } catch (error) { + console.error('API Save error:', error) + throw error + } + }, + { + connection: { + host: process.env.REDIS_HOST || 'localhost', + port: parseInt(process.env.REDIS_PORT || '6379'), + }, + } +) + +export default saveToApiWorker From 6e9d496ab253c2564aa62ade2cf8695fb154e338 Mon Sep 17 00:00:00 2001 From: Christian Landgren Date: Wed, 4 Dec 2024 16:28:30 +0100 Subject: [PATCH 45/82] fix: rename --- src/workers/apiSaveWorker.ts | 41 ------------------------------------ 1 file changed, 41 deletions(-) delete mode 100644 src/workers/apiSaveWorker.ts diff --git a/src/workers/apiSaveWorker.ts b/src/workers/apiSaveWorker.ts deleted file mode 100644 index 0210a71d..00000000 --- a/src/workers/apiSaveWorker.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { Job, Worker } from 'bullmq' -import { DiscordJob } from '../lib/DiscordWorker' - -export interface ApiSaveJob extends Job { - data: { - wikidataId: string - approved: boolean - url?: string - threadId?: string - channelId?: string - messageId?: string - } -} - -export const apiSaveWorker = new Worker( - 'api-save', - async (job: ApiSaveJob) => { - try { - const { wikidataId, approved } = job.data - if (!approved) { - throw new Error('Cannot save unapproved data') - } - - // TODO: Implement actual API save logic here - console.log(`Saving approved data for ${wikidataId} to API`) - - return { success: true } - } catch (error) { - console.error('API Save error:', error) - throw error - } - }, - { - connection: { - host: process.env.REDIS_HOST || 'localhost', - port: parseInt(process.env.REDIS_PORT || '6379'), - }, - } -) - -export default apiSaveWorker From 8e4c0165b2f9f5aa1e4c8d5446338b972a8b876c Mon Sep 17 00:00:00 2001 From: "Christian Landgren (aider)" Date: Wed, 4 Dec 2024 16:30:30 +0100 Subject: [PATCH 46/82] refactor: Move approval logic to saveToApi worker and simplify approve interaction --- src/discord/interactions/approve.ts | 14 -------------- src/workers/saveToApi.ts | 28 ++++++++++++++++++++-------- 2 files changed, 20 insertions(+), 22 deletions(-) diff --git a/src/discord/interactions/approve.ts b/src/discord/interactions/approve.ts index 713ddca4..0b6cd29c 100644 --- a/src/discord/interactions/approve.ts +++ b/src/discord/interactions/approve.ts @@ -1,24 +1,10 @@ import { ButtonInteraction } from 'discord.js' import { DiscordJob } from '../../lib/DiscordWorker' -import { Queue } from 'bullmq' - -const saveToApiQueue = new Queue('api-save', { - connection: { - host: process.env.REDIS_HOST || 'localhost', - port: parseInt(process.env.REDIS_PORT || '6379'), - }, -}) export default { async execute(interaction: ButtonInteraction, job: DiscordJob) { await job.updateData({ ...job.data, approved: true }) - // Add to API save queue - await apiSaveQueue.add('save-approved', { - ...job.data, - approved: true - }) - job.log(`Approving company edit: ${job.data.wikidataId}`) await interaction.reply({ content: `Tack för din granskning, ${interaction?.user?.username}!`, diff --git a/src/workers/saveToApi.ts b/src/workers/saveToApi.ts index fda9e0b7..3b1d7cd4 100644 --- a/src/workers/saveToApi.ts +++ b/src/workers/saveToApi.ts @@ -1,14 +1,16 @@ import { Job, Worker } from 'bullmq' import { DiscordJob } from '../lib/DiscordWorker' +import discord from '../discord' export interface SaveToApiJob extends Job { data: { wikidataId: string - approved: boolean + approved?: boolean url?: string threadId?: string channelId?: string messageId?: string + requiresApproval?: boolean } } @@ -16,15 +18,25 @@ export const saveToApiWorker = new Worker( 'api-save', async (job: SaveToApiJob) => { try { - const { wikidataId, approved } = job.data - if (!approved) { - throw new Error('Cannot save unapproved data') + const { wikidataId, approved, requiresApproval = true, messageId, channelId } = job.data + + // If approval is not required or already approved, proceed with saving + if (!requiresApproval || approved) { + // TODO: Implement actual API save logic here + console.log(`Saving approved data for ${wikidataId} to API`) + return { success: true } + } + + // If approval is required and not yet approved, send approval request + if (messageId && channelId) { + const buttonRow = discord.createButtonRow(job.id) + await discord.sendMessageToChannel(channelId, { + content: `New changes need approval for ${wikidataId}`, + components: [buttonRow] + }) } - // TODO: Implement actual API save logic here - console.log(`Saving approved data for ${wikidataId} to API`) - - return { success: true } + return { awaitingApproval: true } } catch (error) { console.error('API Save error:', error) throw error From d68238e2397ca2c23efcb771ff53ae55b01cb790 Mon Sep 17 00:00:00 2001 From: "Christian Landgren (aider)" Date: Wed, 4 Dec 2024 16:31:01 +0100 Subject: [PATCH 47/82] refactor: Rename saveToApiWorker to saveToAPI in worker files --- src/workers/saveToApi.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/workers/saveToApi.ts b/src/workers/saveToApi.ts index 3b1d7cd4..e6ba780d 100644 --- a/src/workers/saveToApi.ts +++ b/src/workers/saveToApi.ts @@ -14,7 +14,7 @@ export interface SaveToApiJob extends Job { } } -export const saveToApiWorker = new Worker( +export const saveToAPI = new Worker( 'api-save', async (job: SaveToApiJob) => { try { @@ -50,4 +50,4 @@ export const saveToApiWorker = new Worker( } ) -export default saveToApiWorker +export default saveToAPI From 13f821da9525daa754937e6520e7bd23cea46308 Mon Sep 17 00:00:00 2001 From: "Christian Landgren (aider)" Date: Wed, 4 Dec 2024 16:32:16 +0100 Subject: [PATCH 48/82] refactor: Simplify saveToAPI worker by using DiscordWorker methods --- src/workers/saveToApi.ts | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/src/workers/saveToApi.ts b/src/workers/saveToApi.ts index e6ba780d..afd1323e 100644 --- a/src/workers/saveToApi.ts +++ b/src/workers/saveToApi.ts @@ -1,20 +1,15 @@ -import { Job, Worker } from 'bullmq' -import { DiscordJob } from '../lib/DiscordWorker' -import discord from '../discord' +import { Job } from 'bullmq' +import { DiscordJob, DiscordWorker } from '../lib/DiscordWorker' -export interface SaveToApiJob extends Job { - data: { +export interface SaveToApiJob extends DiscordJob { + data: DiscordJob['data'] & { wikidataId: string approved?: boolean - url?: string - threadId?: string - channelId?: string - messageId?: string requiresApproval?: boolean } } -export const saveToAPI = new Worker( +export const saveToAPI = new DiscordWorker( 'api-save', async (job: SaveToApiJob) => { try { @@ -28,13 +23,11 @@ export const saveToAPI = new Worker( } // If approval is required and not yet approved, send approval request - if (messageId && channelId) { - const buttonRow = discord.createButtonRow(job.id) - await discord.sendMessageToChannel(channelId, { - content: `New changes need approval for ${wikidataId}`, - components: [buttonRow] - }) - } + const buttonRow = job.createButtonRow() + await job.sendMessage({ + content: `New changes need approval for ${wikidataId}`, + components: [buttonRow] + }) return { awaitingApproval: true } } catch (error) { From 9726b7a331eee882c539f939a25f7d2bbedebddd Mon Sep 17 00:00:00 2001 From: "Christian Landgren (aider)" Date: Wed, 4 Dec 2024 16:33:57 +0100 Subject: [PATCH 49/82] refactor: Standardize button row creation using Discord helper across workers --- src/workers/saveBiogenic.ts | 1 + src/workers/saveScope3.ts | 1 + src/workers/saveToApi.ts | 3 ++- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/workers/saveBiogenic.ts b/src/workers/saveBiogenic.ts index a5356d9e..64521ce3 100644 --- a/src/workers/saveBiogenic.ts +++ b/src/workers/saveBiogenic.ts @@ -1,4 +1,5 @@ import { DiscordJob, DiscordWorker } from '../lib/DiscordWorker' +import discord from '../discord' import { apiFetch } from '../lib/api' import { defaultMetadata, askDiff } from '../lib/saveUtils' import discord from '../discord' diff --git a/src/workers/saveScope3.ts b/src/workers/saveScope3.ts index c3608fc0..4e802732 100644 --- a/src/workers/saveScope3.ts +++ b/src/workers/saveScope3.ts @@ -4,6 +4,7 @@ import { defaultMetadata, askDiff } from '../lib/saveUtils' const ONE_DAY = 1000 * 60 * 60 * 24 import redis from '../config/redis' +import discord from '../discord' import { getReportingPeriodDates } from '../lib/reportingPeriodDates' export class JobData extends DiscordJob { diff --git a/src/workers/saveToApi.ts b/src/workers/saveToApi.ts index afd1323e..2a7708bd 100644 --- a/src/workers/saveToApi.ts +++ b/src/workers/saveToApi.ts @@ -1,5 +1,6 @@ import { Job } from 'bullmq' import { DiscordJob, DiscordWorker } from '../lib/DiscordWorker' +import discord from '../discord' export interface SaveToApiJob extends DiscordJob { data: DiscordJob['data'] & { @@ -23,7 +24,7 @@ export const saveToAPI = new DiscordWorker( } // If approval is required and not yet approved, send approval request - const buttonRow = job.createButtonRow() + const buttonRow = discord.createButtonRow(job.id) await job.sendMessage({ content: `New changes need approval for ${wikidataId}`, components: [buttonRow] From a46da9be908b9c8c26819b7870caff7cebe41d07 Mon Sep 17 00:00:00 2001 From: Christian Landgren Date: Wed, 4 Dec 2024 16:35:34 +0100 Subject: [PATCH 50/82] refactor: Improve code formatting and add non-null assertion for job.id --- src/workers/saveToApi.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/workers/saveToApi.ts b/src/workers/saveToApi.ts index 2a7708bd..0307c225 100644 --- a/src/workers/saveToApi.ts +++ b/src/workers/saveToApi.ts @@ -14,7 +14,13 @@ export const saveToAPI = new DiscordWorker( 'api-save', async (job: SaveToApiJob) => { try { - const { wikidataId, approved, requiresApproval = true, messageId, channelId } = job.data + const { + wikidataId, + approved, + requiresApproval = true, + messageId, + channelId, + } = job.data // If approval is not required or already approved, proceed with saving if (!requiresApproval || approved) { @@ -24,10 +30,10 @@ export const saveToAPI = new DiscordWorker( } // If approval is required and not yet approved, send approval request - const buttonRow = discord.createButtonRow(job.id) + const buttonRow = discord.createButtonRow(job.id!) await job.sendMessage({ content: `New changes need approval for ${wikidataId}`, - components: [buttonRow] + components: [buttonRow], }) return { awaitingApproval: true } From ccf82882860ef6106b53f39638da2ffbd67428ed Mon Sep 17 00:00:00 2001 From: "Christian Landgren (aider)" Date: Wed, 4 Dec 2024 16:35:37 +0100 Subject: [PATCH 51/82] feat: Add diff support for approval process in save workers --- src/workers/saveBiogenic.ts | 3 ++- src/workers/saveScope12.ts | 3 ++- src/workers/saveScope3.ts | 3 ++- src/workers/saveToApi.ts | 3 ++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/workers/saveBiogenic.ts b/src/workers/saveBiogenic.ts index 64521ce3..27a32e57 100644 --- a/src/workers/saveBiogenic.ts +++ b/src/workers/saveBiogenic.ts @@ -36,7 +36,8 @@ const saveBiogenic = new DiscordWorker('saveBiogenic', async (job) => { ), components: [buttonRow], }) - return await job.moveToDelayed(Date.now() + ONE_DAY) + await job.moveToDelayed(Date.now() + ONE_DAY) + return { diff } } job.editMessage(`🤖 Sparar biogeniska utsläpp...`) diff --git a/src/workers/saveScope12.ts b/src/workers/saveScope12.ts index 5dda738d..e193567f 100644 --- a/src/workers/saveScope12.ts +++ b/src/workers/saveScope12.ts @@ -37,7 +37,8 @@ const saveScope12 = new DiscordWorker('saveScope12', async (job) => { ), components: [buttonRow], }) - return await job.moveToDelayed(Date.now() + ONE_DAY) + await job.moveToDelayed(Date.now() + ONE_DAY) + return { diff } } job.editMessage(`🤖 Sparar utsläppsdata scope 1+2...`) diff --git a/src/workers/saveScope3.ts b/src/workers/saveScope3.ts index 4e802732..4708d28d 100644 --- a/src/workers/saveScope3.ts +++ b/src/workers/saveScope3.ts @@ -36,7 +36,8 @@ const saveScope3 = new DiscordWorker('saveScope3', async (job) => { ), components: [buttonRow], }) - return await job.moveToDelayed(Date.now() + ONE_DAY) + await job.moveToDelayed(Date.now() + ONE_DAY) + return { diff } } job.editMessage(`🤖 Sparar utsläppsdata scope 3...`) diff --git a/src/workers/saveToApi.ts b/src/workers/saveToApi.ts index 0307c225..8d0f7176 100644 --- a/src/workers/saveToApi.ts +++ b/src/workers/saveToApi.ts @@ -7,6 +7,7 @@ export interface SaveToApiJob extends DiscordJob { wikidataId: string approved?: boolean requiresApproval?: boolean + diff?: string } } @@ -32,7 +33,7 @@ export const saveToAPI = new DiscordWorker( // If approval is required and not yet approved, send approval request const buttonRow = discord.createButtonRow(job.id!) await job.sendMessage({ - content: `New changes need approval for ${wikidataId}`, + content: `New changes need approval for ${wikidataId}\n\n${job.data.diff || ''}`, components: [buttonRow], }) From 65c735de73e314cb2cfa569a2697d802ad079206 Mon Sep 17 00:00:00 2001 From: "Christian Landgren (aider)" Date: Wed, 4 Dec 2024 16:37:09 +0100 Subject: [PATCH 52/82] refactor: Modify save workers to use saveToApi with diff and approval handling --- src/workers/saveBiogenic.ts | 48 ++++++++++++---------------------- src/workers/saveScope12.ts | 51 +++++++++++++------------------------ src/workers/saveScope3.ts | 48 ++++++++++++---------------------- 3 files changed, 49 insertions(+), 98 deletions(-) diff --git a/src/workers/saveBiogenic.ts b/src/workers/saveBiogenic.ts index 27a32e57..94d2d7c4 100644 --- a/src/workers/saveBiogenic.ts +++ b/src/workers/saveBiogenic.ts @@ -1,10 +1,5 @@ import { DiscordJob, DiscordWorker } from '../lib/DiscordWorker' -import discord from '../discord' -import { apiFetch } from '../lib/api' import { defaultMetadata, askDiff } from '../lib/saveUtils' -import discord from '../discord' - -const ONE_DAY = 1000 * 60 * 60 * 24 import { getReportingPeriodDates } from '../lib/reportingPeriodDates' export class JobData extends DiscordJob { @@ -22,47 +17,36 @@ const saveBiogenic = new DiscordWorker('saveBiogenic', async (job) => { const metadata = defaultMetadata(url) if (biogenic?.length) { - const existingCompany = await apiFetch(`/companies/${wikidataId}`).catch( - () => null - ) - const diff = await askDiff(existingCompany, { biogenic, fiscalYear }) - - if (diff && !diff.includes('NO_CHANGES')) { - const buttonRow = discord.createButtonRow(job.id!) - await job.sendMessage({ - content: `# ${job.data.companyName}: biogenic emissions\n${diff}`.slice( - 0, - 2000 - ), - components: [buttonRow], - }) - await job.moveToDelayed(Date.now() + ONE_DAY) - return { diff } - } - - job.editMessage(`🤖 Sparar biogeniska utsläpp...`) - return Promise.all( + const data = await Promise.all( biogenic.map(async ({ year, biogenic }) => { const [startDate, endDate] = getReportingPeriodDates( year, fiscalYear.startMonth, fiscalYear.endMonth ) - job.sendMessage(`🤖 Sparar utsläppsdata biogenic för ${year}...`) - job.log(`Saving biogenic for ${year}`) - const body = { + return { startDate, endDate, emissions: { biogenic, }, - metadata, + metadata } - return await apiFetch(`/companies/${wikidataId}/${year}/emissions`, { - body, - }) }) ) + + const diff = await askDiff(null, { biogenic, fiscalYear }) + const requiresApproval = diff && !diff.includes('NO_CHANGES') + + await job.queue.add('api-save', { + ...job.data, + data: data, + diff: diff, + requiresApproval, + wikidataId + }) + + return { data, diff } } return null }) diff --git a/src/workers/saveScope12.ts b/src/workers/saveScope12.ts index e193567f..6c9f1269 100644 --- a/src/workers/saveScope12.ts +++ b/src/workers/saveScope12.ts @@ -1,10 +1,5 @@ import { DiscordJob, DiscordWorker } from '../lib/DiscordWorker' -import { apiFetch } from '../lib/api' import { defaultMetadata, askDiff } from '../lib/saveUtils' -import discord from '../discord' - -const ONE_DAY = 1000 * 60 * 60 * 24 -import redis from '../config/redis' import { getReportingPeriodDates } from '../lib/reportingPeriodDates' export class JobData extends DiscordJob { @@ -22,49 +17,37 @@ const saveScope12 = new DiscordWorker('saveScope12', async (job) => { const metadata = defaultMetadata(url) if (scope12?.length) { - const existingCompany = await apiFetch(`/companies/${wikidataId}`).catch( - () => null - ) - const diff = await askDiff(existingCompany, { scope12, fiscalYear }) - - if (diff && !diff.includes('NO_CHANGES')) { - const buttonRow = discord.createButtonRow(job.id!) - await job.sendMessage({ - content: - `# ${job.data.companyName}: scope 1+2 emissions\n${diff}`.slice( - 0, - 2000 - ), - components: [buttonRow], - }) - await job.moveToDelayed(Date.now() + ONE_DAY) - return { diff } - } - - job.editMessage(`🤖 Sparar utsläppsdata scope 1+2...`) - return Promise.all( + const data = await Promise.all( scope12.map(async ({ year, scope1, scope2 }) => { const [startDate, endDate] = getReportingPeriodDates( - year, + year, fiscalYear.startMonth, fiscalYear.endMonth ) - job.log(`Saving scope1 and scope2 for ${startDate}-${endDate}`) - job.sendMessage(`🤖 Sparar utsläppsdata scope 1+2 för ${year}...`) - const body = { + return { startDate, endDate, emissions: { scope1, scope2, }, - metadata, + metadata } - return await apiFetch(`/companies/${wikidataId}/${year}/emissions`, { - body, - }) }) ) + + const diff = await askDiff(null, { scope12, fiscalYear }) + const requiresApproval = diff && !diff.includes('NO_CHANGES') + + await job.queue.add('api-save', { + ...job.data, + data: data, + diff: diff, + requiresApproval, + wikidataId + }) + + return { data, diff } } return null }) diff --git a/src/workers/saveScope3.ts b/src/workers/saveScope3.ts index 4708d28d..c451869c 100644 --- a/src/workers/saveScope3.ts +++ b/src/workers/saveScope3.ts @@ -1,10 +1,5 @@ import { DiscordJob, DiscordWorker } from '../lib/DiscordWorker' -import { apiFetch } from '../lib/api' import { defaultMetadata, askDiff } from '../lib/saveUtils' - -const ONE_DAY = 1000 * 60 * 60 * 24 -import redis from '../config/redis' -import discord from '../discord' import { getReportingPeriodDates } from '../lib/reportingPeriodDates' export class JobData extends DiscordJob { @@ -22,47 +17,36 @@ const saveScope3 = new DiscordWorker('saveScope3', async (job) => { const metadata = defaultMetadata(url) if (scope3?.length) { - const existingCompany = await apiFetch(`/companies/${wikidataId}`).catch( - () => null - ) - const diff = await askDiff(existingCompany, { scope3, fiscalYear }) - - if (diff && !diff.includes('NO_CHANGES')) { - const buttonRow = discord.createButtonRow(job.id!) - await job.sendMessage({ - content: `# ${job.data.companyName}: scope 3 emissions\n${diff}`.slice( - 0, - 2000 - ), - components: [buttonRow], - }) - await job.moveToDelayed(Date.now() + ONE_DAY) - return { diff } - } - - job.editMessage(`🤖 Sparar utsläppsdata scope 3...`) - return Promise.all( + const data = await Promise.all( scope3.map(async ({ year, scope3 }) => { const [startDate, endDate] = getReportingPeriodDates( year, fiscalYear.startMonth, fiscalYear.endMonth ) - job.sendMessage(`🤖 Sparar utsläppsdata scope 3 för ${year}...`) - job.log(`Saving scope3 for ${year}`) - const body = { + return { startDate, endDate, emissions: { scope3, }, - metadata, + metadata } - return await apiFetch(`/companies/${wikidataId}/${year}/emissions`, { - body, - }) }) ) + + const diff = await askDiff(null, { scope3, fiscalYear }) + const requiresApproval = diff && !diff.includes('NO_CHANGES') + + await job.queue.add('api-save', { + ...job.data, + data: data, + diff: diff, + requiresApproval, + wikidataId + }) + + return { data, diff } } return null }) From 2a75ca28bacc417cbaf56b11dcfa87593f6f0b64 Mon Sep 17 00:00:00 2001 From: Christian Landgren Date: Wed, 4 Dec 2024 16:39:10 +0100 Subject: [PATCH 53/82] refactor: Modify saveScope12 to use saveToAPI and update job data handling --- src/workers/saveScope12.ts | 11 ++++++----- src/workers/saveToApi.ts | 13 ++++--------- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/src/workers/saveScope12.ts b/src/workers/saveScope12.ts index 6c9f1269..c9f4b196 100644 --- a/src/workers/saveScope12.ts +++ b/src/workers/saveScope12.ts @@ -1,6 +1,7 @@ import { DiscordJob, DiscordWorker } from '../lib/DiscordWorker' import { defaultMetadata, askDiff } from '../lib/saveUtils' import { getReportingPeriodDates } from '../lib/reportingPeriodDates' +import saveToAPI from './saveToAPI' export class JobData extends DiscordJob { declare data: DiscordJob['data'] & { @@ -12,7 +13,7 @@ export class JobData extends DiscordJob { } const saveScope12 = new DiscordWorker('saveScope12', async (job) => { - const { url, fiscalYear, wikidata, scope12 = [] } = job.data + const { url, fiscalYear, wikidata, companyName, scope12 = [] } = job.data const wikidataId = wikidata.node const metadata = defaultMetadata(url) @@ -20,7 +21,7 @@ const saveScope12 = new DiscordWorker('saveScope12', async (job) => { const data = await Promise.all( scope12.map(async ({ year, scope1, scope2 }) => { const [startDate, endDate] = getReportingPeriodDates( - year, + year, fiscalYear.startMonth, fiscalYear.endMonth ) @@ -31,7 +32,7 @@ const saveScope12 = new DiscordWorker('saveScope12', async (job) => { scope1, scope2, }, - metadata + metadata, } }) ) @@ -39,12 +40,12 @@ const saveScope12 = new DiscordWorker('saveScope12', async (job) => { const diff = await askDiff(null, { scope12, fiscalYear }) const requiresApproval = diff && !diff.includes('NO_CHANGES') - await job.queue.add('api-save', { + await saveToAPI.queue.add(companyName, { ...job.data, data: data, diff: diff, requiresApproval, - wikidataId + wikidataId, }) return { data, diff } diff --git a/src/workers/saveToApi.ts b/src/workers/saveToApi.ts index 8d0f7176..e5d926c5 100644 --- a/src/workers/saveToApi.ts +++ b/src/workers/saveToApi.ts @@ -1,4 +1,3 @@ -import { Job } from 'bullmq' import { DiscordJob, DiscordWorker } from '../lib/DiscordWorker' import discord from '../discord' @@ -15,13 +14,7 @@ export const saveToAPI = new DiscordWorker( 'api-save', async (job: SaveToApiJob) => { try { - const { - wikidataId, - approved, - requiresApproval = true, - messageId, - channelId, - } = job.data + const { wikidataId, approved, requiresApproval = true } = job.data // If approval is not required or already approved, proceed with saving if (!requiresApproval || approved) { @@ -33,7 +26,9 @@ export const saveToAPI = new DiscordWorker( // If approval is required and not yet approved, send approval request const buttonRow = discord.createButtonRow(job.id!) await job.sendMessage({ - content: `New changes need approval for ${wikidataId}\n\n${job.data.diff || ''}`, + content: `New changes need approval for ${wikidataId}\n\n${ + job.data.diff || '' + }`, components: [buttonRow], }) From fa0296f570598b6d3610a10bfcc1485ee32c5f63 Mon Sep 17 00:00:00 2001 From: "Christian Landgren (aider)" Date: Wed, 4 Dec 2024 16:39:11 +0100 Subject: [PATCH 54/82] refactor: Change saveToApi payload from data to body property --- src/workers/saveBiogenic.ts | 10 ++++++---- src/workers/saveScope12.ts | 10 ++++++---- src/workers/saveScope3.ts | 10 ++++++---- src/workers/saveToApi.ts | 7 ++++--- 4 files changed, 22 insertions(+), 15 deletions(-) diff --git a/src/workers/saveBiogenic.ts b/src/workers/saveBiogenic.ts index 94d2d7c4..734cb47d 100644 --- a/src/workers/saveBiogenic.ts +++ b/src/workers/saveBiogenic.ts @@ -40,10 +40,12 @@ const saveBiogenic = new DiscordWorker('saveBiogenic', async (job) => { await job.queue.add('api-save', { ...job.data, - data: data, - diff: diff, - requiresApproval, - wikidataId + body: { + data: data, + diff: diff, + requiresApproval, + wikidataId + } }) return { data, diff } diff --git a/src/workers/saveScope12.ts b/src/workers/saveScope12.ts index c9f4b196..0830c8cf 100644 --- a/src/workers/saveScope12.ts +++ b/src/workers/saveScope12.ts @@ -42,10 +42,12 @@ const saveScope12 = new DiscordWorker('saveScope12', async (job) => { await saveToAPI.queue.add(companyName, { ...job.data, - data: data, - diff: diff, - requiresApproval, - wikidataId, + body: { + data: data, + diff: diff, + requiresApproval, + wikidataId, + }, }) return { data, diff } diff --git a/src/workers/saveScope3.ts b/src/workers/saveScope3.ts index c451869c..447686bc 100644 --- a/src/workers/saveScope3.ts +++ b/src/workers/saveScope3.ts @@ -40,10 +40,12 @@ const saveScope3 = new DiscordWorker('saveScope3', async (job) => { await job.queue.add('api-save', { ...job.data, - data: data, - diff: diff, - requiresApproval, - wikidataId + body: { + data: data, + diff: diff, + requiresApproval, + wikidataId + } }) return { data, diff } diff --git a/src/workers/saveToApi.ts b/src/workers/saveToApi.ts index e5d926c5..7639773c 100644 --- a/src/workers/saveToApi.ts +++ b/src/workers/saveToApi.ts @@ -2,7 +2,8 @@ import { DiscordJob, DiscordWorker } from '../lib/DiscordWorker' import discord from '../discord' export interface SaveToApiJob extends DiscordJob { - data: DiscordJob['data'] & { + data: DiscordJob['data'] + body?: { wikidataId: string approved?: boolean requiresApproval?: boolean @@ -14,7 +15,7 @@ export const saveToAPI = new DiscordWorker( 'api-save', async (job: SaveToApiJob) => { try { - const { wikidataId, approved, requiresApproval = true } = job.data + const { wikidataId, approved, requiresApproval = true } = job.body || {} // If approval is not required or already approved, proceed with saving if (!requiresApproval || approved) { @@ -27,7 +28,7 @@ export const saveToAPI = new DiscordWorker( const buttonRow = discord.createButtonRow(job.id!) await job.sendMessage({ content: `New changes need approval for ${wikidataId}\n\n${ - job.data.diff || '' + job.body?.diff || '' }`, components: [buttonRow], }) From bb32bb009089488d4d25d05444558184d591e3d6 Mon Sep 17 00:00:00 2001 From: Christian Landgren Date: Wed, 4 Dec 2024 16:46:50 +0100 Subject: [PATCH 55/82] fix: use body instead of data --- src/workers/saveScope12.ts | 63 ++++++++++++++++++-------------------- src/workers/saveScope3.ts | 62 ++++++++++++++++++------------------- src/workers/saveToApi.ts | 14 +++------ 3 files changed, 64 insertions(+), 75 deletions(-) diff --git a/src/workers/saveScope12.ts b/src/workers/saveScope12.ts index 0830c8cf..a739082d 100644 --- a/src/workers/saveScope12.ts +++ b/src/workers/saveScope12.ts @@ -17,42 +17,39 @@ const saveScope12 = new DiscordWorker('saveScope12', async (job) => { const wikidataId = wikidata.node const metadata = defaultMetadata(url) - if (scope12?.length) { - const data = await Promise.all( - scope12.map(async ({ year, scope1, scope2 }) => { - const [startDate, endDate] = getReportingPeriodDates( - year, - fiscalYear.startMonth, - fiscalYear.endMonth - ) - return { - startDate, - endDate, - emissions: { - scope1, - scope2, - }, - metadata, - } - }) - ) + const body = await Promise.all( + scope12.map(async ({ year, scope1, scope2 }) => { + const [startDate, endDate] = getReportingPeriodDates( + year, + fiscalYear.startMonth, + fiscalYear.endMonth + ) + return { + startDate, + endDate, + emissions: { + scope1, + scope2, + }, + metadata, + } + }) + ) - const diff = await askDiff(null, { scope12, fiscalYear }) - const requiresApproval = diff && !diff.includes('NO_CHANGES') + const diff = await askDiff(null, { scope12, fiscalYear }) + const requiresApproval = diff && !diff.includes('NO_CHANGES') - await saveToAPI.queue.add(companyName, { - ...job.data, - body: { - data: data, - diff: diff, - requiresApproval, - wikidataId, - }, - }) + await saveToAPI.queue.add(companyName, { + ...job.data, + data: { + body, + diff, + requiresApproval, + wikidataId, + }, + }) - return { data, diff } - } - return null + return { body, diff, requiresApproval } }) export default saveScope12 diff --git a/src/workers/saveScope3.ts b/src/workers/saveScope3.ts index 447686bc..7d4f00fc 100644 --- a/src/workers/saveScope3.ts +++ b/src/workers/saveScope3.ts @@ -1,6 +1,7 @@ import { DiscordJob, DiscordWorker } from '../lib/DiscordWorker' import { defaultMetadata, askDiff } from '../lib/saveUtils' import { getReportingPeriodDates } from '../lib/reportingPeriodDates' +import saveToAPI from './saveToAPI' export class JobData extends DiscordJob { declare data: DiscordJob['data'] & { @@ -16,41 +17,38 @@ const saveScope3 = new DiscordWorker('saveScope3', async (job) => { const wikidataId = wikidata.node const metadata = defaultMetadata(url) - if (scope3?.length) { - const data = await Promise.all( - scope3.map(async ({ year, scope3 }) => { - const [startDate, endDate] = getReportingPeriodDates( - year, - fiscalYear.startMonth, - fiscalYear.endMonth - ) - return { - startDate, - endDate, - emissions: { - scope3, - }, - metadata - } - }) - ) - - const diff = await askDiff(null, { scope3, fiscalYear }) - const requiresApproval = diff && !diff.includes('NO_CHANGES') - - await job.queue.add('api-save', { - ...job.data, - body: { - data: data, - diff: diff, - requiresApproval, - wikidataId + const body = await Promise.all( + scope3.map(async ({ year, scope3 }) => { + const [startDate, endDate] = getReportingPeriodDates( + year, + fiscalYear.startMonth, + fiscalYear.endMonth + ) + return { + startDate, + endDate, + emissions: { + scope3, + }, + metadata, } }) + ) - return { data, diff } - } - return null + const diff = await askDiff(null, { scope3, fiscalYear }) + const requiresApproval = diff && !diff.includes('NO_CHANGES') + + await saveToAPI.queue.add('api-save', { + ...job.data, + body: { + body: body, + diff: diff, + requiresApproval, + wikidataId, + }, + }) + + return { body, diff, requiresApproval } }) export default saveScope3 diff --git a/src/workers/saveToApi.ts b/src/workers/saveToApi.ts index 7639773c..f73170e9 100644 --- a/src/workers/saveToApi.ts +++ b/src/workers/saveToApi.ts @@ -2,12 +2,12 @@ import { DiscordJob, DiscordWorker } from '../lib/DiscordWorker' import discord from '../discord' export interface SaveToApiJob extends DiscordJob { - data: DiscordJob['data'] - body?: { + data: DiscordJob['data'] & { wikidataId: string approved?: boolean requiresApproval?: boolean diff?: string + body?: any } } @@ -15,7 +15,7 @@ export const saveToAPI = new DiscordWorker( 'api-save', async (job: SaveToApiJob) => { try { - const { wikidataId, approved, requiresApproval = true } = job.body || {} + const { wikidataId, approved, requiresApproval = true, body } = job.data // If approval is not required or already approved, proceed with saving if (!requiresApproval || approved) { @@ -28,7 +28,7 @@ export const saveToAPI = new DiscordWorker( const buttonRow = discord.createButtonRow(job.id!) await job.sendMessage({ content: `New changes need approval for ${wikidataId}\n\n${ - job.body?.diff || '' + job.data.diff || '' }`, components: [buttonRow], }) @@ -38,12 +38,6 @@ export const saveToAPI = new DiscordWorker( console.error('API Save error:', error) throw error } - }, - { - connection: { - host: process.env.REDIS_HOST || 'localhost', - port: parseInt(process.env.REDIS_PORT || '6379'), - }, } ) From aeb58a7acc0af26efb30f30bd79f83fbec3bc5d5 Mon Sep 17 00:00:00 2001 From: "Christian Landgren (aider)" Date: Wed, 4 Dec 2024 16:48:18 +0100 Subject: [PATCH 56/82] refactor: Standardize save workers to use saveToAPI with diff and approval --- src/workers/saveEconomy.ts | 79 ++++++++++++++++++---------------- src/workers/saveGoals.ts | 50 ++++++++++++--------- src/workers/saveIndustry.ts | 50 ++++++++++++--------- src/workers/saveInitiatives.ts | 50 ++++++++++++--------- 4 files changed, 129 insertions(+), 100 deletions(-) diff --git a/src/workers/saveEconomy.ts b/src/workers/saveEconomy.ts index 286cf870..d776f7de 100644 --- a/src/workers/saveEconomy.ts +++ b/src/workers/saveEconomy.ts @@ -1,7 +1,7 @@ import { DiscordJob, DiscordWorker } from '../lib/DiscordWorker' -import { apiFetch } from '../lib/api' -import { defaultMetadata, formatAsReportingPeriods } from '../lib/saveUtils' +import { defaultMetadata, askDiff } from '../lib/saveUtils' import { getReportingPeriodDates } from '../lib/reportingPeriodDates' +import saveToAPI from './saveToAPI' export class JobData extends DiscordJob { declare data: DiscordJob['data'] & { @@ -12,40 +12,45 @@ export class JobData extends DiscordJob { } } -const saveEconomy = new DiscordWorker( - 'saveEconomy', - async (job) => { - const { url, fiscalYear, wikidata, economy = [] } = job.data - const wikidataId = wikidata.node - const metadata = defaultMetadata(url) - - if (economy?.length) { - job.editMessage(`🤖 Sparar ekonomidata...`) - return Promise.all( - economy.map(async ({ year, economy }) => { - const [startDate, endDate] = getReportingPeriodDates( - year, - fiscalYear.startMonth, - fiscalYear.endMonth - ) - job.log(`Saving economy for ${startDate}-${endDate}`) - job.sendMessage(`🤖 Sparar ekonomidata för ${year}...`) - const body = { - startDate, - endDate, - economy, - metadata, - } - - return await apiFetch(`/companies/${wikidataId}/${year}/economy`, { - body, - }) - }) - ) - } - - return null - }, -) +const saveEconomy = new DiscordWorker('saveEconomy', async (job) => { + const { url, fiscalYear, wikidata, companyName, economy = [] } = job.data + const wikidataId = wikidata.node + const metadata = defaultMetadata(url) + + if (economy?.length) { + const body = await Promise.all( + economy.map(async ({ year, economy }) => { + const [startDate, endDate] = getReportingPeriodDates( + year, + fiscalYear.startMonth, + fiscalYear.endMonth + ) + return { + startDate, + endDate, + economy, + metadata, + } + }) + ) + + const diff = await askDiff(null, { economy, fiscalYear }) + const requiresApproval = diff && !diff.includes('NO_CHANGES') + + await saveToAPI.queue.add(companyName, { + ...job.data, + data: { + body, + diff, + requiresApproval, + wikidataId, + }, + }) + + return { body, diff, requiresApproval } + } + + return null +}) export default saveEconomy diff --git a/src/workers/saveGoals.ts b/src/workers/saveGoals.ts index 07db9f75..5ea77e21 100644 --- a/src/workers/saveGoals.ts +++ b/src/workers/saveGoals.ts @@ -1,6 +1,6 @@ import { DiscordJob, DiscordWorker } from '../lib/DiscordWorker' -import { apiFetch } from '../lib/api' -import { defaultMetadata } from '../lib/saveUtils' +import { defaultMetadata, askDiff } from '../lib/saveUtils' +import saveToAPI from './saveToAPI' export class JobData extends DiscordJob { declare data: DiscordJob['data'] & { @@ -10,26 +10,34 @@ export class JobData extends DiscordJob { } } -const saveGoals = new DiscordWorker( - 'saveGoals', - async (job) => { - const { url, wikidata, goals } = job.data - const wikidataId = wikidata.node - const metadata = defaultMetadata(url) - - if (goals) { - job.editMessage(`🤖 Sparar mål...`) - return await apiFetch(`/companies/${wikidataId}/goals`, { - body: { - goals, - metadata, - }, - method: 'POST', - }) +const saveGoals = new DiscordWorker('saveGoals', async (job) => { + const { url, wikidata, companyName, goals } = job.data + const wikidataId = wikidata.node + const metadata = defaultMetadata(url) + + if (goals) { + const body = { + goals, + metadata, } - return null - }, -) + const diff = await askDiff(null, { goals }) + const requiresApproval = diff && !diff.includes('NO_CHANGES') + + await saveToAPI.queue.add(companyName, { + ...job.data, + data: { + body, + diff, + requiresApproval, + wikidataId, + }, + }) + + return { body, diff, requiresApproval } + } + + return null +}) export default saveGoals diff --git a/src/workers/saveIndustry.ts b/src/workers/saveIndustry.ts index 9f397687..6c706918 100644 --- a/src/workers/saveIndustry.ts +++ b/src/workers/saveIndustry.ts @@ -1,6 +1,6 @@ import { DiscordJob, DiscordWorker } from '../lib/DiscordWorker' -import { apiFetch } from '../lib/api' -import { defaultMetadata } from '../lib/saveUtils' +import { defaultMetadata, askDiff } from '../lib/saveUtils' +import saveToAPI from './saveToAPI' export class JobData extends DiscordJob { declare data: DiscordJob['data'] & { @@ -10,26 +10,34 @@ export class JobData extends DiscordJob { } } -const saveIndustry = new DiscordWorker( - 'saveIndustry', - async (job) => { - const { url, wikidata, industry } = job.data - const wikidataId = wikidata.node - const metadata = defaultMetadata(url) - - if (industry) { - job.editMessage(`🤖 Sparar branschdata...`) - return await apiFetch(`/companies/${wikidataId}/industry`, { - body: { - industry, - metadata, - }, - method: 'POST', - }) +const saveIndustry = new DiscordWorker('saveIndustry', async (job) => { + const { url, wikidata, companyName, industry } = job.data + const wikidataId = wikidata.node + const metadata = defaultMetadata(url) + + if (industry) { + const body = { + industry, + metadata, } - return null - }, -) + const diff = await askDiff(null, { industry }) + const requiresApproval = diff && !diff.includes('NO_CHANGES') + + await saveToAPI.queue.add(companyName, { + ...job.data, + data: { + body, + diff, + requiresApproval, + wikidataId, + }, + }) + + return { body, diff, requiresApproval } + } + + return null +}) export default saveIndustry diff --git a/src/workers/saveInitiatives.ts b/src/workers/saveInitiatives.ts index f7a8cc93..d6502154 100644 --- a/src/workers/saveInitiatives.ts +++ b/src/workers/saveInitiatives.ts @@ -1,6 +1,6 @@ import { DiscordJob, DiscordWorker } from '../lib/DiscordWorker' -import { apiFetch } from '../lib/api' -import { defaultMetadata } from '../lib/saveUtils' +import { defaultMetadata, askDiff } from '../lib/saveUtils' +import saveToAPI from './saveToAPI' export class JobData extends DiscordJob { declare data: DiscordJob['data'] & { @@ -10,26 +10,34 @@ export class JobData extends DiscordJob { } } -const saveInitiatives = new DiscordWorker( - 'saveInitiatives', - async (job) => { - const { url, wikidata, initiatives } = job.data - const wikidataId = wikidata.node - const metadata = defaultMetadata(url) - - if (initiatives) { - job.editMessage(`🤖 Sparar initiativ...`) - return await apiFetch(`/companies/${wikidataId}/initiatives`, { - body: { - initiatives, - metadata, - }, - method: 'POST', - }) +const saveInitiatives = new DiscordWorker('saveInitiatives', async (job) => { + const { url, wikidata, companyName, initiatives } = job.data + const wikidataId = wikidata.node + const metadata = defaultMetadata(url) + + if (initiatives) { + const body = { + initiatives, + metadata, } - return null - }, -) + const diff = await askDiff(null, { initiatives }) + const requiresApproval = diff && !diff.includes('NO_CHANGES') + + await saveToAPI.queue.add(companyName, { + ...job.data, + data: { + body, + diff, + requiresApproval, + wikidataId, + }, + }) + + return { body, diff, requiresApproval } + } + + return null +}) export default saveInitiatives From 89d7d7379189b16b8f546e87adeba86d339b3b80 Mon Sep 17 00:00:00 2001 From: Christian Landgren Date: Wed, 4 Dec 2024 17:06:20 +0100 Subject: [PATCH 57/82] chore: refactor diff --- src/lib/saveUtils.ts | 31 +--------------- src/workers/checkDB.ts | 1 + src/workers/saveBiogenic.ts | 70 ++++++++++++++++++++----------------- src/workers/saveEconomy.ts | 69 +++++++++++++++++++----------------- src/workers/saveGoals.ts | 41 ++++++++++------------ 5 files changed, 96 insertions(+), 116 deletions(-) diff --git a/src/lib/saveUtils.ts b/src/lib/saveUtils.ts index 36a56932..7cb42d4d 100644 --- a/src/lib/saveUtils.ts +++ b/src/lib/saveUtils.ts @@ -31,36 +31,7 @@ export const defaultMetadata = (url: string) => ({ comment: 'Parsed by Garbo AI', }) -export const askDiff = async ( - existingCompany: any, - { scope12, scope3, biogenic, fiscalYear }: any -) => { - if (!existingCompany.reportingPeriods?.length) return '' - - const before = { - reportingPeriods: existingCompany.reportingPeriods.map( - ({ startDate, endDate, emissions }) => ({ - startDate, - endDate, - emissions, - }) - ), - } - - const after = { - reportingPeriods: [ - ...(scope12 - ? formatAsReportingPeriods(scope12, fiscalYear, 'emissions') - : []), - ...(scope3 - ? formatAsReportingPeriods(scope3, fiscalYear, 'emissions') - : []), - ...(biogenic - ? formatAsReportingPeriods(biogenic, fiscalYear, 'emissions') - : []), - ], - } - +export const askDiff = async (before: any, after: any) => { return await askPrompt( `What is changed between these two json values? Please respond in clear text with markdown formatting. The purpose is to let an editor approve the changes or suggest changes in Discord. diff --git a/src/workers/checkDB.ts b/src/workers/checkDB.ts index 29a424d2..360548bd 100644 --- a/src/workers/checkDB.ts +++ b/src/workers/checkDB.ts @@ -59,6 +59,7 @@ const checkDB = new DiscordWorker('checkDB', async (job: JobData) => { const base = { name: companyName, data: { + existingCompany, companyName, url, fiscalYear, diff --git a/src/workers/saveBiogenic.ts b/src/workers/saveBiogenic.ts index 734cb47d..f18a6faa 100644 --- a/src/workers/saveBiogenic.ts +++ b/src/workers/saveBiogenic.ts @@ -1,9 +1,11 @@ import { DiscordJob, DiscordWorker } from '../lib/DiscordWorker' import { defaultMetadata, askDiff } from '../lib/saveUtils' import { getReportingPeriodDates } from '../lib/reportingPeriodDates' +import saveToAPI from './saveToAPI' export class JobData extends DiscordJob { declare data: DiscordJob['data'] & { + existingCompany: any companyName: string wikidata: any fiscalYear: any @@ -12,44 +14,48 @@ export class JobData extends DiscordJob { } const saveBiogenic = new DiscordWorker('saveBiogenic', async (job) => { - const { url, fiscalYear, wikidata, biogenic = [] } = job.data + const { + url, + fiscalYear, + wikidata, + existingCompany, + companyName, + biogenic = [], + } = job.data const wikidataId = wikidata.node const metadata = defaultMetadata(url) - if (biogenic?.length) { - const data = await Promise.all( - biogenic.map(async ({ year, biogenic }) => { - const [startDate, endDate] = getReportingPeriodDates( - year, - fiscalYear.startMonth, - fiscalYear.endMonth - ) - return { - startDate, - endDate, - emissions: { - biogenic, - }, - metadata - } - }) - ) - - const diff = await askDiff(null, { biogenic, fiscalYear }) - const requiresApproval = diff && !diff.includes('NO_CHANGES') - - await job.queue.add('api-save', { - ...job.data, - body: { - data: data, - diff: diff, - requiresApproval, - wikidataId + const body = await Promise.all( + biogenic.map(async ({ year, biogenic }) => { + const [startDate, endDate] = getReportingPeriodDates( + year, + fiscalYear.startMonth, + fiscalYear.endMonth + ) + return { + startDate, + endDate, + emissions: { + biogenic, + }, + metadata, } }) + ) + + const diff = await askDiff(existingCompany, { biogenic, fiscalYear }) + const requiresApproval = diff && !diff.includes('NO_CHANGES') + job.log('diff' + diff) + await saveToAPI.queue.add(companyName, { + data: { + ...job.data, + body, + diff, + requiresApproval, + wikidataId, + }, + }) - return { data, diff } - } return null }) diff --git a/src/workers/saveEconomy.ts b/src/workers/saveEconomy.ts index d776f7de..fdbf7ade 100644 --- a/src/workers/saveEconomy.ts +++ b/src/workers/saveEconomy.ts @@ -6,6 +6,7 @@ import saveToAPI from './saveToAPI' export class JobData extends DiscordJob { declare data: DiscordJob['data'] & { companyName: string + existingCompany: any wikidata: any fiscalYear: any economy?: any[] @@ -13,44 +14,48 @@ export class JobData extends DiscordJob { } const saveEconomy = new DiscordWorker('saveEconomy', async (job) => { - const { url, fiscalYear, wikidata, companyName, economy = [] } = job.data + const { + url, + fiscalYear, + wikidata, + existingCompany, + companyName, + economy = [], + } = job.data const wikidataId = wikidata.node const metadata = defaultMetadata(url) - if (economy?.length) { - const body = await Promise.all( - economy.map(async ({ year, economy }) => { - const [startDate, endDate] = getReportingPeriodDates( - year, - fiscalYear.startMonth, - fiscalYear.endMonth - ) - return { - startDate, - endDate, - economy, - metadata, - } - }) - ) - - const diff = await askDiff(null, { economy, fiscalYear }) - const requiresApproval = diff && !diff.includes('NO_CHANGES') - - await saveToAPI.queue.add(companyName, { - ...job.data, - data: { - body, - diff, - requiresApproval, - wikidataId, - }, + const body = await Promise.all( + economy.map(async ({ year, economy }) => { + const [startDate, endDate] = getReportingPeriodDates( + year, + fiscalYear.startMonth, + fiscalYear.endMonth + ) + return { + startDate, + endDate, + economy, + metadata, + } }) + ) - return { body, diff, requiresApproval } - } + const diff = await askDiff(existingCompany, { economy, fiscalYear }) + job.log('diff: ' + diff) + const requiresApproval = diff && !diff.includes('NO_CHANGES') - return null + await saveToAPI.queue.add(companyName, { + data: { + ...job.data, + body, + diff, + requiresApproval, + wikidataId, + }, + }) + + return { body, diff, requiresApproval } }) export default saveEconomy diff --git a/src/workers/saveGoals.ts b/src/workers/saveGoals.ts index 5ea77e21..90a66dd5 100644 --- a/src/workers/saveGoals.ts +++ b/src/workers/saveGoals.ts @@ -5,39 +5,36 @@ import saveToAPI from './saveToAPI' export class JobData extends DiscordJob { declare data: DiscordJob['data'] & { companyName: string + existingCompany: any wikidata: any - goals?: any + goals: any } } const saveGoals = new DiscordWorker('saveGoals', async (job) => { - const { url, wikidata, companyName, goals } = job.data + const { url, wikidata, companyName, existingCompany, goals } = job.data const wikidataId = wikidata.node const metadata = defaultMetadata(url) - if (goals) { - const body = { - goals, - metadata, - } + const body = { + goals, + metadata, + } - const diff = await askDiff(null, { goals }) - const requiresApproval = diff && !diff.includes('NO_CHANGES') + const diff = await askDiff({ goals: existingCompany.goals }, { goals }) + const requiresApproval = diff && !diff.includes('NO_CHANGES') - await saveToAPI.queue.add(companyName, { + await saveToAPI.queue.add(companyName, { + data: { ...job.data, - data: { - body, - diff, - requiresApproval, - wikidataId, - }, - }) - - return { body, diff, requiresApproval } - } - - return null + body, + diff, + requiresApproval, + wikidataId, + }, + }) + + return { body, diff, requiresApproval } }) export default saveGoals From bef2fc66f40e9dcc11d8b8c7d53e13f080d042a8 Mon Sep 17 00:00:00 2001 From: "Christian Landgren (aider)" Date: Wed, 4 Dec 2024 17:07:37 +0100 Subject: [PATCH 58/82] refactor: Consolidate reporting periods workers into single saveReportingPeriods job --- src/workers/extractEmissions.ts | 33 ++------- src/workers/saveReportingPeriods.ts | 101 ++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+), 29 deletions(-) create mode 100644 src/workers/saveReportingPeriods.ts diff --git a/src/workers/extractEmissions.ts b/src/workers/extractEmissions.ts index a2d40624..9bf300a7 100644 --- a/src/workers/extractEmissions.ts +++ b/src/workers/extractEmissions.ts @@ -44,44 +44,19 @@ const extractEmissions = new DiscordWorker( type: JobType.IndustryGics, }, }, - scope12 + (scope12 || scope3 || biogenic || economy) ? { - name: 'saveScope12 ' + companyName, - queueName: 'saveScope12', + name: 'saveReportingPeriods ' + companyName, + queueName: 'saveReportingPeriods', data: { ...base.data, scope12, - }, - } - : null, - scope3 - ? { - name: 'saveScope3 ' + companyName, - queueName: 'saveScope3', - data: { - ...base.data, scope3, - }, - } - : null, - biogenic - ? { - name: 'saveBiogenic ' + companyName, - queueName: 'saveBiogenic', - data: { - ...base.data, biogenic, + economy, }, } : null, - { - ...base, - name: 'economy ' + companyName, - data: { - ...base.data, - type: JobType.Economy, - }, - }, { ...base, name: 'goals ' + companyName, diff --git a/src/workers/saveReportingPeriods.ts b/src/workers/saveReportingPeriods.ts new file mode 100644 index 00000000..a1efa20a --- /dev/null +++ b/src/workers/saveReportingPeriods.ts @@ -0,0 +1,101 @@ +import { DiscordJob, DiscordWorker } from '../lib/DiscordWorker' +import { defaultMetadata, askDiff } from '../lib/saveUtils' +import { getReportingPeriodDates } from '../lib/reportingPeriodDates' +import saveToAPI from './saveToAPI' + +export class JobData extends DiscordJob { + declare data: DiscordJob['data'] & { + companyName: string + wikidata: any + fiscalYear: any + scope12?: any[] + scope3?: any[] + biogenic?: any[] + economy?: any[] + } +} + +const saveReportingPeriods = new DiscordWorker('saveReportingPeriods', async (job) => { + const { url, fiscalYear, wikidata, companyName, scope12 = [], scope3 = [], biogenic = [], economy = [] } = job.data + const wikidataId = wikidata.node + const metadata = defaultMetadata(url) + + const body = await Promise.all([ + ...scope12.map(async ({ year, scope1, scope2 }) => { + const [startDate, endDate] = getReportingPeriodDates( + year, + fiscalYear.startMonth, + fiscalYear.endMonth + ) + return { + startDate, + endDate, + emissions: { + scope1, + scope2, + }, + metadata, + } + }), + ...scope3.map(async ({ year, scope3 }) => { + const [startDate, endDate] = getReportingPeriodDates( + year, + fiscalYear.startMonth, + fiscalYear.endMonth + ) + return { + startDate, + endDate, + emissions: { + scope3, + }, + metadata, + } + }), + ...biogenic.map(async ({ year, biogenic }) => { + const [startDate, endDate] = getReportingPeriodDates( + year, + fiscalYear.startMonth, + fiscalYear.endMonth + ) + return { + startDate, + endDate, + emissions: { + biogenic, + }, + metadata, + } + }), + ...economy.map(async ({ year, economy }) => { + const [startDate, endDate] = getReportingPeriodDates( + year, + fiscalYear.startMonth, + fiscalYear.endMonth + ) + return { + startDate, + endDate, + economy, + metadata, + } + }) + ]) + + const diff = await askDiff(null, { scope12, scope3, biogenic, economy, fiscalYear }) + const requiresApproval = diff && !diff.includes('NO_CHANGES') + + await saveToAPI.queue.add(companyName, { + data: { + ...job.data, + body, + diff, + requiresApproval, + wikidataId, + }, + }) + + return { body, diff, requiresApproval } +}) + +export default saveReportingPeriods From 7e5068b54a442fe1af8536efcd48c68bba953bd5 Mon Sep 17 00:00:00 2001 From: "Christian Landgren (aider)" Date: Wed, 4 Dec 2024 17:10:21 +0100 Subject: [PATCH 59/82] refactor: Consolidate reporting periods from multiple sources with unified data mapping --- src/workers/saveReportingPeriods.ts | 123 ++++++++++++++-------------- 1 file changed, 63 insertions(+), 60 deletions(-) diff --git a/src/workers/saveReportingPeriods.ts b/src/workers/saveReportingPeriods.ts index a1efa20a..8c05c418 100644 --- a/src/workers/saveReportingPeriods.ts +++ b/src/workers/saveReportingPeriods.ts @@ -20,66 +20,69 @@ const saveReportingPeriods = new DiscordWorker('saveReportingPeriods', const wikidataId = wikidata.node const metadata = defaultMetadata(url) - const body = await Promise.all([ - ...scope12.map(async ({ year, scope1, scope2 }) => { - const [startDate, endDate] = getReportingPeriodDates( - year, - fiscalYear.startMonth, - fiscalYear.endMonth - ) - return { - startDate, - endDate, - emissions: { - scope1, - scope2, - }, - metadata, - } - }), - ...scope3.map(async ({ year, scope3 }) => { - const [startDate, endDate] = getReportingPeriodDates( - year, - fiscalYear.startMonth, - fiscalYear.endMonth - ) - return { - startDate, - endDate, - emissions: { - scope3, - }, - metadata, - } - }), - ...biogenic.map(async ({ year, biogenic }) => { - const [startDate, endDate] = getReportingPeriodDates( - year, - fiscalYear.startMonth, - fiscalYear.endMonth - ) - return { - startDate, - endDate, - emissions: { - biogenic, - }, - metadata, - } - }), - ...economy.map(async ({ year, economy }) => { - const [startDate, endDate] = getReportingPeriodDates( - year, - fiscalYear.startMonth, - fiscalYear.endMonth - ) - return { - startDate, - endDate, - economy, - metadata, - } - }) + // Get all unique years from all sources + const years = new Set([ + ...scope12.map(d => d.year), + ...scope3.map(d => d.year), + ...biogenic.map(d => d.year), + ...economy.map(d => d.year) + ]) + + // Create base reporting periods + const reportingPeriods = Array.from(years).map(year => { + const [startDate, endDate] = getReportingPeriodDates( + year, + fiscalYear.startMonth, + fiscalYear.endMonth + ) + return { + startDate, + endDate, + emissions: {}, + economy: undefined, + metadata, + } + }) + + // Fill in data from each source + const body = reportingPeriods.map(period => { + const yearData = { + scope12: scope12.find(d => { + const [s] = getReportingPeriodDates(d.year, fiscalYear.startMonth, fiscalYear.endMonth) + return s === period.startDate + }), + scope3: scope3.find(d => { + const [s] = getReportingPeriodDates(d.year, fiscalYear.startMonth, fiscalYear.endMonth) + return s === period.startDate + }), + biogenic: biogenic.find(d => { + const [s] = getReportingPeriodDates(d.year, fiscalYear.startMonth, fiscalYear.endMonth) + return s === period.startDate + }), + economy: economy.find(d => { + const [s] = getReportingPeriodDates(d.year, fiscalYear.startMonth, fiscalYear.endMonth) + return s === period.startDate + }) + } + + return { + ...period, + emissions: { + ...(yearData.scope12 && { + scope1: yearData.scope12.scope1, + scope2: yearData.scope12.scope2 + }), + ...(yearData.scope3 && { + scope3: yearData.scope3.scope3 + }), + ...(yearData.biogenic && { + biogenic: yearData.biogenic.biogenic + }) + }, + ...(yearData.economy && { + economy: yearData.economy.economy + }) + } ]) const diff = await askDiff(null, { scope12, scope3, biogenic, economy, fiscalYear }) From 7091060a022151424fbb0ca20b3a7a18d05a9e86 Mon Sep 17 00:00:00 2001 From: Christian Landgren Date: Wed, 4 Dec 2024 17:32:09 +0100 Subject: [PATCH 60/82] chore: refactor periods --- src/workers/checkDB.ts | 15 +-- src/workers/saveBiogenic.ts | 62 ------------ src/workers/saveEconomy.ts | 61 ------------ src/workers/saveGoals.ts | 2 +- src/workers/saveIndustry.ts | 41 ++++---- src/workers/saveInitiatives.ts | 22 ++--- src/workers/saveReportingPeriods.ts | 147 +++++++++++++--------------- src/workers/saveScope12.ts | 55 ----------- src/workers/saveScope3.ts | 54 ---------- src/workers/saveToApi.ts | 16 ++- 10 files changed, 114 insertions(+), 361 deletions(-) delete mode 100644 src/workers/saveBiogenic.ts delete mode 100644 src/workers/saveEconomy.ts delete mode 100644 src/workers/saveScope12.ts delete mode 100644 src/workers/saveScope3.ts diff --git a/src/workers/checkDB.ts b/src/workers/checkDB.ts index 360548bd..7dc62e02 100644 --- a/src/workers/checkDB.ts +++ b/src/workers/checkDB.ts @@ -81,15 +81,16 @@ const checkDB = new DiscordWorker('checkDB', async (job: JobData) => { ...base.data, }, children: [ - scope12 || scope3 || biogenic + scope12 || scope3 || biogenic || economy ? { ...base, - queueName: 'saveEmissions', + queueName: 'saveReportingPeriods', data: { ...base.data, scope12, scope3, biogenic, + economy, }, } : null, @@ -103,16 +104,6 @@ const checkDB = new DiscordWorker('checkDB', async (job: JobData) => { }, } : null, - economy - ? { - ...base, - queueName: 'saveEconomy', - data: { - ...base.data, - economy, - }, - } - : null, goals ? { ...base, diff --git a/src/workers/saveBiogenic.ts b/src/workers/saveBiogenic.ts deleted file mode 100644 index f18a6faa..00000000 --- a/src/workers/saveBiogenic.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { DiscordJob, DiscordWorker } from '../lib/DiscordWorker' -import { defaultMetadata, askDiff } from '../lib/saveUtils' -import { getReportingPeriodDates } from '../lib/reportingPeriodDates' -import saveToAPI from './saveToAPI' - -export class JobData extends DiscordJob { - declare data: DiscordJob['data'] & { - existingCompany: any - companyName: string - wikidata: any - fiscalYear: any - biogenic?: any[] - } -} - -const saveBiogenic = new DiscordWorker('saveBiogenic', async (job) => { - const { - url, - fiscalYear, - wikidata, - existingCompany, - companyName, - biogenic = [], - } = job.data - const wikidataId = wikidata.node - const metadata = defaultMetadata(url) - - const body = await Promise.all( - biogenic.map(async ({ year, biogenic }) => { - const [startDate, endDate] = getReportingPeriodDates( - year, - fiscalYear.startMonth, - fiscalYear.endMonth - ) - return { - startDate, - endDate, - emissions: { - biogenic, - }, - metadata, - } - }) - ) - - const diff = await askDiff(existingCompany, { biogenic, fiscalYear }) - const requiresApproval = diff && !diff.includes('NO_CHANGES') - job.log('diff' + diff) - await saveToAPI.queue.add(companyName, { - data: { - ...job.data, - body, - diff, - requiresApproval, - wikidataId, - }, - }) - - return null -}) - -export default saveBiogenic diff --git a/src/workers/saveEconomy.ts b/src/workers/saveEconomy.ts deleted file mode 100644 index fdbf7ade..00000000 --- a/src/workers/saveEconomy.ts +++ /dev/null @@ -1,61 +0,0 @@ -import { DiscordJob, DiscordWorker } from '../lib/DiscordWorker' -import { defaultMetadata, askDiff } from '../lib/saveUtils' -import { getReportingPeriodDates } from '../lib/reportingPeriodDates' -import saveToAPI from './saveToAPI' - -export class JobData extends DiscordJob { - declare data: DiscordJob['data'] & { - companyName: string - existingCompany: any - wikidata: any - fiscalYear: any - economy?: any[] - } -} - -const saveEconomy = new DiscordWorker('saveEconomy', async (job) => { - const { - url, - fiscalYear, - wikidata, - existingCompany, - companyName, - economy = [], - } = job.data - const wikidataId = wikidata.node - const metadata = defaultMetadata(url) - - const body = await Promise.all( - economy.map(async ({ year, economy }) => { - const [startDate, endDate] = getReportingPeriodDates( - year, - fiscalYear.startMonth, - fiscalYear.endMonth - ) - return { - startDate, - endDate, - economy, - metadata, - } - }) - ) - - const diff = await askDiff(existingCompany, { economy, fiscalYear }) - job.log('diff: ' + diff) - const requiresApproval = diff && !diff.includes('NO_CHANGES') - - await saveToAPI.queue.add(companyName, { - data: { - ...job.data, - body, - diff, - requiresApproval, - wikidataId, - }, - }) - - return { body, diff, requiresApproval } -}) - -export default saveEconomy diff --git a/src/workers/saveGoals.ts b/src/workers/saveGoals.ts index 90a66dd5..dcdf0d10 100644 --- a/src/workers/saveGoals.ts +++ b/src/workers/saveGoals.ts @@ -21,7 +21,7 @@ const saveGoals = new DiscordWorker('saveGoals', async (job) => { metadata, } - const diff = await askDiff({ goals: existingCompany.goals }, { goals }) + const diff = await askDiff(existingCompany.goals, goals) const requiresApproval = diff && !diff.includes('NO_CHANGES') await saveToAPI.queue.add(companyName, { diff --git a/src/workers/saveIndustry.ts b/src/workers/saveIndustry.ts index 6c706918..f07e3c4c 100644 --- a/src/workers/saveIndustry.ts +++ b/src/workers/saveIndustry.ts @@ -4,40 +4,37 @@ import saveToAPI from './saveToAPI' export class JobData extends DiscordJob { declare data: DiscordJob['data'] & { + existingCompany: any companyName: string wikidata: any - industry?: any + industry: any } } const saveIndustry = new DiscordWorker('saveIndustry', async (job) => { - const { url, wikidata, companyName, industry } = job.data + const { url, wikidata, companyName, existingCompany, industry } = job.data const wikidataId = wikidata.node const metadata = defaultMetadata(url) - if (industry) { - const body = { - industry, - metadata, - } + const body = { + industry, + metadata, + } - const diff = await askDiff(null, { industry }) - const requiresApproval = diff && !diff.includes('NO_CHANGES') + const diff = await askDiff(existingCompany.industry, industry) + const requiresApproval = diff && !diff.includes('NO_CHANGES') - await saveToAPI.queue.add(companyName, { + await saveToAPI.queue.add(companyName, { + data: { ...job.data, - data: { - body, - diff, - requiresApproval, - wikidataId, - }, - }) - - return { body, diff, requiresApproval } - } - - return null + body, + diff, + requiresApproval, + wikidataId, + }, + }) + + return { body, diff, requiresApproval } }) export default saveIndustry diff --git a/src/workers/saveInitiatives.ts b/src/workers/saveInitiatives.ts index d6502154..af3253af 100644 --- a/src/workers/saveInitiatives.ts +++ b/src/workers/saveInitiatives.ts @@ -5,39 +5,37 @@ import saveToAPI from './saveToAPI' export class JobData extends DiscordJob { declare data: DiscordJob['data'] & { companyName: string + existingCompany: any wikidata: any - initiatives?: any + initiatives: any } } -const saveInitiatives = new DiscordWorker('saveInitiatives', async (job) => { - const { url, wikidata, companyName, initiatives } = job.data - const wikidataId = wikidata.node - const metadata = defaultMetadata(url) +const saveInitiatives = new DiscordWorker( + 'saveInitiatives', + async (job) => { + const { url, companyName, existingCompany, initiatives } = job.data + const metadata = defaultMetadata(url) - if (initiatives) { const body = { initiatives, metadata, } - const diff = await askDiff(null, { initiatives }) + const diff = await askDiff(existingCompany.initiatives, initiatives) const requiresApproval = diff && !diff.includes('NO_CHANGES') await saveToAPI.queue.add(companyName, { - ...job.data, data: { + ...job.data, body, diff, requiresApproval, - wikidataId, }, }) return { body, diff, requiresApproval } } - - return null -}) +) export default saveInitiatives diff --git a/src/workers/saveReportingPeriods.ts b/src/workers/saveReportingPeriods.ts index 8c05c418..7599f862 100644 --- a/src/workers/saveReportingPeriods.ts +++ b/src/workers/saveReportingPeriods.ts @@ -6,6 +6,7 @@ import saveToAPI from './saveToAPI' export class JobData extends DiscordJob { declare data: DiscordJob['data'] & { companyName: string + existingCompany: any wikidata: any fiscalYear: any scope12?: any[] @@ -15,90 +16,80 @@ export class JobData extends DiscordJob { } } -const saveReportingPeriods = new DiscordWorker('saveReportingPeriods', async (job) => { - const { url, fiscalYear, wikidata, companyName, scope12 = [], scope3 = [], biogenic = [], economy = [] } = job.data - const wikidataId = wikidata.node - const metadata = defaultMetadata(url) +const saveReportingPeriods = new DiscordWorker( + 'saveReportingPeriods', + async (job) => { + const { + url, + fiscalYear, + companyName, + wikidata, + existingCompany, + scope12 = [], + scope3 = [], + biogenic = [], + economy = [], + } = job.data + const metadata = defaultMetadata(url) + const wikidataId = wikidata.node - // Get all unique years from all sources - const years = new Set([ - ...scope12.map(d => d.year), - ...scope3.map(d => d.year), - ...biogenic.map(d => d.year), - ...economy.map(d => d.year) - ]) + // Get all unique years from all sources + const years = new Set([ + ...scope12.map((d) => d.year), + ...scope3.map((d) => d.year), + ...biogenic.map((d) => d.year), + ...economy.map((d) => d.year), + ]) - // Create base reporting periods - const reportingPeriods = Array.from(years).map(year => { - const [startDate, endDate] = getReportingPeriodDates( - year, - fiscalYear.startMonth, - fiscalYear.endMonth + // Create base reporting periods + const reportingPeriods = Array.from(years).map((year) => { + const [startDate, endDate] = getReportingPeriodDates( + year, + fiscalYear.startMonth, + fiscalYear.endMonth + ) + return { + year, + startDate, + endDate, + metadata, + } + }) + + // Fill in data from each source + const body = reportingPeriods.map( + ({ year, startDate, endDate, metadata }) => { + return { + emissions: { + scope1: scope12.find((d) => d.year === year)?.scope1, + scope2: scope12.find((d) => d.year === year)?.scope2, + scope3: scope3.find((d) => d.year === year)?.scope3, + biogenic: biogenic.find((d) => d.year === year)?.biogenic, + }, + economy: economy.find((d) => d.year === year)?.economy, + startDate, + endDate, + metadata, + } + } ) - return { - startDate, - endDate, - emissions: {}, - economy: undefined, - metadata, - } - }) - // Fill in data from each source - const body = reportingPeriods.map(period => { - const yearData = { - scope12: scope12.find(d => { - const [s] = getReportingPeriodDates(d.year, fiscalYear.startMonth, fiscalYear.endMonth) - return s === period.startDate - }), - scope3: scope3.find(d => { - const [s] = getReportingPeriodDates(d.year, fiscalYear.startMonth, fiscalYear.endMonth) - return s === period.startDate - }), - biogenic: biogenic.find(d => { - const [s] = getReportingPeriodDates(d.year, fiscalYear.startMonth, fiscalYear.endMonth) - return s === period.startDate - }), - economy: economy.find(d => { - const [s] = getReportingPeriodDates(d.year, fiscalYear.startMonth, fiscalYear.endMonth) - return s === period.startDate - }) - } + const diff = await askDiff(existingCompany.reportingPeriods, body) + job.log('diff: ' + diff) + const requiresApproval = diff && !diff.includes('NO_CHANGES') - return { - ...period, - emissions: { - ...(yearData.scope12 && { - scope1: yearData.scope12.scope1, - scope2: yearData.scope12.scope2 - }), - ...(yearData.scope3 && { - scope3: yearData.scope3.scope3 - }), - ...(yearData.biogenic && { - biogenic: yearData.biogenic.biogenic - }) + await saveToAPI.queue.add(companyName, { + data: { + ...job.data, + body, + diff, + requiresApproval, + wikidataId, }, - ...(yearData.economy && { - economy: yearData.economy.economy - }) - } - ]) - - const diff = await askDiff(null, { scope12, scope3, biogenic, economy, fiscalYear }) - const requiresApproval = diff && !diff.includes('NO_CHANGES') + }) - await saveToAPI.queue.add(companyName, { - data: { - ...job.data, - body, - diff, - requiresApproval, - wikidataId, - }, - }) - - return { body, diff, requiresApproval } -}) + return { body, diff, requiresApproval } + } +) export default saveReportingPeriods diff --git a/src/workers/saveScope12.ts b/src/workers/saveScope12.ts deleted file mode 100644 index a739082d..00000000 --- a/src/workers/saveScope12.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { DiscordJob, DiscordWorker } from '../lib/DiscordWorker' -import { defaultMetadata, askDiff } from '../lib/saveUtils' -import { getReportingPeriodDates } from '../lib/reportingPeriodDates' -import saveToAPI from './saveToAPI' - -export class JobData extends DiscordJob { - declare data: DiscordJob['data'] & { - companyName: string - wikidata: any - fiscalYear: any - scope12?: any[] - } -} - -const saveScope12 = new DiscordWorker('saveScope12', async (job) => { - const { url, fiscalYear, wikidata, companyName, scope12 = [] } = job.data - const wikidataId = wikidata.node - const metadata = defaultMetadata(url) - - const body = await Promise.all( - scope12.map(async ({ year, scope1, scope2 }) => { - const [startDate, endDate] = getReportingPeriodDates( - year, - fiscalYear.startMonth, - fiscalYear.endMonth - ) - return { - startDate, - endDate, - emissions: { - scope1, - scope2, - }, - metadata, - } - }) - ) - - const diff = await askDiff(null, { scope12, fiscalYear }) - const requiresApproval = diff && !diff.includes('NO_CHANGES') - - await saveToAPI.queue.add(companyName, { - ...job.data, - data: { - body, - diff, - requiresApproval, - wikidataId, - }, - }) - - return { body, diff, requiresApproval } -}) - -export default saveScope12 diff --git a/src/workers/saveScope3.ts b/src/workers/saveScope3.ts deleted file mode 100644 index 7d4f00fc..00000000 --- a/src/workers/saveScope3.ts +++ /dev/null @@ -1,54 +0,0 @@ -import { DiscordJob, DiscordWorker } from '../lib/DiscordWorker' -import { defaultMetadata, askDiff } from '../lib/saveUtils' -import { getReportingPeriodDates } from '../lib/reportingPeriodDates' -import saveToAPI from './saveToAPI' - -export class JobData extends DiscordJob { - declare data: DiscordJob['data'] & { - companyName: string - wikidata: any - fiscalYear: any - scope3?: any[] - } -} - -const saveScope3 = new DiscordWorker('saveScope3', async (job) => { - const { url, fiscalYear, wikidata, scope3 = [] } = job.data - const wikidataId = wikidata.node - const metadata = defaultMetadata(url) - - const body = await Promise.all( - scope3.map(async ({ year, scope3 }) => { - const [startDate, endDate] = getReportingPeriodDates( - year, - fiscalYear.startMonth, - fiscalYear.endMonth - ) - return { - startDate, - endDate, - emissions: { - scope3, - }, - metadata, - } - }) - ) - - const diff = await askDiff(null, { scope3, fiscalYear }) - const requiresApproval = diff && !diff.includes('NO_CHANGES') - - await saveToAPI.queue.add('api-save', { - ...job.data, - body: { - body: body, - diff: diff, - requiresApproval, - wikidataId, - }, - }) - - return { body, diff, requiresApproval } -}) - -export default saveScope3 diff --git a/src/workers/saveToApi.ts b/src/workers/saveToApi.ts index f73170e9..d89ffbd8 100644 --- a/src/workers/saveToApi.ts +++ b/src/workers/saveToApi.ts @@ -1,5 +1,6 @@ import { DiscordJob, DiscordWorker } from '../lib/DiscordWorker' import discord from '../discord' +import { apiFetch } from '../lib/api' export interface SaveToApiJob extends DiscordJob { data: DiscordJob['data'] & { @@ -8,6 +9,7 @@ export interface SaveToApiJob extends DiscordJob { requiresApproval?: boolean diff?: string body?: any + wikidata: any } } @@ -15,21 +17,27 @@ export const saveToAPI = new DiscordWorker( 'api-save', async (job: SaveToApiJob) => { try { - const { wikidataId, approved, requiresApproval = true, body } = job.data + const { + wikidata, + approved, + requiresApproval = true, + diff, + body, + } = job.data + const wikidataId = wikidata.node // If approval is not required or already approved, proceed with saving if (!requiresApproval || approved) { // TODO: Implement actual API save logic here console.log(`Saving approved data for ${wikidataId} to API`) + await apiFetch('/companies/' + wikidataId, { body }) return { success: true } } // If approval is required and not yet approved, send approval request const buttonRow = discord.createButtonRow(job.id!) await job.sendMessage({ - content: `New changes need approval for ${wikidataId}\n\n${ - job.data.diff || '' - }`, + content: `New changes need approval for ${wikidataId}\n\n${diff || ''}`, components: [buttonRow], }) From 42de40d2c07915a8e67683506b50b3645fdf68ce Mon Sep 17 00:00:00 2001 From: "Christian Landgren (aider)" Date: Wed, 4 Dec 2024 17:33:03 +0100 Subject: [PATCH 61/82] refactor: Rename save* workers to diff* for consistency --- src/workers/extractEmissions.ts | 4 ++-- src/workers/saveGoals.ts | 4 ++-- src/workers/saveIndustry.ts | 4 ++-- src/workers/saveInitiatives.ts | 6 +++--- src/workers/saveReportingPeriods.ts | 6 +++--- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/workers/extractEmissions.ts b/src/workers/extractEmissions.ts index 9bf300a7..459bf356 100644 --- a/src/workers/extractEmissions.ts +++ b/src/workers/extractEmissions.ts @@ -46,8 +46,8 @@ const extractEmissions = new DiscordWorker( }, (scope12 || scope3 || biogenic || economy) ? { - name: 'saveReportingPeriods ' + companyName, - queueName: 'saveReportingPeriods', + name: 'diffReportingPeriods ' + companyName, + queueName: 'diffReportingPeriods', data: { ...base.data, scope12, diff --git a/src/workers/saveGoals.ts b/src/workers/saveGoals.ts index dcdf0d10..24b0f88a 100644 --- a/src/workers/saveGoals.ts +++ b/src/workers/saveGoals.ts @@ -11,7 +11,7 @@ export class JobData extends DiscordJob { } } -const saveGoals = new DiscordWorker('saveGoals', async (job) => { +const diffGoals = new DiscordWorker('diffGoals', async (job) => { const { url, wikidata, companyName, existingCompany, goals } = job.data const wikidataId = wikidata.node const metadata = defaultMetadata(url) @@ -37,4 +37,4 @@ const saveGoals = new DiscordWorker('saveGoals', async (job) => { return { body, diff, requiresApproval } }) -export default saveGoals +export default diffGoals diff --git a/src/workers/saveIndustry.ts b/src/workers/saveIndustry.ts index f07e3c4c..116b938e 100644 --- a/src/workers/saveIndustry.ts +++ b/src/workers/saveIndustry.ts @@ -11,7 +11,7 @@ export class JobData extends DiscordJob { } } -const saveIndustry = new DiscordWorker('saveIndustry', async (job) => { +const diffIndustry = new DiscordWorker('diffIndustry', async (job) => { const { url, wikidata, companyName, existingCompany, industry } = job.data const wikidataId = wikidata.node const metadata = defaultMetadata(url) @@ -37,4 +37,4 @@ const saveIndustry = new DiscordWorker('saveIndustry', async (job) => { return { body, diff, requiresApproval } }) -export default saveIndustry +export default diffIndustry diff --git a/src/workers/saveInitiatives.ts b/src/workers/saveInitiatives.ts index af3253af..95b2a2e4 100644 --- a/src/workers/saveInitiatives.ts +++ b/src/workers/saveInitiatives.ts @@ -11,8 +11,8 @@ export class JobData extends DiscordJob { } } -const saveInitiatives = new DiscordWorker( - 'saveInitiatives', +const diffInitiatives = new DiscordWorker( + 'diffInitiatives', async (job) => { const { url, companyName, existingCompany, initiatives } = job.data const metadata = defaultMetadata(url) @@ -38,4 +38,4 @@ const saveInitiatives = new DiscordWorker( } ) -export default saveInitiatives +export default diffInitiatives diff --git a/src/workers/saveReportingPeriods.ts b/src/workers/saveReportingPeriods.ts index 7599f862..36d414aa 100644 --- a/src/workers/saveReportingPeriods.ts +++ b/src/workers/saveReportingPeriods.ts @@ -16,8 +16,8 @@ export class JobData extends DiscordJob { } } -const saveReportingPeriods = new DiscordWorker( - 'saveReportingPeriods', +const diffReportingPeriods = new DiscordWorker( + 'diffReportingPeriods', async (job) => { const { url, @@ -92,4 +92,4 @@ const saveReportingPeriods = new DiscordWorker( } ) -export default saveReportingPeriods +export default diffReportingPeriods From 2f93f74443576715e6c83b4971c3ef28e15ba16e Mon Sep 17 00:00:00 2001 From: "Christian Landgren (aider)" Date: Wed, 4 Dec 2024 17:33:48 +0100 Subject: [PATCH 62/82] refactor: Update queue names in checkDB to use 'diff' prefix --- src/workers/checkDB.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/workers/checkDB.ts b/src/workers/checkDB.ts index 7dc62e02..4177f598 100644 --- a/src/workers/checkDB.ts +++ b/src/workers/checkDB.ts @@ -84,7 +84,7 @@ const checkDB = new DiscordWorker('checkDB', async (job: JobData) => { scope12 || scope3 || biogenic || economy ? { ...base, - queueName: 'saveReportingPeriods', + queueName: 'diffReportingPeriods', data: { ...base.data, scope12, @@ -97,7 +97,7 @@ const checkDB = new DiscordWorker('checkDB', async (job: JobData) => { industry ? { ...base, - queueName: 'saveIndustry', + queueName: 'diffIndustry', data: { ...base.data, industry, @@ -107,7 +107,7 @@ const checkDB = new DiscordWorker('checkDB', async (job: JobData) => { goals ? { ...base, - queueName: 'saveGoals', + queueName: 'diffGoals', data: { ...base.data, goals, @@ -117,7 +117,7 @@ const checkDB = new DiscordWorker('checkDB', async (job: JobData) => { initiatives ? { ...base, - queueName: 'saveInitiatives', + queueName: 'diffInitiatives', data: { ...base.data, initiatives, From fbb79b1eee64f91f4735ca153c110e0e1afbd5b7 Mon Sep 17 00:00:00 2001 From: Christian Landgren Date: Wed, 4 Dec 2024 17:37:09 +0100 Subject: [PATCH 63/82] chore: rename save -> diff workers --- src/lib/saveUtils.ts | 1 + src/workers/{saveGoals.ts => diffGoals.ts} | 2 +- src/workers/{saveIndustry.ts => diffIndustry.ts} | 2 +- src/workers/{saveInitiatives.ts => diffInitiatives.ts} | 2 +- .../{saveReportingPeriods.ts => diffReportingPeriods.ts} | 2 +- 5 files changed, 5 insertions(+), 4 deletions(-) rename src/workers/{saveGoals.ts => diffGoals.ts} (93%) rename src/workers/{saveIndustry.ts => diffIndustry.ts} (93%) rename src/workers/{saveInitiatives.ts => diffInitiatives.ts} (92%) rename src/workers/{saveReportingPeriods.ts => diffReportingPeriods.ts} (97%) diff --git a/src/lib/saveUtils.ts b/src/lib/saveUtils.ts index 7cb42d4d..3669b919 100644 --- a/src/lib/saveUtils.ts +++ b/src/lib/saveUtils.ts @@ -32,6 +32,7 @@ export const defaultMetadata = (url: string) => ({ }) export const askDiff = async (before: any, after: any) => { + if (!before || !after) return 'NO_CHANGES' return await askPrompt( `What is changed between these two json values? Please respond in clear text with markdown formatting. The purpose is to let an editor approve the changes or suggest changes in Discord. diff --git a/src/workers/saveGoals.ts b/src/workers/diffGoals.ts similarity index 93% rename from src/workers/saveGoals.ts rename to src/workers/diffGoals.ts index 24b0f88a..e6880403 100644 --- a/src/workers/saveGoals.ts +++ b/src/workers/diffGoals.ts @@ -21,7 +21,7 @@ const diffGoals = new DiscordWorker('diffGoals', async (job) => { metadata, } - const diff = await askDiff(existingCompany.goals, goals) + const diff = await askDiff(existingCompany?.goals, goals) const requiresApproval = diff && !diff.includes('NO_CHANGES') await saveToAPI.queue.add(companyName, { diff --git a/src/workers/saveIndustry.ts b/src/workers/diffIndustry.ts similarity index 93% rename from src/workers/saveIndustry.ts rename to src/workers/diffIndustry.ts index 116b938e..bc08b891 100644 --- a/src/workers/saveIndustry.ts +++ b/src/workers/diffIndustry.ts @@ -21,7 +21,7 @@ const diffIndustry = new DiscordWorker('diffIndustry', async (job) => { metadata, } - const diff = await askDiff(existingCompany.industry, industry) + const diff = await askDiff(existingCompany?.industry, industry) const requiresApproval = diff && !diff.includes('NO_CHANGES') await saveToAPI.queue.add(companyName, { diff --git a/src/workers/saveInitiatives.ts b/src/workers/diffInitiatives.ts similarity index 92% rename from src/workers/saveInitiatives.ts rename to src/workers/diffInitiatives.ts index 95b2a2e4..ad431e18 100644 --- a/src/workers/saveInitiatives.ts +++ b/src/workers/diffInitiatives.ts @@ -22,7 +22,7 @@ const diffInitiatives = new DiscordWorker( metadata, } - const diff = await askDiff(existingCompany.initiatives, initiatives) + const diff = await askDiff(existingCompany?.initiatives, initiatives) const requiresApproval = diff && !diff.includes('NO_CHANGES') await saveToAPI.queue.add(companyName, { diff --git a/src/workers/saveReportingPeriods.ts b/src/workers/diffReportingPeriods.ts similarity index 97% rename from src/workers/saveReportingPeriods.ts rename to src/workers/diffReportingPeriods.ts index 36d414aa..592c948e 100644 --- a/src/workers/saveReportingPeriods.ts +++ b/src/workers/diffReportingPeriods.ts @@ -74,7 +74,7 @@ const diffReportingPeriods = new DiscordWorker( } ) - const diff = await askDiff(existingCompany.reportingPeriods, body) + const diff = await askDiff(existingCompany?.reportingPeriods, body) job.log('diff: ' + diff) const requiresApproval = diff && !diff.includes('NO_CHANGES') From 7f65ebad9b7e110cdc92fa0b65bd07e01982801c Mon Sep 17 00:00:00 2001 From: Christian Landgren Date: Wed, 4 Dec 2024 17:40:33 +0100 Subject: [PATCH 64/82] fix: dont diff metadata --- src/lib/saveUtils.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/lib/saveUtils.ts b/src/lib/saveUtils.ts index 3669b919..06cdeb85 100644 --- a/src/lib/saveUtils.ts +++ b/src/lib/saveUtils.ts @@ -31,6 +31,9 @@ export const defaultMetadata = (url: string) => ({ comment: 'Parsed by Garbo AI', }) +const omit = (obj: any, keys: string[]) => + Object.fromEntries(Object.entries(obj).filter(([key]) => !keys.includes(key))) + export const askDiff = async (before: any, after: any) => { if (!before || !after) return 'NO_CHANGES' return await askPrompt( @@ -40,6 +43,9 @@ Be as brief as possible. Never be technical - meaning no comments about structur Focus only on the actual values that have changed. When handling years and ambiguous dates, always use the last year in the period (e.g. startDate: 2020 - endDate: 2021 should be referred to as 2021). NEVER REPEAT UNCHANGED VALUES OR UNCHANGED YEARS! If nothing important has changed, just write "NO_CHANGES".`, - JSON.stringify({ before, after }) + JSON.stringify({ + before: omit(before, ['metadata']), + after: omit(after, ['metadata']), + }) ) } From a1910de12e01ab5c85f7819022ad401d2817b7dd Mon Sep 17 00:00:00 2001 From: Samuel Plumppu <6125097+Greenheart@users.noreply.github.com> Date: Thu, 5 Dec 2024 08:08:15 +0100 Subject: [PATCH 65/82] Fix import --- src/discord.ts | 2 +- src/workers/{saveToApi.ts => saveToAPI.ts} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename src/workers/{saveToApi.ts => saveToAPI.ts} (100%) diff --git a/src/discord.ts b/src/discord.ts index 63eebfc6..e88bc4bb 100644 --- a/src/discord.ts +++ b/src/discord.ts @@ -16,7 +16,7 @@ import config from './config/discord' import approve from './discord/interactions/approve' import reject from './discord/interactions/reject' import { Queue } from 'bullmq' -import { SaveToApiJob } from './workers/saveToApi' +import { SaveToApiJob } from './workers/saveToAPI' const apiSaveQueue = new Queue('api-save', { connection: { diff --git a/src/workers/saveToApi.ts b/src/workers/saveToAPI.ts similarity index 100% rename from src/workers/saveToApi.ts rename to src/workers/saveToAPI.ts From 163e742a0fa0b01e4c715384c66479285f2fa021 Mon Sep 17 00:00:00 2001 From: Samuel Plumppu <6125097+Greenheart@users.noreply.github.com> Date: Thu, 5 Dec 2024 08:08:24 +0100 Subject: [PATCH 66/82] Fix missing config import --- src/workers/checkDB.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/workers/checkDB.ts b/src/workers/checkDB.ts index 4177f598..97594a48 100644 --- a/src/workers/checkDB.ts +++ b/src/workers/checkDB.ts @@ -1,6 +1,7 @@ import { FlowProducer } from 'bullmq' import { DiscordJob, DiscordWorker } from '../lib/DiscordWorker' import { apiFetch } from '../lib/api' +import redis from '../config/redis' export class JobData extends DiscordJob { declare data: DiscordJob['data'] & { From 5b63aa82ad45cba9fc6826ac04adf075a48ce6f8 Mon Sep 17 00:00:00 2001 From: Samuel Plumppu <6125097+Greenheart@users.noreply.github.com> Date: Thu, 5 Dec 2024 08:13:44 +0100 Subject: [PATCH 67/82] Revert extractEmissions to remove wrong AI-generated code --- src/workers/extractEmissions.ts | 52 ++++++++++++++++++++------------- 1 file changed, 32 insertions(+), 20 deletions(-) diff --git a/src/workers/extractEmissions.ts b/src/workers/extractEmissions.ts index 459bf356..6a427453 100644 --- a/src/workers/extractEmissions.ts +++ b/src/workers/extractEmissions.ts @@ -44,41 +44,53 @@ const extractEmissions = new DiscordWorker( type: JobType.IndustryGics, }, }, - (scope12 || scope3 || biogenic || economy) - ? { - name: 'diffReportingPeriods ' + companyName, - queueName: 'diffReportingPeriods', - data: { - ...base.data, - scope12, - scope3, - biogenic, - economy, - }, - } - : null, { ...base, - name: 'goals ' + companyName, + name: 'scope1+2 ' + companyName, data: { ...base.data, - apiSubEndpoint: 'goals', - type: JobType.Goals, + type: JobType.Scope12, }, }, { ...base, - name: 'initiatives ' + companyName, + name: 'scope3 ' + companyName, data: { ...base.data, - type: JobType.Initiatives, + type: JobType.Scope3, }, }, { - name: 'askDiff ' + companyName, - queueName: 'askDiff', + ...base, + name: 'biogenic ' + companyName, data: { ...base.data, + type: JobType.Biogenic, + }, + }, + { + ...base, + name: 'economy ' + companyName, + data: { + ...base.data, + type: JobType.Economy, + }, + }, + { + ...base, + name: 'goals ' + companyName, + data: { + ...base.data, + apiSubEndpoint: 'goals', + type: JobType.Goals, + }, + }, + { + ...base, + name: 'initiatives ' + companyName, + data: { + ...base.data, + type: JobType.Initiatives, }, }, ], From 6745c0fd1944e39e02a20fc1822b14fc51396688 Mon Sep 17 00:00:00 2001 From: Samuel Plumppu <6125097+Greenheart@users.noreply.github.com> Date: Thu, 5 Dec 2024 08:16:19 +0100 Subject: [PATCH 68/82] Use correct saveToAPI queue --- src/discord.ts | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/src/discord.ts b/src/discord.ts index e88bc4bb..0c935152 100644 --- a/src/discord.ts +++ b/src/discord.ts @@ -15,17 +15,9 @@ import commands from './discord/commands' import config from './config/discord' import approve from './discord/interactions/approve' import reject from './discord/interactions/reject' -import { Queue } from 'bullmq' -import { SaveToApiJob } from './workers/saveToAPI' +import saveToAPI, { SaveToApiJob } from './workers/saveToAPI' -const apiSaveQueue = new Queue('api-save', { - connection: { - host: process.env.REDIS_HOST || 'localhost', - port: parseInt(process.env.REDIS_PORT || '6379'), - }, -}) - -const getJob = (jobId: string) => apiSaveQueue.getJob(jobId) +const getJob = (jobId: string) => saveToAPI.queue.getJob(jobId) export class Discord { client: Client From 49a7dc6abf29f4fc745b9ae5aa7888753568be88 Mon Sep 17 00:00:00 2001 From: Samuel Plumppu <6125097+Greenheart@users.noreply.github.com> Date: Thu, 5 Dec 2024 08:32:23 +0100 Subject: [PATCH 69/82] Use consistent names for job types, to reflect the type actually includes more than just the data --- src/workers/checkDB.ts | 4 +-- src/workers/diffGoals.ts | 4 +-- src/workers/diffIndustry.ts | 47 +++++++++++++++-------------- src/workers/diffInitiatives.ts | 4 +-- src/workers/diffReportingPeriods.ts | 4 +-- src/workers/extractEmissions.ts | 4 +-- src/workers/followUp.ts | 6 ++-- src/workers/guessWikidata.ts | 6 ++-- src/workers/indexMarkdown.ts | 4 +-- src/workers/nlmExtractTables.ts | 4 +-- src/workers/precheck.ts | 4 +-- src/workers/sendCompanyLink.ts | 6 ++-- 12 files changed, 50 insertions(+), 47 deletions(-) diff --git a/src/workers/checkDB.ts b/src/workers/checkDB.ts index 97594a48..cebe5fe5 100644 --- a/src/workers/checkDB.ts +++ b/src/workers/checkDB.ts @@ -3,7 +3,7 @@ import { DiscordJob, DiscordWorker } from '../lib/DiscordWorker' import { apiFetch } from '../lib/api' import redis from '../config/redis' -export class JobData extends DiscordJob { +export class CheckDBJob extends DiscordJob { declare data: DiscordJob['data'] & { companyName: string description?: string @@ -16,7 +16,7 @@ export class JobData extends DiscordJob { const flow = new FlowProducer({ connection: redis }) -const checkDB = new DiscordWorker('checkDB', async (job: JobData) => { +const checkDB = new DiscordWorker('checkDB', async (job: CheckDBJob) => { const { companyName, description, diff --git a/src/workers/diffGoals.ts b/src/workers/diffGoals.ts index e6880403..7d9a7aa6 100644 --- a/src/workers/diffGoals.ts +++ b/src/workers/diffGoals.ts @@ -2,7 +2,7 @@ import { DiscordJob, DiscordWorker } from '../lib/DiscordWorker' import { defaultMetadata, askDiff } from '../lib/saveUtils' import saveToAPI from './saveToAPI' -export class JobData extends DiscordJob { +export class DiffGoalsJob extends DiscordJob { declare data: DiscordJob['data'] & { companyName: string existingCompany: any @@ -11,7 +11,7 @@ export class JobData extends DiscordJob { } } -const diffGoals = new DiscordWorker('diffGoals', async (job) => { +const diffGoals = new DiscordWorker('diffGoals', async (job) => { const { url, wikidata, companyName, existingCompany, goals } = job.data const wikidataId = wikidata.node const metadata = defaultMetadata(url) diff --git a/src/workers/diffIndustry.ts b/src/workers/diffIndustry.ts index bc08b891..5735522e 100644 --- a/src/workers/diffIndustry.ts +++ b/src/workers/diffIndustry.ts @@ -2,7 +2,7 @@ import { DiscordJob, DiscordWorker } from '../lib/DiscordWorker' import { defaultMetadata, askDiff } from '../lib/saveUtils' import saveToAPI from './saveToAPI' -export class JobData extends DiscordJob { +export class DiffIndustryJob extends DiscordJob { declare data: DiscordJob['data'] & { existingCompany: any companyName: string @@ -11,30 +11,33 @@ export class JobData extends DiscordJob { } } -const diffIndustry = new DiscordWorker('diffIndustry', async (job) => { - const { url, wikidata, companyName, existingCompany, industry } = job.data - const wikidataId = wikidata.node - const metadata = defaultMetadata(url) +const diffIndustry = new DiscordWorker( + 'diffIndustry', + async (job) => { + const { url, wikidata, companyName, existingCompany, industry } = job.data + const wikidataId = wikidata.node + const metadata = defaultMetadata(url) - const body = { - industry, - metadata, - } + const body = { + industry, + metadata, + } - const diff = await askDiff(existingCompany?.industry, industry) - const requiresApproval = diff && !diff.includes('NO_CHANGES') + const diff = await askDiff(existingCompany?.industry, industry) + const requiresApproval = diff && !diff.includes('NO_CHANGES') - await saveToAPI.queue.add(companyName, { - data: { - ...job.data, - body, - diff, - requiresApproval, - wikidataId, - }, - }) + await saveToAPI.queue.add(companyName, { + data: { + ...job.data, + body, + diff, + requiresApproval, + wikidataId, + }, + }) - return { body, diff, requiresApproval } -}) + return { body, diff, requiresApproval } + } +) export default diffIndustry diff --git a/src/workers/diffInitiatives.ts b/src/workers/diffInitiatives.ts index ad431e18..de3ab168 100644 --- a/src/workers/diffInitiatives.ts +++ b/src/workers/diffInitiatives.ts @@ -2,7 +2,7 @@ import { DiscordJob, DiscordWorker } from '../lib/DiscordWorker' import { defaultMetadata, askDiff } from '../lib/saveUtils' import saveToAPI from './saveToAPI' -export class JobData extends DiscordJob { +export class DiffInitiativesJob extends DiscordJob { declare data: DiscordJob['data'] & { companyName: string existingCompany: any @@ -11,7 +11,7 @@ export class JobData extends DiscordJob { } } -const diffInitiatives = new DiscordWorker( +const diffInitiatives = new DiscordWorker( 'diffInitiatives', async (job) => { const { url, companyName, existingCompany, initiatives } = job.data diff --git a/src/workers/diffReportingPeriods.ts b/src/workers/diffReportingPeriods.ts index 592c948e..5b795405 100644 --- a/src/workers/diffReportingPeriods.ts +++ b/src/workers/diffReportingPeriods.ts @@ -3,7 +3,7 @@ import { defaultMetadata, askDiff } from '../lib/saveUtils' import { getReportingPeriodDates } from '../lib/reportingPeriodDates' import saveToAPI from './saveToAPI' -export class JobData extends DiscordJob { +export class DiffReportingPeriodsJob extends DiscordJob { declare data: DiscordJob['data'] & { companyName: string existingCompany: any @@ -16,7 +16,7 @@ export class JobData extends DiscordJob { } } -const diffReportingPeriods = new DiscordWorker( +const diffReportingPeriods = new DiscordWorker( 'diffReportingPeriods', async (job) => { const { diff --git a/src/workers/extractEmissions.ts b/src/workers/extractEmissions.ts index 6a427453..984aa58a 100644 --- a/src/workers/extractEmissions.ts +++ b/src/workers/extractEmissions.ts @@ -3,7 +3,7 @@ import redis from '../config/redis' import { DiscordJob, DiscordWorker } from '../lib/DiscordWorker' import { JobType } from '../types' -class JobData extends DiscordJob { +class ExtractEmissionsJob extends DiscordJob { declare data: DiscordJob['data'] & { companyName: string type: JobType @@ -12,7 +12,7 @@ class JobData extends DiscordJob { const flow = new FlowProducer({ connection: redis }) -const extractEmissions = new DiscordWorker( +const extractEmissions = new DiscordWorker( 'extractEmissions', async (job) => { const { companyName } = job.data diff --git a/src/workers/followUp.ts b/src/workers/followUp.ts index da8a263a..447d3918 100644 --- a/src/workers/followUp.ts +++ b/src/workers/followUp.ts @@ -6,7 +6,7 @@ import { zodResponseFormat } from 'openai/helpers/zod' import { resolve } from 'path' import { vectorDB } from '../lib/vectordb' -class JobData extends DiscordJob { +class FollowUpJob extends DiscordJob { declare data: DiscordJob['data'] & { documentId: string apiSubEndpoint: string @@ -15,9 +15,9 @@ class JobData extends DiscordJob { } } -const followUp = new DiscordWorker( +const followUp = new DiscordWorker( 'followUp', - async (job: JobData) => { + async (job: FollowUpJob) => { const { type, url, previousAnswer } = job.data const { diff --git a/src/workers/guessWikidata.ts b/src/workers/guessWikidata.ts index c81ab588..b46189d4 100644 --- a/src/workers/guessWikidata.ts +++ b/src/workers/guessWikidata.ts @@ -5,15 +5,15 @@ import { zodResponseFormat } from 'openai/helpers/zod' import { DiscordJob, DiscordWorker } from '../lib/DiscordWorker' import wikidata from '../prompts/wikidata' -class JobData extends DiscordJob { +class GuessWikidataJob extends DiscordJob { declare data: DiscordJob['data'] & { companyName: string } } -const guessWikidata = new DiscordWorker( +const guessWikidata = new DiscordWorker( 'guessWikidata', - async (job: JobData) => { + async (job: GuessWikidataJob) => { const { companyName } = job.data if (!companyName) throw new Error('No company name was provided') diff --git a/src/workers/indexMarkdown.ts b/src/workers/indexMarkdown.ts index c0457791..4d17d39a 100644 --- a/src/workers/indexMarkdown.ts +++ b/src/workers/indexMarkdown.ts @@ -2,11 +2,11 @@ import { CHUNK_SIZE } from '../config' import { DiscordWorker, DiscordJob } from '../lib/DiscordWorker' import { vectorDB } from '../lib/vectordb' -class JobData extends DiscordJob {} +class IndexMarkdownJob extends DiscordJob {} const indexMarkdown = new DiscordWorker( 'indexMarkdown', - async (job: JobData) => { + async (job: IndexMarkdownJob) => { const { url } = job.data const childrenValues = await job.getChildrenEntries() const { markdown }: { markdown: string } = childrenValues diff --git a/src/workers/nlmExtractTables.ts b/src/workers/nlmExtractTables.ts index 29eeea59..8c5aa267 100644 --- a/src/workers/nlmExtractTables.ts +++ b/src/workers/nlmExtractTables.ts @@ -9,7 +9,7 @@ import { jsonToMarkdown } from '../lib/jsonExtraction' import { openai } from '../lib/openai' import { ParsedDocument } from '../lib/nlm-ingestor-schema' -class JobData extends DiscordJob { +class NLMExtractTablesJob extends DiscordJob { declare data: DiscordJob['data'] & { json: ParsedDocument } @@ -79,7 +79,7 @@ const searchTerms = [ ] const nlmExtractTables = new DiscordWorker( 'nlmExtractTables', - async (job: JobData) => { + async (job: NLMExtractTablesJob) => { const { json, url } = job.data try { diff --git a/src/workers/precheck.ts b/src/workers/precheck.ts index 3f10f9c8..95471e39 100644 --- a/src/workers/precheck.ts +++ b/src/workers/precheck.ts @@ -6,7 +6,7 @@ import { zodResponseFormat } from 'openai/helpers/zod' import { DiscordJob, DiscordWorker } from '../lib/DiscordWorker' import { JobType } from '../types' -class JobData extends DiscordJob { +class PrecheckJob extends DiscordJob { declare data: DiscordJob['data'] & { cachedMarkdown?: string companyName?: string @@ -16,7 +16,7 @@ class JobData extends DiscordJob { const flow = new FlowProducer({ connection: redis }) -const precheck = new DiscordWorker('precheck', async (job: JobData) => { +const precheck = new DiscordWorker('precheck', async (job: PrecheckJob) => { const { cachedMarkdown, type, ...baseData } = job.data const { markdown = cachedMarkdown } = await job.getChildrenEntries() diff --git a/src/workers/sendCompanyLink.ts b/src/workers/sendCompanyLink.ts index 4e521ce0..aa62cdae 100644 --- a/src/workers/sendCompanyLink.ts +++ b/src/workers/sendCompanyLink.ts @@ -1,13 +1,13 @@ import { DiscordJob, DiscordWorker } from '../lib/DiscordWorker' -export class JobData extends DiscordJob { +export class SendCompanyLinkJob extends DiscordJob { declare data: DiscordJob['data'] & { companyName: string wikidata: any } } -const sendCompanyLink = new DiscordWorker( +const sendCompanyLink = new DiscordWorker( 'sendCompanyLink', async (job) => { const { companyName, wikidata } = job.data @@ -27,7 +27,7 @@ const sendCompanyLink = new DiscordWorker( await job.sendMessage(`✅ Företaget har sparats! Se resultatet här: ${url}`) return { url } - }, + } ) export default sendCompanyLink From 8be2df571712aeb0e5f2bf73f68407993981ce9d Mon Sep 17 00:00:00 2001 From: Samuel Plumppu <6125097+Greenheart@users.noreply.github.com> Date: Thu, 5 Dec 2024 09:58:48 +0100 Subject: [PATCH 70/82] Omit metadata when diffing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Hugo Björk --- src/lib/saveUtils.ts | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/src/lib/saveUtils.ts b/src/lib/saveUtils.ts index 06cdeb85..2d361b5d 100644 --- a/src/lib/saveUtils.ts +++ b/src/lib/saveUtils.ts @@ -31,8 +31,29 @@ export const defaultMetadata = (url: string) => ({ comment: 'Parsed by Garbo AI', }) -const omit = (obj: any, keys: string[]) => - Object.fromEntries(Object.entries(obj).filter(([key]) => !keys.includes(key))) +/** + * Recusrively remove the provided keys from all levels of the object. + * + * Handles circular references. + * Source: https://stackoverflow.com/a/72493889 + */ +const recursiveOmit = ( + obj: T, + keys: Set, + visitedIn?: Set +): T => { + if (obj == null || typeof obj !== 'object') return obj + const visited = visitedIn ?? new Set() + visited.add(obj) + Object.entries(obj).forEach(([key, val]) => { + if (keys.has(key)) { + delete obj[key as keyof T] + } else if (typeof val === 'object' && !visited.has(val)) { + recursiveOmit(val, keys, visited) + } + }) + return obj +} export const askDiff = async (before: any, after: any) => { if (!before || !after) return 'NO_CHANGES' @@ -44,8 +65,8 @@ Focus only on the actual values that have changed. When handling years and ambiguous dates, always use the last year in the period (e.g. startDate: 2020 - endDate: 2021 should be referred to as 2021). NEVER REPEAT UNCHANGED VALUES OR UNCHANGED YEARS! If nothing important has changed, just write "NO_CHANGES".`, JSON.stringify({ - before: omit(before, ['metadata']), - after: omit(after, ['metadata']), + before: recursiveOmit(structuredClone(before), new Set(['metadata'])), + after: recursiveOmit(structuredClone(after), new Set(['metadata'])), }) ) } From a4aebe7d46ab01a84e513d95f3efc00cdb6d742b Mon Sep 17 00:00:00 2001 From: Samuel Plumppu <6125097+Greenheart@users.noreply.github.com> Date: Thu, 5 Dec 2024 10:01:25 +0100 Subject: [PATCH 71/82] Only include the changed data when diffing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Hugo Björk --- src/workers/diffReportingPeriods.ts | 43 ++++++++++++++++++----------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/src/workers/diffReportingPeriods.ts b/src/workers/diffReportingPeriods.ts index 5b795405..f529a16c 100644 --- a/src/workers/diffReportingPeriods.ts +++ b/src/workers/diffReportingPeriods.ts @@ -52,28 +52,39 @@ const diffReportingPeriods = new DiscordWorker( year, startDate, endDate, + reportURL: url, metadata, } }) - // Fill in data from each source - const body = reportingPeriods.map( - ({ year, startDate, endDate, metadata }) => { - return { - emissions: { - scope1: scope12.find((d) => d.year === year)?.scope1, - scope2: scope12.find((d) => d.year === year)?.scope2, - scope3: scope3.find((d) => d.year === year)?.scope3, - biogenic: biogenic.find((d) => d.year === year)?.biogenic, - }, - economy: economy.find((d) => d.year === year)?.economy, - startDate, - endDate, - metadata, - } + // Fill in data from each source, only keeping data that was changed. + const body = reportingPeriods.map(({ year, ...period }) => { + const emissions = { + scope1: scope12.find((d) => d.year === year)?.scope1, + scope2: scope12.find((d) => d.year === year)?.scope2, + scope3: scope3.find((d) => d.year === year)?.scope3, + biogenic: biogenic.find((d) => d.year === year)?.biogenic, } - ) + const economyData = { + ...(economy.find((d) => d.year === year)?.economy ? economy : {}), + } + + const reportingPeriod: any = period + + if (Object.values(emissions).some((value) => value !== undefined)) { + reportingPeriod.emissions = emissions + } + + if (Object.values(economyData).some((value) => value !== undefined)) { + reportingPeriod.economy = economyData + } + + return reportingPeriod + }) + + // NOTE: Maybe only keep properties in existingCompany.reportingPeriods, e.g. the relevant economy properties, or the relevant emissions properties + // This could improve accuracy of the diff const diff = await askDiff(existingCompany?.reportingPeriods, body) job.log('diff: ' + diff) const requiresApproval = diff && !diff.includes('NO_CHANGES') From 1e29c282800ccc0f05847772b6c60a3bf27ef93f Mon Sep 17 00:00:00 2001 From: Samuel Plumppu <6125097+Greenheart@users.noreply.github.com> Date: Thu, 5 Dec 2024 11:24:52 +0100 Subject: [PATCH 72/82] Allow saveToAPI to post to different endpoints, specified by the job data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Hugo Björk --- src/workers/diffGoals.ts | 3 +- src/workers/diffIndustry.ts | 1 + src/workers/diffReportingPeriods.ts | 52 +++++++++++++++++------------ src/workers/extractEmissions.ts | 1 - src/workers/followUp.ts | 1 - src/workers/saveToAPI.ts | 16 ++++----- 6 files changed, 42 insertions(+), 32 deletions(-) diff --git a/src/workers/diffGoals.ts b/src/workers/diffGoals.ts index 7d9a7aa6..b07de483 100644 --- a/src/workers/diffGoals.ts +++ b/src/workers/diffGoals.ts @@ -24,11 +24,12 @@ const diffGoals = new DiscordWorker('diffGoals', async (job) => { const diff = await askDiff(existingCompany?.goals, goals) const requiresApproval = diff && !diff.includes('NO_CHANGES') - await saveToAPI.queue.add(companyName, { + await saveToAPI.queue.add(companyName + ' goals', { data: { ...job.data, body, diff, + apiSubEndpoint: 'goals', requiresApproval, wikidataId, }, diff --git a/src/workers/diffIndustry.ts b/src/workers/diffIndustry.ts index 5735522e..78586fe0 100644 --- a/src/workers/diffIndustry.ts +++ b/src/workers/diffIndustry.ts @@ -33,6 +33,7 @@ const diffIndustry = new DiscordWorker( diff, requiresApproval, wikidataId, + apiSubEndpoint: 'industry', }, }) diff --git a/src/workers/diffReportingPeriods.ts b/src/workers/diffReportingPeriods.ts index f529a16c..4c563941 100644 --- a/src/workers/diffReportingPeriods.ts +++ b/src/workers/diffReportingPeriods.ts @@ -53,47 +53,57 @@ const diffReportingPeriods = new DiscordWorker( startDate, endDate, reportURL: url, - metadata, } }) // Fill in data from each source, only keeping data that was changed. - const body = reportingPeriods.map(({ year, ...period }) => { - const emissions = { - scope1: scope12.find((d) => d.year === year)?.scope1, - scope2: scope12.find((d) => d.year === year)?.scope2, - scope3: scope3.find((d) => d.year === year)?.scope3, - biogenic: biogenic.find((d) => d.year === year)?.biogenic, - } + const updatedReportingPeriods = reportingPeriods.map( + ({ year, ...period }) => { + const emissions = { + scope1: scope12.find((d) => d.year === year)?.scope1, + scope2: scope12.find((d) => d.year === year)?.scope2, + scope3: scope3.find((d) => d.year === year)?.scope3, + biogenic: biogenic.find((d) => d.year === year)?.biogenic, + } - const economyData = { - ...(economy.find((d) => d.year === year)?.economy ? economy : {}), - } + const economyData = { + ...(economy.find((d) => d.year === year)?.economy ? economy : {}), + } - const reportingPeriod: any = period + const reportingPeriod: any = period - if (Object.values(emissions).some((value) => value !== undefined)) { - reportingPeriod.emissions = emissions - } + if (Object.values(emissions).some((value) => value !== undefined)) { + reportingPeriod.emissions = emissions + } - if (Object.values(economyData).some((value) => value !== undefined)) { - reportingPeriod.economy = economyData - } + if (Object.values(economyData).some((value) => value !== undefined)) { + reportingPeriod.economy = economyData + } - return reportingPeriod - }) + return reportingPeriod + } + ) // NOTE: Maybe only keep properties in existingCompany.reportingPeriods, e.g. the relevant economy properties, or the relevant emissions properties // This could improve accuracy of the diff - const diff = await askDiff(existingCompany?.reportingPeriods, body) + const diff = await askDiff( + existingCompany?.reportingPeriods, + updatedReportingPeriods + ) job.log('diff: ' + diff) const requiresApproval = diff && !diff.includes('NO_CHANGES') + const body = { + reportingPeriods: updatedReportingPeriods, + metadata, + } + await saveToAPI.queue.add(companyName, { data: { ...job.data, body, diff, + apiSubEndpoint: 'reporting-periods', requiresApproval, wikidataId, }, diff --git a/src/workers/extractEmissions.ts b/src/workers/extractEmissions.ts index 984aa58a..4c065072 100644 --- a/src/workers/extractEmissions.ts +++ b/src/workers/extractEmissions.ts @@ -81,7 +81,6 @@ const extractEmissions = new DiscordWorker( name: 'goals ' + companyName, data: { ...base.data, - apiSubEndpoint: 'goals', type: JobType.Goals, }, }, diff --git a/src/workers/followUp.ts b/src/workers/followUp.ts index 447d3918..32060455 100644 --- a/src/workers/followUp.ts +++ b/src/workers/followUp.ts @@ -9,7 +9,6 @@ import { vectorDB } from '../lib/vectordb' class FollowUpJob extends DiscordJob { declare data: DiscordJob['data'] & { documentId: string - apiSubEndpoint: string type: JobType previousAnswer: string } diff --git a/src/workers/saveToAPI.ts b/src/workers/saveToAPI.ts index d89ffbd8..8b892f5d 100644 --- a/src/workers/saveToAPI.ts +++ b/src/workers/saveToAPI.ts @@ -4,17 +4,17 @@ import { apiFetch } from '../lib/api' export interface SaveToApiJob extends DiscordJob { data: DiscordJob['data'] & { - wikidataId: string approved?: boolean - requiresApproval?: boolean - diff?: string - body?: any - wikidata: any + requiresApproval: boolean + diff: string + body: any + wikidata: { node: string } + apiSubEndpoint: string } } export const saveToAPI = new DiscordWorker( - 'api-save', + 'saveToAPI', async (job: SaveToApiJob) => { try { const { @@ -23,14 +23,14 @@ export const saveToAPI = new DiscordWorker( requiresApproval = true, diff, body, + apiSubEndpoint, } = job.data const wikidataId = wikidata.node // If approval is not required or already approved, proceed with saving if (!requiresApproval || approved) { - // TODO: Implement actual API save logic here console.log(`Saving approved data for ${wikidataId} to API`) - await apiFetch('/companies/' + wikidataId, { body }) + await apiFetch(`/companies/${wikidataId}/${apiSubEndpoint}`, { body }) return { success: true } } From 774fc25e70e9d77ebc34d7703bb3cc6b54231e62 Mon Sep 17 00:00:00 2001 From: Samuel Plumppu <6125097+Greenheart@users.noreply.github.com> Date: Thu, 5 Dec 2024 11:27:48 +0100 Subject: [PATCH 73/82] Use post endpoints to make updates easier MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Hugo Björk --- src/routes/updateCompanies.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/updateCompanies.ts b/src/routes/updateCompanies.ts index 69db6c8f..3b19538f 100644 --- a/src/routes/updateCompanies.ts +++ b/src/routes/updateCompanies.ts @@ -213,7 +213,7 @@ const industrySchema = z.object({ }), }) -router.put( +router.post( '/:wikidataId/industry', processRequest({ body: industrySchema, params: wikidataIdParamSchema }), async (req, res) => { From 4f16f12b40fb63da79b9d8cb84e96ae3537fc8ec Mon Sep 17 00:00:00 2001 From: Samuel Plumppu <6125097+Greenheart@users.noreply.github.com> Date: Thu, 5 Dec 2024 11:42:03 +0100 Subject: [PATCH 74/82] WIP: Upsert reporting periods, upsert emissions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Hugo Björk --- src/lib/prisma.ts | 27 ++++- src/routes/middlewares.ts | 30 ++---- src/routes/updateCompanies.ts | 192 +++++++++++++++++++++++----------- tests/prisma.test.ts | 15 +-- 4 files changed, 173 insertions(+), 91 deletions(-) diff --git a/src/lib/prisma.ts b/src/lib/prisma.ts index 88e73d2f..19d8b5d5 100644 --- a/src/lib/prisma.ts +++ b/src/lib/prisma.ts @@ -552,7 +552,7 @@ export async function updateIndustry( }) } -export async function ensureReportingPeriodExists( +export async function upsertReportingPeriod( company: Company, metadata: Parameters[0]['data'], { @@ -590,3 +590,28 @@ export async function ensureReportingPeriodExists( }, }) } + +export async function upsertEmissions({ + emissionsId, + year, + companyId, +}: { + emissionsId: number + year: string + companyId: string +}) { + return prisma.emissions.upsert({ + where: { id: emissionsId }, + update: {}, + create: { + reportingPeriod: { + connect: { + reportingPeriodId: { + year, + companyId, + }, + }, + }, + }, + }) +} diff --git a/src/routes/middlewares.ts b/src/routes/middlewares.ts index 0758d95a..f471f7da 100644 --- a/src/routes/middlewares.ts +++ b/src/routes/middlewares.ts @@ -12,7 +12,7 @@ import { validateRequest, validateRequestBody } from './zod-middleware' import { z, ZodError } from 'zod' import cors, { CorsOptionsDelegate } from 'cors' -import { ensureReportingPeriodExists } from '../lib/prisma' +import { upsertEmissions, upsertReportingPeriod } from '../lib/prisma' import { GarboAPIError } from '../lib/garbo-api-error' import apiConfig from '../config/api' @@ -164,11 +164,12 @@ export const reportingPeriod = if (req.method === 'POST' || req.method === 'PATCH') { // TODO: Only allow creating a reporting period when updating other data // TODO: Maybe throw 404 if the reporting period was not found and it is a GET request - const reportingPeriod = await ensureReportingPeriodExists( - company, - metadata, - { startDate, endDate, reportURL, year } - ) + const reportingPeriod = await upsertReportingPeriod(company, metadata, { + startDate, + endDate, + reportURL, + year, + }) res.locals.reportingPeriod = reportingPeriod } @@ -182,19 +183,10 @@ export const ensureEmissionsExists = const reportingPeriod = res.locals.reportingPeriod const emissionsId = res.locals.reportingPeriod.emissionsId ?? 0 - const emissions = await prisma.emissions.upsert({ - where: { id: emissionsId ?? 0 }, - update: {}, - create: { - reportingPeriod: { - connect: { - reportingPeriodId: { - year: reportingPeriod.year, - companyId: reportingPeriod.companyId, - }, - }, - }, - }, + const emissions = await upsertEmissions({ + emissionsId: reportingPeriod.emissionsId ?? 0, + companyId: res.locals.company.wikidataId, + year: reportingPeriod.year, }) res.locals.emissions = emissions diff --git a/src/routes/updateCompanies.ts b/src/routes/updateCompanies.ts index 3b19538f..edf7b350 100644 --- a/src/routes/updateCompanies.ts +++ b/src/routes/updateCompanies.ts @@ -17,6 +17,8 @@ import { updateInitiative, createIndustry, updateIndustry, + upsertReportingPeriod, + upsertEmissions, } from '../lib/prisma' import { createMetadata, @@ -33,8 +35,6 @@ import { wikidataIdParamSchema, wikidataIdSchema } from './companySchemas' import { GarboAPIError } from '../lib/garbo-api-error' const router = express.Router() -const tCO2e = 'tCO2e' -const unit = tCO2e router.use('/', fakeAuth(prisma)) router.use('/', express.json()) @@ -258,62 +258,147 @@ router.post( } ) -router.use( - '/:wikidataId/:year', - validateReportingPeriod(), - reportingPeriod(prisma) -) - -router.use('/:wikidataId/:year/emissions', ensureEmissionsExists(prisma)) -router.use('/:wikidataId/:year/economy', ensureEconomyExists(prisma)) - const statedTotalEmissionsSchema = z.object({ total: z.number() }).optional() -const postEmissionsBodySchema = z.object({ - emissions: z.object({ - scope1: z +export const emissionsSchema = z.object({ + scope1: z + .object({ + total: z.number(), + }) + .optional(), + scope2: z + .object({ + mb: z + .number({ description: 'Market-based scope 2 emissions' }) + .optional(), + lb: z + .number({ description: 'Location-based scope 2 emissions' }) + .optional(), + unknown: z + .number({ description: 'Unspecified Scope 2 emissions' }) + .optional(), + }) + .refine( + ({ mb, lb, unknown }) => + mb !== undefined || lb !== undefined || unknown !== undefined, + { + message: + 'At least one property of `mb`, `lb` and `unknown` must be defined if scope2 is provided', + } + ) + .optional(), + scope3: z + .object({ + categories: z + .array( + z.object({ + category: z.number().int().min(1).max(16), + total: z.number(), + }) + ) + .optional(), + statedTotalEmissions: statedTotalEmissionsSchema, + }) + .optional(), + biogenic: z.object({ total: z.number() }).optional(), + statedTotalEmissions: statedTotalEmissionsSchema, + // TODO: add scope1And2 +}) + +const economySchema = z + .object({ + turnover: z .object({ - total: z.number(), + value: z.number().optional(), + currency: z.string().optional(), }) .optional(), - scope2: z + employees: z .object({ - mb: z - .number({ description: 'Market-based scope 2 emissions' }) - .optional(), - lb: z - .number({ description: 'Location-based scope 2 emissions' }) - .optional(), - unknown: z - .number({ description: 'Unspecified Scope 2 emissions' }) - .optional(), + value: z.number().optional(), + unit: z.string().optional(), + }) + .optional(), + }) + .optional() + +const postReportingPeriodsSchema = z.object({ + reportingPeriods: z.array( + z + .object({ + startDate: z.coerce.date(), + endDate: z.coerce.date(), + reportURL: z.string().optional(), + emissions: emissionsSchema, + economy: economySchema, }) .refine( - ({ mb, lb, unknown }) => - mb !== undefined || lb !== undefined || unknown !== undefined, + ({ startDate, endDate }) => startDate.getTime() < endDate.getTime(), { - message: - 'At least one property of `mb`, `lb` and `unknown` must be defined if scope2 is provided', + message: 'startDate must be earlier than endDate', } ) - .optional(), - scope3: z - .object({ - categories: z - .array( - z.object({ - category: z.number().int().min(1).max(16), - total: z.number(), + ), +}) + +router.post( + '/:wikidataId/reporting-periods', + processRequestBody(postReportingPeriodsSchema), + async (req, res) => { + const { reportingPeriods } = postReportingPeriodsSchema.parse(req.body) + const metadata = res.locals.metadata! + + try { + await Promise.allSettled( + // TODO: Upsert each reporting period + reportingPeriods.map( + async ({ emissions, economy, startDate, endDate, reportURL }) => { + const year = endDate.getFullYear().toString() + const reportingPeriod = await upsertReportingPeriod( + res.locals.company, + metadata, + { + startDate, + endDate, + reportURL, + year, + } + ) + + const dbEmissions = await upsertEmissions({ + emissionsId: reportingPeriod.emissionsId ?? 0, + companyId: res.locals.company.wikidataId, + year, }) - ) - .optional(), - statedTotalEmissions: statedTotalEmissionsSchema, + + // TODO: upsert economy + // TODO: update emissions data + // TODO: update economy data + } + ) + ) + } catch (error) { + throw new GarboAPIError('Failed to update reporting periods', { + original: error, + statusCode: 500, }) - .optional(), - biogenic: z.object({ total: z.number() }).optional(), - statedTotalEmissions: statedTotalEmissionsSchema, - // TODO: add scope1And2 - }), + } + + res.json({ ok: true }) + } +) + +router.use( + '/:wikidataId/:year', + validateReportingPeriod(), + reportingPeriod(prisma) +) + +router.use('/:wikidataId/:year/emissions', ensureEmissionsExists(prisma)) +router.use('/:wikidataId/:year/economy', ensureEconomyExists(prisma)) + +const postEmissionsBodySchema = z.object({ + emissions: emissionsSchema, }) // POST /Q12345/2022-2023/emissions @@ -353,22 +438,7 @@ router.post( ) const postEconomyBodySchema = z.object({ - economy: z - .object({ - turnover: z - .object({ - value: z.number().optional(), - currency: z.string().optional(), - }) - .optional(), - employees: z - .object({ - value: z.number().optional(), - unit: z.string().optional(), - }) - .optional(), - }) - .optional(), + economy: economySchema, }) router.post( diff --git a/tests/prisma.test.ts b/tests/prisma.test.ts index bccd4b8d..52c2e112 100644 --- a/tests/prisma.test.ts +++ b/tests/prisma.test.ts @@ -1,8 +1,4 @@ -import { - prisma, - ensureReportingPeriodExists, - upsertCompany, -} from '../src/lib/prisma' +import { prisma, upsertReportingPeriod, upsertCompany } from '../src/lib/prisma' import { resetDB } from '../src/lib/dev-utils' describe('reporting periods', () => { @@ -57,11 +53,10 @@ describe('reporting periods', () => { const startDate = new Date('2023-01-01') const endDate = new Date('2023-12-31') - const reportingPeriod = await ensureReportingPeriodExists( - company, - metadata, - { startDate, endDate } - ) + const reportingPeriod = await upsertReportingPeriod(company, metadata, { + startDate, + endDate, + }) expect(reportingPeriod.companyId).toBe(company.wikidataId) }, 10000) // Increase timeout to 10 seconds From 37d1e22dbcbefbd17225ea26726ebde42daa1ee5 Mon Sep 17 00:00:00 2001 From: Samuel Plumppu <6125097+Greenheart@users.noreply.github.com> Date: Thu, 5 Dec 2024 11:49:30 +0100 Subject: [PATCH 75/82] WIP: Upsert economy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Hugo Björk --- src/lib/prisma.ts | 25 +++++++++++++++++++++++++ src/routes/middlewares.ts | 27 ++++++++++----------------- src/routes/updateCompanies.ts | 9 +++++++-- 3 files changed, 42 insertions(+), 19 deletions(-) diff --git a/src/lib/prisma.ts b/src/lib/prisma.ts index 19d8b5d5..76e8a76e 100644 --- a/src/lib/prisma.ts +++ b/src/lib/prisma.ts @@ -615,3 +615,28 @@ export async function upsertEmissions({ }, }) } + +export async function upsertEconomy({ + economyId, + companyId, + year, +}: { + economyId: number + companyId: string + year: string +}) { + return prisma.economy.upsert({ + where: { id: economyId }, + update: {}, + create: { + reportingPeriod: { + connect: { + reportingPeriodId: { + year, + companyId, + }, + }, + }, + }, + }) +} diff --git a/src/routes/middlewares.ts b/src/routes/middlewares.ts index f471f7da..6460ba0b 100644 --- a/src/routes/middlewares.ts +++ b/src/routes/middlewares.ts @@ -12,7 +12,11 @@ import { validateRequest, validateRequestBody } from './zod-middleware' import { z, ZodError } from 'zod' import cors, { CorsOptionsDelegate } from 'cors' -import { upsertEmissions, upsertReportingPeriod } from '../lib/prisma' +import { + upsertEconomy, + upsertEmissions, + upsertReportingPeriod, +} from '../lib/prisma' import { GarboAPIError } from '../lib/garbo-api-error' import apiConfig from '../config/api' @@ -181,7 +185,6 @@ export const ensureEmissionsExists = (prisma: PrismaClient) => async (req: Request, res: Response, next: NextFunction) => { const reportingPeriod = res.locals.reportingPeriod - const emissionsId = res.locals.reportingPeriod.emissionsId ?? 0 const emissions = await upsertEmissions({ emissionsId: reportingPeriod.emissionsId ?? 0, @@ -197,21 +200,11 @@ export const ensureEconomyExists = (prisma: PrismaClient) => async (req: Request, res: Response, next: NextFunction) => { const reportingPeriod = res.locals.reportingPeriod - const economyId = res.locals.reportingPeriod.economyId ?? 0 - - const economy = await prisma.economy.upsert({ - where: { id: economyId }, - update: {}, - create: { - reportingPeriod: { - connect: { - reportingPeriodId: { - year: reportingPeriod.year, - companyId: reportingPeriod.companyId, - }, - }, - }, - }, + + const economy = await upsertEconomy({ + economyId: reportingPeriod.economyId ?? 0, + companyId: reportingPeriod.companyId, + year: reportingPeriod.year, }) res.locals.economy = economy diff --git a/src/routes/updateCompanies.ts b/src/routes/updateCompanies.ts index edf7b350..9616a61e 100644 --- a/src/routes/updateCompanies.ts +++ b/src/routes/updateCompanies.ts @@ -19,6 +19,7 @@ import { updateIndustry, upsertReportingPeriod, upsertEmissions, + upsertEconomy, } from '../lib/prisma' import { createMetadata, @@ -350,7 +351,6 @@ router.post( try { await Promise.allSettled( - // TODO: Upsert each reporting period reportingPeriods.map( async ({ emissions, economy, startDate, endDate, reportURL }) => { const year = endDate.getFullYear().toString() @@ -371,7 +371,12 @@ router.post( year, }) - // TODO: upsert economy + const dbEconomy = await upsertEconomy({ + economyId: reportingPeriod.economyId ?? 0, + companyId: res.locals.company.wikidataId, + year, + }) + // TODO: update emissions data // TODO: update economy data } From da28d8949e6d64ccb08a42c007f09524769696e2 Mon Sep 17 00:00:00 2001 From: Samuel Plumppu <6125097+Greenheart@users.noreply.github.com> Date: Thu, 5 Dec 2024 13:38:59 +0100 Subject: [PATCH 76/82] Save emissions and economy data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Hugo Björk --- src/routes/updateCompanies.ts | 169 ++++++++++++++++++++-------------- 1 file changed, 101 insertions(+), 68 deletions(-) diff --git a/src/routes/updateCompanies.ts b/src/routes/updateCompanies.ts index 9616a61e..175d30a1 100644 --- a/src/routes/updateCompanies.ts +++ b/src/routes/updateCompanies.ts @@ -261,50 +261,52 @@ router.post( const statedTotalEmissionsSchema = z.object({ total: z.number() }).optional() -export const emissionsSchema = z.object({ - scope1: z - .object({ - total: z.number(), - }) - .optional(), - scope2: z - .object({ - mb: z - .number({ description: 'Market-based scope 2 emissions' }) - .optional(), - lb: z - .number({ description: 'Location-based scope 2 emissions' }) - .optional(), - unknown: z - .number({ description: 'Unspecified Scope 2 emissions' }) - .optional(), - }) - .refine( - ({ mb, lb, unknown }) => - mb !== undefined || lb !== undefined || unknown !== undefined, - { - message: - 'At least one property of `mb`, `lb` and `unknown` must be defined if scope2 is provided', - } - ) - .optional(), - scope3: z - .object({ - categories: z - .array( - z.object({ - category: z.number().int().min(1).max(16), - total: z.number(), - }) - ) - .optional(), - statedTotalEmissions: statedTotalEmissionsSchema, - }) - .optional(), - biogenic: z.object({ total: z.number() }).optional(), - statedTotalEmissions: statedTotalEmissionsSchema, - // TODO: add scope1And2 -}) +export const emissionsSchema = z + .object({ + scope1: z + .object({ + total: z.number(), + }) + .optional(), + scope2: z + .object({ + mb: z + .number({ description: 'Market-based scope 2 emissions' }) + .optional(), + lb: z + .number({ description: 'Location-based scope 2 emissions' }) + .optional(), + unknown: z + .number({ description: 'Unspecified Scope 2 emissions' }) + .optional(), + }) + .refine( + ({ mb, lb, unknown }) => + mb !== undefined || lb !== undefined || unknown !== undefined, + { + message: + 'At least one property of `mb`, `lb` and `unknown` must be defined if scope2 is provided', + } + ) + .optional(), + scope3: z + .object({ + categories: z + .array( + z.object({ + category: z.number().int().min(1).max(16), + total: z.number(), + }) + ) + .optional(), + statedTotalEmissions: statedTotalEmissionsSchema, + }) + .optional(), + biogenic: z.object({ total: z.number() }).optional(), + statedTotalEmissions: statedTotalEmissionsSchema, + // TODO: add scope1And2 + }) + .optional() const economySchema = z .object({ @@ -352,7 +354,13 @@ router.post( try { await Promise.allSettled( reportingPeriods.map( - async ({ emissions, economy, startDate, endDate, reportURL }) => { + async ({ + emissions = {}, + economy = {}, + startDate, + endDate, + reportURL, + }) => { const year = endDate.getFullYear().toString() const reportingPeriod = await upsertReportingPeriod( res.locals.company, @@ -365,20 +373,42 @@ router.post( } ) - const dbEmissions = await upsertEmissions({ - emissionsId: reportingPeriod.emissionsId ?? 0, - companyId: res.locals.company.wikidataId, - year, - }) - - const dbEconomy = await upsertEconomy({ - economyId: reportingPeriod.economyId ?? 0, - companyId: res.locals.company.wikidataId, - year, - }) - - // TODO: update emissions data - // TODO: update economy data + const [dbEmissions, dbEconomy] = await Promise.all([ + upsertEmissions({ + emissionsId: reportingPeriod.emissionsId ?? 0, + companyId: res.locals.company.wikidataId, + year, + }), + upsertEconomy({ + economyId: reportingPeriod.economyId ?? 0, + companyId: res.locals.company.wikidataId, + year, + }), + ]) + + const { scope1, scope2, scope3, statedTotalEmissions, biogenic } = + emissions + const { turnover, employees } = economy + + // Normalise currency + if (turnover) { + turnover.currency = turnover?.currency?.trim()?.toUpperCase() + } + + await Promise.allSettled([ + scope1 && upsertScope1(dbEmissions, scope1, metadata), + scope2 && upsertScope2(dbEmissions, scope2, metadata), + scope3 && upsertScope3(dbEmissions, scope3, metadata), + statedTotalEmissions && + upsertStatedTotalEmissions( + dbEmissions, + statedTotalEmissions, + metadata + ), + biogenic && upsertBiogenic(dbEmissions, biogenic, metadata), + turnover && upsertTurnover(dbEconomy, turnover, metadata), + employees && upsertEmployees(dbEconomy, employees, metadata), + ]) } ) ) @@ -411,12 +441,11 @@ router.post( '/:wikidataId/:year/emissions', processRequestBody(postEmissionsBodySchema), async (req, res) => { - const { - emissions: { scope1, scope2, scope3, statedTotalEmissions, biogenic }, - } = postEmissionsBodySchema.parse(req.body) + const { emissions = {} } = postEmissionsBodySchema.parse(req.body) + const { scope1, scope2, scope3, statedTotalEmissions, biogenic } = emissions const metadata = res.locals.metadata! - const emissions = res.locals.emissions! + const dbEmissions = res.locals.emissions! try { // Only update if the input contains relevant changes @@ -424,12 +453,16 @@ router.post( // There seems to be a type error in zod which doesn't take into account optional objects. await Promise.allSettled([ - scope1 && upsertScope1(emissions, scope1, metadata), - scope2 && upsertScope2(emissions, scope2, metadata), - scope3 && upsertScope3(emissions, scope3, metadata), + scope1 && upsertScope1(dbEmissions, scope1, metadata), + scope2 && upsertScope2(dbEmissions, scope2, metadata), + scope3 && upsertScope3(dbEmissions, scope3, metadata), statedTotalEmissions && - upsertStatedTotalEmissions(emissions, statedTotalEmissions, metadata), - biogenic && upsertBiogenic(emissions, biogenic, metadata), + upsertStatedTotalEmissions( + dbEmissions, + statedTotalEmissions, + metadata + ), + biogenic && upsertBiogenic(dbEmissions, biogenic, metadata), ]) } catch (error) { throw new GarboAPIError('Failed to update emissions', { From 0c6b3929ac7e6b8d065ce37c361760c7654b6a94 Mon Sep 17 00:00:00 2001 From: Samuel Plumppu <6125097+Greenheart@users.noreply.github.com> Date: Thu, 5 Dec 2024 14:00:30 +0100 Subject: [PATCH 77/82] Improve job names and make sure wikidata is sent all the way MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Hugo Björk --- src/workers/checkDB.ts | 2 +- src/workers/diffGoals.ts | 6 ++---- src/workers/diffIndustry.ts | 8 +++----- src/workers/diffInitiatives.ts | 5 +++-- src/workers/diffReportingPeriods.ts | 7 ++----- 5 files changed, 11 insertions(+), 17 deletions(-) diff --git a/src/workers/checkDB.ts b/src/workers/checkDB.ts index cebe5fe5..cfe0e27a 100644 --- a/src/workers/checkDB.ts +++ b/src/workers/checkDB.ts @@ -7,7 +7,7 @@ export class CheckDBJob extends DiscordJob { declare data: DiscordJob['data'] & { companyName: string description?: string - wikidata: any + wikidata: { node: string } fiscalYear: any childrenValues?: any approved?: boolean diff --git a/src/workers/diffGoals.ts b/src/workers/diffGoals.ts index b07de483..7143f4e5 100644 --- a/src/workers/diffGoals.ts +++ b/src/workers/diffGoals.ts @@ -6,14 +6,13 @@ export class DiffGoalsJob extends DiscordJob { declare data: DiscordJob['data'] & { companyName: string existingCompany: any - wikidata: any + wikidata: { node: string } goals: any } } const diffGoals = new DiscordWorker('diffGoals', async (job) => { - const { url, wikidata, companyName, existingCompany, goals } = job.data - const wikidataId = wikidata.node + const { url, companyName, existingCompany, goals } = job.data const metadata = defaultMetadata(url) const body = { @@ -31,7 +30,6 @@ const diffGoals = new DiscordWorker('diffGoals', async (job) => { diff, apiSubEndpoint: 'goals', requiresApproval, - wikidataId, }, }) diff --git a/src/workers/diffIndustry.ts b/src/workers/diffIndustry.ts index 78586fe0..4b408c61 100644 --- a/src/workers/diffIndustry.ts +++ b/src/workers/diffIndustry.ts @@ -6,7 +6,7 @@ export class DiffIndustryJob extends DiscordJob { declare data: DiscordJob['data'] & { existingCompany: any companyName: string - wikidata: any + wikidata: { node: string } industry: any } } @@ -14,8 +14,7 @@ export class DiffIndustryJob extends DiscordJob { const diffIndustry = new DiscordWorker( 'diffIndustry', async (job) => { - const { url, wikidata, companyName, existingCompany, industry } = job.data - const wikidataId = wikidata.node + const { url, companyName, existingCompany, industry } = job.data const metadata = defaultMetadata(url) const body = { @@ -26,13 +25,12 @@ const diffIndustry = new DiscordWorker( const diff = await askDiff(existingCompany?.industry, industry) const requiresApproval = diff && !diff.includes('NO_CHANGES') - await saveToAPI.queue.add(companyName, { + await saveToAPI.queue.add(companyName + ' industry', { data: { ...job.data, body, diff, requiresApproval, - wikidataId, apiSubEndpoint: 'industry', }, }) diff --git a/src/workers/diffInitiatives.ts b/src/workers/diffInitiatives.ts index de3ab168..10243985 100644 --- a/src/workers/diffInitiatives.ts +++ b/src/workers/diffInitiatives.ts @@ -6,7 +6,7 @@ export class DiffInitiativesJob extends DiscordJob { declare data: DiscordJob['data'] & { companyName: string existingCompany: any - wikidata: any + wikidata: { node: string } initiatives: any } } @@ -25,12 +25,13 @@ const diffInitiatives = new DiscordWorker( const diff = await askDiff(existingCompany?.initiatives, initiatives) const requiresApproval = diff && !diff.includes('NO_CHANGES') - await saveToAPI.queue.add(companyName, { + await saveToAPI.queue.add(companyName + ' initiatives', { data: { ...job.data, body, diff, requiresApproval, + apiSubEndpoint: 'initiatives', }, }) diff --git a/src/workers/diffReportingPeriods.ts b/src/workers/diffReportingPeriods.ts index 4c563941..7119fe9f 100644 --- a/src/workers/diffReportingPeriods.ts +++ b/src/workers/diffReportingPeriods.ts @@ -7,7 +7,7 @@ export class DiffReportingPeriodsJob extends DiscordJob { declare data: DiscordJob['data'] & { companyName: string existingCompany: any - wikidata: any + wikidata: { node: string } fiscalYear: any scope12?: any[] scope3?: any[] @@ -23,7 +23,6 @@ const diffReportingPeriods = new DiscordWorker( url, fiscalYear, companyName, - wikidata, existingCompany, scope12 = [], scope3 = [], @@ -31,7 +30,6 @@ const diffReportingPeriods = new DiscordWorker( economy = [], } = job.data const metadata = defaultMetadata(url) - const wikidataId = wikidata.node // Get all unique years from all sources const years = new Set([ @@ -98,14 +96,13 @@ const diffReportingPeriods = new DiscordWorker( metadata, } - await saveToAPI.queue.add(companyName, { + await saveToAPI.queue.add(companyName + ' reporting-periods', { data: { ...job.data, body, diff, apiSubEndpoint: 'reporting-periods', requiresApproval, - wikidataId, }, }) From 1a14416969e38823dd2aeeeeceecd1dc1fb66c9a Mon Sep 17 00:00:00 2001 From: Samuel Plumppu <6125097+Greenheart@users.noreply.github.com> Date: Thu, 5 Dec 2024 14:14:10 +0100 Subject: [PATCH 78/82] Remove unnecessary wrapper object added by aider MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Hugo Björk --- src/workers/checkDB.ts | 2 +- src/workers/diffGoals.ts | 12 +++++------- src/workers/diffIndustry.ts | 12 +++++------- src/workers/diffInitiatives.ts | 12 +++++------- src/workers/diffReportingPeriods.ts | 12 +++++------- 5 files changed, 21 insertions(+), 29 deletions(-) diff --git a/src/workers/checkDB.ts b/src/workers/checkDB.ts index cfe0e27a..c1fb56c1 100644 --- a/src/workers/checkDB.ts +++ b/src/workers/checkDB.ts @@ -128,7 +128,7 @@ const checkDB = new DiscordWorker('checkDB', async (job: CheckDBJob) => { ].filter((e) => e !== null), }) - return JSON.stringify({ saved: true }, null, 2) + return { saved: true } }) export default checkDB diff --git a/src/workers/diffGoals.ts b/src/workers/diffGoals.ts index 7143f4e5..4d9bdfc6 100644 --- a/src/workers/diffGoals.ts +++ b/src/workers/diffGoals.ts @@ -24,13 +24,11 @@ const diffGoals = new DiscordWorker('diffGoals', async (job) => { const requiresApproval = diff && !diff.includes('NO_CHANGES') await saveToAPI.queue.add(companyName + ' goals', { - data: { - ...job.data, - body, - diff, - apiSubEndpoint: 'goals', - requiresApproval, - }, + ...job.data, + body, + diff, + apiSubEndpoint: 'goals', + requiresApproval, }) return { body, diff, requiresApproval } diff --git a/src/workers/diffIndustry.ts b/src/workers/diffIndustry.ts index 4b408c61..a3ac9bf7 100644 --- a/src/workers/diffIndustry.ts +++ b/src/workers/diffIndustry.ts @@ -26,13 +26,11 @@ const diffIndustry = new DiscordWorker( const requiresApproval = diff && !diff.includes('NO_CHANGES') await saveToAPI.queue.add(companyName + ' industry', { - data: { - ...job.data, - body, - diff, - requiresApproval, - apiSubEndpoint: 'industry', - }, + ...job.data, + body, + diff, + requiresApproval, + apiSubEndpoint: 'industry', }) return { body, diff, requiresApproval } diff --git a/src/workers/diffInitiatives.ts b/src/workers/diffInitiatives.ts index 10243985..4e242e32 100644 --- a/src/workers/diffInitiatives.ts +++ b/src/workers/diffInitiatives.ts @@ -26,13 +26,11 @@ const diffInitiatives = new DiscordWorker( const requiresApproval = diff && !diff.includes('NO_CHANGES') await saveToAPI.queue.add(companyName + ' initiatives', { - data: { - ...job.data, - body, - diff, - requiresApproval, - apiSubEndpoint: 'initiatives', - }, + ...job.data, + body, + diff, + requiresApproval, + apiSubEndpoint: 'initiatives', }) return { body, diff, requiresApproval } diff --git a/src/workers/diffReportingPeriods.ts b/src/workers/diffReportingPeriods.ts index 7119fe9f..db51a6c0 100644 --- a/src/workers/diffReportingPeriods.ts +++ b/src/workers/diffReportingPeriods.ts @@ -97,13 +97,11 @@ const diffReportingPeriods = new DiscordWorker( } await saveToAPI.queue.add(companyName + ' reporting-periods', { - data: { - ...job.data, - body, - diff, - apiSubEndpoint: 'reporting-periods', - requiresApproval, - }, + ...job.data, + body, + diff, + apiSubEndpoint: 'reporting-periods', + requiresApproval, }) return { body, diff, requiresApproval } From f414288d99314d4683355c3921687750bbad0da9 Mon Sep 17 00:00:00 2001 From: Samuel Plumppu <6125097+Greenheart@users.noreply.github.com> Date: Thu, 5 Dec 2024 14:27:09 +0100 Subject: [PATCH 79/82] Improve message formatting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Hugo Björk --- src/workers/saveToAPI.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/workers/saveToAPI.ts b/src/workers/saveToAPI.ts index 8b892f5d..1d1e4fe8 100644 --- a/src/workers/saveToAPI.ts +++ b/src/workers/saveToAPI.ts @@ -37,7 +37,9 @@ export const saveToAPI = new DiscordWorker( // If approval is required and not yet approved, send approval request const buttonRow = discord.createButtonRow(job.id!) await job.sendMessage({ - content: `New changes need approval for ${wikidataId}\n\n${diff || ''}`, + content: `## ${apiSubEndpoint}\n\nNew changes need approval for ${wikidataId}\n\n${ + diff || '' + }`, components: [buttonRow], }) From 4d7989e794e890b48100cd823f826b535933ce9c Mon Sep 17 00:00:00 2001 From: Samuel Plumppu <6125097+Greenheart@users.noreply.github.com> Date: Thu, 5 Dec 2024 14:40:04 +0100 Subject: [PATCH 80/82] Remove duplicate job data passed to saveToAPI --- src/workers/diffGoals.ts | 5 ++++- src/workers/diffIndustry.ts | 5 ++++- src/workers/diffInitiatives.ts | 5 ++++- src/workers/diffReportingPeriods.ts | 8 +++++++- 4 files changed, 19 insertions(+), 4 deletions(-) diff --git a/src/workers/diffGoals.ts b/src/workers/diffGoals.ts index 4d9bdfc6..cc46c856 100644 --- a/src/workers/diffGoals.ts +++ b/src/workers/diffGoals.ts @@ -28,7 +28,10 @@ const diffGoals = new DiscordWorker('diffGoals', async (job) => { body, diff, apiSubEndpoint: 'goals', - requiresApproval, + requiresApproval: Boolean(existingCompany), + + // Remove duplicated job data that should be part of the body from now on + goals: undefined, }) return { body, diff, requiresApproval } diff --git a/src/workers/diffIndustry.ts b/src/workers/diffIndustry.ts index a3ac9bf7..58d79519 100644 --- a/src/workers/diffIndustry.ts +++ b/src/workers/diffIndustry.ts @@ -29,8 +29,11 @@ const diffIndustry = new DiscordWorker( ...job.data, body, diff, - requiresApproval, apiSubEndpoint: 'industry', + requiresApproval: Boolean(existingCompany), + + // Remove duplicated job data that should be part of the body from now on + industry: undefined, }) return { body, diff, requiresApproval } diff --git a/src/workers/diffInitiatives.ts b/src/workers/diffInitiatives.ts index 4e242e32..1d48f309 100644 --- a/src/workers/diffInitiatives.ts +++ b/src/workers/diffInitiatives.ts @@ -29,8 +29,11 @@ const diffInitiatives = new DiscordWorker( ...job.data, body, diff, - requiresApproval, apiSubEndpoint: 'initiatives', + requiresApproval: Boolean(existingCompany), + + // Remove duplicated job data that should be part of the body from now on + initiatives: undefined, }) return { body, diff, requiresApproval } diff --git a/src/workers/diffReportingPeriods.ts b/src/workers/diffReportingPeriods.ts index db51a6c0..3cb42caf 100644 --- a/src/workers/diffReportingPeriods.ts +++ b/src/workers/diffReportingPeriods.ts @@ -101,7 +101,13 @@ const diffReportingPeriods = new DiscordWorker( body, diff, apiSubEndpoint: 'reporting-periods', - requiresApproval, + requiresApproval: Boolean(existingCompany), + + // Remove duplicated job data that should be part of the body from now on + scope12: undefined, + scope3: undefined, + biogenic: undefined, + economy: undefined, }) return { body, diff, requiresApproval } From e7fef7bf26aa95631c21f1c265f2316dcdafbbf6 Mon Sep 17 00:00:00 2001 From: Samuel Plumppu <6125097+Greenheart@users.noreply.github.com> Date: Thu, 5 Dec 2024 16:15:36 +0100 Subject: [PATCH 81/82] Fix saving of economy data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Hugo Björk --- src/routes/updateCompanies.ts | 11 ++++++----- src/workers/diffReportingPeriods.ts | 4 +--- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/routes/updateCompanies.ts b/src/routes/updateCompanies.ts index 175d30a1..8a499b98 100644 --- a/src/routes/updateCompanies.ts +++ b/src/routes/updateCompanies.ts @@ -350,6 +350,7 @@ router.post( async (req, res) => { const { reportingPeriods } = postReportingPeriodsSchema.parse(req.body) const metadata = res.locals.metadata! + const company = res.locals.company try { await Promise.allSettled( @@ -363,7 +364,7 @@ router.post( }) => { const year = endDate.getFullYear().toString() const reportingPeriod = await upsertReportingPeriod( - res.locals.company, + company, metadata, { startDate, @@ -376,12 +377,12 @@ router.post( const [dbEmissions, dbEconomy] = await Promise.all([ upsertEmissions({ emissionsId: reportingPeriod.emissionsId ?? 0, - companyId: res.locals.company.wikidataId, + companyId: company.wikidataId, year, }), upsertEconomy({ economyId: reportingPeriod.economyId ?? 0, - companyId: res.locals.company.wikidataId, + companyId: company.wikidataId, year, }), ]) @@ -391,8 +392,8 @@ router.post( const { turnover, employees } = economy // Normalise currency - if (turnover) { - turnover.currency = turnover?.currency?.trim()?.toUpperCase() + if (turnover?.currency) { + turnover.currency = turnover.currency.trim().toUpperCase() } await Promise.allSettled([ diff --git a/src/workers/diffReportingPeriods.ts b/src/workers/diffReportingPeriods.ts index 3cb42caf..b4a33eff 100644 --- a/src/workers/diffReportingPeriods.ts +++ b/src/workers/diffReportingPeriods.ts @@ -64,9 +64,7 @@ const diffReportingPeriods = new DiscordWorker( biogenic: biogenic.find((d) => d.year === year)?.biogenic, } - const economyData = { - ...(economy.find((d) => d.year === year)?.economy ? economy : {}), - } + const economyData = economy.find((d) => d.year === year)?.economy ?? {} const reportingPeriod: any = period From 26f9c1017d365cd6548745faf638dbe6125463b2 Mon Sep 17 00:00:00 2001 From: Samuel Plumppu <6125097+Greenheart@users.noreply.github.com> Date: Thu, 5 Dec 2024 16:15:51 +0100 Subject: [PATCH 82/82] Cleanup unused code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Hugo Björk --- src/lib/DiscordWorker.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/lib/DiscordWorker.ts b/src/lib/DiscordWorker.ts index c6386916..4c5eaf98 100644 --- a/src/lib/DiscordWorker.ts +++ b/src/lib/DiscordWorker.ts @@ -1,4 +1,4 @@ -import { Worker, WorkerOptions, Job, Queue, Processor } from 'bullmq' +import { Worker, WorkerOptions, Job, Queue } from 'bullmq' import { Message, TextChannel } from 'discord.js' import redis from '../config/redis' import discord from '../discord' @@ -8,7 +8,6 @@ export class DiscordJob extends Job { url: string threadId: string channelId: string - wikidataId?: string messageId?: string }