Publish to prod (#265)

* Merge patches from main (#253)

* ghcr.io/klimatbyran/garbo:3.1.6

* fix: always return a value

* 3.1.7

* ghcr.io/klimatbyran/garbo:3.1.7

* 3.1.8

* ghcr.io/klimatbyran/garbo:3.1.8

---------

Co-authored-by: fluxcdbot <[email protected]>
Co-authored-by: Christian Landgren <[email protected]>
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

* Fix #144 - Update goals prompt and clarify to use climate goals (#254)

* Enable stricter TS checks and force workers to always return an explicit result. (#255)

* Enable stricter TS checks and force workers to always return an explicit result.

For longer-running jobs, this seems to make a difference to mark them as completed. Shorter jobs don't seem to need this, but it doesn't hurt to be explicit.

* Consistently parse env vars and crash if something is missing.

* Fix type errors for Discord wrapper

* Fix type errors in API

* Fix some type errors

* Ensure scope3 exists before adding categories

* Remove old, unused code

* Add missing return for precheck job

* Remove outdated job searchVectors since this was replaced by nlmParsePDF

* Update README to reflect latest updates to Garbo

* Show image again

* Docs: Describe how to restore local DB backups (#256)

* Update local DB name to expected DB name from backups

* docs: Describe how to restore local DB backups

* Clarify gics extraction to use 8 digits for the subIndustryCode. (#258)

Fix #93

* chore: Bump GitHub action versions (#257)

* fix: Simplify config and loading of env variables. (#263)

Also add missing env var API_BASE_URL

* fix: remove unused console.log

---------

Co-authored-by: fluxcdbot <[email protected]>
Co-authored-by: Christian Landgren <[email protected]>
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

.env.example

@@ -3,7 +3,7 @@
 
 NODE_ENV=development
 
-DATABASE_URL=postgresql://postgres:mysecretpassword@localhost:5432/companies
+DATABASE_URL=postgresql://postgres:mysecretpassword@localhost:5432/garbo
 API_TOKENS=garbo:auth-123,alex:auth-abc
 
 OPENAI_API_KEY=

.github/workflows/ci.yml

@@ -23,7 +23,7 @@ jobs:
 
     steps:
       - name: 🛎️ Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: 🎫 Update patch version
         run: |
@@ -48,10 +48,10 @@ jobs:
 
     steps:
       - name: 🛎️ Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - id: imagename
-        uses: ASzc/change-string-case-action@v2
+        uses: ASzc/change-string-case-action@v6
         with:
           string: ${{ github.repository }}
 
@@ -71,14 +71,14 @@ jobs:
         uses: martinbeentjes/npm-get-version-action@main
 
       - name: 🔐 Login to Docker Registry
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: 🔧 Build and push Docker Image
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v6
         with:
           push: true
           tags: |

.github/workflows/production.yml

@@ -16,17 +16,17 @@ jobs:
 
     steps:
       - name: 🛎️ Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: 🔐 Login to Docker Registry
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: 🔧 Build and push Docker Image
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v6
         with:
           push: true
           tags: |

README.md

@@ -25,8 +25,8 @@ flowchart TB
     Tables[Extract Tables]
     Emissions[Extract Emissions]
 
-    Industry[Extract Industry]
-    Goals[Extract Climate Goals]
+    Industry[Industry]
+    Goals[Climate Goals]
     Review[Discord Review]
 
     Precheck --> GuessWikidata --> Emissions
@@ -44,11 +44,15 @@ flowchart TB
                                            CheckDB --(no)--> API.Emissions
     Emissions --(followUp)--> Scope3 --> CheckDB --(yes)--> Review --> API.Emissions
                                            CheckDB --(no)--> API.Emissions
+    Emissions --(followUp)--> Biogenic --> CheckDB --(yes)--> Review --> API.Emissions
+                                           CheckDB --(no)--> API.Emissions
     Emissions --(followUp)--> Goals --> CheckDB --(yes)--> Review --> API.Goals
                                            CheckDB --(no)--> API.Goals
     Emissions --(followUp)--> Initiatives --> CheckDB --(yes)--> Review --> API.Initiatives
                                            CheckDB --(no)--> API.Initiatives
     Emissions --(followUp)--> Turnover --> CheckDB --(yes)--> Review --> API.Economy
+                                           CheckDB --(no)--> API.Initiatives
+    Emissions --(followUp)--> Employees --> CheckDB --(yes)--> Review --> API.Economy
                                            CheckDB --(no)--> API.Economy
 ```
 
@@ -122,6 +126,30 @@ To start the workers responsible for doing the actual work, which can be scaled
 npm run dev-workers
 ```
 
+### Restoring a DB backup locally
+
+These steps can be useful to test DB migrations, or develop with data that is similar to the that in the production environment.
+
+1. Optional: Create a local test DB.
+
+```sh
+docker run -d -p 5432:5432 --name garbo_test_postgres -e POSTGRES_PASSWORD=mysecretpassword postgres
+```
+
+Alternatively, make sure your local postgres container is running.
+
+2. Download the DB dump file. Ask someone from Klimatkollen to get one.
+
+3. Restore the backup. This will initially connect to the default `postgres` database without making any modifications and then create any databases if they do not exist
+
+```sh
+docker exec -i container_name pg_restore -U postgres -C -v -d postgres < ~/Downloads/backup_garbo_XYZ.dump
+```
+
+4. Apply DB migrations with `npm run prisma migrate dev`.
+
+5. Restart the Garbo API and workers.
+
 ### Testing
 
 To run the tests, use the following command:

k8s/api.yaml

@@ -45,6 +45,8 @@ spec:
                 secretKeyRef:
                   name: env
                   key: API_TOKENS
+            - name: API_BASE_URL
+              value: http://garbo/api
             - name: POSTGRES_PASSWORD
               valueFrom:
                 secretKeyRef:

k8s/worker.yaml

@@ -43,6 +43,8 @@ spec:
                 secretKeyRef:
                   name: env
                   key: API_TOKENS
+            - name: API_BASE_URL
+              value: http://garbo/api
             - name: OPENAI_API_KEY
               valueFrom:
                 secretKeyRef:

src/lib/env.ts → src/config/api.ts

@@ -6,6 +6,14 @@ const envSchema = z.object({
    * NOTE: This is only relevant during import with alex data, and then we switch to proper auth tokens.
    */
   API_TOKENS: z.string().transform((tokens) => tokens.split(',')),
+  API_BASE_URL: z.string().default('http://localhost:3000/api'),
+  PORT: z.coerce.number().default(3000),
 })
 
-export const ENV = envSchema.parse(process.env)
+const env = envSchema.parse(process.env)
+
+export default {
+  tokens: env.API_TOKENS,
+  baseURL: env.API_BASE_URL,
+  port: env.PORT,
+}

src/config/chromadb.ts

@@ -1,9 +1,18 @@
+import { z } from 'zod'
+
+const envSchema = z.object({
+  CHROMA_HOST: z.string().default('http://127.0.0.1:8000'),
+  CHROMA_TOKEN: z.string().optional(),
+})
+
+const env = envSchema.parse(process.env)
+
 export default {
-  path: process.env.CHROMA_HOST || 'http://127.0.0.1:8000',
-  auth: process.env.CHROMA_TOKEN
+  path: env.CHROMA_HOST,
+  auth: env.CHROMA_TOKEN
     ? {
         provider: 'token',
-        credentials: process.env.CHROMA_TOKEN || '',
+        credentials: env.CHROMA_TOKEN,
       }
     : undefined,
 }

src/config/discord.ts

@@ -1,7 +1,21 @@
+import { z } from 'zod'
+
+const envSchema = z.object({
+  DISCORD_TOKEN: z.string(),
+  DISCORD_APPLICATION_ID: z.string(),
+  DISCORD_SERVER_ID: z.string(),
+  DISCORD_CHANNEL_ID: z.string().default('1201463851447758879'), // defaults to the channel `rapporter-att-granska` on the klimatkollen Discord server
+})
+
+const env = envSchema.parse(process.env)
+
 export default {
-  token: process.env.DISCORD_TOKEN,
-  clientId: process.env.DISCORD_APPLICATION_ID,
-  guildId: process.env.DISCORD_SERVER_ID,
-  worker: process.argv[1].includes('startWorker'), // since the discord is a singleton, we need to know if we are in a worker or not
-  channelId: process.env.DISCORD_CHANNEL_ID || '1201463851447758879', // set to chanel rapporter-att-granska
+  token: env.DISCORD_TOKEN,
+  clientId: env.DISCORD_APPLICATION_ID,
+  guildId: env.DISCORD_SERVER_ID,
+  /**
+   * Since our Discord wrapper is a singleton, we need to know if we are in a worker or not
+   */
+  worker: process.argv[1].includes('startWorker'),
+  channelId: env.DISCORD_CHANNEL_ID,
 }

src/config/nlmIngestor.ts

@@ -1,3 +1,11 @@
+import { z } from 'zod'
+
+const envSchema = z.object({
+  NLM_INGESTOR_URL: z.string().default('http://0.0.0.0:5001'),
+})
+
+export const env = envSchema.parse(process.env)
+
 export default {
-  url: process.env.NLM_INGESTOR_URL || 'http://localhost:5001',
+  url: env.NLM_INGESTOR_URL,
 }

src/config/openai.ts

@@ -1,6 +1,18 @@
+import { z } from 'zod'
+
+const envSchema = z.object({
+  OPENAI_API_KEY: z.string(),
+  OPENAI_ORG_ID: z.string(),
+})
+
+const env = envSchema.parse(process.env)
+
 export default {
-  openai_api_key: process.env.OPENAI_API_KEY,
-  openai_organization_id: process.env.OPENAI_ORGANIZATION_ID,
-  organization: process.env['OPENAI_ORGANIZATION_ID'],
-  apiKey: process.env['OPENAI_API_KEY'], // This is the default and can be omitted
+  // Used by the `OpenAIEmbeddingFunction` from `chromadb`:
+  openai_api_key: env.OPENAI_API_KEY,
+  openai_organization_id: env.OPENAI_ORG_ID,
+
+  // Used by the `openai` Node.js API:
+  organization: env.OPENAI_ORG_ID,
+  apiKey: env.OPENAI_API_KEY,
 }

src/config/redis.ts

@@ -1,5 +1,15 @@
+import { z } from 'zod'
+
+const envSchema = z.object({
+  REDIS_HOST: z.string().default('localhost'),
+  REDIS_PORT: z.coerce.number().default(6379),
+  REDIS_PASSWORD: z.string().optional(),
+})
+
+const env = envSchema.parse(process.env)
+
 export default {
-  host: process.env.REDIS_HOST || 'localhost',
-  port: +process.env.REDIS_PORT || 6379,
-  password: process.env.REDIS_PASSWORD || undefined,
+  host: env.REDIS_HOST,
+  port: env.REDIS_PORT,
+  password: env.REDIS_PASSWORD,
 }

src/discord.ts

@@ -17,7 +17,7 @@ import approve from './discord/interactions/approve'
 import reject from './discord/interactions/reject'
 import saveToAPI, { JobData as SaveToApiJob } from './workers/saveToAPI'
 
-const getJob = (jobId) => saveToAPI.queue.getJob(jobId)
+const getJob = (jobId: string) => saveToAPI.queue.getJob(jobId)
 
 export class Discord {
   client: Client<boolean>
@@ -53,6 +53,13 @@ export class Discord {
           const command = commands.find(
             (command) => command.data.name === interaction.commandName
           )
+          if (!command) {
+            console.error(
+              `Discord error: Command "${interaction.commandName}" not found`
+            )
+            return
+          }
+
           try {
             await command.execute(interaction as ChatInputCommandInteraction)
           } catch (error) {
@@ -147,7 +154,7 @@ export class Discord {
       return thread.send(msg)
     } catch (e) {
       console.error('Error sending message to thread', e)
-      return undefined
+      return null
     }
   }
 
@@ -185,9 +192,15 @@ export class Discord {
     threadId?: string
     messageId: string
   }) {
-    const channel = (await this.client.channels.fetch(
-      threadId || channelId
-    )) as TextChannel
+    const id = threadId || channelId
+    if (!id) {
+      console.error(`Discord error: Unable to find message - no id provided:`, {
+        channelId,
+        threadId,
+      })
+      return null
+    }
+    const channel = (await this.client.channels.fetch(id)) as TextChannel
     const message = await channel.messages.fetch(messageId)
     return message
   }
@@ -200,9 +213,9 @@ export class Discord {
     return await channel?.send(message)
   }
 
-  async lockThread(channelId) {
+  async lockThread(channelId: string) {
     const channel = await this.client.channels.fetch(channelId)
-    if (channel.isThread()) {
+    if (channel?.isThread()) {
       await channel.setLocked(true)
       //await channel.setArchived(true);
     } else {

src/index.ts

@@ -4,8 +4,9 @@ import queue from './queue'
 
 import discord from './discord'
 import api from './api'
+import apiConfig from './config/api'
 
-const port = process.env.PORT || 3000
+const port = apiConfig.port
 const app = express()
 
 app.get('/favicon.ico', express.static('public/favicon.png'))