asd

db/apply.sh

@@ -1,9 +1,4 @@
-# atlas schema apply \
-#   -u "postgres://postgres:password@localhost/nyheter?sslmode=disable" \
-#   --to file://schema.sql \
-#   --dev-url "docker://postgres/15/test" \
-
 atlas schema apply \
   -u "postgres://postgres:password@ai:9001/postgres?sslmode=disable" \
   --to file://schema.sql \
-  --dev-url "docker://postgres/15/test"
+  --dev-url "docker+postgres://ankane/pgvector/dev"

web/src/pages/[lang]/[slug].tsx

@@ -175,9 +175,9 @@ export default function Page(
       </div>
       <div className="mb-6  max-w-2xl mx-auto px-2">
         <article>
-          <div>
+          {/* <div>
             <AudioPlayer audioUrl={article.audioUrl} />
-          </div>
+          </div> */}
           <p className="text-gray-500 text-sm py-2 lg:py-4 mb-4">
             This news was first reported by:{' '}
             <a
@@ -189,10 +189,24 @@ export default function Page(
               Sveriges Radio: {article.sverigesRadioTitle}
             </a>{' '}
           </p>
-          <div className="prose lg:prose-xl font-serif">
+          <div className="prose lg:prose-xl font-serif relative">
             {article.body?.split('\n').map((paragraph, index) => {
               return <p key={index}>{paragraph}</p>;
             })}
+            <div className="absolute w-full h-full bg-black top-14 p-8 text-white font-mono uppercase text-center">
+              <div className="relative">
+                <p className="block">Blocked (for now) by Swedish Radio.</p>
+                <p>
+                  <a
+                    href="https://www.linkedin.com/feed/update/urn:li:activity:7127704087374016512/"
+                    target="_blank"
+                    className="cursor-pointer text-white underline hover:text-gray-300"
+                  >
+                    More info
+                  </a>
+                </p>
+              </div>
+            </div>
           </div>
           <div className="mt-3"></div>
         </article>

worker/package.json

@@ -16,6 +16,7 @@
     "codegen": "kysely-codegen --dialect postgres --camel-case --out-file ./src/utils/kysely-types.d.ts",
     "backfill-images": "ts-node src/scripts/move-images-to-article-images-table.ts",
     "social-media": "ts-node src/50-social-media.ts",
+    "embeddings": "ts-node src/6-embeddings.ts",
     "tmpbackfill": "ts-node src/tmp-backfill-translations.ts",
     "backup": "ts-node src/100-backup.ts",
     "test": "echo \"Error: no test specified\" && exit 1"

worker/src/6-embeddings.ts

@@ -1,8 +1,41 @@
+import { db } from './utils/db';
+import { openai } from './utils/openai';
+
 async function main() {
-  // get all articles that does not have embeddings
-  // get the article's english article
-  // generate embeddings
-  // store the embeddings on the articles table
+  const articles = await db
+    .selectFrom('articleTranslations as at')
+    .innerJoin('articles as a', 'a.id', 'at.articleId')
+    .select(['a.id as articleId', 'at.title', 'at.body'])
+    .where('a.embedding', 'is', null)
+    .where('at.language', '=', 'en')
+    .orderBy('a.id', 'desc')
+    .execute();
+
+  for (const article of articles) {
+    console.log(`Generate embedding for ${article.title}`);
+
+    const text = article.title + ' ' + article.body;
+
+    const res = await openai.embeddings.create({
+      input: text,
+      model: 'text-embedding-ada-002',
+    });
+
+    if (res.data.length === 0) {
+      console.log('no embeddings received from openai');
+      continue;
+    }
+
+    const { embedding } = res.data[0];
+
+    await db
+      .updateTable('articles')
+      .set({
+        embedding: `[${embedding.toString()}]`,
+      })
+      .where('id', '=', article.articleId)
+      .execute();
+  }
 }
 
 main();

worker/src/utils/kysely-types.d.ts

@@ -37,6 +37,7 @@ export interface Articles {
   isPublishedOnSocialMedia: Generated<boolean | null>;
   pageViews: Generated<number | null>;
   articleImageId: number | null;
+  embedding: string | null;
 }
 
 export interface ArticleSocialMediaHooks {