feat: convert images to flashcards

src/controllers/SettingsController/SettingsController.test.ts

@@ -21,15 +21,18 @@ class FakeSettingsService implements IServiceSettings {
   }
 }
 
+function testDefaultSettings(
+  type: 'client' | 'server',
+  expectedOptions: Record<string, string>
+) {
+  const settingsController = new SettingsController(new FakeSettingsService());
+  const defaultOptions = settingsController.getDefaultSettingsCardOptions(type);
+  expect(defaultOptions).toStrictEqual(expectedOptions);
+}
+
 describe('SettingsController', () => {
   test('returns default settings for client', () => {
-    const settingsController = new SettingsController(
-      new FakeSettingsService()
-    );
-    const defaultOptions =
-      settingsController.getDefaultSettingsCardOptions('client');
-
-    expect(defaultOptions).toStrictEqual({
+    testDefaultSettings('client', {
       'add-notion-link': 'false',
       'use-notion-id': 'true',
       all: 'true',
@@ -48,17 +51,12 @@ describe('SettingsController', () => {
       'perserve-newlines': 'true',
       'vertex-ai-pdf-questions': 'false',
       'disable-indented-bullets': 'false',
+      'image-quiz-html-to-anki': 'false',
     });
   });
 
   test('returns default settings for server', () => {
-    const settingsController = new SettingsController(
-      new FakeSettingsService()
-    );
-    const defaultOptions =
-      settingsController.getDefaultSettingsCardOptions('server');
-
-    expect(defaultOptions).toStrictEqual({
+    testDefaultSettings('server', {
       'add-notion-link': 'false',
       'use-notion-id': 'true',
       all: 'true',
@@ -74,6 +72,7 @@ describe('SettingsController', () => {
       'max-one-toggle-per-card': 'true',
       'perserve-newlines': 'false',
       'page-emoji': 'first-emoji',
+      'image-quiz-html-to-anki': 'false',
     });
   });
 });

src/controllers/SettingsController/supportedOptions.ts

@@ -130,6 +130,12 @@ const supportedOptions = (): CardOption[] => {
       'Disable indented bullets from becoming separate cards. This applies to bullet lists.',
       false
     ),
+    new CardOption(
+      'image-quiz-html-to-anki',
+      'Convert Image Quiz HTML to Anki Cards',
+      'Use OCR to extract images and answers from HTML quizzes and convert them into Anki flashcards for review. This is a premium feature.',
+      false
+    ),
   ];
 
   return v.filter(Boolean);

src/lib/parser/PrepareDeck.ts

@@ -1,10 +1,11 @@
 import getDeckFilename from '../anki/getDeckFilename';
 import { DeckParser, DeckParserInput } from './DeckParser';
 import Deck from './Deck';
-import { isPDFFile, isPPTFile } from '../storage/checks';
+import { isImageFile, isPDFFile, isPPTFile } from '../storage/checks';
 import { convertPDFToHTML } from './experimental/VertexAPI/convertPDFToHTML';
 import { convertPDFToImages } from '../pdf/convertPDFToImages';
 import { convertPPTToPDF } from '../pdf/ConvertPPTToPDF';
+import { convertImageToHTML } from './experimental/VertexAPI/convertImageToHTML';
 
 interface PrepareDeckResult {
   name: string;
@@ -16,8 +17,21 @@ export async function PrepareDeck(
   input: DeckParserInput
 ): Promise<PrepareDeckResult> {
   for (const file of input.files) {
-    if ((!isPDFFile(file.name) && !isPPTFile(file.name)) || !file.contents)
+    if (!file.contents) {
       continue;
+    }
+
+    if (
+      isImageFile(file.name) &&
+      input.settings.imageQuizHtmlToAnki &&
+      input.noLimits
+    ) {
+      file.contents = await convertImageToHTML(
+        file.contents?.toString('base64')
+      );
+    }
+
+    if (!isPDFFile(file.name) && !isPPTFile(file.name)) continue;
 
     if (
       isPDFFile(file.name) &&

src/lib/parser/Settings/Settings.ts

@@ -69,6 +69,8 @@ export class Settings {
   readonly vertexAIPDFQuestions: boolean;
   readonly disableIndentedBulletPoints: boolean;
 
+  readonly imageQuizHtmlToAnki: boolean;
+
   constructor(input: { [key: string]: string }) {
     this.deckName = input.deckName;
     if (this.deckName && !this.deckName.trim()) {
@@ -103,6 +105,7 @@ export class Settings {
     this.vertexAIPDFQuestions = input['vertex-ai-pdf-questions'] === 'true';
     this.disableIndentedBulletPoints =
       input['disable-indented-bullets'] === 'true';
+    this.imageQuizHtmlToAnki = input['image-quiz-html-to-anki'] === 'true';
     /* Is this really needed? */
     if (this.parentBlockId) {
       this.addNotionLink = true;
@@ -143,6 +146,7 @@ export class Settings {
       'max-one-toggle-per-card': 'true',
       'perserve-newlines': 'false',
       'page-emoji': 'first-emoji',
+      'image-quiz-html-to-anki': 'false',
     };
   }
 }

src/lib/parser/experimental/VertexAPI/constants.ts

@@ -0,0 +1,20 @@
+import { HarmBlockThreshold, HarmCategory } from '@google-cloud/vertexai';
+
+export const SAFETY_SETTINGS = [
+  {
+    category: HarmCategory.HARM_CATEGORY_HATE_SPEECH,
+    threshold: HarmBlockThreshold.BLOCK_NONE,
+  },
+  {
+    category: HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT,
+    threshold: HarmBlockThreshold.BLOCK_NONE,
+  },
+  {
+    category: HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT,
+    threshold: HarmBlockThreshold.BLOCK_NONE,
+  },
+  {
+    category: HarmCategory.HARM_CATEGORY_HARASSMENT,
+    threshold: HarmBlockThreshold.BLOCK_NONE,
+  },
+];

src/lib/parser/experimental/VertexAPI/convertImageToHTML.ts

@@ -0,0 +1,76 @@
+import { VertexAI } from '@google-cloud/vertexai';
+import { SAFETY_SETTINGS } from './constants';
+
+export const convertImageToHTML = async (
+  imageData: string
+): Promise<string> => {
+  const vertexAI = new VertexAI({
+    project: 'notion-to-anki',
+    location: 'europe-west3',
+  });
+  const model = 'gemini-1.5-flash-002';
+
+  const generativeModel = vertexAI.preview.getGenerativeModel({
+    model: model,
+    generationConfig: {
+      maxOutputTokens: 8192,
+      temperature: 1,
+      topP: 0.95,
+    },
+    safetySettings: SAFETY_SETTINGS,
+  });
+
+  const text1 = {
+    text: `Convert the text in this image to the following format: 
+
+        <ul class=\"toggle\">
+          <li>
+           <details>
+            <summary>
+                n) question
+            </summary>
+        <p>A) ..., </p>
+        <p>B)... </p>
+        etc. 
+        <p>and finally Answer: D</p>
+           </details>
+          </li>
+          </ul>
+
+        —
+        - Extra rules: n=is the number for the question, question=the question text
+    - Add newline between the options
+    - If you are not able to detect the pattern above, try converting this into a question and answer format`,
+  };
+
+  const image1 = {
+    inlineData: {
+      mimeType: 'image/png',
+      data: imageData,
+    },
+  };
+
+  const req = {
+    contents: [{ role: 'user', parts: [text1, image1] }],
+  };
+
+  let htmlContent = '';
+  try {
+    const streamingResp = await generativeModel.generateContentStream(req);
+    for await (const item of streamingResp.stream) {
+      if (
+        item.candidates &&
+        item.candidates[0].content &&
+        item.candidates[0].content.parts
+      ) {
+        htmlContent += item.candidates[0].content.parts
+          .map((part) => part.text)
+          .join('');
+      }
+    }
+  } catch (error) {
+    console.error('Error generating content stream:', error);
+  }
+
+  return htmlContent;
+};

src/lib/parser/experimental/VertexAPI/convertPDFToHTML.ts

@@ -1,12 +1,8 @@
 import path from 'path';
 import fs from 'fs';
 
-import {
-  GenerateContentRequest,
-  HarmBlockThreshold,
-  HarmCategory,
-  VertexAI,
-} from '@google-cloud/vertexai';
+import { GenerateContentRequest, VertexAI } from '@google-cloud/vertexai';
+import { SAFETY_SETTINGS } from './constants';
 
 export const convertPDFToHTML = async (pdf: string): Promise<string> => {
   const vertexAI = new VertexAI({
@@ -21,24 +17,7 @@ export const convertPDFToHTML = async (pdf: string): Promise<string> => {
       temperature: 1,
       topP: 0.95,
     },
-    safetySettings: [
-      {
-        category: HarmCategory.HARM_CATEGORY_HATE_SPEECH,
-        threshold: HarmBlockThreshold.BLOCK_NONE,
-      },
-      {
-        category: HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT,
-        threshold: HarmBlockThreshold.BLOCK_NONE,
-      },
-      {
-        category: HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT,
-        threshold: HarmBlockThreshold.BLOCK_NONE,
-      },
-      {
-        category: HarmCategory.HARM_CATEGORY_HARASSMENT,
-        threshold: HarmBlockThreshold.BLOCK_NONE,
-      },
-    ],
+    safetySettings: SAFETY_SETTINGS,
   });
 
   const document1 = {

src/lib/storage/checks.ts

@@ -42,3 +42,12 @@ export const isPotentialZipFile = (
   }
   return filename.trim().endsWith('.') || !filename.includes('.');
 };
+
+export const isImageFile = (name: string) =>
+  isImageFileEmbedable(name) &&
+  (name.toLowerCase().endsWith('.png') ||
+    name.toLowerCase().endsWith('.jpg') ||
+    name.toLowerCase().endsWith('.jpeg') ||
+    name.toLowerCase().endsWith('.gif') ||
+    name.toLowerCase().endsWith('.bmp') ||
+    name.toLowerCase().endsWith('.svg'));