[IMP] Clipboard: support images in the clipboard

This commit adds the support of images in the clipboard. Namely,
- Pasting an image from the OS clipboard will automatically upload it to
  the filestore and dispatch a CREATE_IMAGE accordingly.
- Copying a chart or an image from the spreadsheet will add it to the
  oOS clipboard. It can then be pasted somewhere else, like a discuss
app, a text editor maybe?.

TODO
- support multi type of images (not just png) It technically works
  because we never check agains the actual type of the file we upload
but it's wrong. Maybe use a generic `image/*` mimetype instead of using
png. We still need to purge out dangerous mimetypes like image/svg(+xml)
- Currently uploads the image from the clipboard THEN only checks if the
  we are copy pasting from the same spreadsheet clipboard (potential
useless upload)
- Not supported by some apps like Office 365 online (word & powerpoint tested). Supposedly because our
  clipboard has some text/* types along the image/* so it ignores the
later. for a fully integrated feature, we should probably decorrelate it
from the standard copy paste and make a  menu "copy image to os
clipboard" ? tbd
-works on discord at least
- No test whatsoever

demo/file_store.js

@@ -21,4 +21,9 @@ export class FileStore {
   async delete(path) {
     console.warn("cannot delete file. Not implemented");
   }
+
+  async getFile(path) {
+    const response = await fetch(path);
+    return await response.blob();
+  }
 }

src/actions/edit_actions.ts

@@ -29,7 +29,7 @@ export const copy: ActionSpec = {
   isReadonlyAllowed: true,
   execute: async (env) => {
     env.model.dispatch("COPY");
-    await env.clipboard.write(env.model.getters.getClipboardContent());
+    await env.clipboard.write(await env.model.getters.getOsClipboardContentAsync());
   },
   icon: "o-spreadsheet-Icon.CLIPBOARD",
 };
@@ -39,7 +39,7 @@ export const cut: ActionSpec = {
   description: "Ctrl+X",
   execute: async (env) => {
     interactiveCut(env);
-    await env.clipboard.write(env.model.getters.getClipboardContent());
+    await env.clipboard.write(await env.model.getters.getOsClipboardContentAsync());
   },
   icon: "o-spreadsheet-Icon.CUT",
 };

src/actions/menu_items_actions.ts

@@ -55,12 +55,13 @@ async function paste(env: SpreadsheetChildEnv, pasteOption?: ClipboardPasteOptio
   const osClipboard = await env.clipboard.read();
   switch (osClipboard.status) {
     case "ok":
-      const clipboardContent = parseOSClipboardContent(osClipboard.content);
-      const clipboardId = clipboardContent.data?.clipboardId;
+      const clipboardId = env.model.getters.getClipboardId();
+      const clipboardContent = await parseOSClipboardContent(env, osClipboard.content, clipboardId);
+      const contentClipboardId = clipboardContent.data?.clipboardId;
 
       const target = env.model.getters.getSelectedZones();
 
-      if (env.model.getters.getClipboardId() !== clipboardId) {
+      if (clipboardId !== contentClipboardId) {
         interactivePasteFromOS(env, target, clipboardContent, pasteOption);
       } else {
         interactivePaste(env, target, pasteOption);

src/components/grid/grid.ts

@@ -38,6 +38,7 @@ import { Store, useStore } from "../../store_engine";
 import { DOMFocusableElementStore } from "../../stores/DOM_focus_store";
 import { ArrayFormulaHighlight } from "../../stores/array_formula_highlight";
 import { HighlightStore } from "../../stores/highlight_store";
+import { AllowedImageMimeTypes } from "../../types/image";
 import {
   Align,
   CellValueType,
@@ -610,11 +611,8 @@ export class Grid extends Component<Props, SpreadsheetChildEnv> {
     } else {
       this.env.model.dispatch("COPY");
     }
-    const content = this.env.model.getters.getClipboardContent();
-    const clipboardData = ev.clipboardData;
-    for (const type in content) {
-      clipboardData?.setData(type, content[type]);
-    }
+    const osContent = await this.env.model.getters.getOsClipboardContentAsync();
+    await this.env.clipboard.write(osContent);
     ev.preventDefault();
   }
 
@@ -629,20 +627,31 @@ export class Grid extends Component<Props, SpreadsheetChildEnv> {
     if (!clipboardData) {
       return;
     }
-
+    const image = [...clipboardData?.files]?.find((file) =>
+      AllowedImageMimeTypes.includes(file.type as (typeof AllowedImageMimeTypes)[number])
+    );
     const osClipboard = {
       content: {
         [ClipboardMIMEType.PlainText]: clipboardData?.getData(ClipboardMIMEType.PlainText),
         [ClipboardMIMEType.Html]: clipboardData?.getData(ClipboardMIMEType.Html),
       },
     };
+    if (image) {
+      // TODO: support import of multiple images
+      osClipboard.content[image.type] = image;
+    }
 
     const target = this.env.model.getters.getSelectedZones();
     const isCutOperation = this.env.model.getters.isCutOperation();
 
-    const clipboardContent = parseOSClipboardContent(osClipboard.content);
-    const clipboardId = clipboardContent.data?.clipboardId;
-    if (this.env.model.getters.getClipboardId() === clipboardId) {
+    const clipboardId = this.env.model.getters.getClipboardId();
+    const clipboardContent = await parseOSClipboardContent(
+      this.env,
+      osClipboard.content,
+      clipboardId
+    );
+    const contentClipboardId = clipboardContent.data?.clipboardId;
+    if (clipboardId === contentClipboardId) {
       interactivePaste(this.env, target);
     } else {
       interactivePasteFromOS(this.env, target, clipboardContent);

src/helpers/clipboard/clipboard_helpers.ts

@@ -3,9 +3,11 @@ import {
   ClipboardMIMEType,
   OSClipboardContent,
   ParsedOSClipboardContent,
+  SpreadsheetChildEnv,
   UID,
   Zone,
 } from "../../types";
+import { AllowedImageMimeTypes } from "../../types/image";
 import { mergeOverlappingZones, positions } from "../zones";
 
 export function getClipboardDataPositions(sheetId: UID, zones: Zone[]): ClipboardCellData {
@@ -62,7 +64,11 @@ export function getPasteZones<T>(target: Zone[], content: T[][]): Zone[] {
   return target.map((t) => splitZoneForPaste(t, width, height)).flat();
 }
 
-export function parseOSClipboardContent(content: OSClipboardContent): ParsedOSClipboardContent {
+export async function parseOSClipboardContent(
+  env: SpreadsheetChildEnv,
+  content: OSClipboardContent,
+  clipboardId: string
+): Promise<ParsedOSClipboardContent> {
   if (!content[ClipboardMIMEType.Html]) {
     return {
       text: content[ClipboardMIMEType.PlainText],
@@ -76,8 +82,25 @@ export function parseOSClipboardContent(content: OSClipboardContent): ParsedOSCl
     .querySelector("div")
     ?.getAttribute("data-osheet-clipboard");
   const spreadsheetContent = oSheetClipboardData && JSON.parse(oSheetClipboardData);
-  return {
-    text: content[ClipboardMIMEType.PlainText],
-    data: spreadsheetContent,
-  };
+  const contentClipboardId = spreadsheetContent?.clipboardId;
+  if (contentClipboardId !== clipboardId) {
+    const clipboardContent: ParsedOSClipboardContent = {
+      text: content[ClipboardMIMEType.PlainText],
+      data: spreadsheetContent,
+    };
+    for (const type of AllowedImageMimeTypes) {
+      if (content[type]) {
+        // TODO: support multiple import
+        const imageData = await env.imageProvider?.upload(content[type]!);
+        clipboardContent.imageData = imageData;
+        break;
+      }
+    }
+    return clipboardContent;
+  } else {
+    return {
+      text: content[ClipboardMIMEType.PlainText],
+      data: spreadsheetContent,
+    };
+  }
 }

src/helpers/clipboard/navigator_clipboard_wrapper.ts

@@ -1,3 +1,4 @@
+import { AllowedImageMimeTypes } from "../../types/image";
 import { ClipboardMIMEType, OSClipboardContent } from "./../../types/clipboard";
 
 export type ClipboardReadResult =
@@ -64,7 +65,12 @@ class WebClipboardWrapper implements ClipboardInterface {
         for (const item of clipboardItems) {
           for (const type of item.types) {
             const blob = await item.getType(type);
-            clipboardContent[type as ClipboardMIMEType] = await blob.text();
+            if (type in AllowedImageMimeTypes) {
+              clipboardContent[type] = blob;
+            } else {
+              const text = await blob.text();
+              clipboardContent[type] = text;
+            }
           }
         }
         return { status: "ok", content: clipboardContent };
@@ -86,11 +92,16 @@ class WebClipboardWrapper implements ClipboardInterface {
     const clipboardItemData = {
       [ClipboardMIMEType.PlainText]: this.getBlob(content, ClipboardMIMEType.PlainText),
       [ClipboardMIMEType.Html]: this.getBlob(content, ClipboardMIMEType.Html),
+      [ClipboardMIMEType.Image]: this.getBlob(content, ClipboardMIMEType.Image),
     };
     return [new ClipboardItem(clipboardItemData)];
   }
 
   private getBlob(clipboardContent: OSClipboardContent, type: ClipboardMIMEType): Blob {
+    const content = clipboardContent[type];
+    if (content instanceof Blob) {
+      return content;
+    }
     return new Blob([clipboardContent[type] || ""], {
       type,
     });

src/helpers/figures/charts/chart_ui_common.ts

@@ -72,6 +72,59 @@ export function chartToImage(
   return undefined;
 }
 
+export async function chartToImageBlob(
+  runtime: ChartRuntime,
+  figure: Figure,
+  type: string
+): Promise<Blob | null> {
+  // wrap the canvas in a div with a fixed size because chart.js would
+  // fill the whole page otherwise
+  const div = document.createElement("div");
+  div.style.width = `${figure.width}px`;
+  div.style.height = `${figure.height}px`;
+  const canvas = document.createElement("canvas");
+  div.append(canvas);
+  canvas.setAttribute("width", figure.width.toString());
+  canvas.setAttribute("height", figure.height.toString());
+  let finalContent: Blob | null = null;
+  // we have to add the canvas to the DOM otherwise it won't be rendered
+  document.body.append(div);
+  if ("chartJsConfig" in runtime) {
+    const config = deepCopy(runtime.chartJsConfig);
+    config.plugins = [backgroundColorChartJSPlugin];
+    const chart = new window.Chart(canvas, config);
+    const imgContent = chart.toBase64Image() as string;
+    finalContent = base64ToBlob(imgContent, "image/png");
+    chart.destroy();
+    div.remove();
+  } else if (type === "scorecard") {
+    const design = getScorecardConfiguration(figure, runtime as ScorecardChartRuntime);
+    drawScoreChart(design, canvas);
+    finalContent = await new Promise((resolve) => canvas.toBlob(resolve, "image/png"));
+    div.remove();
+  } else if (type === "gauge") {
+    drawGaugeChart(canvas, runtime as GaugeChartRuntime);
+    finalContent = await new Promise((resolve) => canvas.toBlob(resolve, "image/png"));
+    div.remove();
+  }
+  return finalContent;
+}
+
+function base64ToBlob(base64: string, mimeType: string): Blob {
+  // Remove the data URL part if present
+  const byteCharacters = atob(base64.split(",")[1]);
+
+  const byteNumbers = new Array(byteCharacters.length);
+  for (let i = 0; i < byteCharacters.length; i++) {
+    byteNumbers[i] = byteCharacters.charCodeAt(i);
+  }
+
+  const byteArray = new Uint8Array(byteNumbers);
+
+  // Create a Blob object from the byteArray
+  return new Blob([byteArray], { type: mimeType });
+}
+
 /**
  * Custom chart.js plugin to set the background color of the canvas
  * https://github.com/chartjs/Chart.js/blob/8fdf76f8f02d31684d34704341a5d9217e977491/docs/configuration/canvas-background.md

src/helpers/figures/images/image_provider.ts

@@ -16,6 +16,12 @@ export class ImageProvider implements ImageProviderInterface {
     return { path, size, mimetype: file.type };
   }
 
+  async upload(file: File): Promise<Image> {
+    const path = await this.fileStore.upload(file);
+    const size = await this.getImageOriginalSize(path);
+    return { path, size, mimetype: file.type };
+  }
+
   private getImageFromUser(): Promise<File> {
     return new Promise((resolve, reject) => {
       const input = document.createElement("input");

src/model.ts

@@ -441,6 +441,7 @@ export class Model extends EventBus<any> implements CommandDispatcher {
       session: this.session,
       defaultCurrency: this.config.defaultCurrency,
       customColors: this.config.customColors || [],
+      external: this.config.external,
     };
   }
 

src/plugins/ui_plugin.ts

@@ -28,6 +28,7 @@ export interface UIPluginConfig {
   readonly session: Session;
   readonly defaultCurrency?: Partial<Currency>;
   readonly customColors: Color[];
+  readonly external: ModelConfig["external"];
 }
 
 export interface UIPluginConstructor {