[IMP] Clipboard: support images in the clipboard

This commit adds the support of images in the clipboard. Namely,
- Pasting an image from the OS clipboard will automatically upload it to
  the filestore and dispatch a CREATE_IMAGE accordingly.
- Copying a chart or an image from the spreadsheet will add it to the
  oOS clipboard. It can then be pasted somewhere else, like a discuss
app, a text editor, etc. as long as they support this type of import.

Task: 4235104

demo/file_store.js

@@ -21,4 +21,9 @@ export class FileStore {
   async delete(path) {
     console.warn("cannot delete file. Not implemented");
   }
+
+  async getFile(path) {
+    const response = await fetch(path);
+    return await response.blob();
+  }
 }

src/actions/edit_actions.ts

@@ -29,7 +29,7 @@ export const copy: ActionSpec = {
   isReadonlyAllowed: true,
   execute: async (env) => {
     env.model.dispatch("COPY");
-    await env.clipboard.write(env.model.getters.getClipboardContent());
+    await env.clipboard.write(await env.model.getters.getOsClipboardContentAsync());
   },
   icon: "o-spreadsheet-Icon.CLIPBOARD",
 };
@@ -39,7 +39,7 @@ export const cut: ActionSpec = {
   description: "Ctrl+X",
   execute: async (env) => {
     interactiveCut(env);
-    await env.clipboard.write(env.model.getters.getClipboardContent());
+    await env.clipboard.write(await env.model.getters.getOsClipboardContentAsync());
   },
   icon: "o-spreadsheet-Icon.CUT",
 };

src/actions/menu_items_actions.ts

@@ -55,12 +55,13 @@ async function paste(env: SpreadsheetChildEnv, pasteOption?: ClipboardPasteOptio
   const osClipboard = await env.clipboard.read();
   switch (osClipboard.status) {
     case "ok":
-      const clipboardContent = parseOSClipboardContent(osClipboard.content);
-      const clipboardId = clipboardContent.data?.clipboardId;
+      const clipboardId = env.model.getters.getClipboardId();
+      const clipboardContent = await parseOSClipboardContent(env, osClipboard.content, clipboardId);
+      const contentClipboardId = clipboardContent.data?.clipboardId;
 
       const target = env.model.getters.getSelectedZones();
 
-      if (env.model.getters.getClipboardId() !== clipboardId) {
+      if (clipboardId !== contentClipboardId) {
         interactivePasteFromOS(env, target, clipboardContent, pasteOption);
       } else {
         interactivePaste(env, target, pasteOption);

src/components/grid/grid.ts

@@ -38,6 +38,7 @@ import { Store, useStore } from "../../store_engine";
 import { DOMFocusableElementStore } from "../../stores/DOM_focus_store";
 import { ArrayFormulaHighlight } from "../../stores/array_formula_highlight";
 import { HighlightStore } from "../../stores/highlight_store";
+import { AllowedImageMimeTypes } from "../../types/image";
 import {
   Align,
   CellValueType,
@@ -610,11 +611,8 @@ export class Grid extends Component<Props, SpreadsheetChildEnv> {
     } else {
       this.env.model.dispatch("COPY");
     }
-    const content = this.env.model.getters.getClipboardContent();
-    const clipboardData = ev.clipboardData;
-    for (const type in content) {
-      clipboardData?.setData(type, content[type]);
-    }
+    const osContent = await this.env.model.getters.getOsClipboardContentAsync();
+    await this.env.clipboard.write(osContent);
     ev.preventDefault();
   }
 
@@ -629,20 +627,31 @@ export class Grid extends Component<Props, SpreadsheetChildEnv> {
     if (!clipboardData) {
       return;
     }
-
+    const image = [...clipboardData?.files]?.find((file) =>
+      AllowedImageMimeTypes.includes(file.type as (typeof AllowedImageMimeTypes)[number])
+    );
     const osClipboard = {
       content: {
         [ClipboardMIMEType.PlainText]: clipboardData?.getData(ClipboardMIMEType.PlainText),
         [ClipboardMIMEType.Html]: clipboardData?.getData(ClipboardMIMEType.Html),
       },
     };
+    if (image) {
+      // TODO: support import of multiple images
+      osClipboard.content[image.type] = image;
+    }
 
     const target = this.env.model.getters.getSelectedZones();
     const isCutOperation = this.env.model.getters.isCutOperation();
 
-    const clipboardContent = parseOSClipboardContent(osClipboard.content);
-    const clipboardId = clipboardContent.data?.clipboardId;
-    if (this.env.model.getters.getClipboardId() === clipboardId) {
+    const clipboardId = this.env.model.getters.getClipboardId();
+    const clipboardContent = await parseOSClipboardContent(
+      this.env,
+      osClipboard.content,
+      clipboardId
+    );
+    const contentClipboardId = clipboardContent.data?.clipboardId;
+    if (clipboardId === contentClipboardId) {
       interactivePaste(this.env, target);
     } else {
       interactivePasteFromOS(this.env, target, clipboardContent);

src/helpers/clipboard/clipboard_helpers.ts

@@ -1,11 +1,14 @@
+import { SpreadsheetClipboardData } from "../../plugins/ui_stateful";
 import {
   ClipboardCellData,
   ClipboardMIMEType,
   OSClipboardContent,
   ParsedOSClipboardContent,
+  SpreadsheetChildEnv,
   UID,
   Zone,
 } from "../../types";
+import { AllowedImageMimeTypes } from "../../types/image";
 import { mergeOverlappingZones, positions } from "../zones";
 
 export function getClipboardDataPositions(sheetId: UID, zones: Zone[]): ClipboardCellData {
@@ -62,22 +65,42 @@ export function getPasteZones<T>(target: Zone[], content: T[][]): Zone[] {
   return target.map((t) => splitZoneForPaste(t, width, height)).flat();
 }
 
-export function parseOSClipboardContent(content: OSClipboardContent): ParsedOSClipboardContent {
-  if (!content[ClipboardMIMEType.Html]) {
+export async function parseOSClipboardContent(
+  env: SpreadsheetChildEnv,
+  content: OSClipboardContent,
+  clipboardId: string
+): Promise<ParsedOSClipboardContent> {
+  let contentClipboardId: string | undefined;
+  let spreadsheetContent: SpreadsheetClipboardData | undefined = undefined;
+  if (content[ClipboardMIMEType.Html]) {
+    const htmlDocument = new DOMParser().parseFromString(
+      content[ClipboardMIMEType.Html],
+      "text/html"
+    );
+    const oSheetClipboardData = htmlDocument
+      .querySelector("div")
+      ?.getAttribute("data-osheet-clipboard");
+    spreadsheetContent = oSheetClipboardData && JSON.parse(oSheetClipboardData);
+    contentClipboardId = spreadsheetContent?.clipboardId;
+  }
+  if (contentClipboardId !== clipboardId) {
+    const clipboardContent: ParsedOSClipboardContent = {
+      text: content[ClipboardMIMEType.PlainText],
+      data: spreadsheetContent,
+    };
+    for (const type of AllowedImageMimeTypes) {
+      if (content[type]) {
+        // TODO: support multiple import
+        const imageData = await env.imageProvider?.upload(content[type]!);
+        clipboardContent.imageData = imageData;
+        break;
+      }
+    }
+    return clipboardContent;
+  } else {
     return {
       text: content[ClipboardMIMEType.PlainText],
+      data: spreadsheetContent,
     };
   }
-  const htmlDocument = new DOMParser().parseFromString(
-    content[ClipboardMIMEType.Html],
-    "text/html"
-  );
-  const oSheetClipboardData = htmlDocument
-    .querySelector("div")
-    ?.getAttribute("data-osheet-clipboard");
-  const spreadsheetContent = oSheetClipboardData && JSON.parse(oSheetClipboardData);
-  return {
-    text: content[ClipboardMIMEType.PlainText],
-    data: spreadsheetContent,
-  };
 }

src/helpers/clipboard/navigator_clipboard_wrapper.ts

@@ -1,3 +1,4 @@
+import { AllowedImageMimeTypes } from "../../types/image";
 import { ClipboardMIMEType, OSClipboardContent } from "./../../types/clipboard";
 
 export type ClipboardReadResult =
@@ -64,7 +65,12 @@ class WebClipboardWrapper implements ClipboardInterface {
         for (const item of clipboardItems) {
           for (const type of item.types) {
             const blob = await item.getType(type);
-            clipboardContent[type as ClipboardMIMEType] = await blob.text();
+            if (type in AllowedImageMimeTypes) {
+              clipboardContent[type] = blob;
+            } else {
+              const text = await blob.text();
+              clipboardContent[type] = text;
+            }
           }
         }
         return { status: "ok", content: clipboardContent };
@@ -83,14 +89,18 @@ class WebClipboardWrapper implements ClipboardInterface {
   }
 
   private getClipboardItems(content: OSClipboardContent): ClipboardItems {
-    const clipboardItemData = {
-      [ClipboardMIMEType.PlainText]: this.getBlob(content, ClipboardMIMEType.PlainText),
-      [ClipboardMIMEType.Html]: this.getBlob(content, ClipboardMIMEType.Html),
-    };
+    const clipboardItemData = {};
+    for (const type of Object.keys(content)) {
+      clipboardItemData[type] = this.getBlob(content, type);
+    }
     return [new ClipboardItem(clipboardItemData)];
   }
 
-  private getBlob(clipboardContent: OSClipboardContent, type: ClipboardMIMEType): Blob {
+  private getBlob(clipboardContent: OSClipboardContent, type: string): Blob {
+    const content = clipboardContent[type];
+    if (content instanceof Blob) {
+      return content;
+    }
     return new Blob([clipboardContent[type] || ""], {
       type,
     });

src/helpers/figures/charts/chart_ui_common.ts

@@ -72,6 +72,59 @@ export function chartToImage(
   return undefined;
 }
 
+export async function chartToImageBlob(
+  runtime: ChartRuntime,
+  figure: Figure,
+  type: string
+): Promise<Blob | null> {
+  // wrap the canvas in a div with a fixed size because chart.js would
+  // fill the whole page otherwise
+  const div = document.createElement("div");
+  div.style.width = `${figure.width}px`;
+  div.style.height = `${figure.height}px`;
+  const canvas = document.createElement("canvas");
+  div.append(canvas);
+  canvas.setAttribute("width", figure.width.toString());
+  canvas.setAttribute("height", figure.height.toString());
+  let finalContent: Blob | null = null;
+  // we have to add the canvas to the DOM otherwise it won't be rendered
+  document.body.append(div);
+  if ("chartJsConfig" in runtime) {
+    const config = deepCopy(runtime.chartJsConfig);
+    config.plugins = [backgroundColorChartJSPlugin];
+    const chart = new window.Chart(canvas, config);
+    const imgContent = chart.toBase64Image() as string;
+    finalContent = base64ToBlob(imgContent, "image/png");
+    chart.destroy();
+    div.remove();
+  } else if (type === "scorecard") {
+    const design = getScorecardConfiguration(figure, runtime as ScorecardChartRuntime);
+    drawScoreChart(design, canvas);
+    finalContent = await new Promise((resolve) => canvas.toBlob(resolve, "image/png"));
+    div.remove();
+  } else if (type === "gauge") {
+    drawGaugeChart(canvas, runtime as GaugeChartRuntime);
+    finalContent = await new Promise((resolve) => canvas.toBlob(resolve, "image/png"));
+    div.remove();
+  }
+  return finalContent;
+}
+
+function base64ToBlob(base64: string, mimeType: string): Blob {
+  // Remove the data URL part if present
+  const byteCharacters = atob(base64.split(",")[1]);
+
+  const byteNumbers = new Array(byteCharacters.length);
+  for (let i = 0; i < byteCharacters.length; i++) {
+    byteNumbers[i] = byteCharacters.charCodeAt(i);
+  }
+
+  const byteArray = new Uint8Array(byteNumbers);
+
+  // Create a Blob object from the byteArray
+  return new Blob([byteArray], { type: mimeType });
+}
+
 /**
  * Custom chart.js plugin to set the background color of the canvas
  * https://github.com/chartjs/Chart.js/blob/8fdf76f8f02d31684d34704341a5d9217e977491/docs/configuration/canvas-background.md

src/helpers/figures/images/image_provider.ts

@@ -16,6 +16,12 @@ export class ImageProvider implements ImageProviderInterface {
     return { path, size, mimetype: file.type };
   }
 
+  async upload(file: File): Promise<Image> {
+    const path = await this.fileStore.upload(file);
+    const size = await this.getImageOriginalSize(path);
+    return { path, size, mimetype: file.type };
+  }
+
   private getImageFromUser(): Promise<File> {
     return new Promise((resolve, reject) => {
       const input = document.createElement("input");
@@ -34,7 +40,7 @@ export class ImageProvider implements ImageProviderInterface {
 
   getImageOriginalSize(path: string): Promise<FigureSize> {
     return new Promise((resolve, reject) => {
-      const image = new Image();
+      const image = new window.Image();
       image.src = path;
       image.addEventListener("load", () => {
         const size = { width: image.width, height: image.height };

src/model.ts

@@ -441,6 +441,7 @@ export class Model extends EventBus<any> implements CommandDispatcher {
       session: this.session,
       defaultCurrency: this.config.defaultCurrency,
       customColors: this.config.customColors || [],
+      external: this.config.external,
     };
   }
 

src/plugins/ui_plugin.ts

@@ -28,6 +28,7 @@ export interface UIPluginConfig {
   readonly session: Session;
   readonly defaultCurrency?: Partial<Currency>;
   readonly customColors: Color[];
+  readonly external: ModelConfig["external"];
 }
 
 export interface UIPluginConstructor {