✨ make zip file generation work

owid · Oct 4, 2024 · dc5bd2c · dc5bd2c
1 parent 8da1b1a
commit dc5bd2c
Show file tree

Hide file tree

Showing 2 changed files with 70 additions and 51 deletions.
diff --git a/functions/_common/grapherRenderer.ts b/functions/_common/grapherRenderer.ts
@@ -17,6 +17,7 @@ import { constructReadme } from "./readmeTools"
 import { svg2png, initialize as initializeSvg2Png } from "svg2png-wasm"
 import { TimeLogger } from "./timeLogger"
 import { png, StatusError } from "itty-router"
+import JSZip from "jszip"
 
 import svg2png_wasm from "../../node_modules/svg2png-wasm/svg2png_wasm_bg.wasm"
 
@@ -314,23 +315,7 @@ async function initGrapher(
     return grapher
 }
 
-export async function fetchMetadataForGrapher(
-    identifier: GrapherIdentifier,
-    env: Env,
-    searchParams?: URLSearchParams
-) {
-    console.log("Initializing grapher")
-    const grapher = await initGrapher(
-        identifier,
-        TWITTER_OPTIONS,
-        searchParams ?? new URLSearchParams(""),
-        env
-    )
-
-    await grapher.downloadLegacyDataFromOwidVariableIds()
-
-    //const useShortNames = searchParams.get("useColumnShortNames") === "true"
-
+function assembleMetadata(grapher: Grapher, searchParams: URLSearchParams) {
     const columnsToIgnore = new Set(
         [
             OwidTableSlugs.entityId,
@@ -383,7 +368,6 @@ export async function fetchMetadataForGrapher(
         console.log("mapping col", col.name)
         const {
             descriptionShort,
-            descriptionFromProducer,
             descriptionKey,
             descriptionProcessing,
             additionalInfo,
@@ -392,19 +376,16 @@ export async function fetchMetadataForGrapher(
             timespan,
             tolerance,
             type,
-            display,
-            presentation,
             origins,
             sourceLink,
             sourceName,
-            catalogPath,
             owidVariableId,
             shortName,
         } = col.def as OwidColumnDef
         const lastUpdated = getLastUpdatedFromVariable(col.def)
         const nextUpdate = getNextUpdateFromVariable(col.def)
 
-        let consensedOrigins:
+        let condensedOrigins:
             | Partial<
                   Pick<
                       OwidOrigin,
@@ -436,8 +417,8 @@ export async function fetchMetadataForGrapher(
             }
         })
 
-        if (!consensedOrigins || consensedOrigins.length === 0) {
-            consensedOrigins = [
+        if (!condensedOrigins || condensedOrigins.length === 0) {
+            condensedOrigins = [
                 {
                     attribution: sourceName,
                     urlMain: sourceLink,
@@ -514,6 +495,29 @@ export async function fetchMetadataForGrapher(
         columns: fromPairs(columns),
     }
 
+    return fullMetadata
+}
+
+export async function fetchMetadataForGrapher(
+    identifier: GrapherIdentifier,
+    env: Env,
+    searchParams?: URLSearchParams
+) {
+    console.log("Initializing grapher")
+    const grapher = await initGrapher(
+        identifier,
+        TWITTER_OPTIONS,
+        searchParams ?? new URLSearchParams(""),
+        env
+    )
+
+    await grapher.downloadLegacyDataFromOwidVariableIds()
+
+    const fullMetadata = assembleMetadata(
+        grapher,
+        searchParams ?? new URLSearchParams("")
+    )
+
     return Response.json(fullMetadata)
 }
 
@@ -522,32 +526,43 @@ export async function fetchZipForGrapher(
     env: Env,
     searchParams?: URLSearchParams
 ) {
+    console.log("preparing to generate zip file")
     const grapher = await initGrapher(
         identifier,
         TWITTER_OPTIONS,
         searchParams ?? new URLSearchParams(""),
         env
     )
     await grapher.downloadLegacyDataFromOwidVariableIds()
-    const defs = grapher.inputTable
-        .getColumns(grapher.inputTable.columnNames)
-        .map((col) => col.def)
-    const table =
-        searchParams.get("csvType") === "filtered"
-            ? grapher.transformedTable
-            : grapher.inputTable
-    const json = JSON.stringify(defs)
+    const metadata = assembleMetadata(grapher, searchParams)
+    const readme = assembleReadme(grapher)
+    const csv = assembleCsv(grapher, searchParams)
+    console.log("Fetched the parts, creating zip file")
     const zip = new JSZip()
-    zip.file("metadata.json", json)
-    zip.file("data.csv", table.toPrettyCsv())
-    const content = await zip.generateAsync({ type: "blob" })
+    zip.file(
+        `${identifier.id}.metadata.json`,
+        JSON.stringify(metadata, undefined, 2)
+    )
+    zip.file(`${identifier.id}.csv`, csv)
+    zip.file("readme.md", readme)
+    const content = await zip.generateAsync({ type: "arraybuffer" })
+    console.log("Generated content, returning response")
     return new Response(content, {
         headers: {
             "Content-Type": "application/zip",
         },
     })
 }
 
+function assembleCsv(grapher: Grapher, searchParams: URLSearchParams): string {
+    const useShortNames = searchParams.get("useColumnShortNames") === "true"
+    const table =
+        searchParams.get("csvType") === "filtered"
+            ? grapher.transformedTable
+            : grapher.inputTable
+    return table.toPrettyCsv(useShortNames)
+}
+
 export async function fetchCsvForGrapher(
     identifier: GrapherIdentifier,
     env: Env,
@@ -560,18 +575,13 @@ export async function fetchCsvForGrapher(
         env
     )
     await grapher.downloadLegacyDataFromOwidVariableIds()
-    const useShortNames = searchParams.get("useColumnShortNames") === "true"
-    const table =
-        searchParams.get("csvType") === "filtered"
-            ? grapher.transformedTable
-            : grapher.inputTable
-    return new Response(table.toPrettyCsv(useShortNames), {
+    const csv = assembleCsv(grapher, searchParams ?? new URLSearchParams(""))
+    return new Response(csv, {
         headers: {
             "Content-Type": "text/csv",
         },
     })
 }
-
 export async function fetchReadmeForGrapher(
     identifier: GrapherIdentifier,
     env: Env,
@@ -587,6 +597,15 @@ export async function fetchReadmeForGrapher(
 
     await grapher.downloadLegacyDataFromOwidVariableIds()
 
+    const readme = assembleReadme(grapher)
+    return new Response(readme, {
+        headers: {
+            "Content-Type": "text/markdown",
+        },
+    })
+}
+
+function assembleReadme(grapher: Grapher): string {
     const columnsToIgnore = new Set(
         [
             OwidTableSlugs.entityId,
@@ -605,12 +624,7 @@ export async function fetchReadmeForGrapher(
 
     const columns = grapher.inputTable.getColumns(columnsToGet)
 
-    const readme = constructReadme(grapher, columns)
-    return new Response(readme, {
-        headers: {
-            "Content-Type": "text/markdown",
-        },
-    })
+    return constructReadme(grapher, columns)
 }
 async function fetchAndRenderGrapherToSvg(
     identifier: GrapherIdentifier,

diff --git a/functions/tsconfig.json b/functions/tsconfig.json
@@ -3,7 +3,12 @@
         "target": "esnext",
         "module": "esnext",
         "moduleResolution": "nodenext",
-        "lib": ["esnext"],
-        "types": ["@cloudflare/workers-types"]
+        "lib": [
+            "esnext"
+        ],
+        "types": [
+            "@cloudflare/workers-types"
+        ],
+        "esModuleInterop": true,
     }
-}
+}