From dc5bd2c1c248575b547d398e041d9b8d64452318 Mon Sep 17 00:00:00 2001 From: Daniel Bachler Date: Tue, 17 Sep 2024 01:19:57 +0200 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20make=20zip=20file=20generation=20wo?= =?UTF-8?q?rk?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- functions/_common/grapherRenderer.ts | 110 +++++++++++++++------------ functions/tsconfig.json | 11 ++- 2 files changed, 70 insertions(+), 51 deletions(-) diff --git a/functions/_common/grapherRenderer.ts b/functions/_common/grapherRenderer.ts index 987d0237f99..a0d0f020373 100644 --- a/functions/_common/grapherRenderer.ts +++ b/functions/_common/grapherRenderer.ts @@ -17,6 +17,7 @@ import { constructReadme } from "./readmeTools" import { svg2png, initialize as initializeSvg2Png } from "svg2png-wasm" import { TimeLogger } from "./timeLogger" import { png, StatusError } from "itty-router" +import JSZip from "jszip" import svg2png_wasm from "../../node_modules/svg2png-wasm/svg2png_wasm_bg.wasm" @@ -314,23 +315,7 @@ async function initGrapher( return grapher } -export async function fetchMetadataForGrapher( - identifier: GrapherIdentifier, - env: Env, - searchParams?: URLSearchParams -) { - console.log("Initializing grapher") - const grapher = await initGrapher( - identifier, - TWITTER_OPTIONS, - searchParams ?? new URLSearchParams(""), - env - ) - - await grapher.downloadLegacyDataFromOwidVariableIds() - - //const useShortNames = searchParams.get("useColumnShortNames") === "true" - +function assembleMetadata(grapher: Grapher, searchParams: URLSearchParams) { const columnsToIgnore = new Set( [ OwidTableSlugs.entityId, @@ -383,7 +368,6 @@ export async function fetchMetadataForGrapher( console.log("mapping col", col.name) const { descriptionShort, - descriptionFromProducer, descriptionKey, descriptionProcessing, additionalInfo, @@ -392,19 +376,16 @@ export async function fetchMetadataForGrapher( timespan, tolerance, type, - display, - presentation, origins, sourceLink, sourceName, - catalogPath, owidVariableId, shortName, } = col.def as OwidColumnDef const lastUpdated = getLastUpdatedFromVariable(col.def) const nextUpdate = getNextUpdateFromVariable(col.def) - let consensedOrigins: + let condensedOrigins: | Partial< Pick< OwidOrigin, @@ -436,8 +417,8 @@ export async function fetchMetadataForGrapher( } }) - if (!consensedOrigins || consensedOrigins.length === 0) { - consensedOrigins = [ + if (!condensedOrigins || condensedOrigins.length === 0) { + condensedOrigins = [ { attribution: sourceName, urlMain: sourceLink, @@ -514,6 +495,29 @@ export async function fetchMetadataForGrapher( columns: fromPairs(columns), } + return fullMetadata +} + +export async function fetchMetadataForGrapher( + identifier: GrapherIdentifier, + env: Env, + searchParams?: URLSearchParams +) { + console.log("Initializing grapher") + const grapher = await initGrapher( + identifier, + TWITTER_OPTIONS, + searchParams ?? new URLSearchParams(""), + env + ) + + await grapher.downloadLegacyDataFromOwidVariableIds() + + const fullMetadata = assembleMetadata( + grapher, + searchParams ?? new URLSearchParams("") + ) + return Response.json(fullMetadata) } @@ -522,6 +526,7 @@ export async function fetchZipForGrapher( env: Env, searchParams?: URLSearchParams ) { + console.log("preparing to generate zip file") const grapher = await initGrapher( identifier, TWITTER_OPTIONS, @@ -529,18 +534,19 @@ export async function fetchZipForGrapher( env ) await grapher.downloadLegacyDataFromOwidVariableIds() - const defs = grapher.inputTable - .getColumns(grapher.inputTable.columnNames) - .map((col) => col.def) - const table = - searchParams.get("csvType") === "filtered" - ? grapher.transformedTable - : grapher.inputTable - const json = JSON.stringify(defs) + const metadata = assembleMetadata(grapher, searchParams) + const readme = assembleReadme(grapher) + const csv = assembleCsv(grapher, searchParams) + console.log("Fetched the parts, creating zip file") const zip = new JSZip() - zip.file("metadata.json", json) - zip.file("data.csv", table.toPrettyCsv()) - const content = await zip.generateAsync({ type: "blob" }) + zip.file( + `${identifier.id}.metadata.json`, + JSON.stringify(metadata, undefined, 2) + ) + zip.file(`${identifier.id}.csv`, csv) + zip.file("readme.md", readme) + const content = await zip.generateAsync({ type: "arraybuffer" }) + console.log("Generated content, returning response") return new Response(content, { headers: { "Content-Type": "application/zip", @@ -548,6 +554,15 @@ export async function fetchZipForGrapher( }) } +function assembleCsv(grapher: Grapher, searchParams: URLSearchParams): string { + const useShortNames = searchParams.get("useColumnShortNames") === "true" + const table = + searchParams.get("csvType") === "filtered" + ? grapher.transformedTable + : grapher.inputTable + return table.toPrettyCsv(useShortNames) +} + export async function fetchCsvForGrapher( identifier: GrapherIdentifier, env: Env, @@ -560,18 +575,13 @@ export async function fetchCsvForGrapher( env ) await grapher.downloadLegacyDataFromOwidVariableIds() - const useShortNames = searchParams.get("useColumnShortNames") === "true" - const table = - searchParams.get("csvType") === "filtered" - ? grapher.transformedTable - : grapher.inputTable - return new Response(table.toPrettyCsv(useShortNames), { + const csv = assembleCsv(grapher, searchParams ?? new URLSearchParams("")) + return new Response(csv, { headers: { "Content-Type": "text/csv", }, }) } - export async function fetchReadmeForGrapher( identifier: GrapherIdentifier, env: Env, @@ -587,6 +597,15 @@ export async function fetchReadmeForGrapher( await grapher.downloadLegacyDataFromOwidVariableIds() + const readme = assembleReadme(grapher) + return new Response(readme, { + headers: { + "Content-Type": "text/markdown", + }, + }) +} + +function assembleReadme(grapher: Grapher): string { const columnsToIgnore = new Set( [ OwidTableSlugs.entityId, @@ -605,12 +624,7 @@ export async function fetchReadmeForGrapher( const columns = grapher.inputTable.getColumns(columnsToGet) - const readme = constructReadme(grapher, columns) - return new Response(readme, { - headers: { - "Content-Type": "text/markdown", - }, - }) + return constructReadme(grapher, columns) } async function fetchAndRenderGrapherToSvg( identifier: GrapherIdentifier, diff --git a/functions/tsconfig.json b/functions/tsconfig.json index 0998a9f102a..4f46142d55d 100644 --- a/functions/tsconfig.json +++ b/functions/tsconfig.json @@ -3,7 +3,12 @@ "target": "esnext", "module": "esnext", "moduleResolution": "nodenext", - "lib": ["esnext"], - "types": ["@cloudflare/workers-types"] + "lib": [ + "esnext" + ], + "types": [ + "@cloudflare/workers-types" + ], + "esModuleInterop": true, } -} +} \ No newline at end of file