Fix multidimensional data pages baking logic

* Don't delete mdims in GrapherBaker
* Delete old mdims in MultiDimBaker
* Improve the console output
* Document issue with TSConfig and getting ESNext types

adminSiteServer/multiDim.ts

@@ -231,6 +231,8 @@ async function cleanUpOrphanedChartConfigs(
     }
 }
 
+// TODO: Do we need to manually delete unpublished multi-dim pages?
+//  - from R2?
 export async function createMultiDimConfig(
     knex: db.KnexReadWriteTransaction,
     slug: string,

baker/GrapherBaker.tsx

@@ -58,6 +58,7 @@ import { logErrorAndMaybeCaptureInSentry } from "../serverUtils/errorLog.js"
 import { getTagToSlugMap } from "./GrapherBakingUtils.js"
 import { knexRaw } from "../db/db.js"
 import { getRelatedChartsForVariable } from "../db/model/Chart.js"
+import { getAllMultiDimDataPageSlugs } from "../db/model/MultiDimDataPage.js"
 import pMap from "p-map"
 
 const renderDatapageIfApplicable = async (
@@ -308,7 +309,6 @@ const bakeGrapherPageAndVariablesPngAndSVGIfChanged = async (
             imageMetadataDictionary
         )
     )
-    console.log(outPath)
 
     const variableIds = lodash.uniq(
         grapher.dimensions?.map((d) => d.variableId)
@@ -411,7 +411,6 @@ export const bakeAllChangedGrapherPagesVariablesPngSvgAndDeleteRemovedGraphers =
                 `
         )
 
-        const newSlugs = chartsToBake.map((row) => row.slug)
         await fs.mkdirp(bakedSiteDir + "/grapher")
 
         // Prefetch imageMetadata instead of each grapher page fetching
@@ -432,7 +431,7 @@ export const bakeAllChangedGrapherPagesVariablesPngSvgAndDeleteRemovedGraphers =
         )
 
         const progressBar = new ProgressBar(
-            "bake grapher page [:bar] :current/:total :elapseds :rate/s :etas :name\n",
+            "bake grapher page [:bar] :current/:total :elapseds :rate/s :name\n",
             {
                 width: 20,
                 total: chartsToBake.length + 1,
@@ -451,11 +450,18 @@ export const bakeAllChangedGrapherPagesVariablesPngSvgAndDeleteRemovedGraphers =
                     async (knex) => await bakeSingleGrapherChart(job, knex),
                     db.TransactionCloseMode.KeepOpen
                 )
-                progressBar.tick({ name: `slug ${job.slug}` })
+                progressBar.tick({ name: job.slug })
             },
             { concurrency: 10 }
         )
 
-        await deleteOldGraphers(bakedSiteDir, excludeUndefined(newSlugs))
+        // Multi-dim data pages are baked into the same directory as graphers
+        // and they are handled separately.
+        const multiDimSlugs = await getAllMultiDimDataPageSlugs(knex)
+        const newSlugs = excludeUndefined([
+            ...chartsToBake.map((row) => row.slug),
+            ...multiDimSlugs,
+        ])
+        await deleteOldGraphers(bakedSiteDir, newSlugs)
         progressBar.tick({ name: `✅ Deleted old graphers` })
     }

baker/GrapherImageBaker.tsx

@@ -41,9 +41,7 @@ export async function bakeGrapherToSvgAndPng(
     if (optimizeSvgs) svgCode = await optimizeSvg(svgCode)
 
     return Promise.all([
-        fs
-            .writeFile(`${outPath}.svg`, svgCode)
-            .then(() => console.log(`${outPath}.svg`)),
+        fs.writeFile(`${outPath}.svg`, svgCode),
         sharp(Buffer.from(grapher.staticSVG), { density: 144 })
             .png()
             .resize(grapher.defaultBounds.width, grapher.defaultBounds.height)

baker/MultiDimBaker.tsx

@@ -1,5 +1,7 @@
+import { glob } from "glob"
 import fs from "fs-extra"
 import path from "path"
+import ProgressBar from "progress"
 import {
     ImageMetadata,
     MultiDimDataPageConfigPreProcessed,
@@ -32,8 +34,10 @@ import {
     resolveFaqsForVariable,
 } from "./DatapageHelpers.js"
 import { logErrorAndMaybeCaptureInSentry } from "../serverUtils/errorLog.js"
+import { getAllPublishedChartSlugs } from "../db/model/Chart.js"
 import {
     getAllMultiDimDataPages,
+    getAllPublishedMultiDimDataPageSlugs,
     getMultiDimDataPageBySlug,
 } from "../db/model/MultiDimDataPage.js"
 
@@ -219,12 +223,42 @@ export const bakeMultiDimDataPage = async (
     await fs.writeFile(outPath, renderedHtml)
 }
 
+async function deleteOldMultiDimPages(
+    bakedSiteDir: string,
+    newSlugs: Set<string>
+) {
+    const oldSlugs = new Set(
+        glob
+            .sync(`${bakedSiteDir}/grapher/*.html`)
+            .map((slug) =>
+                slug
+                    .replace(`${bakedSiteDir}/grapher/`, "")
+                    .replace(".html", "")
+            )
+    )
+    const toRemove = oldSlugs.difference(newSlugs)
+    for (const slug of toRemove) {
+        console.log(`DELETING ${slug}`)
+        const path = `${bakedSiteDir}/grapher/${slug}.html`
+        await fs.unlink(path)
+        console.log(path)
+    }
+}
+
 export const bakeAllMultiDimDataPages = async (
     knex: db.KnexReadonlyTransaction,
     bakedSiteDir: string,
     imageMetadata: Record<string, ImageMetadata>
 ) => {
     const multiDimsBySlug = await getAllMultiDimDataPages(knex)
+    const progressBar = new ProgressBar(
+        "bake multi-dim page [:bar] :current/:total :elapseds :rate/s :name\n",
+        {
+            width: 20,
+            total: multiDimsBySlug.size + 1,
+            renderThrottle: 0,
+        }
+    )
     for (const [slug, row] of multiDimsBySlug.entries()) {
         await bakeMultiDimDataPage(
             knex,
@@ -233,5 +267,11 @@ export const bakeAllMultiDimDataPages = async (
             row.config,
             imageMetadata
         )
+        progressBar.tick({ name: slug })
     }
+    const publishedSlugs = await getAllPublishedMultiDimDataPageSlugs(knex)
+    const chartSlugs = await getAllPublishedChartSlugs(knex)
+    const newSlugs = new Set([...publishedSlugs, ...chartSlugs])
+    await deleteOldMultiDimPages(bakedSiteDir, newSlugs)
+    progressBar.tick({ name: `✅ Deleted old multi-dim pages` })
 }

db/model/Chart.ts

@@ -673,3 +673,20 @@ export const getRedirectsByChartId = async (
         ORDER BY id ASC`,
         [chartId]
     )
+
+export async function getAllPublishedChartSlugs(
+    knex: db.KnexReadonlyTransaction
+): Promise<string[]> {
+    const rows = await db.knexRaw<{
+        slug: string
+    }>(
+        knex,
+        `-- sql
+            SELECT cc.slug
+            FROM charts c
+            JOIN chart_configs cc ON c.configId = cc.id
+            WHERE cc.full->>"$.isPublished" = "true"
+        `
+    )
+    return rows.map((row) => row.slug)
+}

db/model/MultiDimDataPage.ts

@@ -84,6 +84,24 @@ export async function getAllLinkedPublishedMultiDimDataPages(
     return rows.map(enrichRow)
 }
 
+export async function getAllMultiDimDataPageSlugs(
+    knex: KnexReadonlyTransaction
+): Promise<string[]> {
+    const rows = await knex<DbPlainMultiDimDataPage>(
+        MultiDimDataPagesTableName
+    ).select("slug")
+    return rows.map((row) => row.slug)
+}
+
+export async function getAllPublishedMultiDimDataPageSlugs(
+    knex: KnexReadonlyTransaction
+): Promise<string[]> {
+    const rows = await knex<DbPlainMultiDimDataPage>(MultiDimDataPagesTableName)
+        .select("slug")
+        .where({ published: true })
+    return rows.map((row) => row.slug)
+}
+
 export const getMultiDimDataPageBySlug = async (
     knex: KnexReadonlyTransaction,
     slug: string,

devTools/tsconfigs/tsconfig.base.json

@@ -9,7 +9,23 @@
         "declaration": true,
         "declarationMap": true,
 
-        "lib": ["dom", "dom.iterable", "es2020", "es2021", "es2022", "es2023"],
+        // To get newer APIs like Set.prototype.intersection(), we need to use
+        // ESNext lib on Node 22 and current version of TypeScript (5.7.2).
+        // However, using the ESNext option doesn't work with
+        // @types/[email protected], which we are currently pinned to, to fix an
+        // unrelated bug with types in Cloudflare Functions, thus we only use
+        // ESNext.Collection.
+        // https://github.com/microsoft/TypeScript/issues/59919
+        // https://developers.cloudflare.com/workers/languages/typescript/#transitive-loading-of-typesnode-overrides-cloudflareworkers-types
+        "lib": [
+            "dom",
+            "dom.iterable",
+            "es2020",
+            "es2021",
+            "es2022",
+            "es2023",
+            "ESNext.Collection"
+        ],
         // Using es2022 as a `target` caused the following error in wrangler:
         // "Uncaught TypeError: PointVector is not a constructor".
         // This seems to be related to a change in how classes are compiled in