Enhance, debug, and clean up scripts for CSS migration (BL-12857)

custom-css-analysis-and-migration/README.md

@@ -1,33 +1,35 @@
 In Bloom 5.7, we introduce a modernized, parameterized page layout CSS system, known as "Appearance". This new system conflicts with the arbitrary custom css that was used in prior Bloom versions, mostly in the area of margins.
 
-As of this writing, we are only looking at `customBookStyles.css`, but `customCollectionsStyles.css` have the same problem.
+As of this writing, we are looking at both `customBookStyles.css` and `customCollectionsStyles.css`, but not handling having both files with content, and not really dealing with `customCollectionStyles.css` being a collection wide settings file.
 
 When Bloom encounters a pre-5.7 book with a `customBookStyles.css` that would conflict with the new system, it goes into a kind of "safe mode" that keeps things working by using a legacy `basePage.css`. Better, though, is to migrate the old css to the new system. Conceivably, a reliable program could be written to do this automatically. However at the moment what we do is to write the migrations using the utilities here, and then have a dev evaluate individual migrations and then copy them over to the shipping Bloom. So when Bloom encounters one of these books, we may already have a tested migration for it.
 
 # How to use this system
 
 ⚠️ Be careful what you commit to an open source repo. We do not want to expose emails or other private data.
 
-1.  As of this writing, bun only works on linux. If you are on windows, just install Windows Subsystem for Linux (not as big of a deal as it sounds), then run in a WSL terminal in VSCODE.
+1. As of this writing, bun works only on linux. If you are on windows, just install Windows Subsystem for Linux (not as big of a deal as it sounds), then run in a WSL terminal in VSCODE.
 
-1.  Get [bun](https://bun.sh/) installed
+2. Get [bun](https://bun.sh/) installed
 
-1.  Install dependencies: `bun install`
-
-1.  Download all the `customBookStyles.css` from blorg
+3. Choose (or create) a folder for working with CSS files and download all the `customBookStyles.css` and `customCollectionStyles.css` files from bloomlibrary.org.  This will take some time.
 
     `./some-path/BloomBulkDownloader.exe --include "*/*/custom*Styles.css" --syncfolder ./output/downloads --bucket production customBookStyles-files`
 
+4. Download all the `meta.json` files from bloomlibrary.org.  Again, this will take some time.
+
+    `./some-path/BloomBulkDownloader.exe --include "*/*/meta.json" --syncfolder ./output/downloads --bucket production customBookStyles-files`
+
 Each of the following take an optional argument that will limit the number of records processed.
 
-5.  Process those, grouping them into bins of duplicate stylesheets
+5. Process those, grouping them into bins of duplicate stylesheets
 
-    `bun run group-stylesheets.ts 13`
+    `bun run /path-to/group-stylesheets.ts 13`
 
-1.  Process those groups, discarding ones that don't need migration
+6. Process those groups, discarding ones that don't need migration
 
-    `bun run filter-stylesheets.ts 7`
+    `bun run /path-to/filter-stylesheets.ts 7`
 
-1.  Create draft migration files for each one
+7. Create draft migration files for each one
 
-    `bun run create-migrations.ts 3`
+    `bun run /path-to/create-migrations.ts 3`

custom-css-analysis-and-migration/create-migrations.ts

@@ -36,25 +36,47 @@ for (const record of records) {
   if (record.css === undefined) continue;
 
   ++count;
-  if (Bun.argv.length > 2 && count >= Number.parseInt(Bun.argv[2])) break;
-  console.log(`css = ${record.css}`);
+  if (Bun.argv.length > 2 && count > Number.parseInt(Bun.argv[2])) break;
+  //console.log(`css = ${record.css}`);
   const checksum = crypto.createHash("md5").update(record.css).digest("hex");
   const folder = `./output/migration/${count} ${checksum}`;
+  console.log(`output folder=${folder}`);
   fs.mkdirSync(folder);
 
+  let cssPath = `${folder}/customBookStyles.css`;
+
+  const migration = migrateCssToAppearance(record.css);
+
   fs.writeFileSync(
-    `${folder}/customBookStyles.css`,
-    migrateCssToAppearance(record.css) +
-      `\n\n /*  ----- ORIGINAL ---- */\n/*${record.css.replaceAll(
-        "*/",
-        "#/"
-      )}*/`
+    cssPath,
+    migration.modifiedCss +
+      `
+
+/*  ----- ORIGINAL ---- */
+/*${record.css.replaceAll("*/", "#/")}*/
+`
   );
 
+  const brandingSet = new Set();
+  for (let i = 0; i < record.paths.length; ++i) {
+    const metaPath = "./output/downloads/" + record.paths[i] + "/meta.json";
+    try {
+      const metaString: string = fs.readFileSync(metaPath, "utf8");
+      const metadata = JSON.parse(metaString);
+      const brandingProject = metadata.brandingProjectName as string;
+      if (brandingProject &&
+          brandingProject.toLowerCase() !== "default" &&
+          brandingProject.toLowerCase() !== "local-community")
+        brandingSet.add(brandingProject);
+    } catch (e) {
+      console.log("Could not extract brandingProjectName from " + metaPath + ": " + e);
+    }
+  }
+
   const uniqueUploaders = [
     ...new Set(
       record.paths.map((path: string) => {
-        const email = path.split("/")[2];
+        const email = path.split("/")[0];
         const emailParts = email.split("@");
         const obfuscatedEmail =
           emailParts[0].slice(0, -2) + "..." + "@" + emailParts[1];
@@ -63,15 +85,25 @@ for (const record of records) {
     ),
   ].slice(0, 3); // first 3 are enough to give a sense of who uploaded these books
 
-  fs.writeFileSync(
-    `${folder}/appearance.json`,
+  const brandings = [...brandingSet];
+  const date = new Date();
+  const outputString: string =
     `// Matches customBookStyles.css with checksum ${checksum}
-// This was used by ${record.book_count} books (${record.unique_named_books} unique).` +
-      // enhance: +`// Affected branding projects include "Mali-ACR-2020-Soninke".`
-      `// Uploaders included ${JSON.stringify(uniqueUploaders)}.
+// On ${date.toDateString()} this was used by ${record.book_count} books (${record.unique_named_books} unique).
+` +
+    (brandings && brandings.length > 0
+      ? `// Affected branding projects included ${JSON.stringify(brandings)}.
+`
+      : ``) +
+    `// Uploaders included ${JSON.stringify(uniqueUploaders)}.
 // Example Book: ${record.first_book}
 // A replacement customBookStyles.css has been generated.
 {
-}`
-  );
+  "cssThemeName": "default",
+}
+`;
+  let appearancePath = `${folder}/appearance.json`;
+  if (fs.existsSync(appearancePath))
+    appearancePath = `${folder}/appearance2.json`;
+  fs.writeFileSync(appearancePath, outputString);
 }

custom-css-analysis-and-migration/filter-stylesheets.ts

@@ -15,7 +15,7 @@ const count = records.reduce((acc, record) => acc + record.paths.length, 0);
 console.write(`total books with custom css rules: ${count}\r\n`);
 console.write(`total unique css files: ${records.length}\r\n`);
 
-const kProbablyWillInterfere = `\\.marginBox\\s*\\{[^\\}]*?(?<![-\\w])(padding-|left:|top:|right:|bottom:|margin-|width:|height:)[^\\}]*\\}`;
+const kProbablyWillInterfere = `\\.marginBox\\s*\\{[^\\}]*?(?<![-\\w])(padding[-:]|left:|top:|right:|bottom:|margin[-:]|width:|height:)[^\\}]*\\}`;
 const kProbablyWillInterfereRegex = new RegExp(kProbablyWillInterfere, "gi");
 
 const max = Bun.argv.length > 2 ? Number.parseInt(Bun.argv[2]) : 10000000;
@@ -34,7 +34,7 @@ const recordsWithUniqueifiedPaths = filteredRecords.map((record) => {
   const uniquePaths = uniqueFilenames.map((filename) => {
     return paths.find((path) => path.endsWith(filename));
   });
-  const instanceId = paths[0].split("/")[2];
+  const instanceId = paths[0].split("/")[1];
   return {
     book_count: paths.length,
     unique_named_books: uniquePaths.length,
@@ -75,12 +75,13 @@ console.write(
   `, covering ${filteredRecords.reduce(
     (acc, record) => acc + record.paths.length,
     0
-  )} books, `
+  )} books`
 );
-
 // console.write(
-//   `${recordsWithUniqueifiedPaths.reduce(
+//   `, ${recordsWithUniqueifiedPaths.reduce(
 //     (acc, record) => acc + record.uniqueified_paths.length,
 //     0
 //   )} of which have unique names (should remove most rebrands).\r\n`
 // );
+console.write(`
+`);

custom-css-analysis-and-migration/group-stylesheets.ts

@@ -11,35 +11,38 @@ interface FileData {
 }
 
 const boilerplate =
-  '/*  Some books may need control over aspects of layout that cannot yet be adjusted\r\n    from the Bloom interface. In those cases, Bloom provides this "under the hood" method\r\n    of creating style rules using the underlying "Cascading Stylesheets" system.\r\n    These rules are then applied to all books in this collection.  EDIT THIS FILE ONLY\r\n    IN THE COLLECTION FOLDER:  changes made to a copy found in the book folder will be\r\n    lost the next time the book is edited with Bloom!\r\n\r\n Note: you can also add a file named "customBookStyles.css" in the book folder,\r\n    to limit the effects of the rules to just that one book.\r\n\r\n    You can learn about CSS from hundreds of books, or online. However chances are, if\r\n    you need this customization, you will need an expert to create a version of this file\r\n    for you, or give you rules that you can paste in below this line. */';
+  /\/\*  Some books may need control over aspects of layout that cannot yet be adjusted(.|[\r\n])*?\*\//;
 
-const boilerplate2 =
-  '/*  Some books may need control over aspects of layout that cannot yet be adjusted\r\n    from the Bloom interface. In those cases, Bloom provides this "under the hood" method\r\n    of creating style rules using the underlying "Cascading Stylesheets" system. \r\n    These rules are then applied to all books in this collection.\r\n\r\n Note: you can also add a file named "customBookStyles.css" in the book folder,\r\n    to limit the effects of the rules to just that one book.\r\n \r\n    You can learn about CSS from hundreds of books, or online. However chances are, if\r\n    you need this customization, you will need an expert to create a version of this file\r\n    for you, or give you rules that you can paste in below this line. */';
 let count = 0;
 function readFilesRecursively(
   dir: string,
   fileMap: Map<string, FileData>
 ): void {
   const files = fs.readdirSync(dir);
+  let cssCount = 0;
   for (const file of files) {
     const filePath = path.join(dir, file);
     if (Bun.argv.length > 2 && count >= Number.parseInt(Bun.argv[2])) return;
     if (fs.statSync(filePath).isDirectory()) {
       readFilesRecursively(filePath, fileMap);
     } else {
+      if (file === "meta.json") continue;
       const content = fs
         .readFileSync(filePath, "utf8")
         .replace(boilerplate, "")
-        .replace(boilerplate2, "")
         .trim();
       if (content === "") continue;
+      ++cssCount;
 
       const fileData: FileData = fileMap.get(content) || { content, paths: [] };
-      fileData.paths.push(dir.replace("./output/downloads", ""));
+      fileData.paths.push(dir.replace(/^(\.\/)?output\/downloads\//, ""));
       fileMap.set(content, fileData);
-      console.log(++count + " " + dir);
+      console.log(++count + " " + filePath);
     }
   }
+  if (cssCount > 1)
+    console.log(`INFO: customCollectionStyles.css and customBookStyles.css both have content in ${dir}`);
+  cssCount = 0;
 }
 
 const sourceDir = "./output/downloads";