handle errors in supporter images during 11ty build

- `canvas` now checks for errors and discards supporters, which will avoid problems during postbuild step - output debug info in build - remove `image-size` in lieu of `canvas` which is now a direct dev dependency
irrationnelle · Jul 30, 2020 · cf736fe · cf736fe
1 parent f966c94
commit cf736fe
Show file tree

Hide file tree

Showing 4 changed files with 63 additions and 51 deletions.
diff --git a/docs/_data/supporters.js b/docs/_data/supporters.js
@@ -17,11 +17,11 @@
 
 'use strict';
 
-const {writeFile, mkdir} = require('fs').promises;
+const {loadImage} = require('canvas');
+const {writeFile, mkdir, rmdir} = require('fs').promises;
 const {resolve} = require('path');
 const debug = require('debug')('mocha:docs:data:supporters');
 const needle = require('needle');
-const imageSize = require('image-size');
 const blocklist = new Set(require('./blocklist.json'));
 
 /**
@@ -36,7 +36,12 @@ const BLOCKED_STRINGS = /(?:vpn|[ck]a[sz]ino|seo|slots|gambl(?:e|ing)|crypto)/i;
  */
 const API_ENDPOINT = 'https://api.opencollective.com/graphql/v2';
 
-const query = `query account($limit: Int, $offset: Int, $slug: String) {
+const SPONSOR_TIER = 'sponsor';
+const BACKER_TIER = 'backer';
+
+const SUPPORTER_IMAGE_PATH = resolve(__dirname, '../images/supporters');
+
+const SUPPORTER_QUERY = `query account($limit: Int, $offset: Int, $slug: String) {
   account(slug: $slug) {
     orders(limit: $limit, offset: $offset) {
       limit
@@ -61,7 +66,9 @@ const query = `query account($limit: Int, $offset: Int, $slug: String) {
   }
 }`;
 
-const graphqlPageSize = 1000;
+const GRAPHQL_PAGE_SIZE = 1000;
+
+const invalidSupporters = [];
 
 const nodeToSupporter = node => ({
   id: node.fromAccount.id,
@@ -75,6 +82,30 @@ const nodeToSupporter = node => ({
   type: node.fromAccount.type
 });
 
+const fetchImage = async supporter => {
+  try {
+    const {avatar: url} = supporter;
+    const {body: imageBuf} = await needle('get', url);
+    debug('fetched %s', url);
+    const canvasImage = await loadImage(imageBuf);
+    debug('ok %s', url);
+    supporter.dimensions = {
+      width: canvasImage.width,
+      height: canvasImage.height
+    };
+    debug('dimensions %s %dw %dh', url, canvasImage.width, canvasImage.height);
+    const filePath = resolve(SUPPORTER_IMAGE_PATH, supporter.id + '.png');
+    await writeFile(filePath, imageBuf);
+    debug('wrote %s', filePath);
+  } catch (err) {
+    console.error(
+      `failed to load ${supporter.avatar}; will discard ${supporter.tier} "${supporter.name} (${supporter.slug}). reason:\n`,
+      err
+    );
+    invalidSupporters.push(supporter);
+  }
+};
+
 /**
  * Retrieves donation data from OC
  *
@@ -84,26 +115,26 @@ const nodeToSupporter = node => ({
  */
 const getAllOrders = async (slug = 'mochajs') => {
   let allOrders = [];
-  const variables = {limit: graphqlPageSize, offset: 0, slug};
+  const variables = {limit: GRAPHQL_PAGE_SIZE, offset: 0, slug};
 
   // Handling pagination if necessary (2 pages for ~1400 results in May 2019)
   while (true) {
     const result = await needle(
       'post',
       API_ENDPOINT,
-      {query, variables},
+      {query: SUPPORTER_QUERY, variables},
       {json: true}
     );
     const orders = result.body.data.account.orders.nodes;
     allOrders = [...allOrders, ...orders];
-    variables.offset += graphqlPageSize;
-    if (orders.length < graphqlPageSize) {
+    variables.offset += GRAPHQL_PAGE_SIZE;
+    if (orders.length < GRAPHQL_PAGE_SIZE) {
       debug('retrieved %d orders', allOrders.length);
       return allOrders;
     } else {
       debug(
         'loading page %d of orders...',
-        Math.floor(variables.offset / graphqlPageSize)
+        Math.floor(variables.offset / GRAPHQL_PAGE_SIZE)
       );
     }
   }
@@ -143,7 +174,8 @@ const getSupporters = async () => {
               ...supporters.backers,
               {
                 ...supporter,
-                avatar: encodeURI(supporter.imgUrlSmall)
+                avatar: encodeURI(supporter.imgUrlSmall),
+                tier: BACKER_TIER
               }
             ];
           }
@@ -152,7 +184,8 @@ const getSupporters = async () => {
             ...supporters.sponsors,
             {
               ...supporter,
-              avatar: encodeURI(supporter.imgUrlMed)
+              avatar: encodeURI(supporter.imgUrlMed),
+              tier: SPONSOR_TIER
             }
           ];
         }
@@ -164,38 +197,34 @@ const getSupporters = async () => {
       }
     );
 
-  const supporterImagePath = resolve(__dirname, '../images/supporters');
-
-  await mkdir(supporterImagePath, {recursive: true});
+  await rmdir(SUPPORTER_IMAGE_PATH, {recursive: true});
+  debug('blasted %s', SUPPORTER_IMAGE_PATH);
+  await mkdir(SUPPORTER_IMAGE_PATH, {recursive: true});
+  debug('created %s', SUPPORTER_IMAGE_PATH);
 
   // Fetch images for sponsors and save their image dimensions
-  await Promise.all(
-    supporters.sponsors.map(async sponsor => {
-      const filePath = resolve(supporterImagePath, `${sponsor.id}.png`);
-      const {body} = await needle('get', sponsor.avatar);
-      sponsor.dimensions = imageSize(body);
-      await writeFile(filePath, body);
-    })
-  );
-
-  // Fetch images for backers and save their image dimensions
-  await Promise.all(
-    supporters.backers.map(async backer => {
-      const filePath = resolve(supporterImagePath, `${backer.id}.png`);
-      const {body} = await needle('get', backer.avatar);
-      await writeFile(filePath, body);
-    })
-  );
+  await Promise.all([
+    ...supporters.sponsors.map(fetchImage),
+    ...supporters.backers.map(fetchImage)
+  ]);
+
+  invalidSupporters.forEach(supporter => {
+    supporters[supporter.tier].splice(
+      supporters[supporter.tier].indexOf(supporter),
+      1
+    );
+  });
 
   const backerCount = supporters.backers.length;
   const sponsorCount = supporters.sponsors.length;
   const totalValidSupportersCount = backerCount + sponsorCount;
 
   debug(
-    'found %d valid backers and %d valid sponsors (of %d total; %d blocked)',
+    'found %d valid backers and %d valid sponsors (%d total; %d invalid; %d blocked)',
     backerCount,
     sponsorCount,
     totalValidSupportersCount,
+    invalidSupporters.length,
     uniqueSupporters.size - totalValidSupportersCount
   );
   return supporters;

diff --git a/netlify.toml b/netlify.toml
@@ -13,6 +13,7 @@
   command = "npm start docs.production"
 
 [build.environment]
+  DEBUG = "mocha:docs*"
   NODE_VERSION = "12"
   RUBY_VERSION = "2.7.1"
 

diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -90,6 +90,7 @@
     "assetgraph-builder": "^8.0.1",
     "autoprefixer": "^9.7.4",
     "babel-eslint": "^10.1.0",
+    "canvas": "^2.6.1",
     "chai": "^4.2.0",
     "coffee-script": "^1.12.7",
     "core-js": "^3.6.5",
@@ -108,7 +109,6 @@
     "fs-extra": "^9.0.0",
     "husky": "^4.2.3",
     "hyperlink": "^4.4.3",
-    "image-size": "^0.8.3",
     "jsdoc": "^3.6.3",
     "karma": "^4.4.1",
     "karma-chrome-launcher": "^3.1.0",