import adblock-resources repo as a library

lib/adBlockRustUtils.js

@@ -1,5 +1,7 @@
 import { Engine, FilterSet, uBlockResources } from 'adblock-rs'
 
+import { listCatalog, readResources } from 'adblock-resources'
+
 import path from 'path'
 import { promises as fs } from 'fs'
 
@@ -10,33 +12,12 @@ const uBlockWebAccessibleResources = path.join(uBlockLocalRoot, 'src/web_accessi
 const uBlockRedirectEngine = path.join(uBlockLocalRoot, 'src/js/redirect-resources.js')
 const uBlockScriptlets = path.join(uBlockLocalRoot, 'assets/resources/scriptlets.js')
 
-const braveResourcesUrl = 'https://raw.githubusercontent.com/brave/adblock-resources/master/dist/resources.json'
-
-const listCatalogUrl = 'https://raw.githubusercontent.com/brave/adblock-resources/master/filter_lists/list_catalog.json'
-
 const regionalCatalogComponentId = 'gkboaolpopklhgplhaaiboijnklogmbc'
 const regionalCatalogPubkey = 'MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsAnb1lw5UA1Ww4JIVE8PjKNlPogAdFoie+Aczk6ppQ4OrHANxz6oAk1xFuT2W3uhGOc3b/1ydIUMqOIdRFvMdEDUvKVeFyNAVXNSouFF7EBLEzcZfFtqoxeIbwEplVISUm+WUbsdVB9MInY3a4O3kNNuUijY7bmHzAqWMTrBfenw0Lqv38OfREXCiNq/+Jm/gt7FhyBd2oviXWEGp6asUwNavFnj8gQDGVvCf+dse8HRMJn00QH0MOypsZSWFZRmF08ybOu/jTiUo/TuIaHL1H8y9SR970LqsUMozu3ioSHtFh/IVgq7Nqy4TljaKsTE+3AdtjiOyHpW9ZaOkA7j2QIDAQAB'
 
 const resourcesComponentId = 'mfddibmblmbccpadfndgakiopmmhebop'
 const resourcesPubkey = 'MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA7Qk6xtml8Siq8RD6cCbdJpArt0kMci82W/KYw3KR96y67MZAsKJa8rOV2WC1BIpW539Qgl5b5lMS04cjw+sSB7f2ZKM1WOqKNij24nvEKVubunP32u8tbjtzQk9VYNcM2MZMs330eqk7iuBRTvRViSMSeE3ymqp03HFpUGsdtjEBh1A5lroCg41eVnMn1I4GKPvuhT/Qc9Yem5gzXT/3n7H6vOGQ2dVBHz44mhgwtiDcsduh+Det6lCE2TgHOhHPdCewklgcoiNXP4zfXxfpPy1jbwb4w5KUnHSRelhfDnt+jI3jgHsD4IXdVNE5H5ZAnmcOJttbkRiT8kOVS0rJXwIDAQAB'
 
-/**
- * Returns a promise that which resolves with the body parsed as JSON
- *
- * @param url The URL to fetch from
- * @return a promise that resolves with the content of the list or rejects with an error message.
- */
-const requestJSON = (url) => {
-  return fetch(url).then(response => {
-    if (response.status !== 200) {
-      throw new Error(`Error status ${response.status} ${response.statusText} returned for URL: ${url}`)
-    }
-    return response.json()
-  }).catch(error => {
-    throw new Error(`Error when fetching ${url}: ${error.message}`)
-  })
-}
-
 const lazyInit = (fn) => {
   let prom
   return () => {
@@ -45,19 +26,11 @@ const lazyInit = (fn) => {
   }
 }
 
-const getListCatalog = lazyInit(async () => {
-  return requestJSON(listCatalogUrl)
-})
-
 // Legacy logic requires a distinction between default and regional lists.
 // This can be removed once DAT support is no longer needed by iOS.
 const isDefaultList = entry => entry.default_enabled && entry.hidden
-const getDefaultLists = () => getListCatalog().then(catalog => {
-  return catalog.filter(isDefaultList)
-})
-const getRegionalLists = () => getListCatalog().then(catalog => {
-  return catalog.filter(entry => !isDefaultList(entry))
-})
+const getDefaultLists = () => listCatalog.filter(isDefaultList)
+const getRegionalLists = () => listCatalog.filter(entry => !isDefaultList(entry))
 
 // Wraps new template scriptlets with the older "numbered template arg" format and any required dependency code
 const wrapScriptletArgFormat = (fnString, dependencyPrelude) => `{
@@ -114,7 +87,7 @@ const generateResources = lazyInit(async () => {
     uBlockRedirectEngine
   )
 
-  const braveResources = await requestJSON(braveResourcesUrl)
+  const braveResources = readResources()
   resourceData.push(...braveResources)
   resourceData.push(...transformedUboBuiltins)
   return JSON.stringify(resourceData)
@@ -315,7 +288,6 @@ export {
   resourcesPubkey,
   sanityCheckList,
   generateResourcesFile,
-  getListCatalog,
   getDefaultLists,
   getRegionalLists
 }

package.json

@@ -9,6 +9,7 @@
     "@mongodb-js/zstd": "^1.2.0",
     "@sentry/node": "7.119.0",
     "adblock-rs": "0.8.12",
+    "adblock-resources": "github:brave/adblock-resources#master",
     "brave-site-specific-scripts": "github:brave/brave-site-specific-scripts",
     "playlist-component": "github:brave/playlist-component",
     "recursive-readdir-sync": "1.0.6",

scripts/generateAdBlockRustDataFiles.js

@@ -4,14 +4,14 @@
 
 import {
   generateResourcesFile,
-  getListCatalog,
   getDefaultLists,
   getRegionalLists,
   preprocess,
   resourcesComponentId,
   regionalCatalogComponentId,
   sanityCheckList
 } from '../lib/adBlockRustUtils.js'
+import { listCatalog } from 'adblock-resources'
 import Sentry from '../lib/sentry.js'
 import util from '../lib/util.js'
 import path from 'path'
@@ -120,27 +120,28 @@ const generateDataFilesForCatalogEntry = (entry) => {
  */
 const generateDataFilesForAllRegions = () => {
   console.log('Processing per region list updates...')
-  return getRegionalLists().then(regions => {
-    return new Promise((resolve, reject) => {
+  const regions = getRegionalLists()
+  return new Promise((resolve, reject) => {
+    {
       const catalogString = JSON.stringify(regions)
       fs.writeFileSync(getOutPath('regional_catalog.json', regionalCatalogComponentId), catalogString)
-      getListCatalog().then(listCatalog => {
-        const catalogString = JSON.stringify(listCatalog)
-        fs.writeFileSync(getOutPath('list_catalog.json', regionalCatalogComponentId), catalogString)
-        resolve()
-      })
-    }).then(() => Promise.all(regions.map(region =>
-      generateDataFilesForCatalogEntry(region)
-    )))
-  })
+    }
+    {
+      const catalogString = JSON.stringify(listCatalog)
+      fs.writeFileSync(getOutPath('list_catalog.json', regionalCatalogComponentId), catalogString)
+    }
+    resolve()
+  }).then(() => Promise.all(regions.map(region =>
+    generateDataFilesForCatalogEntry(region)
+  )))
 }
 
 const generateDataFilesForResourcesComponent = () => {
   return generateResourcesFile(getOutPath('resources.json', resourcesComponentId))
 }
 
-const generateDataFilesForDefaultAdblock = () => getDefaultLists()
-  .then(defaultLists => Promise.all(defaultLists.map(list => generateDataFilesForCatalogEntry(list))))
+const generateDataFilesForDefaultAdblock = () =>
+  Promise.all(getDefaultLists().map(list => generateDataFilesForCatalogEntry(list)))
 
 generateDataFilesForDefaultAdblock()
   .then(generateDataFilesForResourcesComponent)