Support unified adblock catalog

lib/adBlockRustUtils.js

@@ -10,8 +10,7 @@ const uBlockScriptlets = path.join(uBlockLocalRoot, 'assets/resources/scriptlets
 
 const braveResourcesUrl = 'https://raw.githubusercontent.com/brave/adblock-resources/master/dist/resources.json'
 
-const defaultListsUrl = 'https://raw.githubusercontent.com/brave/adblock-resources/master/filter_lists/default.json'
-const regionalListsUrl = 'https://raw.githubusercontent.com/brave/adblock-resources/master/filter_lists/regional.json'
+const listCatalogUrl = 'https://raw.githubusercontent.com/brave/adblock-resources/master/filter_lists/list_catalog.json'
 
 const regionalCatalogComponentId = 'gkboaolpopklhgplhaaiboijnklogmbc'
 const regionalCatalogPubkey = 'MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsAnb1lw5UA1Ww4JIVE8PjKNlPogAdFoie+Aczk6ppQ4OrHANxz6oAk1xFuT2W3uhGOc3b/1ydIUMqOIdRFvMdEDUvKVeFyNAVXNSouFF7EBLEzcZfFtqoxeIbwEplVISUm+WUbsdVB9MInY3a4O3kNNuUijY7bmHzAqWMTrBfenw0Lqv38OfREXCiNq/+Jm/gt7FhyBd2oviXWEGp6asUwNavFnj8gQDGVvCf+dse8HRMJn00QH0MOypsZSWFZRmF08ybOu/jTiUo/TuIaHL1H8y9SR970LqsUMozu3ioSHtFh/IVgq7Nqy4TljaKsTE+3AdtjiOyHpW9ZaOkA7j2QIDAQAB'
@@ -36,9 +35,6 @@ const requestJSON = (url) => {
   })
 }
 
-const getDefaultLists = requestJSON.bind(null, defaultListsUrl)
-const getRegionalLists = requestJSON.bind(null, regionalListsUrl)
-
 const lazyInit = (fn) => {
   let prom
   return () => {
@@ -47,6 +43,20 @@ const lazyInit = (fn) => {
   }
 }
 
+const getListCatalog = lazyInit(async () => {
+  return requestJSON(listCatalogUrl)
+})
+
+// Legacy logic requires a distinction between default and regional lists.
+// This can be removed once DAT support is no longer needed by iOS.
+const isDefaultList = entry => entry.default_enabled && entry.hidden
+const getDefaultLists = () => getListCatalog().then(catalog => {
+  return catalog.filter(isDefaultList)
+})
+const getRegionalLists = () => getListCatalog().then(catalog => {
+  return catalog.filter(entry => !isDefaultList(entry))
+})
+
 // Wraps new template scriptlets with the older "numbered template arg" format and any required dependency code
 const wrapScriptletArgFormat = (fnString, dependencyPrelude) => `{
   const args = ['{{1}}', '{{2}}', '{{3}}', '{{4}}', '{{5}}', '{{6}}', '{{7}}', '{{8}}', '{{9}}'];
@@ -123,6 +133,7 @@ export {
   resourcesComponentId,
   resourcesPubkey,
   generateResourcesFile,
+  getListCatalog,
   getDefaultLists,
   getRegionalLists
 }

lib/util.js

@@ -386,6 +386,13 @@ const addCommonScriptOptions = (command) => {
     .option('-r, --region <region>', 'The AWS region to use', 'us-west-2')
 }
 
+const escapeStringForJSON = str => {
+  if (typeof str !== 'string') {
+    throw new Error('Not a string: ' + JSON.stringify(str))
+  }
+  return JSON.stringify(str).slice(1, -1)
+}
+
 export default {
   fetchTextFromURL,
   createTableIfNotExists,
@@ -398,5 +405,6 @@ export default {
   parseManifest,
   uploadCRXFile,
   updateDBForCRXFile,
-  addCommonScriptOptions
+  addCommonScriptOptions,
+  escapeStringForJSON
 }

scripts/generateAdBlockRustDataFiles.js

@@ -3,7 +3,7 @@
  * You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 import { Engine, FilterFormat, FilterSet, RuleTypes } from 'adblock-rs'
-import { generateResourcesFile, getDefaultLists, getRegionalLists, resourcesComponentId, regionalCatalogComponentId } from '../lib/adBlockRustUtils.js'
+import { generateResourcesFile, getListCatalog, getDefaultLists, getRegionalLists, resourcesComponentId, regionalCatalogComponentId } from '../lib/adBlockRustUtils.js'
 import util from '../lib/util.js'
 import path from 'path'
 import fs from 'fs'
@@ -113,7 +113,11 @@ const generateDataFilesForAllRegions = () => {
       const catalogString = JSON.stringify(regions)
       fs.writeFileSync(getOutPath('regional_catalog.json', 'default'), catalogString)
       fs.writeFileSync(getOutPath('regional_catalog.json', regionalCatalogComponentId), catalogString)
-      resolve()
+      getListCatalog().then(listCatalog => {
+        const catalogString = JSON.stringify(listCatalog)
+        fs.writeFileSync(getOutPath('list_catalog.json', regionalCatalogComponentId), catalogString)
+        resolve()
+      })
     }).then(() => Promise.all(regions.map(region =>
       generateDataFilesForCatalogEntry(region)
     )))

scripts/generateManifestForRustAdblock.js

@@ -5,39 +5,11 @@
 import { promises as fs } from 'fs'
 import path from 'path'
 
-import { getRegionalLists, regionalCatalogComponentId, regionalCatalogPubkey, resourcesComponentId, resourcesPubkey } from '../lib/adBlockRustUtils.js'
+import { getListCatalog, regionalCatalogComponentId, regionalCatalogPubkey, resourcesComponentId, resourcesPubkey } from '../lib/adBlockRustUtils.js'
+import util from '../lib/util.js'
 
 const outPath = path.join('build', 'ad-block-updater')
 
-const defaultAdblockBase64PublicKey =
-    'MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAs0qzJmHSgIiw7IGFCxij' +
-    '1NnB5hJ5ZQ1LKW9htL4EBOaMJvmqaDs/wfq0nw/goBHWsqqkMBynRTu2Hxxirvdb' +
-    'cugn1Goys5QKPgAvKwDHJp9jlnADWm5xQvPQ4GE1mK1/I3ka9cEOCzPW6GI+wGLi' +
-    'VPx9VZrxHHsSBIJRaEB5Tyi5bj0CZ+kcfMnRTsXIBw3C6xJgCVKISQUkd8mawVvG' +
-    'vqOhBOogCdb9qza5eJ1Cgx8RWKucFfaWWxKLOelCiBMT1Hm1znAoVBHG/blhJJOD' +
-    '5HcH/heRrB4MvrE1J76WF3fvZ03aHVcnlLtQeiNNOZ7VbBDXdie8Nomf/QswbBGa' +
-    'VwIDAQAB'
-
-const defaultPlaintextComponentId = 'iodkpdagapdfkphljnddpjlldadblomo'
-const defaultPlaintextPubkey =
-    'MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsD/B/MGdz0gh7WkcFARn' +
-    'ZTBX9KAw2fuGeogijoI+fET38IK0L+P/trCT2NshqhRNmrDpLzV2+Dmes6PvkA+O' +
-    'dQkUV6VbChJG+baTfr3Oo5PdE0WxmP9Xh8XD7p85DQrk0jJilKuElxpK7Yq0JhcT' +
-    'Sc3XNHeTwBVqCnHwWZZ+XysYQfjuDQ0MgQpS/s7U04OZ63NIPe/iCQm32stvS/pE' +
-    'ya7KdBZXgRBQ59U6M1n1Ikkp3vfECShbBld6VrrmNrl59yKWlEPepJ9oqUc2Wf2M' +
-    'q+SDNXROG554RnU4BnDJaNETTkDTZ0Pn+rmLmp1qY5Si0yGsfHkrv3FS3vdxVozO' +
-    'PQIDAQAB'
-
-const exceptionPlaintextComponentId = 'adcocjohghhfpidemphmcmlmhnfgikei'
-const exceptionPlaintextPubkey =
-    'MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtvmLp4MOseThuH/vFSc7' +
-    'kjr+CDCzR/ieGI8TJZyFQhzA1SKWRl4y0wB+HGkmoq0KPOzKNZq6hxK7jdm/r/nx' +
-    'xOjqutPoUEL+ysxePErMTse2XeWu3psGSTEjPFdQTPEwH8MF2SwXXneOraD0V/GS' +
-    'iCCvlx8yKIXNX7V9ujMo+QoD6hPGslKUZQJAg+OaZ7pAfq5cOuWXNN6jv12UL0eM' +
-    't6Dhl31yEu4kZWeTkiccHqdlB/KvPiqXTrV+qd3Tjvsk6kmUlexu3/zlOwVDz5H/' +
-    'kPuOGvW7kYaW22NWQ9TH6fjffgVcSgHDbZETDiP8fHd76kyi1SZ5YJ09XHTE+i9i' +
-    'kQIDAQAB'
-
 const generateManifestFile = async (name, base64PublicKey, uuid) => {
   const manifest = '{\n' +
                  '  "description": "Brave Ad Block Updater extension",\n' +
@@ -51,37 +23,26 @@ const generateManifestFile = async (name, base64PublicKey, uuid) => {
   return fs.writeFile(filePath, manifest)
 }
 
-const generateManifestFileForDefaultAdblock =
-  generateManifestFile.bind(null, 'Default', defaultAdblockBase64PublicKey, 'default')
-
-const generateManifestFileForDefaultPlaintextAdblock =
-  generateManifestFile.bind(null, 'Default (plaintext)', defaultPlaintextPubkey, defaultPlaintextComponentId)
-
-const generateManifestFileForExceptionAdblock =
-  generateManifestFile.bind(null, 'Exception-exceptions (plaintext)', exceptionPlaintextPubkey, exceptionPlaintextComponentId)
-
 const generateManifestFileForRegionalCatalog =
   generateManifestFile.bind(null, 'Regional Catalog', regionalCatalogPubkey, regionalCatalogComponentId)
 
 const generateManifestFileForResources =
   generateManifestFile.bind(null, 'Resources', resourcesPubkey, resourcesComponentId)
 
-const generateManifestFilesForAllRegions = async () => {
-  const regionalLists = await getRegionalLists()
-  return Promise.all(regionalLists.map(async region => {
-    await generateManifestFile(region.title, region.base64_public_key, region.uuid, region)
-    if (region.list_text_component) {
-      await generateManifestFile(region.title + ' (plaintext)', region.list_text_component.base64_public_key, region.list_text_component.component_id)
+const generateManifestFilesForAllLists = async () => {
+  const catalog = await getListCatalog()
+  return Promise.all(catalog.map(async entry => {
+    const title = util.escapeStringForJSON(entry.title)
+    await generateManifestFile(title, entry.base64_public_key, entry.uuid)
+    if (entry.list_text_component) {
+      await generateManifestFile(title + ' (plaintext)', entry.list_text_component.base64_public_key, entry.list_text_component.component_id)
     }
   }))
 }
 
-generateManifestFileForDefaultAdblock()
-  .then(generateManifestFileForDefaultPlaintextAdblock)
-  .then(generateManifestFileForExceptionAdblock)
-  .then(generateManifestFileForRegionalCatalog)
+generateManifestFileForRegionalCatalog()
   .then(generateManifestFileForResources)
-  .then(generateManifestFilesForAllRegions)
+  .then(generateManifestFilesForAllLists)
   .then(() => {
     console.log('Thank you for updating the data files, don\'t forget to upload them too!')
   })