From b66a80ac50fa81cd811dd111b6b91d6f3b4f2370 Mon Sep 17 00:00:00 2001 From: Anton Lazarev Date: Wed, 30 Aug 2023 12:19:48 -0700 Subject: [PATCH] support unified adblock catalog --- lib/adBlockRustUtils.js | 21 ++++++-- lib/util.js | 10 +++- scripts/generateAdBlockRustDataFiles.js | 8 ++- scripts/generateManifestForRustAdblock.js | 61 ++++------------------- 4 files changed, 42 insertions(+), 58 deletions(-) diff --git a/lib/adBlockRustUtils.js b/lib/adBlockRustUtils.js index 3fe9eb12..1adc62b0 100644 --- a/lib/adBlockRustUtils.js +++ b/lib/adBlockRustUtils.js @@ -10,8 +10,7 @@ const uBlockScriptlets = path.join(uBlockLocalRoot, 'assets/resources/scriptlets const braveResourcesUrl = 'https://raw.githubusercontent.com/brave/adblock-resources/master/dist/resources.json' -const defaultListsUrl = 'https://raw.githubusercontent.com/brave/adblock-resources/master/filter_lists/default.json' -const regionalListsUrl = 'https://raw.githubusercontent.com/brave/adblock-resources/master/filter_lists/regional.json' +const listCatalogUrl = 'https://raw.githubusercontent.com/brave/adblock-resources/master/filter_lists/list_catalog.json' const regionalCatalogComponentId = 'gkboaolpopklhgplhaaiboijnklogmbc' const regionalCatalogPubkey = 'MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsAnb1lw5UA1Ww4JIVE8PjKNlPogAdFoie+Aczk6ppQ4OrHANxz6oAk1xFuT2W3uhGOc3b/1ydIUMqOIdRFvMdEDUvKVeFyNAVXNSouFF7EBLEzcZfFtqoxeIbwEplVISUm+WUbsdVB9MInY3a4O3kNNuUijY7bmHzAqWMTrBfenw0Lqv38OfREXCiNq/+Jm/gt7FhyBd2oviXWEGp6asUwNavFnj8gQDGVvCf+dse8HRMJn00QH0MOypsZSWFZRmF08ybOu/jTiUo/TuIaHL1H8y9SR970LqsUMozu3ioSHtFh/IVgq7Nqy4TljaKsTE+3AdtjiOyHpW9ZaOkA7j2QIDAQAB' @@ -36,9 +35,6 @@ const requestJSON = (url) => { }) } -const getDefaultLists = requestJSON.bind(null, defaultListsUrl) -const getRegionalLists = requestJSON.bind(null, regionalListsUrl) - const lazyInit = (fn) => { let prom return () => { @@ -47,6 +43,20 @@ const lazyInit = (fn) => { } } +const getListCatalog = lazyInit(async () => { + return requestJSON(listCatalogUrl) +}) + +// Legacy logic requires a distinction between default and regional lists. +// This can be removed once DAT support is no longer needed by iOS. +const isDefaultList = entry => entry.default_enabled && entry.hidden +const getDefaultLists = () => getListCatalog().then(catalog => { + return catalog.filter(isDefaultList) +}) +const getRegionalLists = () => getListCatalog().then(catalog => { + return catalog.filter(entry => !isDefaultList(entry)) +}) + // Wraps new template scriptlets with the older "numbered template arg" format and any required dependency code const wrapScriptletArgFormat = (fnString, dependencyPrelude) => `{ const args = ['{{1}}', '{{2}}', '{{3}}', '{{4}}', '{{5}}', '{{6}}', '{{7}}', '{{8}}', '{{9}}']; @@ -123,6 +133,7 @@ export { resourcesComponentId, resourcesPubkey, generateResourcesFile, + getListCatalog, getDefaultLists, getRegionalLists } diff --git a/lib/util.js b/lib/util.js index fc7d47b2..babf80fd 100644 --- a/lib/util.js +++ b/lib/util.js @@ -386,6 +386,13 @@ const addCommonScriptOptions = (command) => { .option('-r, --region ', 'The AWS region to use', 'us-west-2') } +const escapeStringForJSON = str => { + if (typeof str !== 'string') { + throw new Error('Not a string: ' + JSON.stringify(str)) + } + return JSON.stringify(str).slice(1, -1) +} + export default { fetchTextFromURL, createTableIfNotExists, @@ -398,5 +405,6 @@ export default { parseManifest, uploadCRXFile, updateDBForCRXFile, - addCommonScriptOptions + addCommonScriptOptions, + escapeStringForJSON } diff --git a/scripts/generateAdBlockRustDataFiles.js b/scripts/generateAdBlockRustDataFiles.js index 07877032..890e50cb 100644 --- a/scripts/generateAdBlockRustDataFiles.js +++ b/scripts/generateAdBlockRustDataFiles.js @@ -3,7 +3,7 @@ * You can obtain one at http://mozilla.org/MPL/2.0/. */ import { Engine, FilterFormat, FilterSet, RuleTypes } from 'adblock-rs' -import { generateResourcesFile, getDefaultLists, getRegionalLists, resourcesComponentId, regionalCatalogComponentId } from '../lib/adBlockRustUtils.js' +import { generateResourcesFile, getListCatalog, getDefaultLists, getRegionalLists, resourcesComponentId, regionalCatalogComponentId } from '../lib/adBlockRustUtils.js' import util from '../lib/util.js' import path from 'path' import fs from 'fs' @@ -113,7 +113,11 @@ const generateDataFilesForAllRegions = () => { const catalogString = JSON.stringify(regions) fs.writeFileSync(getOutPath('regional_catalog.json', 'default'), catalogString) fs.writeFileSync(getOutPath('regional_catalog.json', regionalCatalogComponentId), catalogString) - resolve() + getListCatalog().then(listCatalog => { + const catalogString = JSON.stringify(listCatalog) + fs.writeFileSync(getOutPath('list_catalog.json', regionalCatalogComponentId), catalogString) + resolve() + }) }).then(() => Promise.all(regions.map(region => generateDataFilesForCatalogEntry(region) ))) diff --git a/scripts/generateManifestForRustAdblock.js b/scripts/generateManifestForRustAdblock.js index 1dabb4bc..3fa7af10 100644 --- a/scripts/generateManifestForRustAdblock.js +++ b/scripts/generateManifestForRustAdblock.js @@ -5,39 +5,11 @@ import { promises as fs } from 'fs' import path from 'path' -import { getRegionalLists, regionalCatalogComponentId, regionalCatalogPubkey, resourcesComponentId, resourcesPubkey } from '../lib/adBlockRustUtils.js' +import { getListCatalog, regionalCatalogComponentId, regionalCatalogPubkey, resourcesComponentId, resourcesPubkey } from '../lib/adBlockRustUtils.js' +import util from '../lib/util.js' const outPath = path.join('build', 'ad-block-updater') -const defaultAdblockBase64PublicKey = - 'MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAs0qzJmHSgIiw7IGFCxij' + - '1NnB5hJ5ZQ1LKW9htL4EBOaMJvmqaDs/wfq0nw/goBHWsqqkMBynRTu2Hxxirvdb' + - 'cugn1Goys5QKPgAvKwDHJp9jlnADWm5xQvPQ4GE1mK1/I3ka9cEOCzPW6GI+wGLi' + - 'VPx9VZrxHHsSBIJRaEB5Tyi5bj0CZ+kcfMnRTsXIBw3C6xJgCVKISQUkd8mawVvG' + - 'vqOhBOogCdb9qza5eJ1Cgx8RWKucFfaWWxKLOelCiBMT1Hm1znAoVBHG/blhJJOD' + - '5HcH/heRrB4MvrE1J76WF3fvZ03aHVcnlLtQeiNNOZ7VbBDXdie8Nomf/QswbBGa' + - 'VwIDAQAB' - -const defaultPlaintextComponentId = 'iodkpdagapdfkphljnddpjlldadblomo' -const defaultPlaintextPubkey = - 'MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsD/B/MGdz0gh7WkcFARn' + - 'ZTBX9KAw2fuGeogijoI+fET38IK0L+P/trCT2NshqhRNmrDpLzV2+Dmes6PvkA+O' + - 'dQkUV6VbChJG+baTfr3Oo5PdE0WxmP9Xh8XD7p85DQrk0jJilKuElxpK7Yq0JhcT' + - 'Sc3XNHeTwBVqCnHwWZZ+XysYQfjuDQ0MgQpS/s7U04OZ63NIPe/iCQm32stvS/pE' + - 'ya7KdBZXgRBQ59U6M1n1Ikkp3vfECShbBld6VrrmNrl59yKWlEPepJ9oqUc2Wf2M' + - 'q+SDNXROG554RnU4BnDJaNETTkDTZ0Pn+rmLmp1qY5Si0yGsfHkrv3FS3vdxVozO' + - 'PQIDAQAB' - -const exceptionPlaintextComponentId = 'adcocjohghhfpidemphmcmlmhnfgikei' -const exceptionPlaintextPubkey = - 'MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtvmLp4MOseThuH/vFSc7' + - 'kjr+CDCzR/ieGI8TJZyFQhzA1SKWRl4y0wB+HGkmoq0KPOzKNZq6hxK7jdm/r/nx' + - 'xOjqutPoUEL+ysxePErMTse2XeWu3psGSTEjPFdQTPEwH8MF2SwXXneOraD0V/GS' + - 'iCCvlx8yKIXNX7V9ujMo+QoD6hPGslKUZQJAg+OaZ7pAfq5cOuWXNN6jv12UL0eM' + - 't6Dhl31yEu4kZWeTkiccHqdlB/KvPiqXTrV+qd3Tjvsk6kmUlexu3/zlOwVDz5H/' + - 'kPuOGvW7kYaW22NWQ9TH6fjffgVcSgHDbZETDiP8fHd76kyi1SZ5YJ09XHTE+i9i' + - 'kQIDAQAB' - const generateManifestFile = async (name, base64PublicKey, uuid) => { const manifest = '{\n' + ' "description": "Brave Ad Block Updater extension",\n' + @@ -51,37 +23,26 @@ const generateManifestFile = async (name, base64PublicKey, uuid) => { return fs.writeFile(filePath, manifest) } -const generateManifestFileForDefaultAdblock = - generateManifestFile.bind(null, 'Default', defaultAdblockBase64PublicKey, 'default') - -const generateManifestFileForDefaultPlaintextAdblock = - generateManifestFile.bind(null, 'Default (plaintext)', defaultPlaintextPubkey, defaultPlaintextComponentId) - -const generateManifestFileForExceptionAdblock = - generateManifestFile.bind(null, 'Exception-exceptions (plaintext)', exceptionPlaintextPubkey, exceptionPlaintextComponentId) - const generateManifestFileForRegionalCatalog = generateManifestFile.bind(null, 'Regional Catalog', regionalCatalogPubkey, regionalCatalogComponentId) const generateManifestFileForResources = generateManifestFile.bind(null, 'Resources', resourcesPubkey, resourcesComponentId) -const generateManifestFilesForAllRegions = async () => { - const regionalLists = await getRegionalLists() - return Promise.all(regionalLists.map(async region => { - await generateManifestFile(region.title, region.base64_public_key, region.uuid, region) - if (region.list_text_component) { - await generateManifestFile(region.title + ' (plaintext)', region.list_text_component.base64_public_key, region.list_text_component.component_id) +const generateManifestFilesForAllLists = async () => { + const catalog = await getListCatalog() + return Promise.all(catalog.map(async entry => { + const title = util.escapeStringForJSON(entry.title) + await generateManifestFile(title, entry.base64_public_key, entry.uuid) + if (entry.list_text_component) { + await generateManifestFile(title + ' (plaintext)', entry.list_text_component.base64_public_key, entry.list_text_component.component_id) } })) } -generateManifestFileForDefaultAdblock() - .then(generateManifestFileForDefaultPlaintextAdblock) - .then(generateManifestFileForExceptionAdblock) - .then(generateManifestFileForRegionalCatalog) +generateManifestFileForRegionalCatalog() .then(generateManifestFileForResources) - .then(generateManifestFilesForAllRegions) + .then(generateManifestFilesForAllLists) .then(() => { console.log('Thank you for updating the data files, don\'t forget to upload them too!') })