Add FXIOS-10594 - Add attachments support to RS and pull `tracking-pr…

…otection-lists-ios` (#23199)

* chore: change config to account for attachments

* chore: update remote settings script to pull in attachments

* chore: commit all changes in RemoteSettingsData

* fix: move changed files from tmp dir

* fix: adjust path for attachments

* chore: manually run script

* chore: update swift schema and fix failing tests

* fix: linting issue

Co-authored-by: Nishant Bhasin <[email protected]>

* fix: linting issue

Co-authored-by: Nishant Bhasin <[email protected]>

* fix: linting issue

Co-authored-by: Nishant Bhasin <[email protected]>

* fix: another linting issue

* fix: do not format json files

---------

Co-authored-by: Nishant Bhasin <[email protected]>

.github/workflows/firefox-ios-update_remote_settings_data_script.yml

@@ -44,7 +44,7 @@ jobs:
       - name: Add changes
         run: |
           git diff
-          git diff --quiet || git add firefox-ios/Client/Assets/RemoteSettingsData/RemotePasswordRules.json
+          git diff --quiet || git add firefox-ios/Client/Assets/RemoteSettingsData/
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 

firefox-ios/Client/Application/RemoteSettings/RemoteSettingsFetchConfig.swift

@@ -5,19 +5,23 @@
 import Foundation
 
 struct RemoteSettingsFetchConfig: Codable, Equatable {
-    let rules: [Rule]
+    let collections: [Collection]
 
-    struct Rule: Codable, Equatable {
+    struct Collection: Codable, Equatable {
         let name: String
         let url: String
-        let file: String
+        var file: String?
         let bucketID: String
-        let collectionsID: String
+        let collectionID: String
+        var fetchAttachments: Bool?
+        var saveRecords: Bool?
 
         enum CodingKeys: String, CodingKey {
             case name, url, file
             case bucketID = "bucket_id"
-            case collectionsID = "collections_id"
+            case collectionID = "collection_id"
+            case fetchAttachments = "fetch_attachments"
+            case saveRecords = "save_records"
         }
     }
 

firefox-ios/Client/Assets/RemoteSettingsData/RemoteSettingsFetchConfig.json

@@ -1,11 +1,19 @@
 {
-  "rules": [
+  "collections": [
     {
       "name": "Password Rules",
-      "url": "https://firefox.settings.services.mozilla.com/v1/buckets/main/collections/password-rules/records",
+      "url": "https://firefox.settings.services.mozilla.com/v1",
       "file": "./firefox-ios/Client/Assets/RemoteSettingsData/RemotePasswordRules.json",
       "bucket_id": "main",
-      "collections_id": "password-rules"
+      "collection_id": "password-rules"
+    },
+    {
+      "name": "Tracking Protection Lists",
+      "url": "https://remote-settings-dev.allizom.org/v1",
+      "bucket_id": "main-preview",
+      "collection_id": "tracking-protection-lists-ios",
+      "fetch_attachments": true,
+      "save_records": false
     }
   ]
 }

firefox-ios/Client/Assets/RemoteSettingsData/Update_Remote_Settings.py

@@ -3,23 +3,38 @@
 import shutil
 import os
 import json
+import mimetypes
 
 CONFIG_FILE = "./firefox-ios/Client/Assets/RemoteSettingsData/RemoteSettingsFetchConfig.json"
-GITHUB_ACTIONS_PATH = "./firefox-ios/Client/Assets/RemoteSettingsData/"
-GITHUB_ACTIONS_TMP_PATH = f"{GITHUB_ACTIONS_PATH}tmp/"
+RS_DATA_PATH = "./firefox-ios/Client/Assets/RemoteSettingsData/"
+GITHUB_ACTIONS_TMP_PATH = f"{RS_DATA_PATH}tmp/"
 
-def fetch_rules(url):
-    response = requests.get(url)
-    if response.status_code == 200:
-        return response.json()["data"]
-    else:
-        print(f"Failed to fetch rules from {url}: {response.status_code}")
+def fetch(url):
+    try:
+        response = requests.get(url)
+        response.raise_for_status()
+        return response
+    except requests.exceptions.RequestException as e:
+        print(f"Failed to fetch data from {url}: {e}")
         return None
 
-def save_tmp_rules(rules, tmp_file_path):
-    with open(tmp_file_path, 'w') as tmp_file:
-        json.dump(rules, tmp_file, indent=4)
-    print(f"Saved temporary rules to {tmp_file_path}")
+def save_content(file_content, file_path, file_mimetype = None):
+    if file_mimetype == "application/json":
+        mode, data = 'w', json.dumps(file_content, indent=4)
+    else:
+        mode, data = 'w', str(file_content)
+    os.makedirs(os.path.dirname(file_path), exist_ok=True)
+    with open(file_path, mode) as file:
+        file.write(data)
+    print(f"Saved file: {file_path}")
+
+def replace_file_with_dirs(tmp_file_path, target_file_path):
+    # Ensure the parent directory of the target path exists
+    target_dir = os.path.dirname(target_file_path)
+    os.makedirs(target_dir, exist_ok=True)
+    # Replace the file
+    os.replace(tmp_file_path, target_file_path)
+
 
 def update_settings_file(tmp_file_path, target_file_path, name):
     if os.path.exists(target_file_path):
@@ -28,34 +43,85 @@ def update_settings_file(tmp_file_path, target_file_path, name):
             os.remove(tmp_file_path)
             return False
         else:
-            os.replace(tmp_file_path, target_file_path)
+            replace_file_with_dirs(tmp_file_path, target_file_path)
             print(f"Updated {target_file_path} with new rules for {name}.")
             return True
     else:
-        os.replace(tmp_file_path, target_file_path)
+        replace_file_with_dirs(tmp_file_path, target_file_path)
         print(f"Created new rules file {target_file_path} for {name}.")
         return True
 
+
+def fetch_records_attachments(records, collection, base_url):
+    changes_detected = False
+    for record in records:
+        attachment = record.get("attachment", None)
+        if attachment:
+            attachment_subdir =  os.path.join("attachments", collection["collection_id"])
+            attachment_extension = mimetypes.guess_extension(attachment["mimetype"])
+            attachment_file_name = f"{record['name']}{attachment_extension}"
+            attachment_file_tmp_path = os.path.join(GITHUB_ACTIONS_TMP_PATH, attachment_subdir, attachment_file_name)
+            attachment_file_path = os.path.join(RS_DATA_PATH, attachment_subdir, attachment_file_name)
+            attachment_url = f"{base_url}{attachment['location']}"
+            attachment_content = fetch(attachment_url).text
+
+            if attachment_content:
+                save_content(attachment_content, attachment_file_tmp_path)
+                if update_settings_file(attachment_file_tmp_path, attachment_file_path, record['name']):
+                    changes_detected = True
+    return changes_detected
+
 def main():
     if not os.path.exists(GITHUB_ACTIONS_TMP_PATH):
         os.makedirs(GITHUB_ACTIONS_TMP_PATH, exist_ok=True)
 
     with open(CONFIG_FILE, 'r') as config_file:
         config = json.load(config_file)
-    
+
     changes_detected = False
-
-    for rule in config["rules"]:
-        print(f"Fetching rules for {rule['name']} from {rule['url']}")
-        rules = fetch_rules(rule["url"])
-        if rules:
-            tmp_file_path = os.path.join(GITHUB_ACTIONS_TMP_PATH, os.path.basename(rule["file"]))
-            save_tmp_rules(rules, tmp_file_path)
-            if update_settings_file(tmp_file_path, rule["file"], rule["name"]):
+
+    for collection in config["collections"]:
+        print(f"Fetching rules for {collection['name']} from {collection['url']}")
+        records_url = f"{collection['url']}/buckets/{collection['bucket_id']}/collections/{collection['collection_id']}/records"
+        response = fetch(records_url).json()
+        records = response.get("data", None)
+
+        # If no records found, skip to next collection
+        if not records:
+            print(f"No records found for {collection['name']}.")
+            continue
+
+        # Fetch attachments only if config has `fetch_attachments` is True
+        # 1. Get base url from the server url = .capabilities.attachments.base_url
+        # 2. For each record if it has an .attachment, fetch it = base_url + attachment.location.url
+        # 3. Use record.name as the file name = record.name + type of attachment ( from mimetype )
+        # 4. Save attachments in attachments/{collection_id}
+        fetch_attachments = collection.get("fetch_attachments", False)
+        base_url = None
+        if fetch_attachments:
+            base_url = (
+                fetch(collection["url"])
+                .json()
+                .get("capabilities", {})
+                .get("attachments", {})
+                .get("base_url", None)
+            )
+
+            if not base_url:
+                print(f"Attachment base URL not found for {collection['url']}")
+                continue
+            changes_detected |= fetch_records_attachments(records, collection, base_url)
+
+        # Don't save records if config has `save_records` set to False
+        save_records = collection.get("save_records", True)
+        if save_records:
+            tmp_file_path = os.path.join(GITHUB_ACTIONS_TMP_PATH, os.path.basename(collection["file"]))
+            save_content(records, tmp_file_path, "application/json")
+            if update_settings_file(tmp_file_path, collection["file"], collection["name"]):
                 changes_detected = True
-    
+
     if not changes_detected:
-        print("No changes detected in any rules.")
+        print("No changes detected in any collections.")
 
 if __name__ == "__main__":
     main()