chore(ci): add i18n checks, json lint, file license check plus a few …

…json fixes (#498)

Description
Add Json lint
Add i18n checks
Add file license check
Plus some JSon fixes found

Motivation and Context
Improve i18n process
Catch json errors

How Has This Been Tested?
Build and run in local fork

.github/workflows/ci.yml

@@ -125,6 +125,71 @@ jobs:
         run: |
           cargo check --release --all-targets --workspace --locked
 
+  file-licenses:
+    # disable for now
+    if: ${{ false }}
+    name: file-licenses
+    runs-on: [ ubuntu-latest ]
+    steps:
+      - name: checkout
+        uses: actions/checkout@v4
+      - name: install ripgrep
+        run: |
+          # https://github.com/BurntSushi/ripgrep/releases/download/14.1.1/ripgrep_14.1.1-1_amd64.deb.sha256
+          wget -v https://github.com/BurntSushi/ripgrep/releases/download/14.1.1/ripgrep_14.1.1-1_amd64.deb
+          sudo dpkg -i ripgrep_14.1.1-1_amd64.deb
+          rg --version || exit 1
+      - name: run the license check
+        run: ./scripts/file_license_check.sh
+
+  i18n-checks:
+    name: i18n-checks
+    runs-on: [ ubuntu-latest ]
+    steps:
+      - name: checkout
+        uses: actions/checkout@v4
+
+      - name: install jsonlint
+        run: |
+          sudo apt-get update
+          sudo apt-get install --no-install-recommends --assume-yes \
+            python3-demjson
+
+      - name: basic jsonlint
+        run: |
+          find public -iname '*.json'  -print0 | \
+            xargs -0 -n1 jsonlint -v
+
+      - name: setup folder for logs
+        run: |
+          mkdir -p ${{ runner.temp }}/i18n_logs
+
+      - name: i18n compare
+        continue-on-error: true
+        working-directory: ./public/locales
+        run: |
+          python3 ../../scripts/i18n_checker.py \
+            compare --en-locale-path en \
+            --base-path . \
+            --search-path .. \
+            --output-dir ${{ runner.temp }}/i18n_logs
+
+      - name: i18n unused
+        continue-on-error: true
+        working-directory: ./public/locales
+        run: |
+          python3 ../../scripts/i18n_checker.py \
+            unused --en-locale-path en \
+            --base-path . \
+            --search-path .. \
+            --output-dir ${{ runner.temp }}/i18n_logs
+
+      - name: Artifact upload for i18n checks
+        uses: actions/upload-artifact@v4
+        with:
+          name: i18n-logs
+          path: ${{ runner.temp }}/i18n_logs
+
   tauri-build:
     name: tauri-build
     runs-on: [ ubuntu-latest ]

public/locales/en/sidebar.json

@@ -6,3 +6,4 @@
     "setup-scheduler": "Setup Scheduler",
     "mining-schedules": "Mining Schedules",
     "mining-schedules-description": "Schedule your mining activity"
+}

public/locales/pl/setup-view.json

@@ -4,7 +4,7 @@
     "dont-worry": "Nie martw się, następnym razem nie zajmie to tak długo",
     "title": {
         "starting-up": "Uruchamianie",
-        "checking-latest-version-mmproxy": "Sprawdzanie najnowszej wersji xtrgpuminer",
+        "checking-latest-version-gpuminer": "Sprawdzanie najnowszej wersji xtrgpuminer",
         "checking-latest-version-node": "Sprawdzanie najnowszej wersji węzła",
         "checking-latest-version-mmproxy": "Sprawdzanie najnowszej wersji mmproxy",
         "checking-latest-version-wallet": "Sprawdzanie najnowszej wersji portfela",

scripts/file_license_check.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+#
+# Must be run from the repo root
+#
+
+set -e
+
+diffparms=${diffparms:-"-u --suppress-blank-empty --strip-trailing-cr --color=never"}
+rgTemp=${rgTemp:-$(mktemp)}
+
+# Exclude files without extensions as well as those with extensions that are not in the list
+#
+rg -i "Copyright.*The Tari Project" --files-without-match \
+    -g '!*.{Dockerfile,asc,bat,config,config.js,css,csv,drawio,env,gitkeep,hbs,html,ini,iss,json,lock,md,min.js,ps1,py,rc,scss,sh,sql,svg,toml,txt,yml,vue,ts,tsx}' . \
+    | while IFS= read -r file; do
+        if [[ -n $(basename "$file" | grep -E '\.') ]]; then
+            echo "$file"
+        fi
+    done | sort > ${rgTemp}
+
+# Sort the .license.ignore file as sorting seems to behave differently on different platforms
+licenseIgnoreTemp=${licenseIgnoreTemp:-$(mktemp)}
+cat .license.ignore | sort > ${licenseIgnoreTemp}
+
+DIFFS=$( diff ${diffparms} ${licenseIgnoreTemp} ${rgTemp} || true )
+
+# clean up
+rm -vf ${rgTemp}
+rm -vf ${licenseIgnoreTemp}
+
+if [ -n "${DIFFS}" ]; then
+    echo "New files detected that either need copyright/license identifiers added, "
+    echo "or they need to be added to .license.ignore"
+    echo "NB: The ignore file must be sorted alphabetically!"
+
+    echo "Diff:"
+    echo "${DIFFS}"
+    exit 1
+fi

scripts/i18n_checker.py

@@ -0,0 +1,164 @@
+
+# https://gist.github.com/metalaureate/b4bc4f11c1f47a8100e319884c4edd70
+
+import os
+import json
+import csv
+import argparse
+
+# Function to recursively find JSON files
+def find_json_files(base_directory):
+    json_files = []
+    for root, _, files in os.walk(base_directory):
+        for file in files:
+            if file.endswith(".json"):
+                json_files.append(os.path.join(root, file))
+    return json_files
+
+# Function to load a JSON file and return its content along with the file name
+def load_json(file_path):
+    with open(file_path, 'r', encoding='utf-8') as f:
+        return json.load(f), os.path.basename(file_path)
+
+# Function to load keys from all English JSON files
+def load_en_keys(en_json_files):
+    all_keys = set()
+    for json_file in en_json_files:
+        with open(json_file, 'r', encoding='utf-8') as f:
+            data = json.load(f)
+            all_keys.update(data.keys())
+    return all_keys
+
+# Function to compare the English keys with other locale keys
+def compare_keys(en_data, other_locale_data):
+    en_keys = set(en_data.keys())
+    other_keys = set(other_locale_data.keys())
+
+    missing_keys = en_keys - other_keys
+    extraneous_keys = other_keys - en_keys
+
+    return missing_keys, extraneous_keys
+
+# Function to search for keys in all files recursively under the search path
+def search_for_key_in_files(key, search_path):
+    for root, _, files in os.walk(search_path):
+        for file in files:
+            file_path = os.path.join(root, file)
+            try:
+                with open(file_path, 'r', encoding='utf-8', errors='ignore') as f:
+                    contents = f.read()
+                    if key in contents:
+                        return True  # Key is found
+            except:
+                pass  # Skip unreadable files
+    return False  # Key not found
+
+# Function to write a CSV for English labels
+def write_english_labels_csv(en_data, output_file):
+    with open(output_file, mode='w', newline='', encoding='utf-8') as file:
+        writer = csv.writer(file, quoting=csv.QUOTE_ALL)
+        writer.writerow(["label_key", "value", "json_file"])
+        for (label_key, json_file), value in en_data.items():
+            writer.writerow([label_key, value, json_file])
+
+# Function to write a consolidated comparison CSV for all locales
+def write_consolidated_comparison_csv(comparison_data, output_file):
+    with open(output_file, mode='w', newline='', encoding='utf-8') as file:
+        writer = csv.writer(file, quoting=csv.QUOTE_ALL)
+        writer.writerow(["locale", "status", "label_key", "json_file"])
+        for locale, data in comparison_data.items():
+            for key, json_file in data['missing_keys']:
+                writer.writerow([locale, "missing", key, json_file])
+            for key, json_file in data['extraneous_keys']:
+                writer.writerow([locale, "extraneous", key, json_file])
+
+# Function to write unused keys to a CSV file
+def write_unused_keys_to_csv(unused_keys, output_file):
+    with open(output_file, mode='w', newline='', encoding='utf-8') as file:
+        writer = csv.writer(file, quoting=csv.QUOTE_ALL)
+        writer.writerow(["unused_key"])
+        for key in unused_keys:
+            writer.writerow([key])
+
+# Unused keys function
+def find_unused_keys(en_locale_path, search_base_path, output_dir):
+    en_json_files = find_json_files(en_locale_path)
+    all_en_keys = load_en_keys(en_json_files)
+
+    unused_keys = []
+    for key in all_en_keys:
+        print(f"Searching for key: {key}")
+        if not search_for_key_in_files(key, search_base_path):
+            unused_keys.append(key)
+
+    output_file = os.path.join(output_dir, 'unused_keys.csv')
+    write_unused_keys_to_csv(unused_keys, output_file)
+    print(f"Unused keys written to {output_file}")
+
+# Key comparison function
+def compare_keys_in_locales(base_path, en_path, output_dir):
+    en_files = find_json_files(en_path)
+    all_en_data = {}
+    for en_file in en_files:
+        en_data, json_file = load_json(en_file)
+        all_en_data.update({(key, json_file): value for key, value in en_data.items()})
+
+    english_labels_output_file = os.path.join(output_dir, 'english_labels.csv')
+    write_english_labels_csv(all_en_data, english_labels_output_file)
+
+    comparison_data = {}
+    other_locales = [d for d in os.listdir(base_path) if os.path.isdir(os.path.join(base_path, d)) and d != 'en']
+
+    for locale in other_locales:
+        locale_path = os.path.join(base_path, locale)
+        locale_files = find_json_files(locale_path)
+        locale_data = {}
+        for locale_file in locale_files:
+            locale_json_data, json_file = load_json(locale_file)
+            locale_data.update({(key, json_file): value for key, value in locale_json_data.items()})
+
+        en_keys = set(key for key, _ in all_en_data.keys())
+        locale_keys = set(key for key, _ in locale_data.keys())
+        missing_keys = en_keys - locale_keys
+        extraneous_keys = locale_keys - en_keys
+
+        missing_keys_with_file = [(key, file) for (key, file) in all_en_data.keys() if key in missing_keys]
+        extraneous_keys_with_file = [(key, file) for (key, file) in locale_data.keys() if key in extraneous_keys]
+
+        comparison_data[locale] = {
+            'missing_keys': missing_keys_with_file,
+            'extraneous_keys': extraneous_keys_with_file
+        }
+
+    consolidated_output_file = os.path.join(output_dir, 'locale_key_comparison_consolidated.csv')
+    write_consolidated_comparison_csv(comparison_data, consolidated_output_file)
+    print(f"Comparison CSV written to {consolidated_output_file}")
+
+# Main function to parse arguments and invoke appropriate functions
+def main():
+    parser = argparse.ArgumentParser(description="CLI tool for comparing locale JSON keys and finding unused keys.")
+    parser.add_argument("mode", choices=["compare", "unused"], help="Mode of operation: 'compare' or 'unused'.")
+    parser.add_argument("--en-locale-path", required=True, help="Path to the English locale directory.")
+    parser.add_argument("--base-path", required=True, help="Base path for all locales (for comparison).")
+    parser.add_argument("--output-dir", required=True, help="Directory to store the output CSV files.")
+    parser.add_argument("--search-path", required=False, help="Path to search for unused keys (required for 'unused' mode).")
+
+    args = parser.parse_args()
+
+    if args.mode == "compare":
+        compare_keys_in_locales(args.base_path, args.en_locale_path, args.output_dir)
+    elif args.mode == "unused":
+        if not args.search_path:
+            print("Error: --search-path is required for 'unused' mode.")
+            return
+        find_unused_keys(args.en_locale_path, args.search_path, args.output_dir)
+
+if __name__ == "__main__":
+    main()
+
+# python tool.py compare --en-locale-path /path/to/en --base-path /path/to/locales --output-dir /path/to/output
+# python tool.py unused --en-locale-path /path/to/en --base-path /path/to/locales --search-path /path/to/src --output-dir /path/to/output
+
+# Example usage:
+# python i18n_checker.py compare --en-locale-path /Users/possum/Projects/tari/universe/public/locales/en --base-path /Users/possum/Projects/tari/universe/public/locales --output-dir ~/Downloads
+# python i18n_checker.py unused --en-locale-path /Users/possum/Projects/tari/universe/public/locales/en --base-path /Users/possum/Projects/tari/universe/public/locales --search-path /Users/possum/Projects/tari/universe/src --output-dir ~/Downloads