dedupers: allow deduping all lists recursively

.github/workflows/build-and-release.yml

@@ -11,29 +11,28 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: [2.7.17, 3.6.7, 3.8.0]
+        include:
+          - python: python3
+            pip: pip3
+          - python: python2
+            pip: pip
     steps:
       - name: Checkout
         uses: actions/checkout@v2
         with:
           python-version: ${{ matrix.python-version }}
           fetch-depth: 0
 
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
-          architecture: x64
-
       - name: Install python dependencies
         run: |
-          python -m pip install --upgrade pip
-          pip install setuptools wheel
-          pip install -e .[tests,elasticsearch5]
+          ${{ matrix.pip }} install --user --upgrade pip
+          ${{ matrix.pip }} --no-cache-dir install --user setuptools wheel "urllib3==1.25.11"
+          ${{ matrix.pip }} --no-cache-dir install --user -e .[tests,elasticsearch5]
 
       - name: Show python dependencies
         run: |
-          pip freeze
+          ${{ matrix.python }} --version
+          ${{ matrix.pip }} freeze
 
       - name: Run tests
         run: |

inspire_utils/dedupers.py

@@ -68,3 +68,30 @@ def _freeze(o):
             seen.add(f)
 
     return result
+
+
+def dedupe_all_lists(obj, exclude_keys=(), dedupe_top_level=True):
+    """Recursively remove duplicates from all lists.
+
+    Args:
+        obj: collection to deduplicate
+        exclude_keys (Container[str]): key names to ignore for deduplication
+        dedupe_top_level (bool): whether the top-level list should be deduplicated too
+    """
+    squared_dedupe_len = 10
+    if isinstance(obj, dict):
+        new_obj = {}
+        for key, value in obj.items():
+            dedupe_top_level = key not in exclude_keys
+            new_obj[key] = dedupe_all_lists(value, exclude_keys, dedupe_top_level)
+        return new_obj
+    elif isinstance(obj, (list, tuple, set)):
+        new_obj = [dedupe_all_lists(v, exclude_keys) for v in obj]
+        if dedupe_top_level:
+            if len(new_obj) < squared_dedupe_len:
+                new_obj = dedupe_list(new_obj)
+            else:
+                new_obj = dedupe_list_of_dicts(new_obj)
+        return type(obj)(new_obj)
+    else:
+        return obj

setup.cfg

@@ -19,4 +19,4 @@ include = inspire_utils/*.py
 addopts = --cov=inspire_utils --cov-report=term-missing:skip-covered
 
 [flake8]
-ignore = *.py E501 FI12 FI14 FI15 FI16 FI17 FI50 FI51 FI53 W504 FI18
+ignore = E501 FI12 FI14 FI15 FI16 FI17 FI50 FI51 FI53 W504 FI18

tests/test_dedupers.py

@@ -22,7 +22,7 @@
 
 from __future__ import absolute_import, division, print_function
 
-from inspire_utils.dedupers import dedupe_list, dedupe_list_of_dicts
+from inspire_utils.dedupers import dedupe_list, dedupe_list_of_dicts, dedupe_all_lists
 
 
 def test_dedupe_list():
@@ -45,3 +45,47 @@ def test_dedupe_list_of_dicts():
     result = dedupe_list_of_dicts(list_of_dicts_with_duplicates)
 
     assert expected == result
+
+
+def test_dedupe_all_lists():
+    obj = {
+        "l0": list(range(10)) + list(range(10)),
+        "o1": [{"foo": "bar"}] * 10,
+        "o2": [{"foo": [1, 2]}, {"foo": [1, 1, 2]}] * 10,
+    }
+
+    expected = {"l0": list(range(10)), "o1": [{"foo": "bar"}], "o2": [{"foo": [1, 2]}]}
+
+    assert dedupe_all_lists(obj) == expected
+
+
+def test_dedupe_all_lists_honors_exclude_keys():
+    obj = {
+        "l0": list(range(10)) + list(range(10)),
+        "o1": [{"foo": "bar"}] * 10,
+        "o2": [{"foo": [1, 2]}, {"foo": [1, 1, 2]}] * 10,
+    }
+
+    expected = {
+        "l0": list(range(10)),
+        "o1": [{"foo": "bar"}] * 10,
+        "o2": [{"foo": [1, 2]}],
+    }
+
+    assert dedupe_all_lists(obj, exclude_keys=["o1"]) == expected
+
+
+def test_dedupe_all_lists_dedupes_under_excluded_keys():
+    obj = {
+        "l0": list(range(10)) + list(range(10)),
+        "o1": [{"foo": "bar"}] * 10,
+        "o2": [{"foo": [1, 2]}, {"foo": [1, 1, 2]}] * 10,
+    }
+
+    expected = {
+        "l0": list(range(10)),
+        "o1": [{"foo": "bar"}],
+        "o2": [{"foo": [1, 2]}] * 20,
+    }
+
+    assert dedupe_all_lists(obj, exclude_keys=["o2"]) == expected