From 1f0d67f07e7c5aee8d8d4cb6f5c2f7492614dc05 Mon Sep 17 00:00:00 2001
From: Kshitij Sobti <kshitij@sobti.in>
Date: Wed, 3 Jun 2020 03:17:51 +0530
Subject: [PATCH 1/3] Backport migration code to v1

---
 completion_aggregator/__init__.py             |   2 +-
 completion_aggregator/api/common.py           |   4 +-
 .../management/commands/migrate_progress.py   |  56 +-----
 .../tasks/aggregation_tasks.py                | 124 +++++---------
 requirements/base.in                          |   8 +-
 requirements/constraints.txt                  |   1 +
 requirements/dev.txt                          | 162 ++++++++++--------
 requirements/doc.txt                          | 108 ++++++------
 requirements/quality.in                       |   4 +-
 requirements/quality.txt                      |  51 +++---
 requirements/test.in                          |   1 +
 requirements/test.txt                         | 112 +++++++-----
 requirements/travis.txt                       |  43 +++--
 .../migrations/0003_coursemodulecompletion.py |  35 ++++
 test_utils/test_app/models.py                 |  17 ++
 tests/test_tasks.py                           | 118 +++++++++++++
 16 files changed, 501 insertions(+), 345 deletions(-)
 create mode 100644 requirements/constraints.txt
 create mode 100644 test_utils/test_app/migrations/0003_coursemodulecompletion.py
 create mode 100644 tests/test_tasks.py

diff --git a/completion_aggregator/__init__.py b/completion_aggregator/__init__.py
index 9de75b2e..bf66eb23 100644
--- a/completion_aggregator/__init__.py
+++ b/completion_aggregator/__init__.py
@@ -4,6 +4,6 @@
 
 from __future__ import absolute_import, unicode_literals
 
-__version__ = '1.5.25'
+__version__ = '1.6.0'
 
 default_app_config = 'completion_aggregator.apps.CompletionAggregatorAppConfig'  # pylint: disable=invalid-name
diff --git a/completion_aggregator/api/common.py b/completion_aggregator/api/common.py
index 2a10d6b7..b6e24baf 100644
--- a/completion_aggregator/api/common.py
+++ b/completion_aggregator/api/common.py
@@ -93,9 +93,9 @@ def authentication_classes(self):  # pragma: no cover
         """
         from openedx.core.lib.api import authentication  # pylint: disable=import-error
         try:
-            from edx_rest_framework_extensions.auth.jwt.authentication import JwtAuthentication  # pylint: disable=import-error
+            from edx_rest_framework_extensions.auth.jwt.authentication import JwtAuthentication
         except ImportError:
-            from edx_rest_framework_extensions.authentication import JwtAuthentication  # pylint: disable=import-error
+            from edx_rest_framework_extensions.authentication import JwtAuthentication
 
         return [
             JwtAuthentication,
diff --git a/completion_aggregator/management/commands/migrate_progress.py b/completion_aggregator/management/commands/migrate_progress.py
index 2daadbd3..b46e66fc 100644
--- a/completion_aggregator/management/commands/migrate_progress.py
+++ b/completion_aggregator/management/commands/migrate_progress.py
@@ -3,19 +3,11 @@
 """
 
 import logging
-import time
 
-from django.core.management.base import BaseCommand, CommandError
+from django.core.management.base import BaseCommand
 
 from ...tasks import aggregation_tasks
 
-try:
-    from progress.models import CourseModuleCompletion
-    PROGRESS_IMPORTED = True
-except ImportError:
-    PROGRESS_IMPORTED = False
-
-
 log = logging.getLogger(__name__)
 
 
@@ -38,60 +30,28 @@ def add_arguments(self, parser):
             default=10000,
             type=int,
         )
-        parser.add_argument(
-            '--start-index',
-            help='Offset from which to start processing CourseModuleCompletions. (default: 0)',
-            default=0,
-            type=int,
-        )
-        parser.add_argument(
-            '--stop-index',
-            help='Offset at which to stop processing CourseModuleCompletions. (default: process all)',
-            default=0,
-            type=int,
-        )
         parser.add_argument(
             '--delay-between-tasks',
             help='Amount of time to wait between submitting tasks in seconds.  (default: 0.0)',
             default=0.0,
             type=float,
         )
-        parser.add_argument(
-            '--ids',
-            help='Migrate specific CourseModuleCompletion IDs',
-        )
 
     def handle(self, *args, **options):
-        if not PROGRESS_IMPORTED:
-            raise CommandError("Unable to import progress models.  Aborting")
         self._configure_logging(options)
         task_options = self.get_task_options(options)
 
-        if options['ids']:
-            migrate_ids = [int(id_) for id_ in options['ids'].split(',')]
-            migrate_ids.sort()
-            for index in migrate_ids:
-                aggregation_tasks.migrate_batch.apply_async(
-                    kwargs={'start': index, 'stop': index + 1},
-                    **task_options
-                )
-                time.sleep(options['delay_between_tasks'])
-        else:
-            cmc_max_id = CourseModuleCompletion.objects.all().order_by('-id')[:1][0].id
-            cmc_min_id = CourseModuleCompletion.objects.all().order_by('id')[:1][0].id
-            start = max(options['start_index'], cmc_min_id)
-            stop = min(cmc_max_id + 1, options['stop_index'] or float('inf'))
-            for index in range(start, stop, options['batch_size']):
-                aggregation_tasks.migrate_batch.apply_async(
-                    kwargs={'start': index, 'stop': min(stop, index + options['batch_size'])},
-                    **task_options
-                )
-                time.sleep(options['delay_between_tasks'])
+        aggregation_tasks.migrate_batch.apply_async(
+            kwargs={
+                'batch_size': options['batch_size'],
+                'delay_between_tasks': options['delay_between_tasks'],
+            },
+            **task_options
+        )
 
     def get_task_options(self, options):
         """
         Return task options for generated celery tasks.
-
         Currently, this adds a routing key, if provided.
         """
         opts = {}
diff --git a/completion_aggregator/tasks/aggregation_tasks.py b/completion_aggregator/tasks/aggregation_tasks.py
index 22377f80..2b26027e 100644
--- a/completion_aggregator/tasks/aggregation_tasks.py
+++ b/completion_aggregator/tasks/aggregation_tasks.py
@@ -5,10 +5,10 @@
 from __future__ import absolute_import, division, print_function, unicode_literals
 
 import logging
+import time
 
 from celery import shared_task
 from celery_utils.logged_task import LoggedTask
-from opaque_keys import InvalidKeyError
 from opaque_keys.edx.keys import CourseKey, UsageKey
 
 from django.contrib.auth.models import User
@@ -17,38 +17,16 @@
 from .. import core
 from ..models import StaleCompletion
 
-try:
-    from progress.models import CourseModuleCompletion
-    PROGRESS_IMPORTED = True
-except ImportError:
-    PROGRESS_IMPORTED = False
-
-
-# SQLite doesn't support the ON DUPLICATE KEY syntax.  INSERT OR REPLACE will
-# have a similar effect, but uses new primary keys.  The drawbacks of this are:
-# * It will consume the available keyspace more quickly.
-# * It will not preserve foreign keys pointing to our table.
-# SQLite is only used in testing environments, so neither of these drawbacks
-# poses an actual problem.
-
-INSERT_OR_UPDATE_MYSQL = """
-    INSERT INTO completion_blockcompletion
-        (user_id, course_key, block_key, block_type, completion)
-    VALUES
-        (%s, %s, %s, %s, 1.0)
-    ON DUPLICATE KEY UPDATE
-        completion=VALUES(completion);
+UPDATE_SQL = """
+UPDATE completion_blockcompletion completion, progress_coursemodulecompletion progress
+   SET completion.created = progress.created,
+       completion.modified = progress.modified
+ WHERE completion.user_id = progress.user_id
+   AND completion.block_key = progress.content_id
+   AND completion.course_key = progress.course_id
+   AND completion.id IN %(ids)s;
 """
 
-INSERT_OR_UPDATE_SQLITE = """
-    INSERT OR REPLACE
-    INTO completion_blockcompletion
-        (user_id, course_key, block_key, block_type, completion)
-    VALUES
-        (%s, %s, %s, %s, 1.0);
-"""
-
-
 log = logging.getLogger(__name__)
 
 
@@ -80,65 +58,49 @@ def update_aggregators(username, course_key, block_keys=(), force=False):
 
     course_key = CourseKey.from_string(course_key)
     block_keys = set(UsageKey.from_string(key).map_into_course(course_key) for key in block_keys)
-    log.info("Updating aggregators in %s for %s. Changed blocks: %s", course_key, user.username, block_keys)
+    log.info(
+        "Updating aggregators in %s for %s. Changed blocks: %s", course_key, user.username, block_keys,
+    )
     return core.update_aggregators(user, course_key, block_keys, force)
 
 
 @shared_task
-def migrate_batch(start, stop):  # Cannot pass a queryset to a task.
+def migrate_batch(batch_size, delay_between_tasks):
     """
-    Convert a batch of CourseModuleCompletions to BlockCompletions.
+    Wraps _migrate_batch to simplify testing.
+    """
+    _migrate_batch(batch_size, delay_between_tasks)
 
-    Given a starting ID and a stopping ID, this task will:
 
+def _migrate_batch(batch_size, delay_between_tasks):
+    """
+    Convert a batch of CourseModuleCompletions to BlockCompletions.
+    Given a starting ID and a stopping ID, this task will:
     * Fetch all CourseModuleCompletions with an ID in range(start_id, stop_id).
     * Update the BlockCompletion table with those CourseModuleCompletion
       records.
     """
-    if not PROGRESS_IMPORTED:
-        log.error("Cannot perform migration: CourseModuleCompletion not importable.")
-
-    queryset = CourseModuleCompletion.objects.all().select_related('user')
-    course_module_completions = queryset.filter(id__gte=start, id__lt=stop)
-
-    processed = {}  # Dict has format: {course: {user: [blocks]}
-    insert_params = []
-    for cmc in course_module_completions:
-        try:
-            course_key = CourseKey.from_string(cmc.course_id)
-            block_key = UsageKey.from_string(cmc.content_id).map_into_course(course_key)
-            block_type = block_key.block_type
-        except InvalidKeyError:
-            log.exception(
-                "Could not migrate CourseModuleCompletion with values: %s",
-                cmc.__dict__,
-            )
-            continue
-        if course_key not in processed:
-            processed[course_key] = set()
-        if cmc.user not in processed[course_key]:
-            processed[course_key].add(cmc.user)
-        # Param order: (user_id, course_key, block_key, block_type)
-        insert_params.append((cmc.user_id, cmc.course_id, cmc.content_id, block_type))
-    if connection.vendor == 'mysql':
-        sql = INSERT_OR_UPDATE_MYSQL
-    else:
-        sql = INSERT_OR_UPDATE_SQLITE
-    with connection.cursor() as cur:
-        cur.executemany(sql, insert_params)
-    # Create aggregators later.
-    stale_completions = []
-    for course_key in processed:
-        for user in processed[course_key]:
-            stale_completions.append(
-                StaleCompletion(
-                    username=user.username,
-                    course_key=course_key,
-                    block_key=None,
-                    force=True
+
+    def get_next_id_batch():
+        while True:
+            with connection.cursor() as cur:
+                count = cur.execute(
+                    """
+                    SELECT id
+                    FROM completion_blockcompletion
+                    WHERE NOT completion_blockcompletion.modified
+                    LIMIT %(batch_size)s;
+                    """,
+                    {"batch_size": batch_size},
                 )
-            )
-    StaleCompletion.objects.bulk_create(
-        stale_completions,
-    )
-    log.info("Completed progress migration batch from %s to %s", start, stop)
+                ids = [row[0] for row in cur.fetchall()]
+                if count == 0:
+                    break
+            yield ids
+
+    with connection.cursor() as cur:
+        count = 0
+        for ids in get_next_id_batch():
+            count = cur.execute(UPDATE_SQL, {"ids": ids},)
+            time.sleep(delay_between_tasks)
+        log.info("Completed progress updation batch of %s objects", count)
diff --git a/requirements/base.in b/requirements/base.in
index 4fd17e19..58e2889e 100644
--- a/requirements/base.in
+++ b/requirements/base.in
@@ -5,9 +5,11 @@ edx-celeryutils<0.2      # Custom task extensions
 Django>=1.8              # Web application framework
 django-oauth-toolkit<1.0
 djangorestframework>=3.0,<3.7     # API tools
-django-model-utils        # Provides TimeStampedModel abstract base class
-edx-opaque-keys>=0.4.2    # Provides CourseKey and UsageKey
+django-model-utils<3.2        # Provides TimeStampedModel abstract base class
+edx-opaque-keys<2,>=0.4.2    # Provides CourseKey and UsageKey
 edx-completion>=1.0.3,<2
 mysqlclient               # For connecting to MySQL
 six
-XBlock
+XBlock<1.3
+jsonfield<2.1
+django-braces<1.14
diff --git a/requirements/constraints.txt b/requirements/constraints.txt
new file mode 100644
index 00000000..0835179a
--- /dev/null
+++ b/requirements/constraints.txt
@@ -0,0 +1 @@
+futures; python_version < "3"
diff --git a/requirements/dev.txt b/requirements/dev.txt
index 78fb651f..aec63478 100644
--- a/requirements/dev.txt
+++ b/requirements/dev.txt
@@ -2,100 +2,114 @@
 # This file is autogenerated by pip-compile
 # To update, run:
 #
-#    pip-compile --no-index --output-file requirements/dev.txt requirements/base.in requirements/dev.in requirements/quality.in
+#    pip-compile --no-index --output-file=requirements/dev.txt requirements/base.in requirements/dev.in requirements/quality.in
 #
 amqp==1.4.9               # via kombu
 anyjson==0.3.3            # via kombu
-appdirs==1.4.3            # via fs
+appdirs==1.4.4            # via fs, virtualenv
 argparse==1.4.0           # via caniusepython3
-astroid==1.5.3            # via edx-lint, pylint, pylint-celery
-backports.functools-lru-cache==1.5  # via caniusepython3
+astroid==1.5.3            # via pylint, pylint-celery
+backports.functools-lru-cache==1.6.1  # via astroid, caniusepython3, isort, pylint
+backports.os==0.1.1       # via fs, path.py
 billiard==3.3.0.23        # via celery
-bleach==3.1.0             # via readme-renderer
-caniusepython3==7.0.0
-celery==3.1.18
-certifi==2019.3.9         # via requests
+bleach==3.1.5             # via readme-renderer
+caniusepython3==7.2.0     # via -r requirements/quality.in
+celery==3.1.18            # via -r requirements/base.in, edx-celeryutils
+certifi==2020.4.5.1       # via requests
 chardet==3.0.4            # via requests
-click-log==0.1.8          # via edx-lint
-click==7.0                # via click-log, edx-lint, pip-tools
-diff-cover==1.0.6
-distlib==0.2.8            # via caniusepython3
-django-braces==1.13.0     # via django-oauth-toolkit
-django-model-utils==3.1.2
-django-oauth-toolkit==0.12.0
-django-waffle==0.15.1     # via edx-django-utils, edx-drf-extensions
-django==1.10.8
+click-log==0.3.2          # via edx-lint
+click==7.1.2              # via click-log, edx-lint, pip-tools
+configparser==4.0.2       # via importlib-metadata, pydocstyle, pylint
+contextlib2==0.6.0.post1  # via importlib-metadata, importlib-resources, virtualenv, zipp
+diff-cover==2.6.1         # via -r requirements/dev.in
+distlib==0.3.0            # via caniusepython3, virtualenv
+django-braces==1.13.0     # via -r requirements/base.in, django-oauth-toolkit
+django-model-utils==3.1.2  # via -r requirements/base.in, edx-celeryutils, edx-completion
+django-oauth-toolkit==0.12.0  # via -r requirements/base.in
+django-waffle==0.20.0     # via edx-django-utils, edx-drf-extensions
+django==1.10.8            # via -r requirements/base.in, django-model-utils, django-oauth-toolkit, edx-celeryutils, edx-completion, edx-django-utils, edx-drf-extensions, edx-i18n-tools, edx-opaque-keys, jsonfield, rest-condition
 djangorestframework-jwt==1.11.0  # via edx-drf-extensions
-djangorestframework==3.6.4
-docutils==0.14            # via readme-renderer
-edx-celeryutils==0.1.5
-edx-completion==1.0.3
-edx-django-utils==1.0.3   # via edx-drf-extensions
-edx-drf-extensions==2.0.1  # via edx-completion
-edx-i18n-tools==0.4.8
-edx-lint==1.1.1
-edx-opaque-keys[django]==0.4.4
-filelock==3.0.10          # via tox
-fs==2.4.4                 # via xblock
-future==0.17.1            # via pyjwkest
-idna==2.8                 # via requests
-importlib-metadata==0.8   # via path.py
-inflect==2.1.0            # via jinja2-pluralize
-isort==4.3.15
+djangorestframework==3.6.4  # via -r requirements/base.in, edx-completion, edx-drf-extensions, rest-condition
+docutils==0.16            # via readme-renderer
+edx-celeryutils==0.1.5    # via -r requirements/base.in
+edx-completion==1.0.3     # via -r requirements/base.in
+edx-django-utils==3.2.2   # via edx-drf-extensions
+edx-drf-extensions==2.4.6  # via edx-completion
+edx-i18n-tools==0.5.3     # via -r requirements/dev.in
+edx-lint==1.1.2           # via -r requirements/dev.in, -r requirements/quality.in
+edx-opaque-keys[django]==1.0.1  # via -r requirements/base.in, edx-completion, edx-drf-extensions
+enum34==1.1.10            # via astroid, fs
+filelock==3.0.12          # via tox, virtualenv
+fs==2.4.11                # via xblock
+future==0.18.2            # via backports.os, pyjwkest
+futures==3.3.0 ; python_version < "3"  # via -c requirements/constraints.txt, caniusepython3, isort
+idna==2.9                 # via requests
+importlib-metadata==1.6.0  # via importlib-resources, inflect, path.py, pluggy, tox, virtualenv
+importlib-resources==1.5.0  # via virtualenv
+inflect==3.0.2            # via jinja2-pluralize
+isort==4.3.21             # via -r requirements/quality.in, pylint
 jinja2-pluralize==0.3.0   # via diff-cover
-jinja2==2.10              # via diff-cover, jinja2-pluralize
-jsonfield==2.0.2          # via edx-celeryutils
+jinja2==2.11.2            # via diff-cover, jinja2-pluralize
+jsonfield==2.0.2          # via -r requirements/base.in, edx-celeryutils
 kombu==3.0.37             # via celery
-lazy-object-proxy==1.3.1  # via astroid
-lxml==4.3.2               # via xblock
+lazy-object-proxy==1.4.3  # via astroid
+lxml==4.5.1               # via xblock
 markupsafe==1.1.1         # via jinja2, xblock
 mccabe==0.6.1             # via pylint
-mysqlclient==1.4.2.post1
-newrelic==4.14.0.115      # via edx-django-utils
+mysqlclient==1.4.6        # via -r requirements/base.in
+newrelic==5.14.0.142      # via edx-django-utils
 oauthlib==2.0.1           # via django-oauth-toolkit
-packaging==19.0           # via caniusepython3
-path.py==11.5.0           # via edx-i18n-tools
-pbr==5.1.3                # via stevedore
-pip-tools==3.5.0
+packaging==20.4           # via bleach, caniusepython3, tox
+path.py==11.5.2           # via edx-i18n-tools
+pathlib2==2.3.5           # via importlib-metadata, importlib-resources, virtualenv
+pbr==5.4.5                # via stevedore
+pip-tools==5.2.0          # via -r requirements/dev.in
 pkginfo==1.5.0.1          # via twine
-pluggy==0.9.0             # via tox
+pluggy==0.13.1            # via diff-cover, tox
 polib==1.1.0              # via edx-i18n-tools
 psutil==1.2.1             # via edx-django-utils, edx-drf-extensions
-py==1.8.0                 # via tox
-pycodestyle==2.5.0
-pycryptodomex==3.7.3      # via pyjwkest
-pydocstyle==3.0.0
-pygments==2.3.1           # via diff-cover, readme-renderer
+py==1.8.1                 # via tox
+pycodestyle==2.6.0        # via -r requirements/quality.in
+pycryptodomex==3.9.7      # via pyjwkest
+pydocstyle==3.0.0         # via -r requirements/quality.in
+pygments==2.5.2           # via diff-cover, readme-renderer
 pyjwkest==1.3.2           # via edx-drf-extensions
 pyjwt==1.7.1              # via djangorestframework-jwt
 pylint-celery==0.3        # via edx-lint
 pylint-django==0.7.2      # via edx-lint
-pylint-plugin-utils==0.5  # via pylint-celery, pylint-django
+pylint-plugin-utils==0.6  # via pylint-celery, pylint-django
 pylint==1.7.6             # via edx-lint, pylint-celery, pylint-django, pylint-plugin-utils
-pymongo==3.7.2            # via edx-opaque-keys
-pyparsing==2.3.1          # via packaging
-python-dateutil==2.8.0    # via edx-drf-extensions, xblock
-pytz==2018.9              # via celery, edx-completion, fs, xblock
-pyyaml==5.1               # via edx-i18n-tools, xblock
-readme-renderer==24.0     # via twine
+pymongo==3.10.1           # via edx-opaque-keys
+pyparsing==2.4.7          # via packaging
+python-dateutil==2.8.1    # via edx-drf-extensions, xblock
+pytz==2020.1              # via celery, edx-completion, fs, xblock
+pyyaml==5.3.1             # via edx-i18n-tools, xblock
+readme-renderer==26.0     # via twine
 requests-toolbelt==0.9.1  # via twine
-requests==2.21.0          # via caniusepython3, edx-drf-extensions, pyjwkest, requests-toolbelt, twine
+requests==2.23.0          # via caniusepython3, edx-drf-extensions, pyjwkest, requests-toolbelt, twine
 rest-condition==1.0.3     # via edx-drf-extensions
-semantic-version==2.6.0   # via edx-drf-extensions
-six==1.12.0
-snowballstemmer==1.2.1    # via pydocstyle
-stevedore==1.30.1         # via edx-opaque-keys
-toml==0.10.0              # via tox
-tox-battery==0.2
-tox==3.7.0
-tqdm==4.31.1              # via twine
-twine==1.13.0
-urllib3==1.24.1           # via requests
-virtualenv==16.4.3        # via tox
-web-fragments==0.3.0      # via xblock
+scandir==1.10.0           # via pathlib2
+semantic-version==2.8.5   # via edx-drf-extensions
+singledispatch==3.4.0.3   # via astroid, importlib-resources, pylint
+six==1.15.0               # via -r requirements/base.in, astroid, bleach, diff-cover, django-oauth-toolkit, django-waffle, edx-drf-extensions, edx-i18n-tools, edx-lint, edx-opaque-keys, fs, packaging, pathlib2, pip-tools, pydocstyle, pyjwkest, pylint, python-dateutil, readme-renderer, singledispatch, stevedore, tox, virtualenv, xblock
+snowballstemmer==2.0.0    # via pydocstyle
+stevedore==1.32.0         # via edx-opaque-keys
+toml==0.10.1              # via tox
+tox-battery==0.2          # via -r requirements/dev.in
+tox==3.15.1               # via -r requirements/dev.in, tox-battery
+tqdm==4.46.0              # via twine
+twine==1.15.0             # via -r requirements/dev.in
+typing==3.7.4.1           # via fs, importlib-resources
+urllib3==1.25.9           # via requests
+virtualenv==20.0.21       # via tox
+web-fragments==0.3.2      # via xblock
 webencodings==0.5.1       # via bleach
-webob==1.8.5              # via xblock
-wheel==0.33.1
-wrapt==1.11.1             # via astroid
-xblock==1.2.2
-zipp==0.3.3               # via importlib-metadata
+webob==1.8.6              # via xblock
+wheel==0.34.2             # via -r requirements/dev.in
+wrapt==1.12.1             # via astroid
+xblock==1.2.9             # via -r requirements/base.in, edx-completion
+zipp==1.2.0               # via importlib-metadata, importlib-resources
+
+# The following packages are considered to be unsafe in a requirements file:
+# pip
+# setuptools
diff --git a/requirements/doc.txt b/requirements/doc.txt
index 50dc2f0d..041e6874 100644
--- a/requirements/doc.txt
+++ b/requirements/doc.txt
@@ -2,69 +2,75 @@
 # This file is autogenerated by pip-compile
 # To update, run:
 #
-#    pip-compile --no-index --output-file requirements/doc.txt requirements/base.in requirements/doc.in
+#    pip-compile --no-index --output-file=requirements/doc.txt requirements/base.in requirements/doc.in
 #
 alabaster==0.7.12         # via sphinx
 amqp==1.4.9               # via kombu
 anyjson==0.3.3            # via kombu
-appdirs==1.4.3            # via fs
-babel==2.6.0              # via sphinx
+appdirs==1.4.4            # via fs
+babel==2.8.0              # via sphinx
+backports.os==0.1.1       # via fs
 billiard==3.3.0.23        # via celery
-bleach==3.1.0             # via readme-renderer
-celery==3.1.18
-certifi==2019.3.9         # via requests
+bleach==3.1.5             # via readme-renderer
+celery==3.1.18            # via -r requirements/base.in, edx-celeryutils
+certifi==2020.4.5.1       # via requests
 chardet==3.0.4            # via doc8, requests
-django-braces==1.13.0     # via django-oauth-toolkit
-django-model-utils==3.1.2
-django-oauth-toolkit==0.12.0
-django-waffle==0.15.1     # via edx-django-utils, edx-drf-extensions
-django==1.10.8
+django-braces==1.13.0     # via -r requirements/base.in, django-oauth-toolkit
+django-model-utils==3.1.2  # via -r requirements/base.in, edx-celeryutils, edx-completion
+django-oauth-toolkit==0.12.0  # via -r requirements/base.in
+django-waffle==0.20.0     # via edx-django-utils, edx-drf-extensions
+django==1.10.8            # via -r requirements/base.in, django-model-utils, django-oauth-toolkit, edx-celeryutils, edx-completion, edx-django-utils, edx-drf-extensions, edx-opaque-keys, jsonfield, rest-condition
 djangorestframework-jwt==1.11.0  # via edx-drf-extensions
-djangorestframework==3.6.4
-doc8==0.8.0
-docutils==0.14            # via doc8, readme-renderer, restructuredtext-lint, sphinx
-edx-celeryutils==0.1.5
-edx-completion==1.0.3
-edx-django-utils==1.0.3   # via edx-drf-extensions
-edx-drf-extensions==2.0.1  # via edx-completion
-edx-opaque-keys[django]==0.4.4
-edx-sphinx-theme==1.4.0
-fs==2.4.4                 # via xblock
-future==0.17.1            # via pyjwkest
-idna==2.8                 # via requests
-imagesize==1.1.0          # via sphinx
-jinja2==2.10              # via sphinx
-jsonfield==2.0.2          # via edx-celeryutils
+djangorestframework==3.6.4  # via -r requirements/base.in, edx-completion, edx-drf-extensions, rest-condition
+doc8==0.8.1               # via -r requirements/doc.in
+docutils==0.16            # via doc8, readme-renderer, restructuredtext-lint, sphinx
+edx-celeryutils==0.1.5    # via -r requirements/base.in
+edx-completion==1.0.3     # via -r requirements/base.in
+edx-django-utils==3.2.2   # via edx-drf-extensions
+edx-drf-extensions==2.4.6  # via edx-completion
+edx-opaque-keys[django]==1.0.1  # via -r requirements/base.in, edx-completion, edx-drf-extensions
+edx-sphinx-theme==1.5.0   # via -r requirements/doc.in
+enum34==1.1.10            # via fs
+fs==2.4.11                # via xblock
+future==0.18.2            # via backports.os, pyjwkest
+idna==2.9                 # via requests
+imagesize==1.2.0          # via sphinx
+jinja2==2.11.2            # via sphinx
+jsonfield==2.0.2          # via -r requirements/base.in, edx-celeryutils
 kombu==3.0.37             # via celery
-lxml==4.3.2               # via xblock
+lxml==4.5.1               # via xblock
 markupsafe==1.1.1         # via jinja2, xblock
-mysqlclient==1.4.2.post1
-newrelic==4.14.0.115      # via edx-django-utils
+mysqlclient==1.4.6        # via -r requirements/base.in
+newrelic==5.14.0.142      # via edx-django-utils
 oauthlib==2.0.1           # via django-oauth-toolkit
-packaging==19.0           # via sphinx
-pbr==5.1.3                # via stevedore
+packaging==20.4           # via bleach, sphinx
+pbr==5.4.5                # via stevedore
 psutil==1.2.1             # via edx-django-utils, edx-drf-extensions
-pycryptodomex==3.7.3      # via pyjwkest
-pygments==2.3.1           # via readme-renderer, sphinx
+pycryptodomex==3.9.7      # via pyjwkest
+pygments==2.5.2           # via doc8, readme-renderer, sphinx
 pyjwkest==1.3.2           # via edx-drf-extensions
 pyjwt==1.7.1              # via djangorestframework-jwt
-pymongo==3.7.2            # via edx-opaque-keys
-pyparsing==2.3.1          # via packaging
-python-dateutil==2.8.0    # via edx-drf-extensions, xblock
-pytz==2018.9              # via babel, celery, edx-completion, fs, xblock
-pyyaml==5.1               # via xblock
-readme-renderer==24.0
-requests==2.21.0          # via edx-drf-extensions, pyjwkest, sphinx
+pymongo==3.10.1           # via edx-opaque-keys
+pyparsing==2.4.7          # via packaging
+python-dateutil==2.8.1    # via edx-drf-extensions, xblock
+pytz==2020.1              # via babel, celery, edx-completion, fs, xblock
+pyyaml==5.3.1             # via xblock
+readme-renderer==26.0     # via -r requirements/doc.in
+requests==2.23.0          # via edx-drf-extensions, pyjwkest, sphinx
 rest-condition==1.0.3     # via edx-drf-extensions
-restructuredtext-lint==1.2.2  # via doc8
-semantic-version==2.6.0   # via edx-drf-extensions
-six==1.12.0
-snowballstemmer==1.2.1    # via sphinx
-sphinx==1.8.5
-sphinxcontrib-websupport==1.1.0  # via sphinx
-stevedore==1.30.1         # via doc8, edx-opaque-keys
-urllib3==1.24.1           # via requests
-web-fragments==0.3.0      # via xblock
+restructuredtext-lint==1.3.0  # via doc8
+semantic-version==2.8.5   # via edx-drf-extensions
+six==1.15.0               # via -r requirements/base.in, bleach, django-oauth-toolkit, django-waffle, doc8, edx-drf-extensions, edx-opaque-keys, edx-sphinx-theme, fs, packaging, pyjwkest, python-dateutil, readme-renderer, sphinx, stevedore, xblock
+snowballstemmer==2.0.0    # via sphinx
+sphinx==1.8.5             # via -r requirements/doc.in, edx-sphinx-theme
+sphinxcontrib-websupport==1.1.2  # via sphinx
+stevedore==1.32.0         # via doc8, edx-opaque-keys
+typing==3.7.4.1           # via fs, sphinx
+urllib3==1.25.9           # via requests
+web-fragments==0.3.2      # via xblock
 webencodings==0.5.1       # via bleach
-webob==1.8.5              # via xblock
-xblock==1.2.2
+webob==1.8.6              # via xblock
+xblock==1.2.9             # via -r requirements/base.in, edx-completion
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools
diff --git a/requirements/quality.in b/requirements/quality.in
index b19fff08..47776cb0 100644
--- a/requirements/quality.in
+++ b/requirements/quality.in
@@ -1,7 +1,9 @@
 # Requirements for code quality checks
 
+-c constraints.txt
+
 caniusepython3            # Additional Python 3 compatibility pylint checks
-edx-lint                  # edX pylint rules and plugins
+edx-lint<1.2              # edX pylint rules and plugins
 isort                     # to standardize order of imports
 pycodestyle               # PEP 8 compliance validation
 pydocstyle                # PEP 257 compliance validation
diff --git a/requirements/quality.txt b/requirements/quality.txt
index e9ee77d7..0420ec41 100644
--- a/requirements/quality.txt
+++ b/requirements/quality.txt
@@ -2,32 +2,39 @@
 # This file is autogenerated by pip-compile
 # To update, run:
 #
-#    pip-compile --no-index --output-file requirements/quality.txt requirements/quality.in
+#    pip-compile --no-index --output-file=requirements/quality.txt requirements/quality.in
 #
 argparse==1.4.0           # via caniusepython3
-astroid==1.5.3            # via edx-lint, pylint, pylint-celery
-backports.functools-lru-cache==1.5  # via caniusepython3
-caniusepython3==7.0.0
-certifi==2019.3.9         # via requests
+astroid==1.5.3            # via pylint, pylint-celery
+backports.functools-lru-cache==1.6.1  # via astroid, caniusepython3, isort, pylint
+caniusepython3==7.2.0     # via -r requirements/quality.in
+certifi==2020.4.5.1       # via requests
 chardet==3.0.4            # via requests
-click-log==0.1.8          # via edx-lint
-click==7.0                # via click-log, edx-lint
-distlib==0.2.8            # via caniusepython3
-edx-lint==1.1.1
-idna==2.8                 # via requests
-isort==4.3.15
-lazy-object-proxy==1.3.1  # via astroid
+click-log==0.3.2          # via edx-lint
+click==7.1.2              # via click-log, edx-lint
+configparser==4.0.2       # via pydocstyle, pylint
+distlib==0.3.0            # via caniusepython3
+edx-lint==1.1.2           # via -r requirements/quality.in
+enum34==1.1.10            # via astroid
+futures==3.3.0 ; python_version < "3"  # via -c requirements/constraints.txt, caniusepython3, isort
+idna==2.9                 # via requests
+isort==4.3.21             # via -r requirements/quality.in, pylint
+lazy-object-proxy==1.4.3  # via astroid
 mccabe==0.6.1             # via pylint
-packaging==19.0           # via caniusepython3
-pycodestyle==2.5.0
-pydocstyle==3.0.0
+packaging==20.4           # via caniusepython3
+pycodestyle==2.6.0        # via -r requirements/quality.in
+pydocstyle==3.0.0         # via -r requirements/quality.in
 pylint-celery==0.3        # via edx-lint
 pylint-django==0.7.2      # via edx-lint
-pylint-plugin-utils==0.5  # via pylint-celery, pylint-django
+pylint-plugin-utils==0.6  # via pylint-celery, pylint-django
 pylint==1.7.6             # via edx-lint, pylint-celery, pylint-django, pylint-plugin-utils
-pyparsing==2.3.1          # via packaging
-requests==2.21.0          # via caniusepython3
-six==1.12.0               # via astroid, edx-lint, packaging, pydocstyle, pylint
-snowballstemmer==1.2.1    # via pydocstyle
-urllib3==1.24.1           # via requests
-wrapt==1.11.1             # via astroid
+pyparsing==2.4.7          # via packaging
+requests==2.23.0          # via caniusepython3
+singledispatch==3.4.0.3   # via astroid, pylint
+six==1.15.0               # via astroid, edx-lint, packaging, pydocstyle, pylint, singledispatch
+snowballstemmer==2.0.0    # via pydocstyle
+urllib3==1.25.9           # via requests
+wrapt==1.12.1             # via astroid
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools
diff --git a/requirements/test.in b/requirements/test.in
index ccc013ea..7fd3e3f6 100644
--- a/requirements/test.in
+++ b/requirements/test.in
@@ -8,3 +8,4 @@ pytest-cov                # pytest extension for code coverage statistics
 pytest-django             # pytest extension for better Django support
 redis
 more-itertools < 6.0.0
+freezegun
diff --git a/requirements/test.txt b/requirements/test.txt
index ee154776..c10c74e4 100644
--- a/requirements/test.txt
+++ b/requirements/test.txt
@@ -2,63 +2,81 @@
 # This file is autogenerated by pip-compile
 # To update, run:
 #
-#    pip-compile --no-index --output-file requirements/test.txt requirements/base.in requirements/test.in
+#    pip-compile --no-index --output-file=requirements/test.txt requirements/base.in requirements/test.in
 #
 amqp==1.4.9               # via kombu
 anyjson==0.3.3            # via kombu
-appdirs==1.4.3            # via fs
-atomicwrites==1.3.0       # via pytest
-attrs==19.1.0             # via pytest
+appdirs==1.4.4            # via fs
+atomicwrites==1.4.0       # via pytest
+attrs==19.3.0             # via pytest
+backports.functools-lru-cache==1.6.1  # via wcwidth
+backports.os==0.1.1       # via fs
 billiard==3.3.0.23        # via celery
-celery==3.1.18
-certifi==2019.3.9         # via requests
+celery==3.1.18            # via -r requirements/base.in, edx-celeryutils
+certifi==2020.4.5.1       # via requests
 chardet==3.0.4            # via requests
-coverage==4.5.3           # via pytest-cov
-ddt==1.2.1
-django-braces==1.13.0     # via django-oauth-toolkit
-django-model-utils==3.1.2
-django-oauth-toolkit==0.12.0
-django-waffle==0.15.1     # via edx-django-utils, edx-drf-extensions
+configparser==4.0.2       # via importlib-metadata
+contextlib2==0.6.0.post1  # via importlib-metadata, zipp
+coverage==5.1             # via pytest-cov
+ddt==1.4.1                # via -r requirements/test.in
+django-braces==1.13.0     # via -r requirements/base.in, django-oauth-toolkit
+django-model-utils==3.1.2  # via -r requirements/base.in, -r requirements/test.in, edx-celeryutils, edx-completion
+django-oauth-toolkit==0.12.0  # via -r requirements/base.in
+django-waffle==0.20.0     # via edx-django-utils, edx-drf-extensions
 djangorestframework-jwt==1.11.0  # via edx-drf-extensions
-djangorestframework==3.6.4
-edx-celeryutils==0.1.5
-edx-completion==1.0.3
-edx-django-utils==1.0.3   # via edx-drf-extensions
-edx-drf-extensions==2.0.1  # via edx-completion
-edx-opaque-keys[django]==0.4.4
-fs==2.4.4                 # via xblock
-future==0.17.1            # via pyjwkest
-idna==2.8                 # via requests
-jsonfield==2.0.2          # via edx-celeryutils
+djangorestframework==3.6.4  # via -r requirements/base.in, edx-completion, edx-drf-extensions, rest-condition
+edx-celeryutils==0.1.5    # via -r requirements/base.in
+edx-completion==1.0.3     # via -r requirements/base.in
+edx-django-utils==3.2.2   # via edx-drf-extensions
+edx-drf-extensions==2.4.6  # via edx-completion
+edx-opaque-keys[django]==1.0.1  # via -r requirements/base.in, edx-completion, edx-drf-extensions
+enum34==1.1.10            # via ddt, fs
+freezegun==0.3.15         # via -r requirements/test.in
+fs==2.4.11                # via xblock
+funcsigs==1.0.2           # via mock, pytest
+future==0.18.2            # via backports.os, pyjwkest
+idna==2.9                 # via requests
+importlib-metadata==1.6.0  # via pluggy, pytest
+jsonfield==2.0.2          # via -r requirements/base.in, edx-celeryutils
 kombu==3.0.37             # via celery
-lxml==4.3.2               # via xblock
+lxml==4.5.1               # via xblock
 markupsafe==1.1.1         # via xblock
-mock==2.0.0
-more-itertools==5.0.0
-mysqlclient==1.4.2.post1
-newrelic==4.14.0.115      # via edx-django-utils
+mock==3.0.5               # via -r requirements/test.in
+more-itertools==5.0.0     # via -r requirements/test.in, pytest
+mysqlclient==1.4.6        # via -r requirements/base.in
+newrelic==5.14.0.142      # via edx-django-utils
 oauthlib==2.0.1           # via django-oauth-toolkit
-pbr==5.1.3                # via mock, stevedore
-pluggy==0.9.0             # via pytest
+packaging==20.4           # via pytest
+pathlib2==2.3.5           # via importlib-metadata, pytest, pytest-django
+pbr==5.4.5                # via stevedore
+pluggy==0.13.1            # via pytest
 psutil==1.2.1             # via edx-django-utils, edx-drf-extensions
-py==1.8.0                 # via pytest
-pycryptodomex==3.7.3      # via pyjwkest
+py==1.8.1                 # via pytest
+pycryptodomex==3.9.7      # via pyjwkest
 pyjwkest==1.3.2           # via edx-drf-extensions
 pyjwt==1.7.1              # via djangorestframework-jwt
-pymongo==3.7.2            # via edx-opaque-keys
-pytest-cov==2.6.1
-pytest-django==3.4.8
-pytest==4.3.1
-python-dateutil==2.8.0    # via edx-drf-extensions, xblock
-pytz==2018.9              # via celery, edx-completion, fs, xblock
-pyyaml==5.1               # via xblock
-redis==3.2.0
-requests==2.21.0          # via edx-drf-extensions, pyjwkest
+pymongo==3.10.1           # via edx-opaque-keys
+pyparsing==2.4.7          # via packaging
+pytest-cov==2.9.0         # via -r requirements/test.in
+pytest-django==3.9.0      # via -r requirements/test.in
+pytest==4.6.10            # via -r requirements/test.in, pytest-cov, pytest-django
+python-dateutil==2.8.1    # via edx-drf-extensions, freezegun, xblock
+pytz==2020.1              # via celery, edx-completion, fs, xblock
+pyyaml==5.3.1             # via xblock
+redis==3.5.3              # via -r requirements/test.in
+requests==2.23.0          # via edx-drf-extensions, pyjwkest
 rest-condition==1.0.3     # via edx-drf-extensions
-semantic-version==2.6.0   # via edx-drf-extensions
-six==1.12.0
-stevedore==1.30.1         # via edx-opaque-keys
-urllib3==1.24.1           # via requests
-web-fragments==0.3.0      # via xblock
-webob==1.8.5              # via xblock
-xblock==1.2.2
+scandir==1.10.0           # via pathlib2
+semantic-version==2.8.5   # via edx-drf-extensions
+six==1.15.0               # via -r requirements/base.in, django-oauth-toolkit, django-waffle, edx-drf-extensions, edx-opaque-keys, freezegun, fs, mock, more-itertools, packaging, pathlib2, pyjwkest, pytest, python-dateutil, stevedore, xblock
+stevedore==1.32.0         # via edx-opaque-keys
+typing==3.7.4.1           # via fs
+urllib3==1.25.9           # via requests
+wcwidth==0.2.3            # via pytest
+web-fragments==0.3.2      # via xblock
+webob==1.8.6              # via xblock
+xblock==1.2.9             # via -r requirements/base.in, edx-completion
+zipp==1.2.0               # via importlib-metadata
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools
diff --git a/requirements/travis.txt b/requirements/travis.txt
index 471539ec..c297660b 100644
--- a/requirements/travis.txt
+++ b/requirements/travis.txt
@@ -2,20 +2,33 @@
 # This file is autogenerated by pip-compile
 # To update, run:
 #
-#    pip-compile --no-index --output-file requirements/travis.txt requirements/travis.in
+#    pip-compile --no-index --output-file=requirements/travis.txt requirements/travis.in
 #
-certifi==2019.3.9         # via requests
+appdirs==1.4.4            # via virtualenv
+certifi==2020.4.5.1       # via requests
 chardet==3.0.4            # via requests
-codecov==2.0.15
-coverage==4.5.3           # via codecov
-filelock==3.0.10          # via tox
-idna==2.8                 # via requests
-pluggy==0.9.0             # via tox
-py==1.8.0                 # via tox
-requests==2.21.0          # via codecov
-six==1.12.0               # via tox
-toml==0.10.0              # via tox
-tox-battery==0.2
-tox==3.7.0
-urllib3==1.24.1           # via requests
-virtualenv==16.4.3        # via tox
+codecov==2.1.3            # via -r requirements/travis.in
+configparser==4.0.2       # via importlib-metadata
+contextlib2==0.6.0.post1  # via importlib-metadata, importlib-resources, virtualenv, zipp
+coverage==5.1             # via codecov
+distlib==0.3.0            # via virtualenv
+filelock==3.0.12          # via tox, virtualenv
+idna==2.9                 # via requests
+importlib-metadata==1.6.0  # via importlib-resources, pluggy, tox, virtualenv
+importlib-resources==1.5.0  # via virtualenv
+packaging==20.4           # via tox
+pathlib2==2.3.5           # via importlib-metadata, importlib-resources, virtualenv
+pluggy==0.13.1            # via tox
+py==1.8.1                 # via tox
+pyparsing==2.4.7          # via packaging
+requests==2.23.0          # via codecov
+scandir==1.10.0           # via pathlib2
+singledispatch==3.4.0.3   # via importlib-resources
+six==1.15.0               # via packaging, pathlib2, tox, virtualenv
+toml==0.10.1              # via tox
+tox-battery==0.2          # via -r requirements/travis.in
+tox==3.15.1               # via -r requirements/travis.in, tox-battery
+typing==3.7.4.1           # via importlib-resources
+urllib3==1.25.9           # via requests
+virtualenv==20.0.21       # via tox
+zipp==1.2.0               # via importlib-metadata, importlib-resources
diff --git a/test_utils/test_app/migrations/0003_coursemodulecompletion.py b/test_utils/test_app/migrations/0003_coursemodulecompletion.py
new file mode 100644
index 00000000..27867ae0
--- /dev/null
+++ b/test_utils/test_app/migrations/0003_coursemodulecompletion.py
@@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.10.8 on 2020-05-18 12:26
+from __future__ import unicode_literals
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+import model_utils.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ('test_app', '0002_auto_20181217_1304'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='CourseModuleCompletion',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')),
+                ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')),
+                ('course_id', models.CharField(db_index=True, max_length=255)),
+                ('content_id', models.CharField(db_index=True, max_length=255)),
+                ('stage', models.CharField(blank=True, max_length=255, null=True)),
+                ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='course_completions', to=settings.AUTH_USER_MODEL)),
+            ],
+            options={
+                'db_table': 'progress_coursemodulecompletion',
+            },
+        ),
+    ]
diff --git a/test_utils/test_app/models.py b/test_utils/test_app/models.py
index ba20c939..bba026bd 100644
--- a/test_utils/test_app/models.py
+++ b/test_utils/test_app/models.py
@@ -9,6 +9,8 @@
 from django.contrib.auth.models import User
 from django.db import models
 
+from model_utils.models import TimeStampedModel
+
 
 class CourseEnrollment(models.Model):
     """
@@ -82,3 +84,18 @@ class CohortMembership(models.Model):
 
     class Meta(object):
         unique_together = (('user', 'course_id'), )
+
+# Copied over from https://github.com/edx-solutions/progress-edx-platform-extensions/blob/master/progress/models.py
+class CourseModuleCompletion(TimeStampedModel):
+    """
+    The CourseModuleCompletion model contains user, course, module information
+    to monitor a user's progression throughout the duration of a course,
+    we need to observe and record completions of the individual course modules.
+    """
+    user = models.ForeignKey(User, db_index=True, related_name="course_completions")
+    course_id = models.CharField(max_length=255, db_index=True)
+    content_id = models.CharField(max_length=255, db_index=True)
+    stage = models.CharField(max_length=255, null=True, blank=True)
+
+    class Meta(object):
+        db_table = 'progress_coursemodulecompletion'
diff --git a/tests/test_tasks.py b/tests/test_tasks.py
new file mode 100644
index 00000000..8e6d95a2
--- /dev/null
+++ b/tests/test_tasks.py
@@ -0,0 +1,118 @@
+# -*- coding: utf-8 -*-
+
+"""
+Tests for the `openedx-completion-aggregator` tasks.
+"""
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import ddt
+import mock
+from freezegun import freeze_time
+from opaque_keys.edx.keys import CourseKey
+
+from django.contrib.auth.models import User
+from django.db import connection
+from django.test import TestCase, override_settings
+
+from completion.models import BlockCompletion
+from completion_aggregator.tasks.aggregation_tasks import _migrate_batch
+from test_utils.compat import StubCompat
+from test_utils.test_app.models import CourseModuleCompletion
+
+
+@ddt.ddt
+@override_settings(
+    COMPLETION_AGGREGATOR_AGGREGATION_LOCK='COMPLETION_AGGREGATOR_AGGREGATION_LOCK',
+    COMPLETION_AGGREGATOR_CLEANUP_LOCK='COMPLETION_AGGREGATOR_CLEANUP_LOCK',
+    COMPLETION_AGGREGATOR_AGGREGATION_LOCK_TIMEOUT_SECONDS=1800,
+    COMPLETION_AGGREGATOR_CLEANUP_LOCK_TIMEOUT_SECONDS=900,
+)
+class MigrateProgressTestCase(TestCase):
+    """
+    Tests of the progress migration code.
+    """
+
+    def setUp(self):
+        super(MigrateProgressTestCase, self).setUp()
+        self.user = user = User.objects.create_user("test", password="test")
+        self.course_key = course_key = CourseKey.from_string('course-v1:edx+course+test')
+        self.block_keys = block_keys = [
+            course_key.make_usage_key('html', 'course-html{}'.format(idx))
+            for idx in range(1, 51)
+        ]
+        stubcompat = StubCompat([course_key.make_usage_key('course', 'course')] + block_keys)
+        for compat_module in 'completion_aggregator.core.compat', 'completion_aggregator.core.compat':
+            patch = mock.patch(compat_module, stubcompat)
+            patch.start()
+            self.addCleanup(patch.stop)
+
+        for idx in range(1, 51):
+            block_key = course_key.make_usage_key('html', 'course-html{}'.format(idx))
+            with freeze_time("2020-02-02T02:02:{}".format(idx)):
+                CourseModuleCompletion.objects.create(
+                    id=idx,
+                    user=user,
+                    course_id=course_key,
+                    content_id=block_key,
+                )
+            with connection.cursor() as cur:
+                cur.execute(
+                    """
+                    INSERT INTO completion_blockcompletion
+                        (user_id, course_key, block_key, block_type, completion, created, modified)
+                    VALUES
+                        (%s, %s, %s, %s, 1.0, %s, %s);
+                    """,
+                    [
+                        user.id,
+                        course_key,
+                        block_key,
+                        block_key.block_type,
+                        "0000-00-00 00:00:00",
+                        "0000-00-00 00:00:00",
+                    ]
+                )
+
+    @mock.patch("time.sleep")
+    def test_migration_updates_created_modified(self, mock_sleep):
+        cmc_count = CourseModuleCompletion.objects.all().count()
+        c_count = BlockCompletion.objects.all().count()
+        self.assertEqual(cmc_count, 50)
+        self.assertEqual(c_count, 50)
+        for block_key in self.block_keys:
+            bc = BlockCompletion.objects.get(
+                user=self.user,
+                course_key=self.course_key,
+                block_key=block_key,
+                block_type=block_key.block_type,
+                completion=1.0,
+            )
+            cmc = CourseModuleCompletion.objects.get(
+                user=self.user,
+                course_id=self.course_key,
+                content_id=block_key,
+            )
+            self.assertNotEqual(bc.created, cmc.created)
+            self.assertNotEqual(bc.modified, cmc.modified)
+        _migrate_batch(11, 0.1)
+        cmc_count = CourseModuleCompletion.objects.all().count()
+        c_count = BlockCompletion.objects.all().count()
+        self.assertEqual(cmc_count, 50)
+        self.assertEqual(c_count, 50)
+        self.assertEqual(mock_sleep.call_count, 5)
+        for block_key in self.block_keys:
+            bc = BlockCompletion.objects.get(
+                user=self.user,
+                course_key=self.course_key,
+                block_key=block_key,
+                block_type=block_key.block_type,
+                completion=1.0,
+            )
+            cmc = CourseModuleCompletion.objects.get(
+                user=self.user,
+                course_id=self.course_key,
+                content_id=block_key,
+            )
+            self.assertEqual(bc.created, cmc.created)
+            self.assertEqual(bc.modified, cmc.modified)

From e4a4d411bff23c23c6016af91a338f9991132734 Mon Sep 17 00:00:00 2001
From: Kshitij Sobti <kshitij@sobti.in>
Date: Wed, 3 Jun 2020 03:23:27 +0530
Subject: [PATCH 2/3] Update settings

---
 test_settings.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/test_settings.py b/test_settings.py
index aa08b490..2345c699 100644
--- a/test_settings.py
+++ b/test_settings.py
@@ -32,6 +32,9 @@ def root(*args):
         'PORT': int(environ.get('EDXAGG_MYSQL_PORT', 3306)),
         'USER': environ.get('EDXAGG_MYSQL_USER', 'root'),
         'PASSWORD': environ.get('EDXAGG_MYSQL_PASSWORD', ''),
+        'OPTIONS': {
+            'init_command': "SET sql_mode='ALLOW_INVALID_DATES'",
+        }
     }
 }
 DEBUG = True

From 6ac7c42082000b7c049c1914b8b65895baa39850 Mon Sep 17 00:00:00 2001
From: Kshitij Sobti <kshitij@sobti.in>
Date: Wed, 3 Jun 2020 10:32:33 +0530
Subject: [PATCH 3/3] Fix requiremetns

---
 requirements/base.in   | 21 +++++++++++++--------
 requirements/dev.in    |  2 ++
 requirements/dev.txt   |  6 +++---
 requirements/doc.in    |  2 ++
 requirements/doc.txt   |  6 +++---
 requirements/test.in   |  2 ++
 requirements/test.txt  |  4 ++--
 requirements/travis.in |  2 ++
 8 files changed, 29 insertions(+), 16 deletions(-)

diff --git a/requirements/base.in b/requirements/base.in
index 58e2889e..5b16b4a8 100644
--- a/requirements/base.in
+++ b/requirements/base.in
@@ -1,15 +1,20 @@
 # Core requirements for using this application
 
-celery==3.1.18            # Asynchronous tasks
-edx-celeryutils<0.2      # Custom task extensions
-Django>=1.8              # Web application framework
+-c constraints.txt
+
+
+celery==3.1.18                      # Asynchronous tasks
+edx-celeryutils<0.2                 # Custom task extensions
+Django>=1.8                         # Web application framework
 django-oauth-toolkit<1.0
-djangorestframework>=3.0,<3.7     # API tools
-django-model-utils<3.2        # Provides TimeStampedModel abstract base class
-edx-opaque-keys<2,>=0.4.2    # Provides CourseKey and UsageKey
+djangorestframework>=3.0,<3.7       # API tools
+django-model-utils<3.2              # Provides TimeStampedModel abstract base class
+edx-opaque-keys<2,>=0.4.2           # Provides CourseKey and UsageKey
 edx-completion>=1.0.3,<2
-mysqlclient               # For connecting to MySQL
-six
+mysqlclient                         # For connecting to MySQL
+six                                 # Python 2/3 compatibility stubs
+# Limit for compatible django version:
 XBlock<1.3
 jsonfield<2.1
 django-braces<1.14
+django-waffle<0.16
diff --git a/requirements/dev.in b/requirements/dev.in
index 0d266175..14739fe4 100644
--- a/requirements/dev.in
+++ b/requirements/dev.in
@@ -1,5 +1,7 @@
 # Additional requirements for development of this application
 
+-c constraints.txt
+
 diff-cover                # Changeset diff test coverage
 edx-lint                  # For updating pylintrc
 edx-i18n-tools            # For i18n_tool dummy
diff --git a/requirements/dev.txt b/requirements/dev.txt
index aec63478..abd0e25f 100644
--- a/requirements/dev.txt
+++ b/requirements/dev.txt
@@ -21,12 +21,12 @@ click-log==0.3.2          # via edx-lint
 click==7.1.2              # via click-log, edx-lint, pip-tools
 configparser==4.0.2       # via importlib-metadata, pydocstyle, pylint
 contextlib2==0.6.0.post1  # via importlib-metadata, importlib-resources, virtualenv, zipp
-diff-cover==2.6.1         # via -r requirements/dev.in
+diff-cover==3.0.0         # via -r requirements/dev.in
 distlib==0.3.0            # via caniusepython3, virtualenv
 django-braces==1.13.0     # via -r requirements/base.in, django-oauth-toolkit
 django-model-utils==3.1.2  # via -r requirements/base.in, edx-celeryutils, edx-completion
 django-oauth-toolkit==0.12.0  # via -r requirements/base.in
-django-waffle==0.20.0     # via edx-django-utils, edx-drf-extensions
+django-waffle==0.15.1     # via -r requirements/base.in, edx-django-utils, edx-drf-extensions
 django==1.10.8            # via -r requirements/base.in, django-model-utils, django-oauth-toolkit, edx-celeryutils, edx-completion, edx-django-utils, edx-drf-extensions, edx-i18n-tools, edx-opaque-keys, jsonfield, rest-condition
 djangorestframework-jwt==1.11.0  # via edx-drf-extensions
 djangorestframework==3.6.4  # via -r requirements/base.in, edx-completion, edx-drf-extensions, rest-condition
@@ -91,7 +91,7 @@ rest-condition==1.0.3     # via edx-drf-extensions
 scandir==1.10.0           # via pathlib2
 semantic-version==2.8.5   # via edx-drf-extensions
 singledispatch==3.4.0.3   # via astroid, importlib-resources, pylint
-six==1.15.0               # via -r requirements/base.in, astroid, bleach, diff-cover, django-oauth-toolkit, django-waffle, edx-drf-extensions, edx-i18n-tools, edx-lint, edx-opaque-keys, fs, packaging, pathlib2, pip-tools, pydocstyle, pyjwkest, pylint, python-dateutil, readme-renderer, singledispatch, stevedore, tox, virtualenv, xblock
+six==1.15.0               # via -r requirements/base.in, astroid, bleach, diff-cover, django-oauth-toolkit, edx-drf-extensions, edx-i18n-tools, edx-lint, edx-opaque-keys, fs, packaging, pathlib2, pip-tools, pydocstyle, pyjwkest, pylint, python-dateutil, readme-renderer, singledispatch, stevedore, tox, virtualenv, xblock
 snowballstemmer==2.0.0    # via pydocstyle
 stevedore==1.32.0         # via edx-opaque-keys
 toml==0.10.1              # via tox
diff --git a/requirements/doc.in b/requirements/doc.in
index 7d47c89e..3a906fbf 100644
--- a/requirements/doc.in
+++ b/requirements/doc.in
@@ -1,5 +1,7 @@
 # Requirements for documentation validation
 
+-c constraints.txt
+
 doc8                      # reStructuredText style checker
 edx_sphinx_theme          # edX theme for Sphinx output
 readme_renderer           # Validates README.rst for usage on PyPI
diff --git a/requirements/doc.txt b/requirements/doc.txt
index 041e6874..1337a3e9 100644
--- a/requirements/doc.txt
+++ b/requirements/doc.txt
@@ -18,7 +18,7 @@ chardet==3.0.4            # via doc8, requests
 django-braces==1.13.0     # via -r requirements/base.in, django-oauth-toolkit
 django-model-utils==3.1.2  # via -r requirements/base.in, edx-celeryutils, edx-completion
 django-oauth-toolkit==0.12.0  # via -r requirements/base.in
-django-waffle==0.20.0     # via edx-django-utils, edx-drf-extensions
+django-waffle==0.15.1     # via -r requirements/base.in, edx-django-utils, edx-drf-extensions
 django==1.10.8            # via -r requirements/base.in, django-model-utils, django-oauth-toolkit, edx-celeryutils, edx-completion, edx-django-utils, edx-drf-extensions, edx-opaque-keys, jsonfield, rest-condition
 djangorestframework-jwt==1.11.0  # via edx-drf-extensions
 djangorestframework==3.6.4  # via -r requirements/base.in, edx-completion, edx-drf-extensions, rest-condition
@@ -58,9 +58,9 @@ pyyaml==5.3.1             # via xblock
 readme-renderer==26.0     # via -r requirements/doc.in
 requests==2.23.0          # via edx-drf-extensions, pyjwkest, sphinx
 rest-condition==1.0.3     # via edx-drf-extensions
-restructuredtext-lint==1.3.0  # via doc8
+restructuredtext-lint==1.3.1  # via doc8
 semantic-version==2.8.5   # via edx-drf-extensions
-six==1.15.0               # via -r requirements/base.in, bleach, django-oauth-toolkit, django-waffle, doc8, edx-drf-extensions, edx-opaque-keys, edx-sphinx-theme, fs, packaging, pyjwkest, python-dateutil, readme-renderer, sphinx, stevedore, xblock
+six==1.15.0               # via -r requirements/base.in, bleach, django-oauth-toolkit, doc8, edx-drf-extensions, edx-opaque-keys, edx-sphinx-theme, fs, packaging, pyjwkest, python-dateutil, readme-renderer, sphinx, stevedore, xblock
 snowballstemmer==2.0.0    # via sphinx
 sphinx==1.8.5             # via -r requirements/doc.in, edx-sphinx-theme
 sphinxcontrib-websupport==1.1.2  # via sphinx
diff --git a/requirements/test.in b/requirements/test.in
index 7fd3e3f6..8d952ec4 100644
--- a/requirements/test.in
+++ b/requirements/test.in
@@ -1,5 +1,7 @@
 # Requirements for test runs.
 
+-c constraints.txt
+
 ddt			  # Provides test parameterization
 django-model-utils        # Provides TimeStampedModel abstract base class
 mock
diff --git a/requirements/test.txt b/requirements/test.txt
index c10c74e4..d4ac3772 100644
--- a/requirements/test.txt
+++ b/requirements/test.txt
@@ -22,7 +22,7 @@ ddt==1.4.1                # via -r requirements/test.in
 django-braces==1.13.0     # via -r requirements/base.in, django-oauth-toolkit
 django-model-utils==3.1.2  # via -r requirements/base.in, -r requirements/test.in, edx-celeryutils, edx-completion
 django-oauth-toolkit==0.12.0  # via -r requirements/base.in
-django-waffle==0.20.0     # via edx-django-utils, edx-drf-extensions
+django-waffle==0.15.1     # via -r requirements/base.in, edx-django-utils, edx-drf-extensions
 djangorestframework-jwt==1.11.0  # via edx-drf-extensions
 djangorestframework==3.6.4  # via -r requirements/base.in, edx-completion, edx-drf-extensions, rest-condition
 edx-celeryutils==0.1.5    # via -r requirements/base.in
@@ -68,7 +68,7 @@ requests==2.23.0          # via edx-drf-extensions, pyjwkest
 rest-condition==1.0.3     # via edx-drf-extensions
 scandir==1.10.0           # via pathlib2
 semantic-version==2.8.5   # via edx-drf-extensions
-six==1.15.0               # via -r requirements/base.in, django-oauth-toolkit, django-waffle, edx-drf-extensions, edx-opaque-keys, freezegun, fs, mock, more-itertools, packaging, pathlib2, pyjwkest, pytest, python-dateutil, stevedore, xblock
+six==1.15.0               # via -r requirements/base.in, django-oauth-toolkit, edx-drf-extensions, edx-opaque-keys, freezegun, fs, mock, more-itertools, packaging, pathlib2, pyjwkest, pytest, python-dateutil, stevedore, xblock
 stevedore==1.32.0         # via edx-opaque-keys
 typing==3.7.4.1           # via fs
 urllib3==1.25.9           # via requests
diff --git a/requirements/travis.in b/requirements/travis.in
index dd3a0c05..f9dc7720 100644
--- a/requirements/travis.in
+++ b/requirements/travis.in
@@ -1,5 +1,7 @@
 # Requirements for running tests in Travis
 
+-c constraints.txt
+
 codecov                   # Code coverage reporting
 tox                       # Virtualenv management for tests
 tox-battery==0.2          # Makes tox aware of requirements file changes; restricted by https://github.com/signalpillar/tox-battery/issues/6