diff --git a/CHANGELOG.md b/CHANGELOG.md index 09f77065e7..a5e0146803 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ Note: Breaking changes between versions are indicated by "💥". ## Unreleased +## v12.0.0 (2021-06-09) + +- 💥[Improvement] Upgrade all services to open-release/lilac.master. +- 💥[Feature] Migrate Android app building and the WebUI frontend away from core Tutor and to dedicated plugins (see [TEP](https://discuss.overhang.io/c/community/tep/9)). The `DOCKER_IMAGE_ANDROID` setting is thus renamed to `ANDROID_DOCKER_IMAGE`. +- [Feature] Run `docker-compose build` as part of `tutor local start`. + ## v11.3.1 (2021-06-08) - [Improvement] Avoid permission issues in Kubernetes/Openshift for users who do not have the rights to edit their namespace. diff --git a/Makefile b/Makefile index ce9e9416f2..7f7ec22740 100644 --- a/Makefile +++ b/Makefile @@ -90,8 +90,7 @@ ci-test-bundle: ## Run basic tests on bundle yes "" | ./dist/tutor config save --interactive ./dist/tutor config save ./dist/tutor plugins list - # ./dist/tutor plugins enable discovery ecommerce figures license minio notes xqueue - ./dist/tutor plugins enable discovery ecommerce license minio notes xqueue + ./dist/tutor plugins enable android discovery ecommerce license mfe minio notes webui xqueue ./dist/tutor plugins list ./dist/tutor license --help diff --git a/README.rst b/README.rst index 3aaf9496cb..f03a57ad24 100644 --- a/README.rst +++ b/README.rst @@ -32,7 +32,7 @@ Tutor: the docker-based Open edX distribution designed for peace of mind :alt: AGPL License :target: https://www.gnu.org/licenses/agpl-3.0.en.html -**Tutor** is a docker-based `Open edX `_ distribution, both for production and local development. The goal of Tutor is to make it easy to deploy, customize, upgrade and scale Open edX. Tutor is reliable, fast, extensible, and it is already used by dozens of Open edX platforms around the world. +**Tutor** is a docker-based `Open edX `_ distribution, both for production and local development. The goal of Tutor is to make it easy to deploy, customize, upgrade and scale Open edX. Tutor is reliable, fast, extensible, and it is already used by hundreds of Open edX platforms around the world. Do you need professional assistance setting up or managing your Open edX platform? Overhang.IO provides online support as part of its `Long Term Support (LTS) offering `__. @@ -42,11 +42,11 @@ Features * 100% `open source `__ * Runs entirely on Docker * World-famous 1-click `installation and upgrades `__ -* Comes with batteries included: `theming `__, `SCORM `__, `HTTPS `__, `web-based administration interface `__, `mobile app `__, `custom translations `__... +* Comes with batteries included: `theming `__, `SCORM `__, `HTTPS `__, `web-based administration interface `__, `mobile app `__, `custom translations `__... * Extensible architecture with `plugins `__ * Works with `Kubernetes `__ * No technical skill required with the `1-click Tutor AWS image `__ -* Amazing plugins available with `Tutor Wizard Edition `__ +* Amazing premium plugins available in the `Tutor Wizard Edition `__ .. _readme_intro_end: @@ -66,6 +66,11 @@ Documentation Extensive documentation is available online: https://docs.tutor.overhang.io/ +Is there a problem? +------------------- + +Please follow the instructions from the `troubleshooting section `__ in the docs. + .. _readme_support_start: Support diff --git a/bin/main.py b/bin/main.py index feaf04c2c7..e259de2840 100755 --- a/bin/main.py +++ b/bin/main.py @@ -3,12 +3,14 @@ # Manually install plugins (this is for creating the bundle) for plugin_name in [ + "android", "discovery", "ecommerce", - # "figures", "license", + "mfe", "minio", "notes", + "webui", "xqueue", ]: try: diff --git a/docs/configuration.rst b/docs/configuration.rst index 57763d8d42..582f674ea3 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -61,7 +61,6 @@ Custom images ************* - ``DOCKER_IMAGE_OPENEDX`` (default: ``"{{ DOCKER_REGISTRY }}overhangio/openedx:{{ TUTOR_VERSION }}"``) -- ``DOCKER_IMAGE_ANDROID`` (default: ``"{{ DOCKER_REGISTRY }}overhangio/openedx-android:{{ TUTOR_VERSION }}"``) - ``DOCKER_IMAGE_FORUM`` (default: ``"{{ DOCKER_REGISTRY }}overhangio/openedx-forum:{{ TUTOR_VERSION }}"``) These configuration parameters define which image to run for each service. By default, the docker image tag matches the Tutor version it was built with. @@ -80,7 +79,7 @@ You may want to pull/push images from/to a custom docker registry. For instance, Open edX customisation ~~~~~~~~~~~~~~~~~~~~~~ -- ``OPENEDX_COMMON_VERSION`` (default: ``"open-release/koa.2"``) +- ``OPENEDX_COMMON_VERSION`` (default: ``"open-release/lilac.1"``) This defines the default version that will be pulled from all Open edX git repositories. @@ -214,8 +213,7 @@ openedx Docker Image build arguments When building the "openedx" Docker image, it is possible to specify a few `arguments `__: - ``EDX_PLATFORM_REPOSITORY`` (default: ``"https://github.com/edx/edx-platform.git"``) -- ``EDX_PLATFORM_VERSION`` (default: ``"open-release/koa.3"``) -- ``EDX_PLATFORM_VERSION_DATE`` (default: ``"20200227"``) +- ``EDX_PLATFORM_VERSION`` (default: ``"open-release/lilac.1"``) - ``NPM_REGISTRY`` (default: ``"https://registry.npmjs.org/"``) These arguments can be specified from the command line, `very much like Docker `__. For instance:: @@ -286,16 +284,16 @@ Note that your edx-platform version must be a fork of the latest release **tag** If you don't create your fork from this tag, you *will* have important compatibility issues with other services. In particular: -- Do not try to run a fork from an older (pre-Koa) version of edx-platform: this will simply not work. +- Do not try to run a fork from an older (pre-Lilac) version of edx-platform: this will simply not work. - Do not try to run a fork from the edx-platform master branch: there is a 99% probability that it will fail. -- Do not try to run a fork from the open-release/koa.master branch: Tutor will attempt to apply security and bug fix patches that might already be included in the open-release/koa.master but which were not yet applied to the latest release tag. Patch application will thus fail if you base your fork from the open-release/koa.master branch. +- Do not try to run a fork from the open-release/lilac.master branch: Tutor will attempt to apply security and bug fix patches that might already be included in the open-release/lilac.master but which were not yet applied to the latest release tag. Patch application will thus fail if you base your fork from the open-release/lilac.master branch. .. _i18n: Adding custom translations ~~~~~~~~~~~~~~~~~~~~~~~~~~ -If you are not running Open edX in English, chances are that some strings will not be properly translated. In most cases, this is because not enough contributors have helped translate Open edX in your language. It happens! With Tutor, available translated languages include those that come bundled with `edx-platform `__ as well as those from `openedx-i18n `__. +If you are not running Open edX in English, chances are that some strings will not be properly translated. In most cases, this is because not enough contributors have helped translate Open edX in your language. It happens! With Tutor, available translated languages include those that come bundled with `edx-platform `__ as well as those from `openedx-i18n `__. Tutor offers a relatively simple mechanism to add custom translations to the openedx Docker image. You should create a folder that corresponds to your language code in the "build/openedx/locale" folder of the Tutor environment. This folder should contain a "LC_MESSAGES" folder. For instance:: @@ -316,9 +314,9 @@ Then, add a "django.po" file there that will contain your custom translations:: .. warning:: Don't forget to specify the file ``Content-Type`` when adding message strings with non-ASCII characters; otherwise a ``UnicodeDecodeError`` will be raised during compilation. -The "String to translate" part should match *exactly* the string that you would like to translate. You cannot make it up! The best way to find this string is to copy-paste it from the `upstream django.po file for the English language `__. +The "String to translate" part should match *exactly* the string that you would like to translate. You cannot make it up! The best way to find this string is to copy-paste it from the `upstream django.po file for the English language `__. -If you cannot find the string to translate in this file, then it means that you are trying to translate a string that is used in some piece of javascript code. Those strings are stored in a different file named "djangojs.po". You can check it out `in the edx-platform repo as well `__. Your custom javascript strings should also be stored in a "djangojs.po" file that should be placed in the same directory. +If you cannot find the string to translate in this file, then it means that you are trying to translate a string that is used in some piece of javascript code. Those strings are stored in a different file named "djangojs.po". You can check it out `in the edx-platform repo as well `__. Your custom javascript strings should also be stored in a "djangojs.po" file that should be placed in the same directory. To recap, here is an example. To translate a few strings in French, both from django.po and djangojs.po, we would have the following file hierarchy:: diff --git a/docs/dev.rst b/docs/dev.rst index 3efae6ceb1..ced2dd1926 100644 --- a/docs/dev.rst +++ b/docs/dev.rst @@ -25,7 +25,7 @@ This ``openedx-dev`` development image differs from the ``openedx`` production i - The user that runs inside the container has the same UID as the user on the host, in order to avoid permission problems inside mounted volumes (and in particular in the edx-platform repository). - Additional python and system requirements are installed for convenient debugging: `ipython `__, `ipdb `__, vim, telnet. -- The edx-platform `development requirements `__ are installed. +- The edx-platform `development requirements `__ are installed. Since the ``openedx-dev`` is based upon the ``openedx`` docker image, it should be re-built every time the ``openedx`` docker image is modified. @@ -137,7 +137,7 @@ Prepare the edx-platform repo If you choose any but the first solution above, you will have to make sure that your fork works with Tutor. -First of all, you should make sure that you are working off the ``open-release/koa.3`` tag. See the :ref:`fork edx-platform section ` for more information. +First of all, you should make sure that you are working off the ``open-release/lilac.1`` tag. See the :ref:`fork edx-platform section ` for more information. Then, you should run the following commands:: diff --git a/docs/extra.rst b/docs/extra.rst deleted file mode 100644 index 5da269bb85..0000000000 --- a/docs/extra.rst +++ /dev/null @@ -1,76 +0,0 @@ -.. _extra: - -Extra features -============== - -.. _webui: - -Web UI ------- - -Tutor comes with a web user interface (UI) that allows you to administer your Open edX platform remotely. It's especially convenient for remote administration of the platform. - -Launching the web UI -~~~~~~~~~~~~~~~~~~~~ - -:: - - tutor webui start - -You can then access the interface at http://localhost:3737, or http://youserverurl:3737. - -.. image:: img/webui.png - -All ``tutor`` commands can be executed from this web UI: you just don't need to prefix the commands with ``tutor``. For instance, to deploy a local Open edX instance, run:: - - local quickstart - -instead of ``tutor local quickstart``. - -Authentication -~~~~~~~~~~~~~~ - -**WARNING** Once you launch the web UI, it is accessible by everyone, which means that your Open edX platform is at risk. If you are planning to leave the web UI up for a long time, you should setup a user and password for authentication:: - - tutor webui configure - -.. _mobile: - -Mobile Android application --------------------------- - -With Tutor, you can build an Android mobile application for your platform. To build the application in debug mode, run:: - - tutor android build debug - -The ``.apk`` file will then be available in ``$(tutor config printroot)/data/android``. Transfer it to an Android phone to install the application. You should be able to sign in and view available courses. - -Building a custom Android app -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The Android app is built from the `official edx-app-android repository `__. To change this repository or the app version, you can simply build a different docker image with:: - - tutor images build \ - --build-arg ANDROID_APP_REPOSITORY=https://github.com/mycustomfork/edx-app-android \ - --build-arg ANDROID_APP_VERSION=master \ - android - -Releasing an Android app -~~~~~~~~~~~~~~~~~~~~~~~~ - -**Note**: this is an untested feature. - -Releasing an Android app on the Play Store requires to build the app in release mode. To do so, edit the ``$TUTOR_ROOT/config.yml`` configuration file and define the following variables:: - - ANDROID_RELEASE_STORE_PASSWORD - ANDROID_RELEASE_KEY_PASSWORD - ANDROID_RELEASE_KEY_ALIAS - -Then, place your keystore file in ``$(tutor config printroot)/env/android/app.keystore``. Finally, build the application with:: - - tutor android build release - -Customising the Android app -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Customising the application, such as the logo or the background image, is currently not supported. If you are interested by this feature, please tell us about it in the Tutor `discussion forums `_. diff --git a/docs/faq.rst b/docs/faq.rst index bf82a7199a..03fcf23383 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -28,7 +28,7 @@ The `native installation ` for remotely installing the platform, :ref:`Kubernetes deployment `, additional languages, etc. You'll discover these differences as you explore Tutor :) +There are also many features that are not included in the native installation, such as a `web user interface `__ for remotely installing the platform, :ref:`Kubernetes deployment `, additional languages, etc. You'll discover these differences as you explore Tutor :) What's the difference with the official devstack? ------------------------------------------------- @@ -45,12 +45,12 @@ What features are missing from Tutor? Tutor tries very hard to support all major Open edX features, notably in the form of :ref:`plugins `. In particular, the discovery and ecommerce services, once unavailable in Tutor, can now be easily installed via plugins. If you are interested in sponsoring the development of a new plugin, please `get in touch `__! -It should be noted that the `Analytics `__ stack is currently unsupported, and will likely stay so in the future, as it would require a tremendous amount of work to containerize all the components. For generating great-looking analytics reports, we recommend the `Figures plugin `__. +It should be noted that the `Analytics `__ stack is currently unsupported, and will likely stay so in the future, as it would require a tremendous amount of work to containerize all the components. We are currently working on a replacement solution. Are there people already running this in production? ---------------------------------------------------- -Yes, many of them. There is no way to count precisely how many running Open edX platforms were deployed with Tutor, but from feedback collected directly from real users, there must be dozens, if not hundreds. Tutor is also used by some Open edX providers who are hosting platforms for their customers. +Yes, many of them. There is no way to count precisely how many running Open edX platforms were deployed with Tutor, but from feedback collected directly from real users, there must be hundreds, if not thousands. Tutor is also used by some Open edX providers who are hosting platforms for their customers. Why should I trust software written by some random guy on the Internet? ----------------------------------------------------------------------- diff --git a/docs/img/webui.png b/docs/img/webui.png deleted file mode 100644 index 4f52c60058..0000000000 Binary files a/docs/img/webui.png and /dev/null differ diff --git a/docs/index.rst b/docs/index.rst index a3fbc6da58..168f7e914a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -23,7 +23,6 @@ run configuration plugins - extra troubleshooting tutor faq @@ -50,6 +49,4 @@ The AGPL license covers the Tutor code, including the Dockerfiles, but not the c The Tutor plugin system is licensed under the terms of the `Apache License, Version 2.0 `__. -The :ref:`Tutor Web UI ` depends on the `Gotty `_ binary, which is provided under the terms of the `MIT license `_. - © 2021 Tutor is a registered trademark of SASU NULI NULI. All Rights Reserved. diff --git a/docs/install.rst b/docs/install.rst index 4d27a0d019..648a638dd5 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -88,9 +88,9 @@ Upgrading With Tutor, it is very easy to upgrade to a more recent Open edX or Tutor release. Just install the latest ``tutor`` version (using either methods above) and run the ``quickstart`` command again. If you have :ref:`customised ` your docker images, you will have to re-build them prior to running ``quickstart``. -``quickstart`` should take care of automatically running the upgrade process. If for some reason you need to *manually* upgrade from an Open edX release to the next, you should run ``tutor local upgrade``. For instance, to upgrade from Juniper to Koa, run:: +``quickstart`` should take care of automatically running the upgrade process. If for some reason you need to *manually* upgrade from an Open edX release to the next, you should run ``tutor local upgrade``. For instance, to upgrade from Koa to Lilac, run:: - tutor local upgrade --from=juniper + tutor local upgrade --from=koa .. _autocomplete: diff --git a/docs/intro.rst b/docs/intro.rst index c6b4a23d85..497f2083e5 100644 --- a/docs/intro.rst +++ b/docs/intro.rst @@ -66,7 +66,6 @@ The Android mobile application for this website can be downloaded at this url: h Urls: * LMS: https://demo.openedx.overhang.io -* Analytics (from the `Figures plugin `__): https://demo.openedx.overhang.io/figures * Studio (CMS): https://studio.demo.openedx.overhang.io The platform is reset every day at 9:00 AM, `Paris (France) time `__, so feel free to try and break things as much as you want. @@ -86,12 +85,12 @@ This command does two things: All these files are stored in a single folder, called the Tutor project root. On Linux, this folder is in ``~/.local/share/tutor``. On Mac OS it is ``~/Library/Application Support/tutor``. The values from ``config.yml`` are used to generate the environment files in ``env/``. As a consequence, **every time the values from** ``config.yml`` **are modified, the environment must be regenerated**. This can be done with:: - + tutor config save - + Another consequence is that **any manual change made to a file in** ``env/`` **will be overwritten by** ``tutor config save`` **commands**. Consider yourself warned! I'm ready, where do I start? ---------------------------- -Right :ref:`here `! \ No newline at end of file +Right :ref:`here `! diff --git a/docs/plugins.rst b/docs/plugins.rst index c6c55e1d68..2f58115e4d 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -20,36 +20,31 @@ Commands -------- List installed plugins:: - + tutor plugins list - + Enable/disable a plugin:: - + tutor plugins enable myplugin tutor plugins disable myplugin - + After enabling or disabling a plugin, the environment should be re-generated with:: - + tutor config save - + .. _existing_plugins: Existing plugins ---------------- -- `Course discovery `__: Deploy an API for interacting with your course catalog -- `Ecommerce `__: Sell courses and products on your Open edX platform -- `Figures `__: Visualize daily stats about course engagement -- `MinIO `__: S3 emulator for object storage and scalable Open edX deployment. -- `Notes `__: Allows students to annotate portions of the courseware. -- `Xqueue `__: for external grading +Officially-supported plugins are listed on the `Overhang.IO `__ website. Plugin development ------------------ - + .. toctree:: :maxdepth: 2 plugins/api plugins/gettingstarted - plugins/examples \ No newline at end of file + plugins/examples diff --git a/docs/plugins/api.rst b/docs/plugins/api.rst index 9ffd3e533e..f8676b9202 100644 --- a/docs/plugins/api.rst +++ b/docs/plugins/api.rst @@ -45,13 +45,13 @@ Plugin patches affect the rendered environment templates. In many places the Tut .. note:: The list of existing patches can be found by searching for `{{ patch(` strings in the Tutor source code:: - + git grep "{{ patch" - + The list of patches can also be browsed online `on Github `__. - + Example:: - + patches = { "local-docker-compose-services": """redis: image: redis:latest""" @@ -76,11 +76,11 @@ Hooks are actions that are run during the lifetime of the platform. For instance The services that will be run during initialisation should be added to the ``init`` hook, for instance for database creation and migrations. Example:: - + hooks = { "init": ["myservice1", "myservice2"] } - + During initialisation, "myservice1" and "myservice2" will be run in sequence with the commands defined in the templates ``myplugin/hooks/myservice1/init`` and ``myplugin/hooks/myservice2/init``. To initialise a "foo" service, Tutor runs the "foo-job" service that is found in the ``env/local/docker-compose.jobs.yml`` file. By default, Tutor comes with a few services in this file: mysql-job, lms-job, cms-job, forum-job. If your plugin requires running custom services during initialisation, you will need to add them to the ``docker-compose.jobs.yml`` template. To do so, just use the "local-docker-compose-jobs-services" patch. @@ -102,13 +102,13 @@ Example:: hooks = { "build-image": {"myimage": "myimage:latest"} } - + With this hook, users will be able to build the ``myimage:latest`` docker image by running:: - + tutor images build myimage or:: - + tutor images build all This assumes that there is a ``Dockerfile`` file in the ``myplugin/build/myimage`` subfolder of the plugin templates directory. @@ -119,18 +119,18 @@ This assumes that there is a ``Dockerfile`` file in the ``myplugin/build/myimage This hook allows pulling/pushing images from/to a docker registry. Example:: - + hooks = { "remote-image": {"myimage": "myimage:latest"}, } With this hook, users will be able to pull and push the ``myimage:latest`` docker image by running:: - + tutor images pull myimage tutor images push myimage or:: - + tutor images pull all tutor images push all @@ -142,20 +142,24 @@ templates In order to define plugin-specific hooks, a plugin should also have a template directory that includes the plugin hooks. The ``templates`` attribute should point to that directory. Example:: - + import os templates = os.path.join(os.path.abspath(os.path.dirname(__file__)), "templates") With the above declaration, you can store plugin-specific templates in the ``templates/myplugin`` folder next to the ``plugin.py`` file. In Tutor, templates are `Jinja2 `__-formatted files that will be rendered in the Tutor environment (the ``$(tutor config printroot)/env`` folder) when running ``tutor config save``. The environment files are overwritten every time the environment is saved. Plugin developers can create templates that make use of the built-in `Jinja2 API `__. In addition, a couple additional filters are added by Tutor: - + * ``common_domain``: Return the longest common name between two domain names. Example: ``{{ "studio.demo.myopenedx.com"|common_domain("lms.demo.myopenedx.com") }}`` is equal to "demo.myopenedx.com". * ``encrypt``: Encrypt an arbitrary string. The encryption process is compatible with `htpasswd `__ verification. * ``list_if``: In a list of ``(value, condition)`` tuples, return the list of ``value`` for which the ``condition`` is true. +* ``long_to_base64``: Base-64 encode a long integer. +* ``iter_values_named``: Yield the values of the configuration settings that match a certain pattern. Example: ``{% for value in iter_values_named(prefix="KEY", suffix="SUFFIX")%}...{% endfor %}``. By default, only non-empty values are yielded. To iterate also on empty values, pass the ``allow_empty=True`` argument. * ``patch``: See :ref:`patches `. * ``random_string``: Return a random string of the given length composed of ASCII letters and digits. Example: ``{{ 8|random_string }}``. * ``reverse_host``: Reverse a domain name (see `reference `__). Example: ``{{ "demo.myopenedx.com"|reverse_host }}`` is equal to "com.myopenedx.demo". +* ``rsa_import_key``: Import a PEM-formatted RSA key and return the corresponding object. +* ``rsa_private_key``: Export an RSA private key in PEM format. * ``walk_templates``: Iterate recursively over the templates of the given folder. For instance:: {% for file in "apps/myplugin"|walk_templates %} @@ -175,26 +179,26 @@ command A plugin can provide custom command line commands. Commands are assumed to be `click.Command `__ objects. Example:: - + import click - + @click.command(help="I'm a plugin command") def command(): click.echo("Hello from myplugin!") Any user who installs the ``myplugin`` plugin can then run:: - + $ tutor myplugin Hello from myplugin! You can even define subcommands by creating `command groups `__:: - + import click - + @click.group(help="I'm a plugin command group") def command(): pass - + @click.command(help="I'm a plugin subcommand") def dosomething(): click.echo("This subcommand is awesome") @@ -203,5 +207,5 @@ This would allow any user to run:: $ tutor myplugin dosomething This subcommand is awesome - + See the official `click documentation `__ for more information. diff --git a/docs/plugins/gettingstarted.rst b/docs/plugins/gettingstarted.rst index 6421a898bb..c6613b7087 100644 --- a/docs/plugins/gettingstarted.rst +++ b/docs/plugins/gettingstarted.rst @@ -9,7 +9,7 @@ YAML file ~~~~~~~~~ YAML files that are stored in the tutor plugins root folder will be automatically considered as plugins. The location of the plugin root can be found by running:: - + tutor plugins printroot On Linux, this points to ``~/.local/share/tutor-plugins``. The location of the plugin root folder can be modified by setting the ``TUTOR_PLUGINS_ROOT`` environment variable. @@ -17,7 +17,7 @@ On Linux, this points to ``~/.local/share/tutor-plugins``. The location of the p YAML plugins need to define two extra keys: "name" and "version". Custom CLI commands are not supported by YAML plugins. Let's create a simple plugin that adds your own `Google Analytics `__ tracking code to your Open edX platform. We need to add the ``GOOGLE_ANALYTICS_ACCOUNT`` and ``GOOGLE_ANALYTICS_TRACKING_ID`` settings to both the LMS and the CMS settings. To do so, we will only have to create the ``openedx-common-settings`` patch, which is shared by the development and the production settings both for the LMS and the CMS. First, create the plugin directory:: - + mkdir "$(tutor plugins printroot)" Then add the following content to the plugin file located at ``$(tutor plugins printroot)/myplugin.yml``:: @@ -31,16 +31,16 @@ Then add the following content to the plugin file located at ``$(tutor plugins p GOOGLE_ANALYTICS_TRACKING_ID = "UA-654321-1" Of course, you should replace your Google Analytics tracking code with your own. You can verify that your plugin is correctly installed, but not enabled yet:: - + $ tutor plugins list googleanalytics@0.1.0 (disabled) - + You can then enable your newly-created plugin:: - + tutor plugins enable googleanalytics Update your environment to apply changes from your plugin:: - + tutor config save You should be able to view your changes in every LMS and CMS settings file:: @@ -48,11 +48,11 @@ You should be able to view your changes in every LMS and CMS settings file:: grep -r googleanalytics "$(tutor config printroot)/env/apps/openedx/settings/" Now just restart your platform to start sending tracking events to Google Analytics:: - + tutor local quickstart That's it! And it's very easy to share your plugins. Just upload them to your Github repo and share the url with other users. They will be able to install your plugin by running:: - + tutor plugins install https://raw.githubusercontent.com/username/yourrepo/master/googleanalytics.yml Python package @@ -61,7 +61,7 @@ Python package Creating a plugin as a Python package allows you to define more complex logic and to store your patches in a more structured way. Python Tutor plugins are regular Python packages that define a specific entrypoint: ``tutor.plugin.v0``. Example:: - + from setuptools import setup setup( ... diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 7d20341aad..fac29acd4d 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -18,7 +18,7 @@ Yes :) This is what happens when you run ``tutor local quickstart``: 2. Configuration files are generated from templates. 3. Docker images are downloaded. 4. Docker containers are provisioned. -5. A full, production-ready Open edX platform (`Koa `__ release) is run with docker-compose. +5. A full, production-ready Open edX platform (`Lilac `__ release) is run with docker-compose. The whole procedure should require less than 10 minutes, on a server with a good bandwidth. Note that your host environment will not be affected in any way, since everything runs inside docker containers. Root access is not even necessary. diff --git a/docs/troubleshooting.rst b/docs/troubleshooting.rst index cc6d5aa856..0b3671d946 100644 --- a/docs/troubleshooting.rst +++ b/docs/troubleshooting.rst @@ -13,7 +13,7 @@ What should you do if you have a problem? 3. Search for your problem in the `open and closed Github issues `_. 4. Search for your problem in the `community forums `__. 5. If, despite all your efforts, you can't solve the problem by yourself, you should discuss it in the `community forums `__. Please give as much details about your problem as possible! As a rule of thumb, **people will not dedicate more time to solving your problem than you took to write your question**. -6. If you are *absolutely* positive that you are facing a technical issue with Tutor, and not with Open edX, not with your server, not your custom configuration, then, and only then, should you open an issue on `Github `__. You *must* follow the instructions from the issue template!!! If you do not follow this procedure, your Github issues will be mercilessly closed 🤯. +6. If you are *absolutely* positive that you are facing a technical issue with Tutor, and not with Open edX, not with your server, not your custom configuration, then, and only then, should you open an issue on `Github `__. You *must* follow the instructions from the issue template!!! If you do not follow this procedure, your Github issues will be mercilessly closed 🤯. Do you need professional assistance with your tutor-managed Open edX platform? Overhang.IO offers online support as part of its `Long Term Support (LTS) offering `__. diff --git a/docs/whatnext.rst b/docs/whatnext.rst index 648450ace4..e1fb699059 100644 --- a/docs/whatnext.rst +++ b/docs/whatnext.rst @@ -23,7 +23,7 @@ Tutor makes it easy to :ref:`develop ` and :ref:`install ` yo Adding features --------------- -Check out the Tutor :ref:`plugins `, :ref:`extra features ` and :ref:`configuration/customization options `. +Check out the Tutor :ref:`plugins ` and :ref:`configuration/customization options `. Hacking into Open edX --------------------- @@ -38,4 +38,4 @@ Yes, Tutor comes with Kubernetes deployment support :ref:`out of the box `. Meeting the community --------------------- -Ask your questions and chat with the Tutor community on the official community forums: https://discuss.overhang.io \ No newline at end of file +Ask your questions and chat with the Tutor community on the official community forums: https://discuss.overhang.io diff --git a/requirements/base.in b/requirements/base.in index b3a3ad9525..97611c2ab3 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -1,6 +1,5 @@ appdirs click -click_repl mypy pycryptodome jinja2 diff --git a/requirements/base.txt b/requirements/base.txt index 9acb263fa4..63dcc83760 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -14,12 +14,8 @@ certifi==2021.5.30 # requests chardet==4.0.0 # via requests -click-repl==0.2.0 - # via -r requirements/base.in click==8.0.1 - # via - # -r requirements/base.in - # click-repl + # via -r requirements/base.in google-auth==1.30.1 # via kubernetes idna==2.10 @@ -36,8 +32,6 @@ mypy==0.812 # via -r requirements/base.in oauthlib==3.1.1 # via requests-oauthlib -prompt-toolkit==3.0.18 - # via click-repl pyasn1-modules==0.2.8 # via google-auth pyasn1==0.4.8 @@ -62,7 +56,6 @@ rsa==4.7.2 # via google-auth six==1.16.0 # via - # click-repl # google-auth # kubernetes # python-dateutil @@ -74,8 +67,6 @@ urllib3==1.26.5 # via # kubernetes # requests -wcwidth==0.2.5 - # via prompt-toolkit websocket-client==1.0.1 # via kubernetes diff --git a/requirements/dev.txt b/requirements/dev.txt index 1ec50faf33..a8964314d5 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -31,13 +31,10 @@ chardet==4.0.0 # via # -r requirements/base.txt # requests -click-repl==0.2.0 - # via -r requirements/base.txt click==8.0.1 # via # -r requirements/base.txt # black - # click-repl # pip-tools colorama==0.4.4 # via twine @@ -98,10 +95,6 @@ pip-tools==6.1.0 # via -r requirements/dev.in pkginfo==1.7.0 # via twine -prompt-toolkit==3.0.18 - # via - # -r requirements/base.txt - # click-repl pyasn1-modules==0.2.8 # via # -r requirements/base.txt @@ -162,7 +155,6 @@ six==1.16.0 # via # -r requirements/base.txt # bleach - # click-repl # google-auth # kubernetes # python-dateutil @@ -189,10 +181,6 @@ urllib3==1.26.5 # -r requirements/base.txt # kubernetes # requests -wcwidth==0.2.5 - # via - # -r requirements/base.txt - # prompt-toolkit webencodings==0.5.1 # via bleach websocket-client==1.0.1 diff --git a/requirements/docs.txt b/requirements/docs.txt index eb47ae7589..bea66ff509 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -23,12 +23,8 @@ chardet==4.0.0 # via # -r requirements/base.txt # requests -click-repl==0.2.0 - # via -r requirements/base.txt click==8.0.1 - # via - # -r requirements/base.txt - # click-repl + # via -r requirements/base.txt docutils==0.16 # via # sphinx @@ -65,10 +61,6 @@ oauthlib==3.1.1 # requests-oauthlib packaging==20.9 # via sphinx -prompt-toolkit==3.0.18 - # via - # -r requirements/base.txt - # click-repl pyasn1-modules==0.2.8 # via # -r requirements/base.txt @@ -111,7 +103,6 @@ rsa==4.7.2 six==1.16.0 # via # -r requirements/base.txt - # click-repl # google-auth # kubernetes # python-dateutil @@ -148,10 +139,6 @@ urllib3==1.26.5 # -r requirements/base.txt # kubernetes # requests -wcwidth==0.2.5 - # via - # -r requirements/base.txt - # prompt-toolkit websocket-client==1.0.1 # via # -r requirements/base.txt diff --git a/requirements/plugins.txt b/requirements/plugins.txt index cb282d810c..da66ffdfe5 100644 --- a/requirements/plugins.txt +++ b/requirements/plugins.txt @@ -1,7 +1,10 @@ -tutor-discovery -tutor-ecommerce -#tutor-figures -tutor-license -tutor-minio -tutor-notes -tutor-xqueue \ No newline at end of file +# change version ranges when upgrading from lilac +tutor-android>=12.0.0,<13.0.0 +tutor-discovery>=12.0.0,<13.0.0 +tutor-ecommerce>=12.0.0,<13.0.0 +tutor-license>=12.0.0,<13.0.0 +tutor-mfe>=12.0.0,<13.0.0 +tutor-minio>=12.0.0,<13.0.0 +tutor-notes>=12.0.0,<13.0.0 +tutor-webui>=12.0.0,<13.0.0 +tutor-xqueue>=12.0.0,<13.0.0 diff --git a/tests/test_env.py b/tests/test_env.py index fe09c04c7c..17528f2691 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -42,6 +42,13 @@ def test_render_str(self) -> None: "hello world", env.render_str({"name": "world"}, "hello {{ name }}") ) + def test_render_unknown(self) -> None: + config: Config = { + "var1": "a", + } + self.assertEqual("ab", env.render_unknown(config, "{{ var1 }}b")) + self.assertEqual({"x": "ac"}, env.render_unknown(config, {"x": "{{ var1 }}c"})) + def test_common_domain(self) -> None: self.assertEqual( "mydomain.com", @@ -176,3 +183,22 @@ def test_renderer_is_reset_on_config_change(self) -> None: self.assertNotIn("plugin1/myplugin.txt", env1.loader.list_templates()) self.assertIn("plugin1/myplugin.txt", env2.loader.list_templates()) + + def test_iter_values_named(self) -> None: + config: Config = { + "something0_test_app": 0, + "something1_test_not_app": 1, + "notsomething_test_app": 2, + "something3_test_app": 3, + } + renderer = env.Renderer.instance(config) + self.assertEqual([2, 3], list(renderer.iter_values_named(suffix="test_app"))) + self.assertEqual([1, 3], list(renderer.iter_values_named(prefix="something"))) + self.assertEqual( + [0, 3], + list( + renderer.iter_values_named( + prefix="something", suffix="test_app", allow_empty=True + ) + ), + ) diff --git a/tutor.spec b/tutor.spec index f089dab80a..bfd63aaa75 100644 --- a/tutor.spec +++ b/tutor.spec @@ -2,16 +2,12 @@ import importlib import os import pkg_resources -import wcwidth block_cipher = None datas = [("./tutor/templates", "./tutor/templates")] hidden_imports = [] -# Fix missing wcwidth/version.json file -datas.append((os.path.dirname(wcwidth.__file__), 'wcwidth')) - # Auto-discover plugins and include patches & templates folders for entrypoint in pkg_resources.iter_entry_points("tutor.plugin.v0"): plugin_name = entrypoint.name diff --git a/tutor/__about__.py b/tutor/__about__.py index 62077c2499..0e21ac941a 100644 --- a/tutor/__about__.py +++ b/tutor/__about__.py @@ -1 +1 @@ -__version__ = "11.3.1" +__version__ = "12.0.0" diff --git a/tutor/commands/android.py b/tutor/commands/android.py deleted file mode 100644 index 90fddf68c1..0000000000 --- a/tutor/commands/android.py +++ /dev/null @@ -1,52 +0,0 @@ -import click - -from .compose import ComposeJobRunner -from .local import docker_compose as local_docker_compose -from .. import config as tutor_config -from .. import env as tutor_env -from .. import fmt -from ..types import Config -from .context import Context - - -@click.group(help="Build an Android app for your Open edX platform [BETA FEATURE]") -def android() -> None: - pass - - -@click.command(help="Build the application") -@click.argument("mode", type=click.Choice(["debug", "release"])) -@click.pass_obj -def build(context: Context, mode: str) -> None: - config = tutor_config.load(context.root) - docker_run(context.root, build_command(config, mode)) - fmt.echo_info( - "The {} APK file is available in {}".format( - mode, tutor_env.data_path(context.root, "android") - ) - ) - - -def build_command(config: Config, target: str) -> str: - gradle_target = { - "debug": "assembleProdDebuggable", - "release": "assembleProdRelease", - }[target] - apk_folder = {"debug": "debuggable", "release": "release"}[target] - - command = """ -sed -i "s/APPLICATION_ID = .*/APPLICATION_ID = \\"{{ LMS_HOST|reverse_host|replace("-", "_") }}\\"/g" constants.gradle -./gradlew {gradle_target} -cp OpenEdXMobile/build/outputs/apk/prod/{apk_folder}/*.apk /openedx/data/""" - command = tutor_env.render_str(config, command) - command = command.format(gradle_target=gradle_target, apk_folder=apk_folder) - return command - - -def docker_run(root: str, command: str) -> None: - config = tutor_config.load(root) - runner = ComposeJobRunner(root, config, local_docker_compose) - runner.run_job("android", command) - - -android.add_command(build) diff --git a/tutor/commands/cli.py b/tutor/commands/cli.py index 1dd9a92c50..ec42c08803 100755 --- a/tutor/commands/cli.py +++ b/tutor/commands/cli.py @@ -3,9 +3,7 @@ import appdirs import click -import click_repl -from .android import android from .config import config_command from .context import Context from .dev import dev @@ -13,8 +11,6 @@ from .k8s import k8s from .local import local from .plugins import plugins_command, add_plugin_commands -from .ui import ui -from .webui import webui from ..__about__ import __version__ from .. import exceptions from .. import fmt @@ -23,15 +19,11 @@ def main() -> None: try: - click_repl.register_repl(cli, name="ui") cli.add_command(images_command) cli.add_command(config_command) cli.add_command(local) cli.add_command(dev) - cli.add_command(android) cli.add_command(k8s) - cli.add_command(ui) - cli.add_command(webui) cli.add_command(print_help) cli.add_command(plugins_command) add_plugin_commands(cli) diff --git a/tutor/commands/compose.py b/tutor/commands/compose.py index 782420def8..be755834a2 100644 --- a/tutor/commands/compose.py +++ b/tutor/commands/compose.py @@ -83,7 +83,10 @@ def run_job(self, service: str, command: str) -> int: ) -@click.command(help="Run all or a selection of configured Open edX services") +@click.command( + short_help="Run all or a selection of services.", + help="Run all or a selection of services. Docker images will be rebuilt where necessary.", +) @click.option("-d", "--detach", is_flag=True, help="Start in daemon mode") @click.argument("services", metavar="service", nargs=-1) @click.pass_obj @@ -93,6 +96,9 @@ def start(context: Context, detach: bool, services: List[str]) -> None: command.append("-d") config = tutor_config.load(context.root) + # Rebuild Docker images with a `build: ...` context. + context.docker_compose(context.root, config, "build") + # Start services context.docker_compose(context.root, config, *command, *services) diff --git a/tutor/commands/images.py b/tutor/commands/images.py index 2936efa0a5..b27a8eef9a 100644 --- a/tutor/commands/images.py +++ b/tutor/commands/images.py @@ -11,7 +11,7 @@ from .. import utils from .context import Context -BASE_IMAGE_NAMES = ["openedx", "forum", "android"] +BASE_IMAGE_NAMES = ["openedx", "forum"] DEV_IMAGE_NAMES = ["openedx-dev"] VENDOR_IMAGES = [ "caddy", diff --git a/tutor/commands/k8s.py b/tutor/commands/k8s.py index c91f2fc5e9..31814c6707 100644 --- a/tutor/commands/k8s.py +++ b/tutor/commands/k8s.py @@ -396,7 +396,10 @@ def wait(context: Context, name: str) -> None: @click.command(help="Upgrade from a previous Open edX named release") @click.option( - "--from", "from_version", default="ironwood", type=click.Choice(["ironwood"]) + "--from", + "from_version", + default="koa", + type=click.Choice(["ironwood", "juniper", "koa"]), ) @click.pass_obj def upgrade(context: Context, from_version: str) -> None: @@ -408,8 +411,13 @@ def upgrade(context: Context, from_version: str) -> None: running_version = "juniper" if running_version == "juniper": + upgrade_from_juniper(config) running_version = "koa" + if running_version == "koa": + upgrade_from_koa(config) + running_version = "lilac" + def upgrade_from_ironwood(config: Config) -> None: if not config["RUN_MONGODB"]: @@ -458,6 +466,26 @@ def upgrade_from_juniper(config: Config) -> None: fmt.echo_info(message) +def upgrade_from_koa(config: Config) -> None: + if not config["RUN_MONGODB"]: + fmt.echo_info( + "You are not running MongDB (RUN_MONGODB=false). It is your " + "responsibility to upgrade your MongoDb instance to v4.0. There is " + "nothing left to do to upgrade to Lilac from Koa." + ) + return + message = """Automatic release upgrade is unsupported in Kubernetes. To upgrade from Koa to Lilac, you should upgrade +your MongoDb cluster from v3.6 to v4.0. You should run something similar to: + + tutor k8s stop + tutor config save --set DOCKER_IMAGE_MONGODB=mongo:4.0 + tutor k8s start + tutor k8s exec mongodb mongo --eval 'db.adminCommand({ setFeatureCompatibilityVersion: "4.0" })' + tutor config save --unset DOCKER_IMAGE_MONGODB + """ + fmt.echo_info(message) + + def kubectl_exec( config: Config, service: str, command: str, attach: bool = False ) -> int: diff --git a/tutor/commands/local.py b/tutor/commands/local.py index fd89807cba..064aebc1f6 100644 --- a/tutor/commands/local.py +++ b/tutor/commands/local.py @@ -87,8 +87,8 @@ def quickstart(context: click.Context, non_interactive: bool, pullimages: bool) @click.option( "--from", "from_version", - default="juniper", - type=click.Choice(["ironwood", "juniper"]), + default="koa", + type=click.Choice(["ironwood", "juniper", "koa"]), ) @click.option("-I", "--non-interactive", is_flag=True, help="Run non-interactively") @click.pass_context @@ -117,6 +117,10 @@ def upgrade(context: click.Context, from_version: str, non_interactive: bool) -> upgrade_from_juniper(context, config) running_version = "koa" + if running_version == "koa": + upgrade_from_koa(context, config) + running_version = "lilac" + def upgrade_from_ironwood(context: click.Context, config: Config) -> None: click.echo(fmt.title("Upgrading from Ironwood")) @@ -129,40 +133,13 @@ def upgrade_from_ironwood(context: click.Context, config: Config) -> None: fmt.echo_info( "You are not running MongDB (RUN_MONGODB=false). It is your " "responsibility to upgrade your MongoDb instance to v3.6. There is " - "nothing left to do to upgrade from Ironwood." + "nothing left to do to upgrade from Ironwood to Juniper." ) return - # Note that the DOCKER_IMAGE_MONGODB value is never saved, because we only save the - # environment, not the configuration. - click.echo(fmt.title("Upgrading MongoDb from v3.2 to v3.4")) - config["DOCKER_IMAGE_MONGODB"] = "mongo:3.4.24" - tutor_env.save(context.obj.root, config) - context.invoke(compose.start, detach=True, services=["mongodb"]) - context.invoke( - compose.execute, - args=[ - "mongodb", - "mongo", - "--eval", - 'db.adminCommand({ setFeatureCompatibilityVersion: "3.4" })', - ], - ) + upgrade_mongodb(context, config, "3.4") context.invoke(compose.stop) - - click.echo(fmt.title("Upgrading MongoDb from v3.4 to v3.6")) - config["DOCKER_IMAGE_MONGODB"] = "mongo:3.6.18" - tutor_env.save(context.obj.root, config) - context.invoke(compose.start, detach=True, services=["mongodb"]) - context.invoke( - compose.execute, - args=[ - "mongodb", - "mongo", - "--eval", - 'db.adminCommand({ setFeatureCompatibilityVersion: "3.6" })', - ], - ) + upgrade_mongodb(context, config, "3.6") context.invoke(compose.stop) @@ -198,6 +175,36 @@ def upgrade_from_juniper(context: click.Context, config: Config) -> None: context.invoke(compose.stop) +def upgrade_from_koa(context: click.Context, config: Config) -> None: + if not config["RUN_MONGODB"]: + fmt.echo_info( + "You are not running MongDB (RUN_MONGODB=false). It is your " + "responsibility to upgrade your MongoDb instance to v4.0. There is " + "nothing left to do to upgrade from Koa to Lilac." + ) + return + upgrade_mongodb(context, config, "4.0") + + +def upgrade_mongodb(context: click.Context, config: Config, to_version: str) -> None: + click.echo(fmt.title("Upgrading MongoDb to v{}".format(to_version))) + # Note that the DOCKER_IMAGE_MONGODB value is never saved, because we only save the + # environment, not the configuration. + config["DOCKER_IMAGE_MONGODB"] = "mongo:{}".format(to_version) + tutor_env.save(context.obj.root, config) + context.invoke(compose.start, detach=True, services=["mongodb"]) + context.invoke( + compose.execute, + args=[ + "mongodb", + "mongo", + "--eval", + 'db.adminCommand({ setFeatureCompatibilityVersion: "%s" })' % to_version, + ], + ) + context.invoke(compose.stop) + + local.add_command(quickstart) local.add_command(upgrade) compose.add_commands(local) diff --git a/tutor/commands/ui.py b/tutor/commands/ui.py deleted file mode 100644 index 6b0605647f..0000000000 --- a/tutor/commands/ui.py +++ /dev/null @@ -1,21 +0,0 @@ -import click -import click_repl - - -@click.command( - short_help="Interactive shell", - help="Launch an interactive shell for launching Tutor commands", -) -def ui() -> None: - click.echo( - """Welcome to the Tutor interactive shell UI! -Type "help" to view all available commands. -Type "local quickstart" to configure and launch a new platform from scratch. -Type to exit.""" - ) - while True: - try: - click_repl.repl(click.get_current_context()) - return # this happens on a ctrl+d - except Exception: # pylint: disable=broad-except - pass diff --git a/tutor/commands/webui.py b/tutor/commands/webui.py deleted file mode 100644 index 9508b960bd..0000000000 --- a/tutor/commands/webui.py +++ /dev/null @@ -1,161 +0,0 @@ -import io -import os -import platform -import subprocess -import sys -import tarfile -from typing import Dict, Optional -from urllib.request import urlopen - -import click - -# Note: it is important that this module does not depend on config, such that -# the web ui can be launched even where there is no configuration. -from .. import fmt -from .. import env as tutor_env -from .. import exceptions -from .. import serialize -from ..types import Config -from .context import Context - - -@click.group( - short_help="Web user interface", help="""Run Tutor commands from a web terminal""" -) -def webui() -> None: - pass - - -@click.command(help="Start the web UI") -@click.option( - "-p", - "--port", - default=3737, - type=int, - show_default=True, - help="Port number to listen", -) -@click.option( - "-h", "--host", default="0.0.0.0", show_default=True, help="Host address to listen" -) -@click.pass_obj -def start(context: Context, port: int, host: str) -> None: - check_gotty_binary(context.root) - fmt.echo_info("Access the Tutor web UI at http://{}:{}".format(host, port)) - while True: - config = load_config(context.root) - user = config["user"] - password = config["password"] - command = [ - gotty_path(context.root), - "--permit-write", - "--address", - host, - "--port", - str(port), - "--title-format", - "Tutor web UI - {{ .Command }} ({{ .Hostname }})", - ] - if user and password: - credential = "{}:{}".format(user, password) - command += ["--credential", credential] - else: - fmt.echo_alert( - "Running web UI without user authentication. Run 'tutor webui configure' to setup authentication" - ) - command += [sys.argv[0], "ui"] - p = subprocess.Popen(command) - while True: - try: - p.wait(timeout=2) - except subprocess.TimeoutExpired: - new_config = load_config(context.root) - if new_config != config: - click.echo( - "WARNING configuration changed. Tutor web UI is now going to restart. Reload this page to continue." - ) - p.kill() - p.wait() - break - - -@click.command(help="Configure authentication") -@click.option("-u", "--user", prompt="User name", help="Authentication user name") -@click.option( - "-p", - "--password", - prompt=True, - hide_input=True, - confirmation_prompt=True, - help="Authentication password", -) -@click.pass_obj -def configure(context: Context, user: str, password: str) -> None: - save_webui_config_file(context.root, {"user": user, "password": password}) - fmt.echo_info( - "The web UI configuration has been updated. " - "If at any point you wish to reset your username and password, " - "just delete the following file:\n\n {}".format(config_path(context.root)) - ) - - -def check_gotty_binary(root: str) -> None: - path = gotty_path(root) - if os.path.exists(path): - return - fmt.echo_info("Downloading gotty to {}...".format(path)) - - # Generate release url - # Note: I don't know how to handle arm - architecture = "amd64" if platform.architecture()[0] == "64bit" else "386" - url = "https://github.com/yudai/gotty/releases/download/v1.0.1/gotty_{system}_{architecture}.tar.gz".format( - system=platform.system().lower(), architecture=architecture - ) - - # Download - response = urlopen(url) - - # Decompress - dirname = os.path.dirname(path) - if not os.path.exists(dirname): - os.makedirs(dirname) - compressed = tarfile.open(fileobj=io.BytesIO(response.read())) - compressed.extract("./gotty", dirname) - - -def load_config(root: str) -> Dict[str, Optional[str]]: - path = config_path(root) - if not os.path.exists(path): - save_webui_config_file(root, {"user": None, "password": None}) - with open(config_path(root)) as f: - config = serialize.load(f) - if not isinstance(config, dict): - raise exceptions.TutorError( - "Invalid webui: expected dict, got {}".format(config.__class__) - ) - return config - - -def save_webui_config_file(root: str, config: Config) -> None: - path = config_path(root) - directory = os.path.dirname(path) - if not os.path.exists(directory): - os.makedirs(directory) - with open(path, "w") as of: - serialize.dump(config, of) - - -def gotty_path(root: str) -> str: - return get_path(root, "gotty") - - -def config_path(root: str) -> str: - return get_path(root, "config.yml") - - -def get_path(root: str, filename: str) -> str: - return tutor_env.pathjoin(root, "webui", filename) - - -webui.add_command(start) -webui.add_command(configure) diff --git a/tutor/config.py b/tutor/config.py index c72de93dad..fda00aa227 100644 --- a/tutor/config.py +++ b/tutor/config.py @@ -102,7 +102,6 @@ def load_required(config: Config, defaults: Config) -> None: "OPENEDX_SECRET_KEY", "MYSQL_ROOT_PASSWORD", "OPENEDX_MYSQL_PASSWORD", - "ANDROID_OAUTH2_SECRET", "ID", "JWT_RSA_PRIVATE_KEY", ]: diff --git a/tutor/env.py b/tutor/env.py index adb79f276e..3e7f492a6a 100644 --- a/tutor/env.py +++ b/tutor/env.py @@ -8,7 +8,7 @@ from . import exceptions, fmt, plugins, utils from .__about__ import __version__ -from .types import Config +from .types import Config, ConfigValue TEMPLATES_ROOT = pkg_resources.resource_filename("tutor", "templates") VERSION_FILENAME = "version" @@ -52,12 +52,13 @@ def __init__( environment.filters["encrypt"] = utils.encrypt environment.filters["list_if"] = utils.list_if environment.filters["long_to_base64"] = utils.long_to_base64 + environment.globals["iter_values_named"] = self.iter_values_named + environment.globals["patch"] = self.patch environment.filters["random_string"] = utils.random_string environment.filters["reverse_host"] = utils.reverse_host + environment.globals["rsa_import_key"] = utils.rsa_import_key environment.filters["rsa_private_key"] = utils.rsa_private_key environment.filters["walk_templates"] = self.walk_templates - environment.globals["patch"] = self.patch - environment.globals["rsa_import_key"] = utils.rsa_import_key environment.globals["TUTOR_VERSION"] = __version__ self.environment = environment @@ -71,6 +72,29 @@ def iter_templates_in(self, *prefix: str) -> Iterable[str]: if template.startswith(full_prefix) and self.is_part_of_env(template): yield template + def iter_values_named( + self, + prefix: Optional[str] = None, + suffix: Optional[str] = None, + allow_empty: bool = False, + ) -> Iterable[ConfigValue]: + """ + Iterate on all config values for which the name match the given pattern. + + Note that here we only iterate on the values, not the key names. Empty + values (those that evaluate to boolean `false`) will not be yielded, unless + `allow_empty` is True. + TODO document this in the plugins API + """ + for var_name, value in self.config.items(): + if prefix is not None and not var_name.startswith(prefix): + continue + if suffix is not None and not var_name.endswith(suffix): + continue + if not allow_empty and not value: + continue + yield value + def walk_templates(self, subdir: str) -> Iterable[str]: """ Iterate on the template files from `templates/`. @@ -110,15 +134,13 @@ def patch(self, name: str, separator: str = "\n", suffix: str = "") -> str: """ patches = [] for plugin, patch in plugins.iter_patches(self.config, name): - patch_template = self.environment.from_string(patch) try: - patches.append(patch_template.render(**self.config)) - except jinja2.exceptions.UndefinedError as e: - raise exceptions.TutorError( - "Missing configuration value: {} in patch '{}' from plugin {}".format( - e.args[0], name, plugin - ) + patches.append(self.render_str(patch)) + except exceptions.TutorError: + fmt.echo_error( + "Error rendering patch '{}' from plugin {}".format(name, plugin) ) + raise rendered = separator.join(patches) if rendered: rendered += suffix @@ -180,13 +202,11 @@ def save(root: str, config: Config) -> None: """ root_env = pathjoin(root) for prefix in [ - "android/", "apps/", "build/", "dev/", "k8s/", "local/", - "webui/", VERSION_FILENAME, "kustomization.yml", ]: @@ -252,27 +272,16 @@ def render_file(config: Config, *path: str) -> Union[str, bytes]: return renderer.render_template(template_name) -def render_dict(config: Config) -> None: +def render_unknown(config: Config, value: Any) -> Any: """ - Render the values from the dict. This is useful for rendering the default - values from config.yml. + Render an unknown `value` object with the selected config. - Args: - config (dict) + If `value` is a dict, its values are also rendered. """ - rendered: Config = {} - for key, value in config.items(): - if isinstance(value, str): - rendered[key] = render_str(config, value) - else: - rendered[key] = value - for k, v in rendered.items(): - config[k] = v - - -def render_unknown(config: Config, value: Any) -> Any: if isinstance(value, str): return render_str(config, value) + elif isinstance(value, dict): + return {k: render_unknown(config, v) for k, v in value.items()} return value diff --git a/tutor/templates/android/edx.properties b/tutor/templates/android/edx.properties deleted file mode 100644 index 69139982d3..0000000000 --- a/tutor/templates/android/edx.properties +++ /dev/null @@ -1,3 +0,0 @@ -edx.android { - configFiles = ['tutor.yaml'] -} diff --git a/tutor/templates/android/gradle.properties b/tutor/templates/android/gradle.properties deleted file mode 100644 index 486a13a5b1..0000000000 --- a/tutor/templates/android/gradle.properties +++ /dev/null @@ -1,4 +0,0 @@ -RELEASE_STORE_FILE=/openedx/config/app.keystore -RELEASE_STORE_PASSWORD={{ ANDROID_RELEASE_STORE_PASSWORD }} -RELEASE_KEY_PASSWORD={{ ANDROID_RELEASE_KEY_PASSWORD }} -RELEASE_KEY_ALIAS={{ ANDROID_RELEASE_KEY_ALIAS }} diff --git a/tutor/templates/android/tutor.yaml b/tutor/templates/android/tutor.yaml deleted file mode 100644 index d5ecc8d298..0000000000 --- a/tutor/templates/android/tutor.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# See docs: https://openedx.atlassian.net/wiki/spaces/LEARNER/pages/48792067/App+Configuration+Flags -API_HOST_URL: "{{ "https" if ENABLE_HTTPS else "http" }}://{{ LMS_HOST }}" -ENVIRONMENT_DISPLAY_NAME: "tutor" -PLATFORM_NAME: "{{ PLATFORM_NAME }}" -PLATFORM_DESTINATION_NAME: "{{ LMS_HOST }}" -FEEDBACK_EMAIL_ADDRESS: "{{ CONTACT_EMAIL }}" -OAUTH_CLIENT_ID: "android" - -COURSE_VIDEOS_ENABLED: true -CERTIFICATES_ENABLED: true -DISCUSSIONS_ENABLED: true -DISCOVERY: - COURSE: - TYPE: native -DOWNLOAD_TO_SD_CARD_ENABLED: true -NEW_LOGISTRATION_ENABLED: true -USER_PROFILES_ENABLED : true -VIDEO_TRANSCRIPT_ENABLED: true \ No newline at end of file diff --git a/tutor/templates/apps/openedx/config/cms.env.json b/tutor/templates/apps/openedx/config/cms.env.json index 06b4b8c80a..32aea1bab7 100644 --- a/tutor/templates/apps/openedx/config/cms.env.json +++ b/tutor/templates/apps/openedx/config/cms.env.json @@ -29,11 +29,6 @@ "ENABLE_COMPREHENSIVE_THEMING": true, "COMPREHENSIVE_THEME_DIRS": ["/openedx/themes"], "STATIC_ROOT_BASE": "/openedx/staticfiles", - "ELASTIC_SEARCH_CONFIG": [{ - {% if ELASTICSEARCH_SCHEME == "https" %}"use_ssl": true,{% endif %} - "host": "{{ ELASTICSEARCH_HOST }}", - "port": {{ ELASTICSEARCH_PORT }} - }], "EMAIL_BACKEND": "django.core.mail.backends.smtp.EmailBackend", "EMAIL_HOST": "{{ SMTP_HOST }}", "EMAIL_PORT": {{ SMTP_PORT }}, diff --git a/tutor/templates/apps/openedx/config/lms.env.json b/tutor/templates/apps/openedx/config/lms.env.json index 2ec8242ed1..15a2b12903 100644 --- a/tutor/templates/apps/openedx/config/lms.env.json +++ b/tutor/templates/apps/openedx/config/lms.env.json @@ -38,11 +38,6 @@ "ENABLE_COMPREHENSIVE_THEMING": true, "COMPREHENSIVE_THEME_DIRS": ["/openedx/themes"], "STATIC_ROOT_BASE": "/openedx/staticfiles", - "ELASTIC_SEARCH_CONFIG": [{ - {% if ELASTICSEARCH_SCHEME == "https" %}"use_ssl": true,{% endif %} - "host": "{{ ELASTICSEARCH_HOST }}", - "port": {{ ELASTICSEARCH_PORT }} - }], "EMAIL_BACKEND": "django.core.mail.backends.smtp.EmailBackend", "EMAIL_HOST": "{{ SMTP_HOST }}", "EMAIL_PORT": {{ SMTP_PORT }}, diff --git a/tutor/templates/apps/openedx/settings/cms/__init__.py b/tutor/templates/apps/openedx/settings/cms/__init__.py index f0c04d958b..e69de29bb2 100644 --- a/tutor/templates/apps/openedx/settings/cms/__init__.py +++ b/tutor/templates/apps/openedx/settings/cms/__init__.py @@ -1 +0,0 @@ -{% include "apps/openedx/settings/partials/pre_common_all.py" %} diff --git a/tutor/templates/apps/openedx/settings/lms/__init__.py b/tutor/templates/apps/openedx/settings/lms/__init__.py index f0c04d958b..e69de29bb2 100644 --- a/tutor/templates/apps/openedx/settings/lms/__init__.py +++ b/tutor/templates/apps/openedx/settings/lms/__init__.py @@ -1 +0,0 @@ -{% include "apps/openedx/settings/partials/pre_common_all.py" %} diff --git a/tutor/templates/apps/openedx/settings/partials/common_all.py b/tutor/templates/apps/openedx/settings/partials/common_all.py index 0226510f2b..1f4186f183 100644 --- a/tutor/templates/apps/openedx/settings/partials/common_all.py +++ b/tutor/templates/apps/openedx/settings/partials/common_all.py @@ -32,6 +32,13 @@ # Behave like memcache when it comes to connection errors DJANGO_REDIS_IGNORE_EXCEPTIONS = True +# Elasticsearch connection parameters +ELASTIC_SEARCH_CONFIG = [{ + {% if ELASTICSEARCH_SCHEME == "https" %}"use_ssl": True,{% endif %} + "host": "{{ ELASTICSEARCH_HOST }}", + "port": {{ ELASTICSEARCH_PORT }}, +}] + DEFAULT_FROM_EMAIL = ENV_TOKENS.get("DEFAULT_FROM_EMAIL", ENV_TOKENS["CONTACT_EMAIL"]) DEFAULT_FEEDBACK_EMAIL = ENV_TOKENS.get("DEFAULT_FEEDBACK_EMAIL", ENV_TOKENS["CONTACT_EMAIL"]) SERVER_EMAIL = ENV_TOKENS.get("SERVER_EMAIL", ENV_TOKENS["CONTACT_EMAIL"]) @@ -86,6 +93,11 @@ "formatter": "standard", } LOGGING["loggers"]["tracking"]["handlers"] = ["console", "local", "tracking"] +# Silence some loggers (note: we must attempt to get rid of these when upgrading from one release to the next) +import warnings +warnings.filterwarnings("ignore", category=DeprecationWarning, module="newrelic.console") +warnings.filterwarnings("ignore", category=DeprecationWarning, module="lms.djangoapps.course_wiki.plugins.markdownedx.wiki_plugin") +warnings.filterwarnings("ignore", category=DeprecationWarning, module="wiki.plugins.links.wiki_plugin") # Email EMAIL_USE_SSL = {{ SMTP_USE_SSL }} @@ -146,5 +158,11 @@ "user": None, } +# Custom features +# LTI 1.3 will be enabled by default after lilac, and it's going to be a big +# deal, so we enable it early. We should remove this once the feature flag is +# deprecated. +FEATURES["LTI_1P3_ENABLED"] = True + {{ patch("openedx-common-settings") }} ######## End of settings common to LMS and CMS diff --git a/tutor/templates/apps/openedx/settings/partials/common_lms.py b/tutor/templates/apps/openedx/settings/partials/common_lms.py index ef9bd6be37..15d9cbcd30 100644 --- a/tutor/templates/apps/openedx/settings/partials/common_lms.py +++ b/tutor/templates/apps/openedx/settings/partials/common_lms.py @@ -7,9 +7,6 @@ REGISTRATION_EXTRA_FIELDS["terms_of_service"] = "required" REGISTRATION_EXTRA_FIELDS["honor_code"] = "hidden" -# This url must not be None and should not be used anywhere -LEARNING_MICROFRONTEND_URL = "http://learn.openedx.org" - # Fix media files paths PROFILE_IMAGE_BACKEND["options"]["location"] = os.path.join( MEDIA_ROOT, "profile-images/" @@ -28,4 +25,4 @@ {{ patch("openedx-lms-common-settings") }} -######## End of common LMS settings \ No newline at end of file +######## End of common LMS settings diff --git a/tutor/templates/apps/openedx/settings/partials/pre_common_all.py b/tutor/templates/apps/openedx/settings/partials/pre_common_all.py deleted file mode 100644 index 282db6331f..0000000000 --- a/tutor/templates/apps/openedx/settings/partials/pre_common_all.py +++ /dev/null @@ -1,10 +0,0 @@ -# Silence overly verbose warnings -import logging -import warnings -from django.utils.deprecation import RemovedInDjango30Warning, RemovedInDjango31Warning -from rest_framework import RemovedInDRF310Warning, RemovedInDRF311Warning -warnings.simplefilter('ignore', RemovedInDjango30Warning) -warnings.simplefilter('ignore', RemovedInDjango31Warning) -warnings.simplefilter('ignore', RemovedInDRF310Warning) -warnings.simplefilter('ignore', RemovedInDRF311Warning) -warnings.simplefilter('ignore', DeprecationWarning) diff --git a/tutor/templates/build/android/Dockerfile b/tutor/templates/build/android/Dockerfile deleted file mode 100644 index b586322431..0000000000 --- a/tutor/templates/build/android/Dockerfile +++ /dev/null @@ -1,40 +0,0 @@ -FROM docker.io/ubuntu:20.04 -MAINTAINER Overhang.io - -ENV DEBIAN_FRONTEND=noninteractive -RUN apt update && \ - apt upgrade -y && \ - apt install -y wget unzip git openjdk-8-jre openjdk-8-jdk - -RUN mkdir /openedx - -# Install Android SDK -# Inspired from https://github.com/LiveXP/docker-android-sdk/blob/master/Dockerfile -ENV ANDROID_SDK_VERSION 6200805 -ENV ANDROID_SDK_PATH /openedx/android-sdk -ENV ANDROID_HOME /openedx/android-sdk -RUN mkdir ${ANDROID_HOME} -WORKDIR /openedx/android-sdk -RUN wget https://dl.google.com/android/repository/commandlinetools-linux-${ANDROID_SDK_VERSION}_latest.zip && \ - unzip commandlinetools-linux-${ANDROID_SDK_VERSION}_latest.zip && \ - rm commandlinetools-linux-${ANDROID_SDK_VERSION}_latest.zip - -# Accept licenses -# https://developer.android.com/studio/command-line/sdkmanager -ARG ANDROID_API_LEVEL=28 -RUN yes | /openedx/android-sdk/tools/bin/sdkmanager --sdk_root=${ANDROID_HOME} --install "platforms;android-$ANDROID_API_LEVEL" 1> /dev/null - -# Install android app repo -ARG ANDROID_APP_REPOSITORY=https://github.com/edx/edx-app-android -ARG ANDROID_APP_VERSION=release/2.23.2 -RUN git clone $ANDROID_APP_REPOSITORY --branch $ANDROID_APP_VERSION /openedx/edx-app-android -WORKDIR /openedx/edx-app-android - -# Install gradle and all dependencies -RUN ./gradlew -v -RUN ./gradlew tasks - -# User-customized config -COPY ./edx.properties ./OpenEdXMobile/edx.properties -RUN mkdir /openedx/config -RUN ln -s /openedx/config/gradle.properties ./OpenEdXMobile/gradle.properties diff --git a/tutor/templates/build/android/edx.properties b/tutor/templates/build/android/edx.properties deleted file mode 100644 index f49574b029..0000000000 --- a/tutor/templates/build/android/edx.properties +++ /dev/null @@ -1 +0,0 @@ -edx.dir = '/openedx/config' diff --git a/tutor/templates/build/forum/bin/docker-entrypoint.sh b/tutor/templates/build/forum/bin/docker-entrypoint.sh index 6ad2f892b1..eec93e3b03 100755 --- a/tutor/templates/build/forum/bin/docker-entrypoint.sh +++ b/tutor/templates/build/forum/bin/docker-entrypoint.sh @@ -1,6 +1,8 @@ #!/bin/sh -e export MONGOHQ_URL="mongodb://$MONGODB_AUTH$MONGODB_HOST:$MONGODB_PORT/cs_comments_service" +# the search server variable was renamed after the upgrade to elasticsearch 7 +export SEARCH_SERVER_ES7="$SEARCH_SERVER" echo "Waiting for mongodb/elasticsearch..." dockerize -wait tcp://$MONGODB_HOST:$MONGODB_PORT -wait $SEARCH_SERVER -wait-retry-interval 5s -timeout 600s diff --git a/tutor/templates/build/openedx/Dockerfile b/tutor/templates/build/openedx/Dockerfile index 63eb428761..76366a84a0 100644 --- a/tutor/templates/build/openedx/Dockerfile +++ b/tutor/templates/build/openedx/Dockerfile @@ -36,29 +36,11 @@ RUN mkdir -p /openedx/edx-platform && \ WORKDIR /openedx/edx-platform {% if patch("openedx-dockerfile-git-patches-default") %} -# Custom edx-platform default patches +# Custom edx-platform patches {{ patch("openedx-dockerfile-git-patches-default") }} {% else %} # Patch edx-platform -# Security patches -# https://github.com/edx/edx-platform/pull/27394 -RUN curl https://github.com/overhangio/edx-platform/commit/a0fdc97f1704659d26e167de3fbf2ab8c371d67b.patch | git apply - -# Django security releases -RUN curl https://github.com/overhangio/edx-platform/commit/67973f2445f667af23f779d5551070835de03efe.patch | git apply - -# Fix video unit completion -RUN curl https://github.com/overhangio/edx-platform/commit/3d489952f7cfd83fed47c700c7cd0b477b68351e.patch | git apply - -# Make it possible to disable learner records globally -# https://github.com/edx/edx-platform/pull/25182 -# https://github.com/overhangio/edx-platform/tree/overhangio/disable-learner-records-from-settings -RUN curl https://github.com/overhangio/edx-platform/commit/bd038bab3cf02df147e754f7743e46b68b43bac8.patch | git apply - -# Fix inconvenient pavelib warning -# https://github.com/edx/edx-platform/pull/25771 -# https://github.com/overhangio/edx-platform/tree/overhangio/fix-paver-warning -RUN curl https://github.com/overhangio/edx-platform/commit/bc0ab09f9945bd14aa6be1dbbf928cce58f079d2.patch | git apply - -# Fix js upload in scorm packages by upgrading django-pipeline -# https://github.com/edx/edx-platform/pull/25957 -# https://github.com/overhangio/edx-platform/tree/overhangio/upgrade-django-pipeline -RUN curl https://github.com/overhangio/edx-platform/commit/1c733e3ba11249cf16358684169e6137a308e796.patch | git apply - +# RUN curl https://github.com/overhangio/edx-platform/commit/.patch | git apply - {% endif %} ###### Download extra locales to /openedx/locale/contrib/locale @@ -91,7 +73,7 @@ RUN pip install setuptools==44.1.0 pip==20.0.2 wheel==0.34.2 RUN pip install -r ./requirements/edx/base.txt # Install scorm xblock -RUN pip install "openedx-scorm-xblock<12.0.0,>=11.0.0" +RUN pip install "openedx-scorm-xblock<13.0.0,>=12.0.0" # Install django-redis for using redis as a django cache RUN pip install django-redis==4.12.1 @@ -112,7 +94,7 @@ FROM python as nodejs-requirements ENV PATH /openedx/nodeenv/bin:/openedx/venv/bin:${PATH} # Install nodeenv with the version provided by edx-platform -RUN pip install nodeenv==1.4.0 +RUN pip install nodeenv==1.6.0 RUN nodeenv /openedx/nodeenv --node=12.13.0 --prebuilt # Install nodejs requirements diff --git a/tutor/templates/build/openedx/bin/site-configuration b/tutor/templates/build/openedx/bin/site-configuration new file mode 100644 index 0000000000..32736d561a --- /dev/null +++ b/tutor/templates/build/openedx/bin/site-configuration @@ -0,0 +1,81 @@ +#! /usr/bin/env python3 +import argparse +import lms.startup + +lms.startup.run() + +from django.conf import settings +from django.contrib.sites.models import Site + +from openedx.core.djangoapps.site_configuration.models import SiteConfiguration + + +def main(): + parser = argparse.ArgumentParser(description="Manage site configuration") + subparsers = parser.add_subparsers() + + # Set command + parser_set = subparsers.add_parser("set", help="Set a site configuration key/value") + parser_set.add_argument( + "-d", "--domain", help="Site domain: by default this will be the LMS domain" + ) + parser_set.add_argument("key", help="Configuration key") + parser_set.add_argument( + "value", + help="Configuration value: 'true' and 'false' will be converted to booleans.", + ) + parser_set.set_defaults(func=set_command) + + # Unset command + parser_unset = subparsers.add_parser( + "unset", help="Remove a site configuration key" + ) + parser_unset.add_argument( + "-d", "--domain", help="Site domain: by default this will be the LMS domain" + ) + parser_unset.add_argument("key", help="Configuration key") + parser_unset.set_defaults(func=unset_command) + + args = parser.parse_args() + if hasattr(args, "func"): + args.func(args) + else: + parser.print_help() + + +def set_command(args): + configuration = get_site_configuration(args.domain) + + value = args.value + if value == "true": + value = True + elif value == "false": + value = False + + configuration.site_values[args.key] = args.value + configuration.save() + + +def get_site_configuration(domain): + domain = domain or settings.LMS_BASE + site, site_created = Site.objects.get_or_create(domain=domain) + if site_created: + site.name = domain + site.save() + configuration, configuration_created = SiteConfiguration.objects.get_or_create(site=site) + if configuration_created: + # Configuration is disabled by default + configuration.enabled = True + configuration.save() + return configuration + + +def unset_command(args): + configuration = get_site_configuration(args.domain) + if args.key in configuration.site_values: + configuration.site_values.pop(args.key) + configuration.save() + + +if __name__ == "__main__": + main() diff --git a/tutor/templates/build/openedx/revisions.yml b/tutor/templates/build/openedx/revisions.yml index a62b76995d..4c167a2b35 100644 --- a/tutor/templates/build/openedx/revisions.yml +++ b/tutor/templates/build/openedx/revisions.yml @@ -1 +1 @@ -EDX_PLATFORM_REVISION: koa \ No newline at end of file +EDX_PLATFORM_REVISION: lilac \ No newline at end of file diff --git a/tutor/templates/config.yml b/tutor/templates/config.yml index 4d3b1c32a5..78aacb4349 100644 --- a/tutor/templates/config.yml +++ b/tutor/templates/config.yml @@ -3,7 +3,6 @@ MYSQL_ROOT_PASSWORD: "{{ 8|random_string }}" OPENEDX_MYSQL_PASSWORD: "{{ 8|random_string }}" OPENEDX_SECRET_KEY: "{{ 24|random_string }}" -ANDROID_OAUTH2_SECRET: "{{ 24|random_string }}" ID: "{{ 24|random_string }}" # This must be defined early @@ -24,21 +23,17 @@ CMS_HOST: "studio.{{ LMS_HOST }}" CONTACT_EMAIL: "contact@{{ LMS_HOST }}" OPENEDX_AWS_ACCESS_KEY: "" OPENEDX_AWS_SECRET_ACCESS_KEY: "" -ANDROID_RELEASE_STORE_PASSWORD: "android store password" -ANDROID_RELEASE_KEY_PASSWORD: "android release key password" -ANDROID_RELEASE_KEY_ALIAS: "android release key alias" DEV_PROJECT_NAME: "tutor_dev" DOCKER_REGISTRY: "docker.io/" DOCKER_IMAGE_OPENEDX: "{{ DOCKER_REGISTRY }}overhangio/openedx:{{ TUTOR_VERSION }}" DOCKER_IMAGE_OPENEDX_DEV: "{{ DOCKER_REGISTRY }}overhangio/openedx-dev:{{ TUTOR_VERSION }}" -DOCKER_IMAGE_ANDROID: "{{ DOCKER_REGISTRY }}overhangio/openedx-android:{{ TUTOR_VERSION }}" -DOCKER_IMAGE_CADDY: "{{ DOCKER_REGISTRY }}caddy:2.2.1" +DOCKER_IMAGE_CADDY: "{{ DOCKER_REGISTRY }}caddy:2.3.0" DOCKER_IMAGE_FORUM: "{{ DOCKER_REGISTRY }}overhangio/openedx-forum:{{ TUTOR_VERSION }}" -DOCKER_IMAGE_MONGODB: "{{ DOCKER_REGISTRY }}mongo:3.6.18" -DOCKER_IMAGE_MYSQL: "{{ DOCKER_REGISTRY }}mysql:5.7.32" -DOCKER_IMAGE_ELASTICSEARCH: "{{ DOCKER_REGISTRY }}elasticsearch:1.5.2" -DOCKER_IMAGE_NGINX: "{{ DOCKER_REGISTRY }}nginx:1.13" -DOCKER_IMAGE_REDIS: "{{ DOCKER_REGISTRY }}redis:6.0.9" +DOCKER_IMAGE_MONGODB: "{{ DOCKER_REGISTRY }}mongo:4.0.23" +DOCKER_IMAGE_MYSQL: "{{ DOCKER_REGISTRY }}mysql:5.7.33" +DOCKER_IMAGE_ELASTICSEARCH: "{{ DOCKER_REGISTRY }}elasticsearch:7.8.1" +DOCKER_IMAGE_NGINX: "{{ DOCKER_REGISTRY }}nginx:1.19.9" +DOCKER_IMAGE_REDIS: "{{ DOCKER_REGISTRY }}redis:6.2.1" DOCKER_IMAGE_SMTP: "{{ DOCKER_REGISTRY }}namshi/smtp:latest" LOCAL_PROJECT_NAME: "tutor_local" ELASTICSEARCH_HOST: "elasticsearch" @@ -57,14 +52,14 @@ MONGODB_DATABASE: "openedx" MONGODB_PORT: 27017 MONGODB_USERNAME: "" MONGODB_PASSWORD: "" +OPENEDX_CACHE_REDIS_DB: 1 +OPENEDX_CELERY_REDIS_DB: 0 OPENEDX_CMS_UWSGI_WORKERS: 2 OPENEDX_LMS_UWSGI_WORKERS: 2 OPENEDX_MYSQL_DATABASE: "openedx" OPENEDX_CSMH_MYSQL_DATABASE: "{{ OPENEDX_MYSQL_DATABASE }}_csmh" OPENEDX_MYSQL_USERNAME: "openedx" -OPENEDX_COMMON_VERSION: "open-release/koa.3" -OPENEDX_CELERY_REDIS_DB: 0 -OPENEDX_CACHE_REDIS_DB: 1 +OPENEDX_COMMON_VERSION: "open-release/lilac.1" MYSQL_HOST: "mysql" MYSQL_PORT: 3306 MYSQL_ROOT_USERNAME: "root" diff --git a/tutor/templates/hooks/forum/init b/tutor/templates/hooks/forum/init index 2e9cf9fef4..e98c05a5e2 100644 --- a/tutor/templates/hooks/forum/init +++ b/tutor/templates/hooks/forum/init @@ -1,2 +1,2 @@ bundle exec rake search:initialize -bundle exec rake search:rebuild_index +bundle exec rake search:rebuild_indices diff --git a/tutor/templates/hooks/lms/init b/tutor/templates/hooks/lms/init index caeb61889a..3b25b853e5 100644 --- a/tutor/templates/hooks/lms/init +++ b/tutor/templates/hooks/lms/init @@ -4,19 +4,6 @@ echo "Loading settings $DJANGO_SETTINGS_MODULE" ./manage.py lms migrate -# Delete obsolete credentials for Android application -./manage.py lms shell -c 'from oauth2_provider.models import get_application_model -get_application_model().objects.filter(name="android").exclude(user__username="login_service_user").delete()' -# Create oauth credentials for Android application -./manage.py lms create_dot_application \ - --client-id android \ - --client-secret {{ ANDROID_OAUTH2_SECRET }} \ - --grant-type password \ - --public \ - --update \ - android \ - login_service_user - # Fix incorrect uploaded file path if [ -d /openedx/data/uploads/ ]; then if [ -n "$(ls -A /openedx/data/uploads/)" ]; then diff --git a/tutor/templates/k8s/deployments.yml b/tutor/templates/k8s/deployments.yml index 8446f4c090..8631e32e27 100644 --- a/tutor/templates/k8s/deployments.yml +++ b/tutor/templates/k8s/deployments.yml @@ -262,12 +262,16 @@ spec: - name: elasticsearch image: {{ DOCKER_IMAGE_ELASTICSEARCH }} env: - - name: ES_JAVA_OPTS - value: "-Xms1g -Xmx1g" - - name: "cluster.name" - value: openedx - - name: "bootstrap.memory_lock" + - name: cluster.name + value: "openedx" + - name: bootstrap.memory_lock value: "true" + - name: discovery.type + value: "single-node" + - name: ES_JAVA_OPTS + value: "-Xms{{ ELASTICSEARCH_HEAP_SIZE }} -Xmx{{ ELASTICSEARCH_HEAP_SIZE }}" + - name: TAKE_FILE_OWNERSHIP + value: "1" ports: - containerPort: 9200 volumeMounts: diff --git a/tutor/templates/local/docker-compose.jobs.yml b/tutor/templates/local/docker-compose.jobs.yml index b7a5eebf60..c57bc6c0b1 100644 --- a/tutor/templates/local/docker-compose.jobs.yml +++ b/tutor/templates/local/docker-compose.jobs.yml @@ -38,10 +38,4 @@ services: MONGODB_PORT: "{{ MONGODB_PORT }}" depends_on: {{ [("elasticsearch", RUN_ELASTICSEARCH), ("mongodb", RUN_MONGODB)]|list_if }} - android-job: - image: {{ DOCKER_IMAGE_ANDROID }} - volumes: - - "../android/:/openedx/config/" - - "../../data/android/:/openedx/data/" - - {{ patch("local-docker-compose-jobs-services")|indent(4) }} \ No newline at end of file + {{ patch("local-docker-compose-jobs-services")|indent(4) }} diff --git a/tutor/templates/local/docker-compose.yml b/tutor/templates/local/docker-compose.yml index c7e7cde90b..6d9ca61a48 100644 --- a/tutor/templates/local/docker-compose.yml +++ b/tutor/templates/local/docker-compose.yml @@ -27,7 +27,12 @@ services: {% if RUN_ELASTICSEARCH %} elasticsearch: image: {{ DOCKER_IMAGE_ELASTICSEARCH }} - command: ["elasticsearch", "-Xms{{ ELASTICSEARCH_HEAP_SIZE }}", "-Xmx{{ ELASTICSEARCH_HEAP_SIZE }}", "--cluster.name=openedx", "--bootstrap.mlockall=true"] + environment: + - cluster.name=openedx + - bootstrap.memory_lock=true + - discovery.type=single-node + - "ES_JAVA_OPTS=-Xms{{ ELASTICSEARCH_HEAP_SIZE }} -Xmx{{ ELASTICSEARCH_HEAP_SIZE }}" + - TAKE_FILE_OWNERSHIP=1 ulimits: memlock: soft: -1