diff --git a/tutor/templates/build/openedx/Dockerfile b/tutor/templates/build/openedx/Dockerfile index 4d7d8b0081..09dec035b4 100644 --- a/tutor/templates/build/openedx/Dockerfile +++ b/tutor/templates/build/openedx/Dockerfile @@ -11,6 +11,13 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ ENV LC_ALL en_US.UTF-8 {{ patch("openedx-dockerfile-minimal") }} +ENV PATH /openedx/edx-platform/node_modules/.bin:/openedx/nodeenv/bin:/openedx/venv/bin:${PATH} +ENV VIRTUAL_ENV /openedx/venv/ +ENV XDG_CACHE_HOME /openedx/.cache +ENV COMPREHENSIVE_THEME_DIRS /openedx/themes +ENV STATIC_ROOT_LMS /openedx/staticfiles +ENV STATIC_ROOT_CMS /openedx/staticfiles/studio + ###### Install python with pyenv in /opt/pyenv and create virtualenv in /openedx/venv FROM minimal as python # https://github.com/pyenv/pyenv/wiki/Common-build-problems#prerequisites @@ -78,9 +85,6 @@ FROM scratch as mnt-{{ name }} ###### Install python requirements in virtualenv FROM python as python-requirements -ENV PATH /openedx/venv/bin:${PATH} -ENV VIRTUAL_ENV /openedx/venv/ -ENV XDG_CACHE_HOME /openedx/.cache RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ --mount=type=cache,target=/var/lib/apt,sharing=locked \ @@ -119,8 +123,7 @@ RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \ {% endfor %} ###### Install nodejs with nodeenv in /openedx/nodeenv -FROM python as nodejs-requirements -ENV PATH /openedx/nodeenv/bin:/openedx/venv/bin:${PATH} +FROM python as nodejs # Install nodeenv with the version provided by edx-platform # https://github.com/openedx/edx-platform/blob/master/requirements/edx/base.txt @@ -128,15 +131,72 @@ RUN pip install nodeenv==1.8.0 RUN nodeenv /openedx/nodeenv --node=18.20.1 --prebuilt # Install nodejs requirements +FROM nodejs as nodejs-requirements ARG NPM_REGISTRY={{ NPM_REGISTRY }} WORKDIR /openedx/edx-platform -RUN --mount=type=bind,from=edx-platform,source=/package.json,target=/openedx/edx-platform/package.json \ - --mount=type=bind,from=edx-platform,source=/package-lock.json,target=/openedx/edx-platform/package-lock.json \ - --mount=type=bind,from=edx-platform,source=/scripts/copy-node-modules.sh,target=/openedx/edx-platform/scripts/copy-node-modules.sh \ - --mount=type=cache,target=/root/.npm,sharing=shared \ +COPY --link --from=edx-platform /package.json package.json +COPY --link --from=edx-platform /package-lock.json package-lock.json +COPY --link --from=edx-platform /scripts/copy-node-modules.sh scripts/copy-node-modules.sh +RUN --mount=type=cache,target=/root/.npm,sharing=shared \ npm clean-install --no-audit --registry=$NPM_REGISTRY -###### Production image with system and python requirements +FROM nodejs-requirements as pre-assets +{{ patch("openedx-dockerfile-pre-assets") }} + +# Gather minimal sources for webpacking JS into bundles +# Copy the entire source tree, and then delete everything that isn't relevant. +FROM pre-assets as js-sources +COPY --link --from=edx-platform / . +RUN find . -type f -a \! \ + \( -path 'node_modules/*' \ + -o -path '*/static/*' \ + -o -path '*/assets/*' \ + -o -path '*/js/*' \ + -o -name '*.underscore' \ + -o -name '*.json' \ + -o -name '*.js' \ + -o -name '*.jsx' \ + -o -name '.babelrc' \ + \) -delete + +# Intermediate image to capture prod JS build +FROM pre-assets as js-production +COPY --link --from=js-sources /openedx/edx-platform /openedx/edx-platform +RUN npm run webpack + +# Intermediate image to capture dev JS build +FROM pre-assets as js-development +COPY --link --from=js-sources /openedx/edx-platform /openedx/edx-platform +RUN npm run webpack-dev + +# Gather minimal requirements and sources for compiling Sass into CSS +FROM pre-assets as css-sources +COPY --link --from=edx-platform /requirements/edx/assets.txt requirements/edx/assets.txt +RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \ + pip install -r requirements/edx/assets.txt +COPY --link --from=edx-platform /scripts/compile_sass.py scripts/compile_sass.py +COPY --link --from=edx-platform /common/static common/static +COPY --link --from=edx-platform /lms/static/sass lms/static/sass +COPY --link --from=edx-platform /lms/static/sass/partials lms/static/sass/partials +COPY --link --from=edx-platform /lms/static/certificates/sass lms/static/certificates/sass +COPY --link --from=edx-platform /cms/static/sass cms/static/sass +COPY --link --from=edx-platform /cms/static/sass/partials cms/static/sass/partials +COPY --link --from=edx-platform /xmodule/assets xmodule/assets +COPY --link --from=edx-platform /lms/static/css/vendor lms/static/css/vendor + +# Intermediate image to capture compressed CSS build +FROM css-sources as css-production +RUN npm run compile-sass -- --skip-themes +COPY --link ./themes/ /openedx/themes +RUN npm run compile-sass -- --skip-default + +# Intermediate image to capture uncompressed CSS build +FROM css-sources as css-development +RUN npm run compile-sass-dev -- --skip-themes +COPY --link ./themes/ /openedx/themes +RUN npm run compile-sass-dev -- --skip-default + +###### Intermediate image shared between production-final and development FROM minimal as production # Install system requirements @@ -160,30 +220,16 @@ USER ${APP_USER_ID} # https://hub.docker.com/r/powerman/dockerize/tags COPY --link --from=docker.io/powerman/dockerize:0.19.0 /usr/local/bin/dockerize /usr/local/bin/dockerize + +# Link in python requirements and node env from intermediate images COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=edx-platform / /openedx/edx-platform COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=python /opt/pyenv /opt/pyenv COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=python-requirements /openedx/venv /openedx/venv COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=python-requirements /mnt /mnt -COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=nodejs-requirements /openedx/nodeenv /openedx/nodeenv -COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=nodejs-requirements /openedx/edx-platform/node_modules /openedx/node_modules - -# Symlink node_modules such that we can bind-mount the edx-platform repository -RUN ln -s /openedx/node_modules /openedx/edx-platform/node_modules - -ENV PATH /openedx/venv/bin:./node_modules/.bin:/openedx/nodeenv/bin:${PATH} -ENV VIRTUAL_ENV /openedx/venv/ -ENV COMPREHENSIVE_THEME_DIRS /openedx/themes -ENV STATIC_ROOT_LMS /openedx/staticfiles -ENV STATIC_ROOT_CMS /openedx/staticfiles/studio +COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=nodejs /openedx/nodeenv /openedx/nodeenv WORKDIR /openedx/edx-platform -{# Install auto-mounted directories as Python packages. #} -{% for name in iter_mounted_directories(MOUNTS, "openedx") %} -COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=mnt-{{ name }} / /mnt/{{ name }} -RUN pip install -e "/mnt/{{ name }}" -{% endfor %} - # We install edx-platform here because it creates an egg-info folder in the current # repo. We need both the source code and the virtualenv to run this command. RUN pip install -e . @@ -198,44 +244,11 @@ ENV REVISION_CFG /openedx/config/revisions.yml COPY --chown=app:app settings/lms/*.py ./lms/envs/tutor/ COPY --chown=app:app settings/cms/*.py ./cms/envs/tutor/ -# Pull latest translations via atlas -RUN make clean_translations -RUN ./manage.py lms --settings=tutor.i18n pull_plugin_translations --verbose --repository='{{ ATLAS_REPOSITORY }}' --revision='{{ ATLAS_REVISION }}' {{ ATLAS_OPTIONS }} -RUN ./manage.py lms --settings=tutor.i18n pull_xblock_translations --repository='{{ ATLAS_REPOSITORY }}' --revision='{{ ATLAS_REVISION }}' {{ ATLAS_OPTIONS }} -RUN atlas pull --repository='{{ ATLAS_REPOSITORY }}' --revision='{{ ATLAS_REVISION }}' {{ ATLAS_OPTIONS }} \ - translations/edx-platform/conf/locale:conf/locale \ - translations/studio-frontend/src/i18n/messages:conf/plugins-locale/studio-frontend -RUN ./manage.py lms --settings=tutor.i18n compile_xblock_translations -RUN ./manage.py cms --settings=tutor.i18n compile_xblock_translations -RUN ./manage.py lms --settings=tutor.i18n compile_plugin_translations -RUN ./manage.py lms --settings=tutor.i18n compilemessages -v1 -RUN ./manage.py lms --settings=tutor.i18n compilejsi18n -RUN ./manage.py cms --settings=tutor.i18n compilejsi18n - # Copy scripts COPY --chown=app:app ./bin /openedx/bin RUN chmod a+x /openedx/bin/* ENV PATH /openedx/bin:${PATH} -{{ patch("openedx-dockerfile-pre-assets") }} - -# Build & collect production assets. By default, only assets from the default theme -# will be processed. This makes the docker image lighter and faster to build. -RUN npm run postinstall # Postinstall artifacts are stuck in nodejs-requirements layer. Create them here too. -RUN npm run compile-sass -- --skip-themes -RUN npm run webpack - -# Now that the default theme is built, build any custom themes -COPY --chown=app:app ./themes/ /openedx/themes -RUN npm run compile-sass -- --skip-default - -# and finally, collect assets for the production image, -# de-duping assets with symlinks. -RUN ./manage.py lms collectstatic --noinput --settings=tutor.assets && \ - ./manage.py cms collectstatic --noinput --settings=tutor.assets && \ - # De-duplicate static assets with symlinks \ - rdfind -makesymlinks true -followsymlinks true /openedx/staticfiles/ - # Create a data directory, which might be used (or not) RUN mkdir /openedx/data @@ -254,7 +267,7 @@ ENV DJANGO_SETTINGS_MODULE lms.envs.tutor.production EXPOSE 8000 -###### Intermediate image with dev/test dependencies +###### Image ready with development packages and uncompressed static assets FROM production as development # Install useful system requirements (as root) @@ -273,22 +286,18 @@ RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \ RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \ pip install ipdb==0.13.13 ipython==8.24.0 -{# Re-install mounted requirements, otherwise they will be superseded by upstream reqs #} +# Install mounted requirements +# Must be done after installing dev requirements, so that local reqs supersede pypi reqs {% for name in iter_mounted_directories(MOUNTS, "openedx") %} COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=mnt-{{ name }} / /mnt/{{ name }} RUN pip install -e "/mnt/{{ name }}" {% endfor %} +{{ patch("openedx-dev-dockerfile-post-python-requirements") }} + # Add ipdb as default PYTHONBREAKPOINT ENV PYTHONBREAKPOINT=ipdb.set_trace -# Recompile static assets: in development mode all static assets are stored in edx-platform, -# and the location of these files is stored in webpack-stats.json. If we don't recompile -# static assets, then production assets will be served instead. -RUN rm -r /openedx/staticfiles && \ - mkdir /openedx/staticfiles && \ - npm run build-dev - {{ patch("openedx-dev-dockerfile-post-python-requirements") }} # Default django settings @@ -296,9 +305,56 @@ ENV DJANGO_SETTINGS_MODULE lms.envs.tutor.development CMD ./manage.py $SERVICE_VARIANT runserver 0.0.0.0:8000 -###### Final image with production cmd +# Link in dev JS requirements and dev assets from intermediate dev images +COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=js-development /openedx/edx-platform/common/static/bundles common/static/bundles +COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=nodejs-requirements /openedx/edx-platform/node_modules node_modules +COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=js-development /openedx/staticfiles /openedx/staticfiles +COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=css-development /openedx/edx-platform/cms/static/css cms/static/css +COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=css-development /openedx/edx-platform/lms/static/css lms/static/css +COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=css-development /openedx/edx-platform/lms/static/certificates/css lms/static/certificates/css +COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=css-development /openedx/themes /openedx/themes +RUN npm run postinstall # Postinstall artifacts are stuck in nodejs-requirements layer. Create them here too. + +###### Final image with production assets and command FROM production as final +{# Install auto-mounted directories as Python packages. #} +{% for name in iter_mounted_directories(MOUNTS, "openedx") %} +COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=mnt-{{ name }} / /mnt/{{ name }} +RUN pip install -e "/mnt/{{ name }}" +{% endfor %} + +# Link in JS requirements and static assets from intermediate images +COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=nodejs-requirements /openedx/edx-platform/node_modules node_modules +COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=js-production /openedx/staticfiles /openedx/staticfiles +COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=js-production /openedx/edx-platform/common/static/bundles common/static/bundles +COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=css-production /openedx/edx-platform/cms/static/css cms/static/css +COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=css-production /openedx/edx-platform/lms/static/css lms/static/css +COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=css-production /openedx/edx-platform/lms/static/certificates/css lms/static/certificates/css +COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=css-production /openedx/themes /openedx/themes +RUN npm run postinstall # Postinstall artifacts are stuck in nodejs-requirements layer. Create them here too. + +# Pull latest translations via atlas +RUN make clean_translations +RUN ./manage.py lms --settings=tutor.i18n pull_plugin_translations --verbose --repository='{{ ATLAS_REPOSITORY }}' --revision='{{ ATLAS_REVISION }}' {{ ATLAS_OPTIONS }} +RUN ./manage.py lms --settings=tutor.i18n pull_xblock_translations --repository='{{ ATLAS_REPOSITORY }}' --revision='{{ ATLAS_REVISION }}' {{ ATLAS_OPTIONS }} +RUN atlas pull --repository='{{ ATLAS_REPOSITORY }}' --revision='{{ ATLAS_REVISION }}' {{ ATLAS_OPTIONS }} \ + translations/edx-platform/conf/locale:conf/locale \ + translations/studio-frontend/src/i18n/messages:conf/plugins-locale/studio-frontend +RUN ./manage.py lms --settings=tutor.i18n compile_xblock_translations +RUN ./manage.py cms --settings=tutor.i18n compile_xblock_translations +RUN ./manage.py lms --settings=tutor.i18n compile_plugin_translations +RUN ./manage.py lms --settings=tutor.i18n compilemessages -v1 +RUN ./manage.py lms --settings=tutor.i18n compilejsi18n +RUN ./manage.py cms --settings=tutor.i18n compilejsi18n + +# and finally, collect assets for the production image, +# de-duping assets with symlinks. +RUN ./manage.py lms collectstatic --noinput --settings=tutor.assets && \ + ./manage.py cms collectstatic --noinput --settings=tutor.assets && \ + # De-duplicate static assets with symlinks \ + rdfind -makesymlinks true -followsymlinks true /openedx/staticfiles/ + # Default amount of uWSGI processes ENV UWSGI_WORKERS=2