From e42e28abe5beb395f233efe602b1cb55564ace97 Mon Sep 17 00:00:00 2001 From: Bilal Qamar <59555732+BilalQamar95@users.noreply.github.com> Date: Fri, 11 Oct 2024 16:09:09 +0500 Subject: [PATCH 1/7] feat: added dockerfile and docker image push workflow for enterprise-catalog --- .../push-enterprise-catalog-image.yaml | 62 +++++++++++ dockerfiles/enterprise-catalog.Dockerfile | 104 ++++++++++++++++++ 2 files changed, 166 insertions(+) create mode 100644 .github/workflows/push-enterprise-catalog-image.yaml create mode 100644 dockerfiles/enterprise-catalog.Dockerfile diff --git a/.github/workflows/push-enterprise-catalog-image.yaml b/.github/workflows/push-enterprise-catalog-image.yaml new file mode 100644 index 0000000..526bfd3 --- /dev/null +++ b/.github/workflows/push-enterprise-catalog-image.yaml @@ -0,0 +1,62 @@ +name: Build and Push Enterprise Catalog Image + +on: + workflow_dispatch: + inputs: + branch: + description: "Target branch from which the source dockerfile from image will be sourced" + + schedule: + - cron: "0 4 * * 1-5" # UTC Time + + pull_request: + branches: + - '**' + +jobs: + build-and-push-image: + runs-on: ubuntu-latest + + steps: + - name: Get tag name + id: get-tag-name + uses: actions/github-script@v5 + with: + script: | + const tagName = "${{ github.event.inputs.branch }}" || 'latest'; + console.log('Will use tag: ' + tagName); + return tagName; + result-encoding: string + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - name: Build and push Dev Docker image + uses: docker/build-push-action@v6 + with: + file: ./dockerfiles/enterprise-catalog.Dockerfile + push: true + target: legacy_devapp + tags: edxops/enterprise-catalog-dev:${{ steps.get-tag-name.outputs.result }} + + - name: Send failure notification + if: failure() + uses: dawidd6/action-send-mail@v3 + with: + server_address: email-smtp.us-east-1.amazonaws.com + server_port: 465 + username: ${{secrets.edx_smtp_username}} + password: ${{secrets.edx_smtp_password}} + subject: Push Image to docker.io/edxops failed in Enterprise Catalog + to: team-cosmonauts@edx.org + from: github-actions + body: Push Image to docker.io/edxops for Enterprise Catalog failed! For details see "github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" diff --git a/dockerfiles/enterprise-catalog.Dockerfile b/dockerfiles/enterprise-catalog.Dockerfile new file mode 100644 index 0000000..d216d77 --- /dev/null +++ b/dockerfiles/enterprise-catalog.Dockerfile @@ -0,0 +1,104 @@ +FROM ubuntu:focal as app +MAINTAINER sre@edx.org + +# Packages installed: +# git +# Used to pull in particular requirements from github rather than pypi, +# and to check the sha of the code checkout. +# language-pack-en locales +# ubuntu locale support so that system utilities have a consistent +# language and time zone. +# python3-pip +# install pip to install application requirements.txt files +# pkg-config +# mysqlclient>=2.2.0 requires this (https://github.com/PyMySQL/mysqlclient/issues/620) +# libssl-dev +# mysqlclient wont install without this. +# libmysqlclient-dev +# to install header files needed to use native C implementation for +# MySQL-python for performance gains. + +ARG PYTHON_VERSION=3.12 +ENV TZ=UTC +ENV TERM=xterm-256color +ENV DEBIAN_FRONTEND=noninteractive + +# If you add a package here please include a comment above describing what it is used for +RUN apt-get update && \ + apt-get install -y software-properties-common && \ + apt-add-repository -y ppa:deadsnakes/ppa + +RUN apt-get update && apt-get -qy install --no-install-recommends \ + build-essential \ + language-pack-en \ + locales \ + curl \ + pkg-config \ + libmysqlclient-dev \ + libssl-dev \ + libffi-dev \ + libsqlite3-dev \ + git \ + wget \ + python3.12 \ + python3.12-dev \ + python3.12-distutils \ + python3-pip + +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python${PYTHON_VERSION} +RUN pip install virtualenv + +# cloning git repo +RUN curl -L https://github.com/openedx/enterprise-catalog/archive/refs/heads/master.tar.gz | tar -xz --strip-components=1 + +ENV VIRTUAL_ENV=/venv +RUN virtualenv -p python$PYTHON_VERSION $VIRTUAL_ENV +ENV PATH="$VIRTUAL_ENV/bin:$PATH" + +RUN pip install pip==24.0 setuptools==69.5.1 + +RUN locale-gen en_US.UTF-8 +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en +ENV LC_ALL en_US.UTF-8 +ENV DJANGO_SETTINGS_MODULE enterprise_catalog.settings.production + +# Prod ports +EXPOSE 8160 +EXPOSE 8161 + +RUN useradd -m --shell /bin/false app + +RUN pip install -r requirements/production.txt + +# Code is owned by root so it cannot be modified by the application user. +# So we copy it before changing users. +USER app + +# Gunicorn 19 does not log to stdout or stderr by default. Once we are past gunicorn 19, the logging to STDOUT need not be specified. +CMD ["gunicorn", "--workers=2", "--name", "enterprise_catalog", "-c", "/edx/app/enterprise_catalog/enterprise_catalog/enterprise_catalog/docker_gunicorn_configuration.py", "--log-file", "-", "--max-requests=1000", "enterprise_catalog.wsgi:application"] + + +############################################################### +# Create newrelic image used by the experimental docker shim. # +############################################################### +# TODO: remove this after we migrate to k8s since it will serve no more purpose. +FROM app as newrelic +RUN pip install newrelic +CMD ["newrelic-admin", "run-program", "gunicorn", "--workers=2", "--name", "enterprise_catalog", "-c", "/edx/app/enterprise_catalog/enterprise_catalog/enterprise_catalog/docker_gunicorn_configuration.py", "--log-file", "-", "--max-requests=1000", "enterprise_catalog.wsgi:application"] + +################################# +# Create image used by devstack # +################################# +# TODO: remove this after we migrate to k8s. It already isn't used today, but just defer changes until absolutely +# necessary for safety. +FROM app as legacy_devapp +# Dev ports +EXPOSE 18160 +EXPOSE 18161 +USER root +RUN pip install -r requirements/dev.txt +USER app +CMD ["gunicorn", "--reload", "--workers=2", "--name", "enterprise_catalog", "-b", ":18160", "-c", "/edx/app/enterprise_catalog/enterprise_catalog/enterprise_catalog/docker_gunicorn_configuration.py", "--log-file", "-", "--max-requests=1000", "enterprise_catalog.wsgi:application"] From 9126817e3c8df4a526948ff1308d20d720ab6b79 Mon Sep 17 00:00:00 2001 From: Bilal Qamar <59555732+BilalQamar95@users.noreply.github.com> Date: Fri, 11 Oct 2024 16:13:25 +0500 Subject: [PATCH 2/7] chore: Remove pull_request trigger from workflow --- .github/workflows/push-enterprise-catalog-image.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/push-enterprise-catalog-image.yaml b/.github/workflows/push-enterprise-catalog-image.yaml index 526bfd3..d6d2698 100644 --- a/.github/workflows/push-enterprise-catalog-image.yaml +++ b/.github/workflows/push-enterprise-catalog-image.yaml @@ -9,10 +9,6 @@ on: schedule: - cron: "0 4 * * 1-5" # UTC Time - pull_request: - branches: - - '**' - jobs: build-and-push-image: runs-on: ubuntu-latest From 5ee7c3b67c32b754e2727e7a830bceddc40a6d74 Mon Sep 17 00:00:00 2001 From: Bilal Qamar <59555732+BilalQamar95@users.noreply.github.com> Date: Tue, 15 Oct 2024 12:11:03 +0500 Subject: [PATCH 3/7] chore: updated mail team for failure notification --- .github/workflows/push-enterprise-catalog-image.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/push-enterprise-catalog-image.yaml b/.github/workflows/push-enterprise-catalog-image.yaml index d6d2698..b411ed1 100644 --- a/.github/workflows/push-enterprise-catalog-image.yaml +++ b/.github/workflows/push-enterprise-catalog-image.yaml @@ -53,6 +53,6 @@ jobs: username: ${{secrets.edx_smtp_username}} password: ${{secrets.edx_smtp_password}} subject: Push Image to docker.io/edxops failed in Enterprise Catalog - to: team-cosmonauts@edx.org + to: team-titans@edx.org from: github-actions body: Push Image to docker.io/edxops for Enterprise Catalog failed! For details see "github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" From 1dd8b4eb03af19a1a4d8ca6246f47830482a43ac Mon Sep 17 00:00:00 2001 From: Bilal Qamar <59555732+BilalQamar95@users.noreply.github.com> Date: Wed, 16 Oct 2024 16:39:24 +0500 Subject: [PATCH 4/7] perf: updated Dockerfile to optimize requirements installation and dependency caching --- .github/workflows/push-enterprise-catalog-image.yaml | 5 +++++ dockerfiles/enterprise-catalog.Dockerfile | 9 +++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/push-enterprise-catalog-image.yaml b/.github/workflows/push-enterprise-catalog-image.yaml index b411ed1..6afd67f 100644 --- a/.github/workflows/push-enterprise-catalog-image.yaml +++ b/.github/workflows/push-enterprise-catalog-image.yaml @@ -9,6 +9,11 @@ on: schedule: - cron: "0 4 * * 1-5" # UTC Time +# Added for testing purposes. Will remove once the PR is finalised + pull_request: + branches: + - '**' + jobs: build-and-push-image: runs-on: ubuntu-latest diff --git a/dockerfiles/enterprise-catalog.Dockerfile b/dockerfiles/enterprise-catalog.Dockerfile index d216d77..7c19bed 100644 --- a/dockerfiles/enterprise-catalog.Dockerfile +++ b/dockerfiles/enterprise-catalog.Dockerfile @@ -50,8 +50,7 @@ RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python${PYTHON_VERSION} RUN pip install virtualenv -# cloning git repo -RUN curl -L https://github.com/openedx/enterprise-catalog/archive/refs/heads/master.tar.gz | tar -xz --strip-components=1 +RUN mkdir -p requirements ENV VIRTUAL_ENV=/venv RUN virtualenv -p python$PYTHON_VERSION $VIRTUAL_ENV @@ -71,8 +70,13 @@ EXPOSE 8161 RUN useradd -m --shell /bin/false app + +RUN curl -L -o requirements/production.txt https://raw.githubusercontent.com/openedx/enterprise-catalog/master/requirements/production.txt RUN pip install -r requirements/production.txt +# Cloning the repository +RUN curl -L https://github.com/openedx/enterprise-catalog/archive/refs/heads/master.tar.gz | tar -xz --strip-components=1 + # Code is owned by root so it cannot be modified by the application user. # So we copy it before changing users. USER app @@ -99,6 +103,7 @@ FROM app as legacy_devapp EXPOSE 18160 EXPOSE 18161 USER root +RUN curl -L -o requirements/dev.txt https://raw.githubusercontent.com/openedx/enterprise-catalog/master/requirements/dev.txt RUN pip install -r requirements/dev.txt USER app CMD ["gunicorn", "--reload", "--workers=2", "--name", "enterprise_catalog", "-b", ":18160", "-c", "/edx/app/enterprise_catalog/enterprise_catalog/enterprise_catalog/docker_gunicorn_configuration.py", "--log-file", "-", "--max-requests=1000", "enterprise_catalog.wsgi:application"] From d98fb9585e423c2d9287019553065a68b0ad268c Mon Sep 17 00:00:00 2001 From: Bilal Qamar <59555732+BilalQamar95@users.noreply.github.com> Date: Wed, 16 Oct 2024 16:43:59 +0500 Subject: [PATCH 5/7] chore: Remove pull_request trigger from workflow --- .github/workflows/push-enterprise-catalog-image.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/push-enterprise-catalog-image.yaml b/.github/workflows/push-enterprise-catalog-image.yaml index 6afd67f..b411ed1 100644 --- a/.github/workflows/push-enterprise-catalog-image.yaml +++ b/.github/workflows/push-enterprise-catalog-image.yaml @@ -9,11 +9,6 @@ on: schedule: - cron: "0 4 * * 1-5" # UTC Time -# Added for testing purposes. Will remove once the PR is finalised - pull_request: - branches: - - '**' - jobs: build-and-push-image: runs-on: ubuntu-latest From cf96c72b032da2f65c5b3e3b7479e6ca3a49a8ba Mon Sep 17 00:00:00 2001 From: Bilal Qamar <59555732+BilalQamar95@users.noreply.github.com> Date: Fri, 18 Oct 2024 14:57:02 +0500 Subject: [PATCH 6/7] refactor: remove redundant curl command --- .github/workflows/push-enterprise-catalog-image.yaml | 5 +++++ dockerfiles/enterprise-catalog.Dockerfile | 1 - 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/push-enterprise-catalog-image.yaml b/.github/workflows/push-enterprise-catalog-image.yaml index b411ed1..6afd67f 100644 --- a/.github/workflows/push-enterprise-catalog-image.yaml +++ b/.github/workflows/push-enterprise-catalog-image.yaml @@ -9,6 +9,11 @@ on: schedule: - cron: "0 4 * * 1-5" # UTC Time +# Added for testing purposes. Will remove once the PR is finalised + pull_request: + branches: + - '**' + jobs: build-and-push-image: runs-on: ubuntu-latest diff --git a/dockerfiles/enterprise-catalog.Dockerfile b/dockerfiles/enterprise-catalog.Dockerfile index 7c19bed..dd81b2f 100644 --- a/dockerfiles/enterprise-catalog.Dockerfile +++ b/dockerfiles/enterprise-catalog.Dockerfile @@ -103,7 +103,6 @@ FROM app as legacy_devapp EXPOSE 18160 EXPOSE 18161 USER root -RUN curl -L -o requirements/dev.txt https://raw.githubusercontent.com/openedx/enterprise-catalog/master/requirements/dev.txt RUN pip install -r requirements/dev.txt USER app CMD ["gunicorn", "--reload", "--workers=2", "--name", "enterprise_catalog", "-b", ":18160", "-c", "/edx/app/enterprise_catalog/enterprise_catalog/enterprise_catalog/docker_gunicorn_configuration.py", "--log-file", "-", "--max-requests=1000", "enterprise_catalog.wsgi:application"] From 4a5405df1a4cc1ac5773d36e1f2515071249b60c Mon Sep 17 00:00:00 2001 From: Bilal Qamar <59555732+BilalQamar95@users.noreply.github.com> Date: Fri, 18 Oct 2024 15:01:22 +0500 Subject: [PATCH 7/7] chore: Remove pull_request trigger from workflow --- .github/workflows/push-enterprise-catalog-image.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/push-enterprise-catalog-image.yaml b/.github/workflows/push-enterprise-catalog-image.yaml index 6afd67f..b411ed1 100644 --- a/.github/workflows/push-enterprise-catalog-image.yaml +++ b/.github/workflows/push-enterprise-catalog-image.yaml @@ -9,11 +9,6 @@ on: schedule: - cron: "0 4 * * 1-5" # UTC Time -# Added for testing purposes. Will remove once the PR is finalised - pull_request: - branches: - - '**' - jobs: build-and-push-image: runs-on: ubuntu-latest