From b8e096beae7868e36a30f2935247f4af63c008dc Mon Sep 17 00:00:00 2001 From: Bilal Qamar <59555732+BilalQamar95@users.noreply.github.com> Date: Thu, 10 Oct 2024 17:51:20 +0500 Subject: [PATCH 1/6] feat: added dockerfile and docker image push workflow for edx-api-notes --- .../workflows/push-edx-api-notes-image.yaml | 57 ++++++++++++ dockerfiles/edx-api-notes.Dockerfile | 89 +++++++++++++++++++ 2 files changed, 146 insertions(+) create mode 100644 .github/workflows/push-edx-api-notes-image.yaml create mode 100644 dockerfiles/edx-api-notes.Dockerfile diff --git a/.github/workflows/push-edx-api-notes-image.yaml b/.github/workflows/push-edx-api-notes-image.yaml new file mode 100644 index 0000000..612edb4 --- /dev/null +++ b/.github/workflows/push-edx-api-notes-image.yaml @@ -0,0 +1,57 @@ +name: Build and Push Edx Api Notes Image + +on: + workflow_dispatch: + inputs: + branch: + description: "Target branch from which the source dockerfile from image will be sourced" + + schedule: + - cron: "0 4 * * 1-5" # UTC Time + +jobs: + build-and-push-image: + runs-on: ubuntu-latest + + steps: + - name: Get tag name + id: get-tag-name + uses: actions/github-script@v5 + with: + script: | + const tagName = "${{ github.event.inputs.branch }}" || 'latest'; + return tagName; + result-encoding: string + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - name: Build and push Dev Docker image + uses: docker/build-push-action@v6 + with: + file: ./dockerfiles/edx-api-notes.Dockerfile + push: true + target: dev + tags: edxops/edx-api-notes-dev:${{ steps.get-tag-name.outputs.result }} + + - name: Send failure notification + if: failure() + uses: dawidd6/action-send-mail@v3 + with: + server_address: email-smtp.us-east-1.amazonaws.com + server_port: 465 + username: ${{secrets.edx_smtp_username}} + password: ${{secrets.edx_smtp_password}} + subject: Push Image to docker.io/edxops failed in Edx Api Notes + to: team-cosmonauts@edx.org + from: github-actions + body: Push Image to docker.io/edxops for Edx Api Notes failed! For details see "github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" diff --git a/dockerfiles/edx-api-notes.Dockerfile b/dockerfiles/edx-api-notes.Dockerfile new file mode 100644 index 0000000..576c20c --- /dev/null +++ b/dockerfiles/edx-api-notes.Dockerfile @@ -0,0 +1,89 @@ +FROM ubuntu:focal as app + +# Packages installed: +# git; Used to pull in particular requirements from github rather than pypi, +# and to check the sha of the code checkout. + +# language-pack-en locales; ubuntu locale support so that system utilities have a consistent +# language and time zone. + +# python3.8-dev; to install python 3.8 +# python3-venv; installs venv module required to create virtual environments + +# libssl-dev; # mysqlclient wont install without this. + +# libmysqlclient-dev; to install header files needed to use native C implementation for +# MySQL-python for performance gains. + +# If you add a package here please include a comment above describing what it is used for + +RUN apt-get update && \ + apt-get install -y software-properties-common && \ + apt-add-repository -y ppa:deadsnakes/ppa && \ + apt-get update && apt-get upgrade -qy && \ + apt-get install \ + language-pack-en \ + locales \ + git \ + libmysqlclient-dev \ + pkg-config \ + libssl-dev \ + build-essential \ + python3.8-dev \ + python3.8-distutils \ + python3-virtualenv -qy && \ + rm -rf /var/lib/apt/lists/* + + +RUN locale-gen en_US.UTF-8 +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en +ENV LC_ALL en_US.UTF-8 + + +# ENV variables lifetime is bound to the container whereas ARGS variables lifetime is bound to the image building process only +# Also ARGS provide us an option of compatibility of Path structure for Tutor and other OpenedX installations +ARG COMMON_CFG_DIR "/edx/etc" +ARG COMMON_APP_DIR="/edx/app" +ARG NOTES_VENV_DIR="${COMMON_APP_DIR}/venvs/notes" + +ENV PATH="$NOTES_VENV_DIR/bin:$PATH" + +RUN useradd -m --shell /bin/false app + +# Install curl +RUN apt-get update && apt-get install -y curl + +# cloning git repo +RUN curl -L https://github.com/openedx/edx-notes-api/archive/refs/heads/master.tar.gz | tar -xz --strip-components=1 + +RUN virtualenv -p python3.8 --always-copy ${NOTES_VENV_DIR} + +# edx_notes_api service config commands below +RUN pip install --no-cache-dir -r requirements/base.txt +RUN pip install --no-cache-dir -r requirements/pip.txt + +RUN mkdir -p /edx/var/log + +EXPOSE 8120 + +FROM app as dev + +ENV DJANGO_SETTINGS_MODULE "notesserver.settings.devstack" + +# Backwards compatibility with devstack +RUN touch "${COMMON_APP_DIR}/edx_notes_api_env" + +CMD while true; do python ./manage.py runserver 0.0.0.0:8120; sleep 2; done + +FROM app as production + +ENV EDXNOTES_CONFIG_ROOT /edx/etc +ENV DJANGO_SETTINGS_MODULE "notesserver.settings.yaml_config" + +# Code is owned by root so it cannot be modified by the application user. +# So we copy it before changing users. +USER app + +# Gunicorn 19 does not log to stdout or stderr by default. Once we are past gunicorn 19, the logging to STDOUT need not be specified. +CMD gunicorn --workers=2 --name notes -c /edx/app/notes/notesserver/docker_gunicorn_configuration.py --log-file - --max-requests=1000 notesserver.wsgi:application From 15e1717ad179816e14064b439c1907dd93559dff Mon Sep 17 00:00:00 2001 From: Bilal Qamar <59555732+BilalQamar95@users.noreply.github.com> Date: Thu, 10 Oct 2024 18:35:01 +0500 Subject: [PATCH 2/6] refactor: updated push image workflow to add platforms --- .github/workflows/push-edx-api-notes-image.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/push-edx-api-notes-image.yaml b/.github/workflows/push-edx-api-notes-image.yaml index 612edb4..2e756e7 100644 --- a/.github/workflows/push-edx-api-notes-image.yaml +++ b/.github/workflows/push-edx-api-notes-image.yaml @@ -42,6 +42,7 @@ jobs: push: true target: dev tags: edxops/edx-api-notes-dev:${{ steps.get-tag-name.outputs.result }} + platforms: linux/amd64,linux/arm64 - name: Send failure notification if: failure() From 1c0535366b3a7a846d7a37a18667273b3e35b0b9 Mon Sep 17 00:00:00 2001 From: Bilal Qamar <59555732+BilalQamar95@users.noreply.github.com> Date: Tue, 15 Oct 2024 12:06:48 +0500 Subject: [PATCH 3/6] chore: updated mail team for failure notification --- .github/workflows/push-edx-api-notes-image.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/push-edx-api-notes-image.yaml b/.github/workflows/push-edx-api-notes-image.yaml index 2e756e7..29137ca 100644 --- a/.github/workflows/push-edx-api-notes-image.yaml +++ b/.github/workflows/push-edx-api-notes-image.yaml @@ -53,6 +53,6 @@ jobs: username: ${{secrets.edx_smtp_username}} password: ${{secrets.edx_smtp_password}} subject: Push Image to docker.io/edxops failed in Edx Api Notes - to: team-cosmonauts@edx.org + to: team-aurora@edx.org from: github-actions body: Push Image to docker.io/edxops for Edx Api Notes failed! For details see "github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" From 1540232c4c380a66b2ff45af2d38533a0f555f7f Mon Sep 17 00:00:00 2001 From: Bilal Qamar <59555732+BilalQamar95@users.noreply.github.com> Date: Wed, 16 Oct 2024 14:52:31 +0500 Subject: [PATCH 4/6] perf: updated Dockerfile to optimize requirements installation and dependency caching --- .../workflows/push-edx-api-notes-image.yaml | 5 +++++ dockerfiles/edx-api-notes.Dockerfile | 21 ++++++++++++------- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/.github/workflows/push-edx-api-notes-image.yaml b/.github/workflows/push-edx-api-notes-image.yaml index 29137ca..9f94a4c 100644 --- a/.github/workflows/push-edx-api-notes-image.yaml +++ b/.github/workflows/push-edx-api-notes-image.yaml @@ -9,6 +9,11 @@ on: schedule: - cron: "0 4 * * 1-5" # UTC Time +# Added for testing purposes. Will remove once the PR is finalised + pull_request: + branches: + - '**' + jobs: build-and-push-image: runs-on: ubuntu-latest diff --git a/dockerfiles/edx-api-notes.Dockerfile b/dockerfiles/edx-api-notes.Dockerfile index 576c20c..9097c34 100644 --- a/dockerfiles/edx-api-notes.Dockerfile +++ b/dockerfiles/edx-api-notes.Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:focal as app # Packages installed: -# git; Used to pull in particular requirements from github rather than pypi, +# git; Used to pull in particular requirements from github rather than pypi, # and to check the sha of the code checkout. # language-pack-en locales; ubuntu locale support so that system utilities have a consistent @@ -12,7 +12,7 @@ FROM ubuntu:focal as app # libssl-dev; # mysqlclient wont install without this. -# libmysqlclient-dev; to install header files needed to use native C implementation for +# libmysqlclient-dev; to install header files needed to use native C implementation for # MySQL-python for performance gains. # If you add a package here please include a comment above describing what it is used for @@ -54,12 +54,15 @@ RUN useradd -m --shell /bin/false app # Install curl RUN apt-get update && apt-get install -y curl -# cloning git repo -RUN curl -L https://github.com/openedx/edx-notes-api/archive/refs/heads/master.tar.gz | tar -xz --strip-components=1 +RUN mkdir -p requirements RUN virtualenv -p python3.8 --always-copy ${NOTES_VENV_DIR} -# edx_notes_api service config commands below +RUN pip install --upgrade pip setuptools + +RUN curl -L -o requirements/base.txt https://raw.githubusercontent.com/openedx/edx-notes-api/master/requirements/base.txt +RUN curl -L -o requirements/pip.txt https://raw.githubusercontent.com/openedx/edx-notes-api/master/requirements/pip.txt + RUN pip install --no-cache-dir -r requirements/base.txt RUN pip install --no-cache-dir -r requirements/pip.txt @@ -72,7 +75,9 @@ FROM app as dev ENV DJANGO_SETTINGS_MODULE "notesserver.settings.devstack" # Backwards compatibility with devstack -RUN touch "${COMMON_APP_DIR}/edx_notes_api_env" +RUN touch "${COMMON_APP_DIR}/edx_notes_api_env" + +RUN curl -L https://github.com/openedx/edx-notes-api/archive/refs/heads/master.tar.gz | tar -xz --strip-components=1 CMD while true; do python ./manage.py runserver 0.0.0.0:8120; sleep 2; done @@ -81,8 +86,8 @@ FROM app as production ENV EDXNOTES_CONFIG_ROOT /edx/etc ENV DJANGO_SETTINGS_MODULE "notesserver.settings.yaml_config" -# Code is owned by root so it cannot be modified by the application user. -# So we copy it before changing users. +RUN curl -L https://github.com/openedx/edx-notes-api/archive/refs/heads/master.tar.gz | tar -xz --strip-components=1 + USER app # Gunicorn 19 does not log to stdout or stderr by default. Once we are past gunicorn 19, the logging to STDOUT need not be specified. From d2f07993e03632b8b5418d5c73086f3c4a2c824e Mon Sep 17 00:00:00 2001 From: Bilal Qamar <59555732+BilalQamar95@users.noreply.github.com> Date: Wed, 16 Oct 2024 15:08:43 +0500 Subject: [PATCH 5/6] chore: Remove pull_request trigger from workflow --- .github/workflows/push-edx-api-notes-image.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/push-edx-api-notes-image.yaml b/.github/workflows/push-edx-api-notes-image.yaml index 9f94a4c..29137ca 100644 --- a/.github/workflows/push-edx-api-notes-image.yaml +++ b/.github/workflows/push-edx-api-notes-image.yaml @@ -9,11 +9,6 @@ on: schedule: - cron: "0 4 * * 1-5" # UTC Time -# Added for testing purposes. Will remove once the PR is finalised - pull_request: - branches: - - '**' - jobs: build-and-push-image: runs-on: ubuntu-latest From 374da6f79c83074d508ddb38a5f53a9eeddd0002 Mon Sep 17 00:00:00 2001 From: Bilal Qamar <59555732+BilalQamar95@users.noreply.github.com> Date: Wed, 16 Oct 2024 16:01:43 +0500 Subject: [PATCH 6/6] chore: updated push workflow --- ...-notes-image.yaml => push-edx-notes-api-image.yaml} | 10 +++++----- ...x-api-notes.Dockerfile => edx-notes-api.Dockerfile} | 0 2 files changed, 5 insertions(+), 5 deletions(-) rename .github/workflows/{push-edx-api-notes-image.yaml => push-edx-notes-api-image.yaml} (87%) rename dockerfiles/{edx-api-notes.Dockerfile => edx-notes-api.Dockerfile} (100%) diff --git a/.github/workflows/push-edx-api-notes-image.yaml b/.github/workflows/push-edx-notes-api-image.yaml similarity index 87% rename from .github/workflows/push-edx-api-notes-image.yaml rename to .github/workflows/push-edx-notes-api-image.yaml index 29137ca..8ad2558 100644 --- a/.github/workflows/push-edx-api-notes-image.yaml +++ b/.github/workflows/push-edx-notes-api-image.yaml @@ -1,4 +1,4 @@ -name: Build and Push Edx Api Notes Image +name: Build and Push Edx Notes Api Image on: workflow_dispatch: @@ -38,10 +38,10 @@ jobs: - name: Build and push Dev Docker image uses: docker/build-push-action@v6 with: - file: ./dockerfiles/edx-api-notes.Dockerfile + file: ./dockerfiles/edx-notes-api.Dockerfile push: true target: dev - tags: edxops/edx-api-notes-dev:${{ steps.get-tag-name.outputs.result }} + tags: edxops/edx-notes-api-dev:${{ steps.get-tag-name.outputs.result }} platforms: linux/amd64,linux/arm64 - name: Send failure notification @@ -52,7 +52,7 @@ jobs: server_port: 465 username: ${{secrets.edx_smtp_username}} password: ${{secrets.edx_smtp_password}} - subject: Push Image to docker.io/edxops failed in Edx Api Notes + subject: Push Image to docker.io/edxops failed in Edx Notes Api to: team-aurora@edx.org from: github-actions - body: Push Image to docker.io/edxops for Edx Api Notes failed! For details see "github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" + body: Push Image to docker.io/edxops for Edx Notes Api failed! For details see "github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" diff --git a/dockerfiles/edx-api-notes.Dockerfile b/dockerfiles/edx-notes-api.Dockerfile similarity index 100% rename from dockerfiles/edx-api-notes.Dockerfile rename to dockerfiles/edx-notes-api.Dockerfile