From 6e0b982baa421d2b7b5477e0a3ab19fd8a731171 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Rame=CC=81?= Date: Mon, 19 Feb 2024 19:59:05 +0100 Subject: [PATCH] aaa --- .github/workflows/ci.yml | 167 ++++++++++++++++++++ .github/workflows/clean-images.yml | 5 + .github/workflows/pr.yml | 49 ++++++ .github/workflows/prepare-runner-image.yaml | 3 + Dockerfile | 20 +++ Dockerfile.clevercloud | 3 + Dockerfile.runner | 20 +++ Makefile | 84 ++++++++++ next.config.js | 2 +- start-and-wait-to-init.sh | 54 +++++++ 10 files changed, 406 insertions(+), 1 deletion(-) create mode 100755 .github/workflows/ci.yml create mode 100644 .github/workflows/clean-images.yml create mode 100644 .github/workflows/pr.yml create mode 100644 .github/workflows/prepare-runner-image.yaml create mode 100644 Dockerfile create mode 100644 Dockerfile.clevercloud create mode 100644 Dockerfile.runner create mode 100755 Makefile create mode 100755 start-and-wait-to-init.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100755 index 0000000..ba5a690 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,167 @@ +name: Continuous Integration +on: + push: + branches: + - main + - dev + pull_request: +env: + APP_NAME: etabli + CONTAINER_REGISTRY: ghcr.io + CONTAINER_IMAGE_FOLDER: ghcr.io/${{ github.repository }} + NODE_OPTIONS: --max_old_space_size=4096 + NODE_VERSION: 18.19.0 + RUBY_VERSION: 3.2.2 + PLAYWRIGHT_BROWSERS_CACHE_FOLDER_SUFFIX: .cache/ms-playwright +concurrency: + # Prevent parallel builds of the same branch + group: cicd-${{ github.ref }} + cancel-in-progress: false +jobs: + requirements: + name: Continuous Integration + runs-on: ubuntu-latest + permissions: + contents: write + issues: write + pull-requests: write + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set environment for branch + run: | + if [[ $GITHUB_REF_NAME == 'main' ]]; then + echo "APP_MODE=prod" >> $GITHUB_ENV + echo "CLEVER_APP_ID=${{ secrets.CLEVER_APP_ID_PRODUCTION }}" >> $GITHUB_ENV + elif [[ $GITHUB_REF_NAME == 'dev' ]]; then + echo "APP_MODE=dev" >> $GITHUB_ENV + echo "CLEVER_APP_ID=${{ secrets.CLEVER_APP_ID_DEVELOPMENT }}" >> $GITHUB_ENV + else + echo "APP_MODE=test" >> $GITHUB_ENV + fi + + - name: Setup Node.js + uses: actions/setup-node@v3 + with: + node-version: ${{ env.NODE_VERSION }} + + - name: Setup Ruby + uses: actions/setup-ruby@v1 + with: + ruby-version: ${{ env.RUBY_VERSION }} + + - name: Export npm store directory as an environment variable + shell: bash + run: | + echo "STORE_PATH=$(npm config get cache)" >> $GITHUB_ENV + + - uses: actions/cache@v3 + name: Setup npm cache + with: + path: ${{ env.STORE_PATH }} + key: ${{ runner.os }}-npm-store-${{ hashFiles('**/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-npm-store- + + - uses: actions/cache@v3 + name: Setup Next.js build cache + with: + path: ${{ github.workspace }}/.next/cache + key: ${{ runner.os }}-nextjs-${{ hashFiles('**/package-lock.json') }}-${{ hashFiles('**.[jt]s', '**.[jt]sx') }} + restore-keys: | + ${{ runner.os }}-nextjs-${{ hashFiles('**/package-lock.json') }}- + + - name: Install dependencies + env: + PLAYWRIGHT_BROWSERS_PATH: ${{ env.STORE_PATH }}/${{ env.PLAYWRIGHT_BROWSERS_CACHE_FOLDER_SUFFIX }} + run: make deps + + - name: Prepare linting + run: make lint-prepare + + - name: Lint + run: make lint + + - name: Format check + run: make format-check + + - name: Prepare tests + run: make test-prepare + + - name: Install `docker-compose` for local CI/CD simulations (https://github.com/nektos/act/issues/112#issuecomment-1387307297) + if: ${{ env.ACT }} + uses: KengoTODA/actions-setup-docker-compose@v1 + with: + version: '2.14.2' + - name: Install `Xvfb` and others to run browsers for end-to-end testing in local CI/CD simulations (https://github.com/nektos/act/issues/1300#issuecomment-1387344639) + if: ${{ env.ACT }} + run: sudo apt-get update && sudo apt-get install -y xvfb && npx playwright install-deps + + - name: Test unit + run: make test-unit + + - name: Build + env: + SENTRY_URL: ${{ secrets.SENTRY_URL }} + SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} + SENTRY_ORG: ${{ secrets.SENTRY_ORG }} + SENTRY_PROJECT: ${{ secrets.SENTRY_PROJECT }} + SENTRY_RELEASE_UPLOAD: true + run: make build + + - name: Test end-to-end + env: + PLAYWRIGHT_BROWSERS_PATH: ${{ env.STORE_PATH }}/${{ env.PLAYWRIGHT_BROWSERS_CACHE_FOLDER_SUFFIX }} + run: make test-e2e + + # # Disabled since too long, need to consider if our Storybook e2e tests are sufficient + # - name: Accessibility with Lighthouse + # run: make accessibility + # env: + # NEXTJS_BUILD_OUTPUT_MODE: export + # LHCI_GITHUB_APP_TOKEN: ${{ secrets.LHCI_GITHUB_APP_TOKEN }} + + - name: Publish to Chromatic + if: ${{ !github.event.act }} + uses: chromaui/action@v1 + env: + CHROMATIC_PROJECT_TOKEN: ${{ secrets.CHROMATIC_PROJECT_TOKEN }} + with: + # Note: since we use `buildScriptName` we have to specify some of those parameters into the underlying `package.json` script named `chromatic` + projectToken: ${{ secrets.CHROMATIC_PROJECT_TOKEN }} + buildScriptName: build + autoAcceptChanges: true + onlyChanged: true + externals: public/** + exitZeroOnChanges: true + + - name: Log in to the Container registry + if: ${{ !github.event.act && (github.ref_name == 'dev' || github.ref_name == 'main') }} + uses: docker/login-action@v2 + with: + registry: ${{ env.CONTAINER_REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push the application Docker image + if: ${{ !github.event.act && (github.ref_name == 'dev' || github.ref_name == 'main') }} + uses: docker/build-push-action@v5 + with: + push: true + tags: ${{ env.CONTAINER_IMAGE_FOLDER }}/${{ env.APP_NAME }}-${{ github.ref_name }}:${{ github.sha }},${{ env.CONTAINER_IMAGE_FOLDER }}/${{ env.APP_NAME }}-${{ github.ref_name }}:latest + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Deploy to Clever Cloud + if: ${{ !github.event.act && (github.ref_name == 'dev' || github.ref_name == 'main') }} + uses: 47ng/actions-clever-cloud@v1.3.1 + with: + appID: ${{ env.CLEVER_APP_ID }} + force: true + quiet: true # disable copying into GitHub Actions all logs from Clever Cloud + env: + CLEVER_TOKEN: ${{ secrets.CLEVER_TOKEN }} + CLEVER_SECRET: ${{ secrets.CLEVER_SECRET }} diff --git a/.github/workflows/clean-images.yml b/.github/workflows/clean-images.yml new file mode 100644 index 0000000..a656c6e --- /dev/null +++ b/.github/workflows/clean-images.yml @@ -0,0 +1,5 @@ +name: Clean old Docker images +on: + schedule: + - cron: '0 7 * * 1' # Every monday at 7am + workflow_dispatch: # Allow triggering this pipeline manually diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml new file mode 100644 index 0000000..4dd3c5f --- /dev/null +++ b/.github/workflows/pr.yml @@ -0,0 +1,49 @@ +name: Pull Request + +on: [pull_request] + +jobs: + build: + runs-on: ubuntu-latest + + strategy: + matrix: + node-version: [16.15.0] + + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Setup Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} + + - uses: pnpm/action-setup@v2 + name: Install pnpm + id: pnpm-install + with: + version: 7.27.1 + run_install: false + + - name: Get pnpm store directory + id: pnpm-cache + shell: bash + run: | + echo "STORE_PATH=$(pnpm store path)" >> $GITHUB_OUTPUT + + - uses: actions/cache@v3 + name: Setup pnpm cache + with: + path: ${{ steps.pnpm-cache.outputs.STORE_PATH }} + key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} + restore-keys: | + ${{ runner.os }}-pnpm-store- + + - run: make deps + - run: make lint + - run: make test + - run: make build + # - run: make coverage diff --git a/.github/workflows/prepare-runner-image.yaml b/.github/workflows/prepare-runner-image.yaml new file mode 100644 index 0000000..0b97155 --- /dev/null +++ b/.github/workflows/prepare-runner-image.yaml @@ -0,0 +1,3 @@ +name: Prepare the Docker runner image +on: + workflow_dispatch: # Allow triggering this pipeline manually diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..10242a0 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,20 @@ +# [IMPORTANT] Must be built from the root of the project for the COPY/paths to work + +ARG PORT=3000 + +FROM ghcr.io/betagouv/etabli/etabli-${BRANCH}:latest + +RUN addgroup --system --gid 1001 nodejs +RUN adduser --system --uid 1001 nextjs + +USER nextjs + +COPY --chown=nextjs:nodejs ".next/standalone" ./ +COPY --chown=nextjs:nodejs ".next/static" "./.next/static" +COPY --chown=nextjs:nodejs "public" "./public" +COPY --chown=nextjs:nodejs "start-and-wait-to-init.sh" ./ + +ENV PORT $PORT +EXPOSE $PORT + +CMD start-and-wait-to-init.sh diff --git a/Dockerfile.clevercloud b/Dockerfile.clevercloud new file mode 100644 index 0000000..d2901a7 --- /dev/null +++ b/Dockerfile.clevercloud @@ -0,0 +1,3 @@ +# We didn't want to rebuild everything in the Clever Cloud pipeline so we pushed an image from GitHub to be retrieved here without much processing + +FROM ghcr.io/betagouv/etabli/etabli-${BRANCH}:latest diff --git a/Dockerfile.runner b/Dockerfile.runner new file mode 100644 index 0000000..e0f5335 --- /dev/null +++ b/Dockerfile.runner @@ -0,0 +1,20 @@ +ARG NODE_VERSION=18.19.0 +ARG RUBY_VERSION=3.2.2-r1 +ARG PYTHON_VERSION= + +FROM node:${NODE_VERSION}-alpine + +RUN apk add --no-cache 'ruby=${RUBY_VERSION}' +RUN apk update + +# Install tools +RUN gem install bundler +RUN bundle --gemfile src/bibliothecary/Gemfile + +# TODO: +# TODO: "Don't run Bundler as root. Installing your bundle as root will break this application for all non-root users on this machine." +# TODO: +# TODO: ruby, python, and only get the final image (the builder should be outside in the GitHub runner) +# TODO: +# TODO: +# TODO: diff --git a/Makefile b/Makefile new file mode 100755 index 0000000..33242ad --- /dev/null +++ b/Makefile @@ -0,0 +1,84 @@ +define HELP_TEXT + + Makefile commands + + make deps - Install dependent programs and libraries + make ... - (see the Makefile) + +endef + +help: + $(info $(HELP_TEXT)) + +build: + npm run build +#--mode prod + +build-dev: + npm run build +#--mode dev + +serve: + npm run dev +#--mode test + +serve-dev: + npm run dev +#--mode dev + +lint-prepare: + npm run lint:prepare + +lint: + npm run lint + +test-prepare: + npm run test:prepare + +test-unit: + npm run test:unit + +test-unit-watch: + npm run test:unit:watch + +test-e2e: + npm run test:e2e:headless + +clean: + npm run clean + +accessibility: + npm run accessibility + +accessibility-open: + npm run accessibility:open + +deps: + npm install + +tools-up: + docker-compose up -d + +tools-down: + docker-compose down + +format: + npm run format + +format-check: + npm run format:check + +simulate-cicd-with-push: +# Install `act` through a package manager to make it working +# +# Notes: +# - there is no way to specify the pipeline branch, you must locally change it +# - you can have weird issues like "unable to get git", try to set your local head to the remote one (with your changes uncommitted) +# - caching: +# - for now the cache does not work and even if there is https://github.com/sp-ricard-valverde/github-act-cache-server it's a bit overheaded for a ponctual simulation +# - using "--bind" is not ideal because `npm` will recreate the whole "node_modules" on the host, so we have to do `npm install` then (it would make sense for a computer dedicated to this) +# - so we use "--reuse" that keeps using the existing docker container if any, to avoid downloading a new time each dependency. If you get weird behavior you can still remove the docker container from `act` and restart the command +# - we inject a meaningful commit SHA into "SOURCE_VERSION" otherwise a Jest process will fail since we use it for Sentry stuff in the Next.js app +# - the default image is missing browsers stuff for e2e tests and `docker-compose`, we added 2 steps in the workflow to not deal with a custom image not officially supported (and the full official image is around 15GB... we don't want that either) +# - `.actrc` breaks `act` commands in specific situations, we avoid using it to factorize commands + act push --reuse --env-file .github/act/.env --env SOURCE_VERSION="$(git rev-parse @{upstream})" --eventpath .github/act/event.json diff --git a/next.config.js b/next.config.js index d937e68..8c24ef3 100644 --- a/next.config.js +++ b/next.config.js @@ -38,7 +38,7 @@ const moduleExports = async () => { let standardModuleExports = { reactStrictMode: true, swcMinify: true, - output: 'standalone', // To debug locally the `next start` comment this line (it will avoid trying to mess with the assembling folders logic of standalone mode) + output: process.env.NEXTJS_BUILD_OUTPUT || 'standalone', // To debug locally the `next start` comment this line (it will avoid trying to mess with the assembling folders logic of standalone mode) env: { // Those will replace `process.env.*` with hardcoded values (useful when the value is calculated during the build time) SENTRY_RELEASE_TAG: appHumanVersion, diff --git a/start-and-wait-to-init.sh b/start-and-wait-to-init.sh new file mode 100755 index 0000000..8bd944c --- /dev/null +++ b/start-and-wait-to-init.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +# +# We trigger an init endpoint to set up all async processes like cron scheduling +# because unfortunately Next.js does not provide a proper way to use a startup callback +# + +echo "Start the server in background" +node server.js & + +# Store the server process ID +next_pid=$! + +stop_server() { + echo "Shut down Next.js server..." + kill $next_pid + exit 0 +} + +# Bind the callback to the SIGINT signal to shut down the background process properly +trap stop_server SIGINT + +check_server_and_init() { + timeout=15 + counter=0 + + while true; do + response=$(curl --write-out %{http_code} --silent --output /dev/null http://localhost:$PORT) + if [ "$response" = "200" ]; then + break + fi + + if [ $counter -eq $timeout ]; then + echo "Error: the Next.js server is not ready within the expected timeframe" + + # Kill the server since it has no reason to continue + kill $next_pid + exit 1 + fi + + sleep 1 + counter=$((counter+1)) + done + + curl http://localhost:$PORT/api/init +} + +# In parallel wait for the server readiness to init some services +check_server_and_init + +# Wait for the Next.js server to return +wait + +echo "The wait instruction has exited normally"