From c228f5edf747427e437c07258d12b05ac3714eb0 Mon Sep 17 00:00:00 2001 From: Peter Baker Date: Thu, 15 Aug 2024 12:55:11 +1000 Subject: [PATCH 01/19] Adds fix and build outputs to turbo config Signed-off-by: Peter Baker --- turbo.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/turbo.json b/turbo.json index 003ba5be7..78fe29ba7 100644 --- a/turbo.json +++ b/turbo.json @@ -2,9 +2,11 @@ "$schema": "https://turbo.build/schema.json", "tasks": { "lint": {}, + "fix": {}, "clean": {}, "build": { - "dependsOn": ["^build"] + "dependsOn": ["^build"], + "outputs": ["build", "dist"] } } } \ No newline at end of file From 919090e216df24e8fe7ad4d08aac904a046a66bf Mon Sep 17 00:00:00 2001 From: Peter Baker Date: Thu, 15 Aug 2024 12:57:28 +1000 Subject: [PATCH 02/19] Adding cache false to clean task Signed-off-by: Peter Baker --- turbo.json | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/turbo.json b/turbo.json index 78fe29ba7..ead77d824 100644 --- a/turbo.json +++ b/turbo.json @@ -1,12 +1,19 @@ { - "$schema": "https://turbo.build/schema.json", - "tasks": { - "lint": {}, - "fix": {}, - "clean": {}, - "build": { - "dependsOn": ["^build"], - "outputs": ["build", "dist"] + "$schema": "https://turbo.build/schema.json", + "tasks": { + "lint": {}, + "fix": {}, + "clean": { + "cache": false + }, + "build": { + "dependsOn": [ + "^build" + ], + "outputs": [ + "build", + "dist" + ] + } } - } } \ No newline at end of file From 9ab7f73f8f3ec46d1df5e7a8f30da8a6f0eccab7 Mon Sep 17 00:00:00 2001 From: Peter Baker Date: Thu, 15 Aug 2024 13:04:22 +1000 Subject: [PATCH 03/19] Adding caching to build-lint, npm i and turbo Signed-off-by: Peter Baker --- .github/workflows/build-lint.yml | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-lint.yml b/.github/workflows/build-lint.yml index ee79ead84..92584c02d 100644 --- a/.github/workflows/build-lint.yml +++ b/.github/workflows/build-lint.yml @@ -24,12 +24,37 @@ jobs: node-version: 20.9.0 cache: 'npm' + - name: Setup node cache + id: node-cache + uses: actions/cache@v3 + with: + path: "**/node_modules" + key: npm-${{ hashFiles('package-lock.json') }}-${{ hashFiles('package.json') }} + restore-keys: npm- + - name: Install dependencies - run: npm install + if: steps.node-cache.outputs.cache-hit != 'true' + run: npm i + - name: Configure Turborepo Remote Cache + env: + TURBO_TEAM: ${{ vars.TURBO_TEAM }} + TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} + run: | + mkdir -p .turbo + echo '{"apiurl": "${{ vars.TURBO_API_URL }}"}' > .turbo/config.json + echo "TURBO_TEAM and TURBO_TOKEN environment variables set" + echo "Temporary turborepo config file created at .turbo/config.json" + - name: Lint + env: + TURBO_TEAM: ${{ vars.TURBO_TEAM }} + TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} run: npm run lint - name: Build + env: + TURBO_TEAM: ${{ vars.TURBO_TEAM }} + TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} run: npm run build \ No newline at end of file From bda75cd8f9ae94570d4fe5a5c35a1cde33718bcb Mon Sep 17 00:00:00 2001 From: Peter Baker Date: Thu, 15 Aug 2024 13:09:53 +1000 Subject: [PATCH 04/19] force rerun to test npm cache Signed-off-by: Peter Baker From 28c0cb25c4f48e9c0c0857d3498c8fad1527deca Mon Sep 17 00:00:00 2001 From: Peter Baker Date: Thu, 15 Aug 2024 13:52:04 +1000 Subject: [PATCH 05/19] Making durable to failure and testing docker build with turbo cache Signed-off-by: Peter Baker --- .github/workflows/build-lint.yml | 14 +++++++++++--- .github/workflows/publish-conductor.yml | 6 +++++- api/BuildDockerfile | 21 +++++++++++++++------ purge.sh => scripts/purge.sh | 0 4 files changed, 31 insertions(+), 10 deletions(-) rename purge.sh => scripts/purge.sh (100%) diff --git a/.github/workflows/build-lint.yml b/.github/workflows/build-lint.yml index 92584c02d..a053071d4 100644 --- a/.github/workflows/build-lint.yml +++ b/.github/workflows/build-lint.yml @@ -42,9 +42,17 @@ jobs: TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} run: | mkdir -p .turbo - echo '{"apiurl": "${{ vars.TURBO_API_URL }}"}' > .turbo/config.json - echo "TURBO_TEAM and TURBO_TOKEN environment variables set" - echo "Temporary turborepo config file created at .turbo/config.json" + if [ -n "${{ vars.TURBO_API_URL }}" ]; then + echo '{"apiurl": "${{ vars.TURBO_API_URL }}"}' > .turbo/config.json + echo "Turborepo config file created at .turbo/config.json" + else + echo "TURBO_API_URL not set. Skipping Turborepo cache configuration." + fi + if [ -n "$TURBO_TEAM" ] && [ -n "$TURBO_TOKEN" ]; then + echo "TURBO_TEAM and TURBO_TOKEN environment variables set" + else + echo "TURBO_TEAM and/or TURBO_TOKEN not set. Remote caching may not be available." + fi - name: Lint env: diff --git a/.github/workflows/publish-conductor.yml b/.github/workflows/publish-conductor.yml index fed20671b..cf8b27846 100644 --- a/.github/workflows/publish-conductor.yml +++ b/.github/workflows/publish-conductor.yml @@ -3,6 +3,7 @@ on: push: branches: - main + - chore/add-build-outputs-to-turbo release: types: [published] @@ -51,9 +52,12 @@ jobs: type=raw,value=latest,enable={{is_default_branch}} type=semver,pattern={{version}},enable=${{ github.event_name == 'release' || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')) }} - - name: Build and push Docker image uses: docker/build-push-action@v4 + env: + TURBO_TEAM: ${{ vars.TURBO_TEAM }} + TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} + TURBO_API_URL: ${{ vars.TURBO_API_URL }} with: context: . file: ./api/BuildDockerfile diff --git a/api/BuildDockerfile b/api/BuildDockerfile index 56263eaa8..479771879 100644 --- a/api/BuildDockerfile +++ b/api/BuildDockerfile @@ -43,14 +43,23 @@ COPY --from=builder /app/out/json/ . # COPY --from=builder /app/out/full/library/data-model/tsconfig.json ./library/data-model/tsconfig.json RUN npm install -# Build the project -COPY --from=builder /app/out/full/ . +# install turbo global +RUN npm install turbo@^2.0.11 -g -# build packages +# Configure Turbo cache if TURBO_API_URL is set +RUN mkdir -p .turbo && \ + if [ -n "$TURBO_API_URL" ]; then \ + echo '{"apiurl": "'${TURBO_API_URL}'"}' > .turbo/config.json && \ + echo "Turborepo config file created at .turbo/config.json"; \ + else \ + echo "TURBO_API_URL not set. Skipping Turborepo cache configuration."; \ + fi -# TODO use turbo build here and filter for app -RUN npm run docker-build-api +# Build the project +COPY --from=builder /app/out/full/ . +# build packages using turbo +RUN turbo run build --filter=@faims3/api # PRODUCTION RUNNER [runner] # ========================== @@ -91,4 +100,4 @@ COPY --from=installer /app/library/data-model/build ./library/data-model/build COPY --from=builder /app/out/json/ . # Run the node app -CMD ["node", "api/build/src/index.js"] \ No newline at end of file +CMD ["node", "api/build/src/index.js"] diff --git a/purge.sh b/scripts/purge.sh similarity index 100% rename from purge.sh rename to scripts/purge.sh From 0bce2fa4104fbe04f6fb780afb2e86340acf6b9e Mon Sep 17 00:00:00 2001 From: Peter Baker Date: Thu, 15 Aug 2024 13:57:18 +1000 Subject: [PATCH 06/19] changing how variables passed in Signed-off-by: Peter Baker --- .github/workflows/publish-conductor.yml | 8 ++++---- api/BuildDockerfile | 11 +++++++++++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/.github/workflows/publish-conductor.yml b/.github/workflows/publish-conductor.yml index cf8b27846..aa129c268 100644 --- a/.github/workflows/publish-conductor.yml +++ b/.github/workflows/publish-conductor.yml @@ -54,13 +54,13 @@ jobs: - name: Build and push Docker image uses: docker/build-push-action@v4 - env: - TURBO_TEAM: ${{ vars.TURBO_TEAM }} - TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} - TURBO_API_URL: ${{ vars.TURBO_API_URL }} with: context: . file: ./api/BuildDockerfile push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + build-args: | + TURBO_TEAM=${{ vars.TURBO_TEAM }} + TURBO_TOKEN=${{ secrets.TURBO_TOKEN }} + TURBO_API_URL=${{ vars.TURBO_API_URL }} diff --git a/api/BuildDockerfile b/api/BuildDockerfile index 479771879..2f81cba63 100644 --- a/api/BuildDockerfile +++ b/api/BuildDockerfile @@ -6,6 +6,17 @@ # Use Node 20 base FROM node:20-alpine AS base +# Declare build arguments with default empty values +ARG TURBO_TEAM="" +ARG TURBO_TOKEN="" +ARG TURBO_API_URL="" + +# Set environment variables using the build arguments, with fallback to empty string +ENV TURBO_TEAM=${TURBO_TEAM} +ENV TURBO_TOKEN=${TURBO_TOKEN} +ENV TURBO_API_URL=${TURBO_API_URL} + + # PRUNE TO GET DEPENDENCIES [builder] # =================================== From 110ebd80dc46f967cbfb9dee2f4a98112a4870a2 Mon Sep 17 00:00:00 2001 From: Peter Baker Date: Thu, 15 Aug 2024 14:04:56 +1000 Subject: [PATCH 07/19] Removing turbo from docker build but keeping build filter Signed-off-by: Peter Baker --- .github/workflows/build-lint.yml | 12 +++--------- .github/workflows/publish-conductor.yml | 6 +----- api/BuildDockerfile | 20 -------------------- 3 files changed, 4 insertions(+), 34 deletions(-) diff --git a/.github/workflows/build-lint.yml b/.github/workflows/build-lint.yml index a053071d4..2dc72cc85 100644 --- a/.github/workflows/build-lint.yml +++ b/.github/workflows/build-lint.yml @@ -11,6 +11,9 @@ jobs: name: Build and Test timeout-minutes: 15 runs-on: ubuntu-latest + env: + TURBO_TEAM: ${{ vars.TURBO_TEAM }} + TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} steps: - name: Check out code @@ -37,9 +40,6 @@ jobs: run: npm i - name: Configure Turborepo Remote Cache - env: - TURBO_TEAM: ${{ vars.TURBO_TEAM }} - TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} run: | mkdir -p .turbo if [ -n "${{ vars.TURBO_API_URL }}" ]; then @@ -55,14 +55,8 @@ jobs: fi - name: Lint - env: - TURBO_TEAM: ${{ vars.TURBO_TEAM }} - TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} run: npm run lint - name: Build - env: - TURBO_TEAM: ${{ vars.TURBO_TEAM }} - TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} run: npm run build \ No newline at end of file diff --git a/.github/workflows/publish-conductor.yml b/.github/workflows/publish-conductor.yml index aa129c268..e2e851368 100644 --- a/.github/workflows/publish-conductor.yml +++ b/.github/workflows/publish-conductor.yml @@ -59,8 +59,4 @@ jobs: file: ./api/BuildDockerfile push: true tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - build-args: | - TURBO_TEAM=${{ vars.TURBO_TEAM }} - TURBO_TOKEN=${{ secrets.TURBO_TOKEN }} - TURBO_API_URL=${{ vars.TURBO_API_URL }} + labels: ${{ steps.meta.outputs.labels }} \ No newline at end of file diff --git a/api/BuildDockerfile b/api/BuildDockerfile index 2f81cba63..c4fec9bf5 100644 --- a/api/BuildDockerfile +++ b/api/BuildDockerfile @@ -6,17 +6,6 @@ # Use Node 20 base FROM node:20-alpine AS base -# Declare build arguments with default empty values -ARG TURBO_TEAM="" -ARG TURBO_TOKEN="" -ARG TURBO_API_URL="" - -# Set environment variables using the build arguments, with fallback to empty string -ENV TURBO_TEAM=${TURBO_TEAM} -ENV TURBO_TOKEN=${TURBO_TOKEN} -ENV TURBO_API_URL=${TURBO_API_URL} - - # PRUNE TO GET DEPENDENCIES [builder] # =================================== @@ -57,15 +46,6 @@ RUN npm install # install turbo global RUN npm install turbo@^2.0.11 -g -# Configure Turbo cache if TURBO_API_URL is set -RUN mkdir -p .turbo && \ - if [ -n "$TURBO_API_URL" ]; then \ - echo '{"apiurl": "'${TURBO_API_URL}'"}' > .turbo/config.json && \ - echo "Turborepo config file created at .turbo/config.json"; \ - else \ - echo "TURBO_API_URL not set. Skipping Turborepo cache configuration."; \ - fi - # Build the project COPY --from=builder /app/out/full/ . From 5772df40bbfe81ff0b28ab0837a688060763f051 Mon Sep 17 00:00:00 2001 From: Peter Baker Date: Thu, 15 Aug 2024 14:30:51 +1000 Subject: [PATCH 08/19] testing bundle script Signed-off-by: Peter Baker --- bundle.sh | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100755 bundle.sh diff --git a/bundle.sh b/bundle.sh new file mode 100755 index 000000000..08a40f9c6 --- /dev/null +++ b/bundle.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +# Check if all arguments are provided +if [ "$#" -ne 3 ]; then + echo "Usage: $0 " + exit 1 +fi + +json_dump_path="$1" +source_path="$2" +output_path="$3" + +# Function to process each directory +process_directory() { + local relative_dir="$1" + + # Check if package.json exists in this directory + if [ -f "$json_dump_path/$relative_dir/package.json" ]; then + echo "Processing $relative_dir" + + # Create the destination directory + mkdir -p "$output_path/$relative_dir" + + # Copy node_modules, dist, and build if they exist + for folder in node_modules dist build views; do + if [ -d "$source_path/$relative_dir/$folder" ]; then + echo "Copying $source_path/$relative_dir/$folder to $output_path/$relative_dir/$folder" + cp -R "$source_path/$relative_dir/$folder" "$output_path/$relative_dir/" + fi + done + fi +} + +# Find all package.json files in the json dump directory and process their directories +find "$json_dump_path" -name "package.json" | while read -r package_json; do + relative_dir=$(realpath --relative-to="$json_dump_path" "$(dirname "$package_json")") + process_directory "$relative_dir" +done + +echo "Copy operation completed." \ No newline at end of file From a8140a3feb283c35766c4d760bd6a22679eb2453 Mon Sep 17 00:00:00 2001 From: Peter Baker Date: Thu, 15 Aug 2024 16:45:37 +1000 Subject: [PATCH 09/19] Making docker build agnostic to package through build args, context aware bundling script, update pipeline Signed-off-by: Peter Baker --- .dockerignore | 1 + .github/workflows/publish-conductor.yml | 14 ++++- api/BuildDockerfile => Dockerfile.build | 59 ++++++++----------- bundle.sh | 58 +++++++++++++++--- .../aws-cdk/lib/components/conductor.ts | 2 +- 5 files changed, 90 insertions(+), 44 deletions(-) rename api/BuildDockerfile => Dockerfile.build (50%) diff --git a/.dockerignore b/.dockerignore index fc7ae1ef0..5f9587c8a 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,4 +3,5 @@ docs/ node_modules/ tests/ .env +Dockerfile.build out \ No newline at end of file diff --git a/.github/workflows/publish-conductor.yml b/.github/workflows/publish-conductor.yml index e2e851368..7fa28f3fe 100644 --- a/.github/workflows/publish-conductor.yml +++ b/.github/workflows/publish-conductor.yml @@ -52,11 +52,21 @@ jobs: type=raw,value=latest,enable={{is_default_branch}} type=semver,pattern={{version}},enable=${{ github.event_name == 'release' || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')) }} + # Build and push Docker image using docker/build-push-action - name: Build and push Docker image uses: docker/build-push-action@v4 with: + # Set the build context to the root of the repository (for monorepo build) context: . - file: ./api/BuildDockerfile + # Specify the location of the Dockerfile + file: Dockerfile.build + # Push the image to the container registry push: true + # Use the tags generated by the docker/metadata-action tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} \ No newline at end of file + # Use the labels generated by the docker/metadata-action + labels: ${{ steps.meta.outputs.labels }} + # Specify build arguments to build the faims3 api project in the monorepo + build-args: | + PACKAGE_NAME=@faims3/api + NODE_RUN_DIR=api/build/src \ No newline at end of file diff --git a/api/BuildDockerfile b/Dockerfile.build similarity index 50% rename from api/BuildDockerfile rename to Dockerfile.build index c4fec9bf5..538bfa331 100644 --- a/api/BuildDockerfile +++ b/Dockerfile.build @@ -6,14 +6,19 @@ # Use Node 20 base FROM node:20-alpine AS base +ARG PACKAGE_NAME +ENV PACKAGE_NAME=${PACKAGE_NAME} +ARG NODE_RUN_PATH +ENV NODE_RUN_PATH=${NODE_RUN_PATH} + # PRUNE TO GET DEPENDENCIES [builder] # =================================== # Use turbo to prune package dependencies FROM base AS builder -RUN apk update -RUN apk add --no-cache libc6-compat + +RUN apk update && apk add --no-cache libc6-compat # work in /app folder WORKDIR /app @@ -25,32 +30,36 @@ RUN npm install turbo@^2.0.11 -g COPY . . # prune using turbo for only faims3 and it's dependencies -RUN turbo prune @faims3/api --docker +RUN turbo prune ${PACKAGE_NAME} --docker # INSTALL PACKAGES [installer] # ============================ # add lockfiles dependencies FROM base AS installer -RUN apk update -RUN apk add --no-cache libc6-compat + +RUN apk update && apk add --no-cache libc6-compat WORKDIR /app # Install dependencies COPY .gitignore .gitignore COPY --from=builder /app/out/json/ . -# COPY --from=builder /app/out/full/library/data-model/tsconfig.json ./library/data-model/tsconfig.json -RUN npm install - # install turbo global RUN npm install turbo@^2.0.11 -g +RUN npm install + # Build the project COPY --from=builder /app/out/full/ . +COPY --from=builder /app/out/json/ json # build packages using turbo -RUN turbo run build --filter=@faims3/api +RUN turbo run build --filter=${PACKAGE_NAME} + +# Package all necessary files to run app into builder out +COPY bundle.sh . +RUN sh bundle.sh json . out && cp -R json/* /app/out/ # PRODUCTION RUNNER [runner] # ========================== @@ -63,32 +72,16 @@ RUN addgroup --system --gid 1001 nodejs RUN adduser --system --uid 1001 faims USER faims -# NOTE: To run API, we need -# 1) all node modules -# 2) all package*.json -# 3) all builds - -# NODE MODULES - -# Node modules for top level -COPY --from=installer /app/node_modules ./node_modules -# Node modules for data model -COPY --from=installer /app/library/data-model/node_modules ./library/data-model/node_modules +# Copy all files which are required to run the app +COPY --from=installer /app/out . -# BUILD FILES +# Run the node app +ENV NODE_RUN_DIR=${NODE_RUN_DIR} -# Build for API -COPY --from=installer /app/api/build ./api/build -# Copy handle bar views into appropriate build path -# TODO set the app root directory better and build into build process more nicely -COPY --from=installer /app/api/views ./views -# Build for data model -COPY --from=installer /app/library/data-model/build ./library/data-model/build +#CMD node ${NODE_RUN_PATH} +#CMD ["sh", "-c", "node $NODE_RUN_PATH"] +WORKDIR /app/${NODE_RUN_DIR} +CMD ["node", "index.js"] -# PACKAGE*.json FILES -# Copy all package json and package lock info from prune (tells node what to do) -COPY --from=builder /app/out/json/ . -# Run the node app -CMD ["node", "api/build/src/index.js"] diff --git a/bundle.sh b/bundle.sh index 08a40f9c6..9c17abb08 100755 --- a/bundle.sh +++ b/bundle.sh @@ -1,7 +1,20 @@ -#!/bin/bash +#!/bin/sh + +# Helper script used for docker builds. + +# bundle.sh: A script to copy specific directories from a source to an output location +# based on the presence of package.json files in a JSON dump directory. + +# Usage: ./bundle.sh +# +# Arguments: +# json_dump_path: Path to the directory containing package.json files +# source_path: Path to the source directory containing files to be copied +# output_path: Path to the output directory where files will be copied # Check if all arguments are provided if [ "$#" -ne 3 ]; then + echo "Error: Incorrect number of arguments" echo "Usage: $0 " exit 1 fi @@ -10,30 +23,59 @@ json_dump_path="$1" source_path="$2" output_path="$3" -# Function to process each directory +# Function: get_relative_path +# Purpose: Calculate the relative path from a base directory to a target directory +# Arguments: +# $1: target directory path +# $2: base directory path +# Returns: The relative path as a string +get_relative_path() { + local target="$1" # Store the target directory path + local base="$2" # Store the base directory path + + # Special case: if target and base are the same, return empty string + if [ "$target" = "$base" ]; then + echo "" + return + fi + + local back="" # Initialize a variable to store the "../" parts of the path + + # Loop while the target doesn't start with the base path + while [ "${target#$base}" = "${target}" ]; do + base="${base%/*}" # Remove the last directory from the base path + back="../$back" # Add "../" to the relative path + done + echo "$back${target#$base/}" +} + +# Function: process_directory +# Purpose: Process a directory by copying specific folders if they exist +# Arguments: +# $1: relative directory path process_directory() { local relative_dir="$1" + local full_path="$json_dump_path/$relative_dir" # Check if package.json exists in this directory - if [ -f "$json_dump_path/$relative_dir/package.json" ]; then - echo "Processing $relative_dir" - + if [ -f "$full_path/package.json" ]; then # Create the destination directory mkdir -p "$output_path/$relative_dir" - # Copy node_modules, dist, and build if they exist + # Copy node_modules, dist, build, and views if they exist for folder in node_modules dist build views; do if [ -d "$source_path/$relative_dir/$folder" ]; then - echo "Copying $source_path/$relative_dir/$folder to $output_path/$relative_dir/$folder" cp -R "$source_path/$relative_dir/$folder" "$output_path/$relative_dir/" fi done fi } +# Main script logic # Find all package.json files in the json dump directory and process their directories find "$json_dump_path" -name "package.json" | while read -r package_json; do - relative_dir=$(realpath --relative-to="$json_dump_path" "$(dirname "$package_json")") + dir=$(dirname "$package_json") + relative_dir=$(get_relative_path "$dir" "$json_dump_path") process_directory "$relative_dir" done diff --git a/infrastructure/aws-cdk/lib/components/conductor.ts b/infrastructure/aws-cdk/lib/components/conductor.ts index f906beb30..e29df76d0 100644 --- a/infrastructure/aws-cdk/lib/components/conductor.ts +++ b/infrastructure/aws-cdk/lib/components/conductor.ts @@ -105,7 +105,7 @@ export class FaimsConductor extends Construct { } else { // Build and bundle with cdk and ECR conductorContainerImage = ecs.ContainerImage.fromAsset(getPathToRoot(), { - file: 'api/BuildDockerfile', + file: 'Dockerfile.build', exclude: ['infrastructure'], }); } From 6975684f22755127ae635c17122c619d7b5b875a Mon Sep 17 00:00:00 2001 From: Peter Baker Date: Thu, 15 Aug 2024 16:48:04 +1000 Subject: [PATCH 10/19] Incorrect arg name Signed-off-by: Peter Baker --- Dockerfile.build | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/Dockerfile.build b/Dockerfile.build index 538bfa331..3e652ed2d 100644 --- a/Dockerfile.build +++ b/Dockerfile.build @@ -8,8 +8,8 @@ FROM node:20-alpine AS base ARG PACKAGE_NAME ENV PACKAGE_NAME=${PACKAGE_NAME} -ARG NODE_RUN_PATH -ENV NODE_RUN_PATH=${NODE_RUN_PATH} +ARG NODE_RUN_DIR +ENV NODE_RUN_DIR=${NODE_RUN_DIR} # PRUNE TO GET DEPENDENCIES [builder] # =================================== @@ -77,10 +77,8 @@ COPY --from=installer /app/out . # Run the node app ENV NODE_RUN_DIR=${NODE_RUN_DIR} - -#CMD node ${NODE_RUN_PATH} -#CMD ["sh", "-c", "node $NODE_RUN_PATH"] WORKDIR /app/${NODE_RUN_DIR} + CMD ["node", "index.js"] From 9f35535112daf8660aad0d12226e814654df4f32 Mon Sep 17 00:00:00 2001 From: Peter Baker Date: Thu, 15 Aug 2024 17:04:31 +1000 Subject: [PATCH 11/19] Cleaning up dockerfile.build and explaining in README Signed-off-by: Peter Baker --- Dockerfile.build | 43 ++++++++++++++++++++++--------------- README.md | 55 ++++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 77 insertions(+), 21 deletions(-) diff --git a/Dockerfile.build b/Dockerfile.build index 3e652ed2d..4b1215233 100644 --- a/Dockerfile.build +++ b/Dockerfile.build @@ -4,21 +4,28 @@ # AND https://github.com/ThaddeusJiang/turborepo-starter/blob/main/apps/api/Dockerfile # Use Node 20 base -FROM node:20-alpine AS base +FROM node:20-alpine AS alpine +# install some dependencies +RUN apk update && apk add --no-cache libc6-compat + +# create this layer to use to avoid repeatedly installing +FROM alpine as base + +# What is the @faims3/... name of the package to build ARG PACKAGE_NAME ENV PACKAGE_NAME=${PACKAGE_NAME} + +# What directory relative to the monorepo will the node build folder containing +# index.js be built to. For example API -> api/build/src ARG NODE_RUN_DIR ENV NODE_RUN_DIR=${NODE_RUN_DIR} -# PRUNE TO GET DEPENDENCIES [builder] +# PRUNE TO GET DEPENDENCIES [pruner] # =================================== # Use turbo to prune package dependencies -FROM base AS builder - - -RUN apk update && apk add --no-cache libc6-compat +FROM base AS pruner # work in /app folder WORKDIR /app @@ -26,24 +33,26 @@ WORKDIR /app # install turbo global RUN npm install turbo@^2.0.11 -g -# copy everything into builder +# copy everything into pruner COPY . . -# prune using turbo for only faims3 and it's dependencies +# prune using turbo for only faims3 and it's dependencies. This produces an +# /app/out folder which has /full (all needed deps) or /json (package*.json +# only) RUN turbo prune ${PACKAGE_NAME} --docker -# INSTALL PACKAGES [installer] + +# INSTALL PACKAGES [builder] # ============================ # add lockfiles dependencies -FROM base AS installer +FROM base AS builder -RUN apk update && apk add --no-cache libc6-compat WORKDIR /app # Install dependencies COPY .gitignore .gitignore -COPY --from=builder /app/out/json/ . +COPY --from=pruner /app/out/json/ . # install turbo global RUN npm install turbo@^2.0.11 -g @@ -51,20 +60,20 @@ RUN npm install # Build the project -COPY --from=builder /app/out/full/ . -COPY --from=builder /app/out/json/ json +COPY --from=pruner /app/out/full/ . +COPY --from=pruner /app/out/json/ json # build packages using turbo RUN turbo run build --filter=${PACKAGE_NAME} -# Package all necessary files to run app into builder out +# Package all necessary files to run app into pruner out COPY bundle.sh . RUN sh bundle.sh json . out && cp -R json/* /app/out/ # PRODUCTION RUNNER [runner] # ========================== -FROM base AS runner +FROM alpine AS runner WORKDIR /app # Don't run production as root @@ -73,7 +82,7 @@ RUN adduser --system --uid 1001 faims USER faims # Copy all files which are required to run the app -COPY --from=installer /app/out . +COPY --from=builder /app/out . # Run the node app ENV NODE_RUN_DIR=${NODE_RUN_DIR} diff --git a/README.md b/README.md index 21eb1c337..2b9020e60 100644 --- a/README.md +++ b/README.md @@ -14,10 +14,9 @@ The repository contains the following: - /library: shared library for the project - /tests: contains the end-to-end tests for the project - ## Local development quick start -Ensure you have uuid installed e.g. +Ensure you have uuid installed e.g. ```bash sudo apt-get install uuid @@ -40,7 +39,7 @@ source ~/.bashrc You can then setup Node v20 and activate it ```bash -nvm install 20 +nvm install 20 nvm use 20 ``` @@ -52,7 +51,6 @@ Now run the script to get a docker service running locally. ./localdev.sh ``` - ## Initial step-by step setup Clone the repository and install node modules (note this only needs to be run from the parent folder) @@ -217,3 +215,52 @@ Run tests inside the conductor instance: docker compose exec conductor npm run test ``` + +## Production docker builds + +This repo contains a `Dockerfile.build` which is designed to build and package a Node.js application in a multi-stage process, optimized for monorepo structures using Turborepo. + +### Stages: + +1. **Base**: Sets up the Alpine Node.js 20 environment with necessary dependencies. +2. **Pruner**: Uses Turborepo to prune the monorepo, isolating the specified package and its dependencies. +3. **Builder**: Installs dependencies, builds the project, and bundles necessary files. +4. **Runner**: Creates a minimal production image to run the built application. + +### Usage: + +To build a package (e.g., `@faims3/api`) with the build output in a specific directory (e.g., `/api/build/src`): + +```bash +docker build -f Dockerfile.build \ + --build-arg PACKAGE_NAME=@faims3/api \ + --build-arg NODE_RUN_DIR=api/build/src \ + -t your-image-name . +``` + +### bundle.sh + +`bundle.sh` is a script which utilises the output of the `turbo prune --docker` command. It leverages the existence of `package.json` files at directories to determine necessary build paths to copy when bundling a Node based Dockerimage. It currently copies the following folder + +- node_modules +- build +- dist +- views (for the API specifically) + +`realpath` could not be used because `alpine` images do not support `--relative-to` meaning a custom function is included which finds relative paths. + +I would not recommend using this script for local work, it is predominately to help the Docker build be able to resolve dependencies without any manual intervention while still building a pruned output image. + +```bash +# Helper script used for docker builds. + +# bundle.sh: A script to copy specific directories from a source to an output location +# based on the presence of package.json files in a JSON dump directory. + +# Usage: ./bundle.sh +# +# Arguments: +# json_dump_path: Path to the directory containing package.json files +# source_path: Path to the source directory containing files to be copied +# output_path: Path to the output directory where files will be copied +``` From de80b865d2ff8489324b1593c1adf5c79cf53ea5 Mon Sep 17 00:00:00 2001 From: Peter Baker Date: Thu, 15 Aug 2024 17:08:59 +1000 Subject: [PATCH 12/19] Moving to alpine layer Signed-off-by: Peter Baker --- Dockerfile.build | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/Dockerfile.build b/Dockerfile.build index 4b1215233..20e98c894 100644 --- a/Dockerfile.build +++ b/Dockerfile.build @@ -6,12 +6,6 @@ # Use Node 20 base FROM node:20-alpine AS alpine -# install some dependencies -RUN apk update && apk add --no-cache libc6-compat - -# create this layer to use to avoid repeatedly installing -FROM alpine as base - # What is the @faims3/... name of the package to build ARG PACKAGE_NAME ENV PACKAGE_NAME=${PACKAGE_NAME} @@ -21,6 +15,12 @@ ENV PACKAGE_NAME=${PACKAGE_NAME} ARG NODE_RUN_DIR ENV NODE_RUN_DIR=${NODE_RUN_DIR} +# install some dependencies +RUN apk update && apk add --no-cache libc6-compat + +# create this layer to use to avoid repeatedly installing +FROM alpine AS base + # PRUNE TO GET DEPENDENCIES [pruner] # =================================== @@ -85,7 +85,6 @@ USER faims COPY --from=builder /app/out . # Run the node app -ENV NODE_RUN_DIR=${NODE_RUN_DIR} WORKDIR /app/${NODE_RUN_DIR} CMD ["node", "index.js"] From 48e3bfd73294d7974e62707c7b501aaca6b205a0 Mon Sep 17 00:00:00 2001 From: Peter Baker Date: Fri, 16 Aug 2024 09:41:01 +1000 Subject: [PATCH 13/19] Testing without variables to ensure graceful failure Signed-off-by: Peter Baker --- .github/workflows/build-lint.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-lint.yml b/.github/workflows/build-lint.yml index 2dc72cc85..c6ed92c59 100644 --- a/.github/workflows/build-lint.yml +++ b/.github/workflows/build-lint.yml @@ -11,9 +11,9 @@ jobs: name: Build and Test timeout-minutes: 15 runs-on: ubuntu-latest - env: - TURBO_TEAM: ${{ vars.TURBO_TEAM }} - TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} + #env: + # TURBO_TEAM: ${{ vars.TURBO_TEAM }} + # TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} steps: - name: Check out code From c4fc8d3a36ff5d997765dc21303c7b39fe4187b1 Mon Sep 17 00:00:00 2001 From: Peter Baker Date: Fri, 16 Aug 2024 09:41:16 +1000 Subject: [PATCH 14/19] Reverting change Signed-off-by: Peter Baker --- .github/workflows/build-lint.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-lint.yml b/.github/workflows/build-lint.yml index c6ed92c59..2dc72cc85 100644 --- a/.github/workflows/build-lint.yml +++ b/.github/workflows/build-lint.yml @@ -11,9 +11,9 @@ jobs: name: Build and Test timeout-minutes: 15 runs-on: ubuntu-latest - #env: - # TURBO_TEAM: ${{ vars.TURBO_TEAM }} - # TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} + env: + TURBO_TEAM: ${{ vars.TURBO_TEAM }} + TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} steps: - name: Check out code From d27cac77cd55a435170bb7156fab35d745e46779 Mon Sep 17 00:00:00 2001 From: Peter Baker Date: Fri, 16 Aug 2024 09:42:36 +1000 Subject: [PATCH 15/19] Didn't trigger build - reverting again Signed-off-by: Peter Baker --- .github/workflows/build-lint.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-lint.yml b/.github/workflows/build-lint.yml index 2dc72cc85..c6ed92c59 100644 --- a/.github/workflows/build-lint.yml +++ b/.github/workflows/build-lint.yml @@ -11,9 +11,9 @@ jobs: name: Build and Test timeout-minutes: 15 runs-on: ubuntu-latest - env: - TURBO_TEAM: ${{ vars.TURBO_TEAM }} - TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} + #env: + # TURBO_TEAM: ${{ vars.TURBO_TEAM }} + # TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} steps: - name: Check out code From f7e9d8b4e452daafc3842891584fc63ca4525336 Mon Sep 17 00:00:00 2001 From: Peter Baker Date: Fri, 16 Aug 2024 09:44:27 +1000 Subject: [PATCH 16/19] Removing entirely to ensure valid syntax Signed-off-by: Peter Baker --- .github/workflows/build-lint.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/build-lint.yml b/.github/workflows/build-lint.yml index c6ed92c59..2705a791a 100644 --- a/.github/workflows/build-lint.yml +++ b/.github/workflows/build-lint.yml @@ -11,9 +11,6 @@ jobs: name: Build and Test timeout-minutes: 15 runs-on: ubuntu-latest - #env: - # TURBO_TEAM: ${{ vars.TURBO_TEAM }} - # TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} steps: - name: Check out code From 19b46a1d7db7dbd2ca299e8936f5f825c78ed6c5 Mon Sep 17 00:00:00 2001 From: Peter Baker Date: Fri, 16 Aug 2024 09:45:35 +1000 Subject: [PATCH 17/19] Restoring functionality Signed-off-by: Peter Baker --- .github/workflows/build-lint.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build-lint.yml b/.github/workflows/build-lint.yml index 2705a791a..2dc72cc85 100644 --- a/.github/workflows/build-lint.yml +++ b/.github/workflows/build-lint.yml @@ -11,6 +11,9 @@ jobs: name: Build and Test timeout-minutes: 15 runs-on: ubuntu-latest + env: + TURBO_TEAM: ${{ vars.TURBO_TEAM }} + TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} steps: - name: Check out code From 738c077aa94471552be60f3211a4cf7847e2c8d5 Mon Sep 17 00:00:00 2001 From: Peter Baker Date: Tue, 20 Aug 2024 13:19:42 +1000 Subject: [PATCH 18/19] Adding ^ to node version to indicate non breaking change v20 compatibility Signed-off-by: Peter Baker --- api/package.json | 2 +- app/package.json | 2 +- designer/package.json | 2 +- infrastructure/aws-cdk/package.json | 2 +- library/data-model/package.json | 2 +- package-lock.json | 12 ++++++------ package.json | 2 +- 7 files changed, 12 insertions(+), 12 deletions(-) diff --git a/api/package.json b/api/package.json index 0058562d7..02c54c7ab 100644 --- a/api/package.json +++ b/api/package.json @@ -5,7 +5,7 @@ "main": "build/src/index.js", "packageManager": "npm@10.7.0", "engines": { - "node": "20.9.0" + "node": "^20.9.0" }, "scripts": { "lint": "gts lint", diff --git a/app/package.json b/app/package.json index 5208205a6..9f88227e9 100644 --- a/app/package.json +++ b/app/package.json @@ -16,7 +16,7 @@ ] }, "engines": { - "node": "20.9.0" + "node": "^20.9.0" }, "scripts": { "start": "vite", diff --git a/designer/package.json b/designer/package.json index 466c43015..b60f753e7 100644 --- a/designer/package.json +++ b/designer/package.json @@ -5,7 +5,7 @@ "version": "0.0.1", "type": "module", "engines": { - "node": "20.9.0" + "node": "^20.9.0" }, "scripts": { "start": "vite", diff --git a/infrastructure/aws-cdk/package.json b/infrastructure/aws-cdk/package.json index de248d03b..e12c6b315 100644 --- a/infrastructure/aws-cdk/package.json +++ b/infrastructure/aws-cdk/package.json @@ -6,7 +6,7 @@ "@faims3/infra-aws-cdk": "bin/@faims3/infra-aws-cdk.js" }, "engines": { - "node": "20.9.0" + "node": "^20.9.0" }, "scripts": { "build": "tsc", diff --git a/library/data-model/package.json b/library/data-model/package.json index 759d70c62..85d617ac5 100644 --- a/library/data-model/package.json +++ b/library/data-model/package.json @@ -5,7 +5,7 @@ "description": "Database access layer for FAIMS3", "main": "./build/src/index.js", "engines": { - "node": "20.9.0" + "node": "^20.9.0" }, "exports": { ".": "./build/src/index.js" diff --git a/package-lock.json b/package-lock.json index 9a51d248a..fd2593c01 100644 --- a/package-lock.json +++ b/package-lock.json @@ -32,7 +32,7 @@ "typescript": "^5.5.4" }, "engines": { - "node": "20.9.0" + "node": "^20.9.0" } }, "api": { @@ -116,7 +116,7 @@ "typescript": "^5.5.4" }, "engines": { - "node": "20.9.0" + "node": "^20.9.0" } }, "api/node_modules/@types/node": { @@ -250,7 +250,7 @@ "type-fest": "^4.20.0" }, "engines": { - "node": "20.9.0" + "node": "^20.9.0" } }, "app/node_modules/@types/markdown-it": { @@ -325,7 +325,7 @@ "webdriverio": "^8.32.2" }, "engines": { - "node": "20.9.0" + "node": "^20.9.0" } }, "designer/node_modules/@emotion/styled": { @@ -717,7 +717,7 @@ "typescript": "^5.5.4" }, "engines": { - "node": "20.9.0" + "node": "^20.9.0" } }, "infrastructure/aws-cdk/node_modules/@types/jest": { @@ -844,7 +844,7 @@ "typescript": "^5.5.4" }, "engines": { - "node": "20.9.0" + "node": "^20.9.0" }, "peerDependencies": { "pouchdb": "^7.3.0" diff --git a/package.json b/package.json index aa6678676..6d8b61e63 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "version": "1.0.0", "main": "index.js", "engines": { - "node": "20.9.0" + "node": "^20.9.0" }, "workspaces": [ "api", From 88fd8a05db99c903c6cbaab899c60c27aa2aaca4 Mon Sep 17 00:00:00 2001 From: Peter Baker Date: Tue, 20 Aug 2024 15:08:50 +1000 Subject: [PATCH 19/19] REmoving unwanted trigger for testing Signed-off-by: Peter Baker --- .github/workflows/publish-conductor.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/publish-conductor.yml b/.github/workflows/publish-conductor.yml index 7fa28f3fe..49f1b5b98 100644 --- a/.github/workflows/publish-conductor.yml +++ b/.github/workflows/publish-conductor.yml @@ -3,7 +3,6 @@ on: push: branches: - main - - chore/add-build-outputs-to-turbo release: types: [published]