Merge pull request #9 from wri/release/rare-redwood

[RELEASE] Rare Redwood

.env.local.sample

@@ -1,5 +1,7 @@
-PHP_PROXY_TARGET=http://host.docker.internal:8080
-USER_SERVICE_PROXY_TARGET=http://host.docker.internal:4010
+DOCKER_HOST=unix://$HOME/.docker/run/docker.sock
+
+USER_SERVICE_PORT=4010
+JOB_SERVICE_PORT=4020
 
 DB_HOST=localhost
 DB_PORT=3360

.github/workflows/deploy-api-gateway.yml

@@ -1,4 +1,5 @@
 name: Api Gateway Deploy
+run-name: 'Api Gateway Deploy [env: ${{ inputs.env }}]'
 
 on:
   workflow_dispatch:
@@ -25,7 +26,7 @@ env:
   PHP_PROXY_TARGET: ${{ vars.PHP_PROXY_TARGET }}
 
 jobs:
-  main:
+  deploy-api-gateway:
     runs-on: ubuntu-latest
     environment: ${{ inputs.env }}
     steps:
@@ -42,6 +43,6 @@ jobs:
       - name: CDK Deploy
         id: cdk-deploy
         run: |
-          cd apps/api-gateway
+          cd cdk/api-gateway
           NODE_ENV=development npm i
           TM_ENV=${{ inputs.env }} npx --yes cdk deploy --require-approval never

.github/workflows/deploy-user-service.yml → .github/workflows/deploy-service.yml

@@ -1,8 +1,16 @@
-name: User Service Deploy
+name: Service Deploy
+run-name: 'Service Deploy [service: ${{ inputs.service }}, env: ${{ inputs.env }}]'
 
 on:
   workflow_dispatch:
     inputs:
+      service:
+        description: 'Service to deploy'
+        type: choice
+        required: true
+        options:
+          - user-service
+          - job-service
       env:
         description: 'Deployment target environment'
         type: choice
@@ -20,13 +28,13 @@ permissions:
 env:
   AWS_REGION: eu-west-1
   AWS_ROLE_TO_ASSUME: arn:aws:iam::603634817705:role/terramatch-microservices-github-actions
-  AWS_ROLE_SESSION_NAME: terramatch-microservices-cicd-user-service
-  ECR_REPOSITORY: terramatch-microservices/user-service-${{ inputs.env }}
+  AWS_ROLE_SESSION_NAME: terramatch-microservices-cicd-${{ inputs.service }}
+  ECR_REPOSITORY: terramatch-microservices/${{ inputs.service }}-${{ inputs.env }}
   ECR_REGISTRY: 603634817705.dkr.ecr.eu-west-1.amazonaws.com
   IMAGE_TAG: ${{ github.sha }}
 
 jobs:
-  main:
+  deploy-service:
     runs-on: ubuntu-latest
     environment: ${{ inputs.env }}
     steps:
@@ -51,15 +59,15 @@ jobs:
           echo "DB_PASSWORD=\"${{ secrets.DB_PASSWORD }}\"" >> .env
           : # Don't build the base image with NODE_ENV because it'll limit the packages that are installed
           docker build -t terramatch-microservices-base:nx-base .
-          USER_SERVICE_IMAGE=$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
-          docker build --build-arg NODE_ENV=production --build-arg BUILD_FLAG=--prod -f apps/user-service/Dockerfile -t $USER_SERVICE_IMAGE .
-          docker push $USER_SERVICE_IMAGE
-          echo "image=$USER_SERVICE_IMAGE"
+          SERVICE_IMAGE=$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
+          docker build --build-arg NODE_ENV=production --build-arg BUILD_FLAG=--prod -f apps/${{ inputs.service }}/Dockerfile -t $SERVICE_IMAGE .
+          docker push $SERVICE_IMAGE
+          echo "image=$SERVICE_IMAGE"
 
       - name: Launch new task definition
         id: launch
         run: |
-          cd apps/user-service/stack
+          cd cdk/service-stack
           npm i
-          IMAGE_TAG=$IMAGE_TAG TM_ENV=${{ inputs.env }} npx --yes cdk deploy --require-approval never
+          IMAGE_TAG=$IMAGE_TAG TM_SERVICE=${{ inputs.service }} TM_ENV=${{ inputs.env }} npx --yes cdk deploy --require-approval never
 

.github/workflows/pull-request.yml

@@ -11,7 +11,7 @@ permissions:
   contents: read
 
 jobs:
-  main:
+  build-lint-test:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -29,15 +29,11 @@ jobs:
           node-version: 20
           cache: 'npm'
 
-      - run: npm ci --legacy-peer-deps && (cd apps/api-gateway; npm ci) && (cd apps/api-gateway/lambda/local-proxy; npm ci)
+      - run: npm ci --legacy-peer-deps
 
       - uses: nrwl/nx-set-shas@v4
 
-      # Distributed execution only works with cacheable builds. The Api Gateway build is not currently
-      # cacheable. Since the codebase is currently small, we can get away without distribution, but once
-      # it grows, we'll want to look into what it will take to make the api gateway build cacheable and remove
-      # NX_CLOUD_DISTRIBUTED_EXECUTION=false from this command.
-      - run: NX_CLOUD_DISTRIBUTED_EXECUTION=false ARCH=X86 npx nx affected -t lint build
+      - run: NX_CLOUD_DISTRIBUTED_EXECUTION=false npx nx affected -t lint build
 
       - uses: KengoTODA/actions-setup-docker-compose@v1
         with:

README.md

@@ -11,27 +11,41 @@ Repository for the Microservices API backend of the TerraMatch service
 
 # Building and starting the apps
  * Copy `.env.local.sample` to `.env`
- * The ApiGateway does not hot-reload and needs to be re-built when there are changes:
-   * `nx build api-gateway` or `nx run-many -t build` (to build all apps)
-   * This will build the local proxy Lambda function and the CDK Stack
-   * Note: The architecture for the local lambda proxy defaults to ARM_64. This will be the fastest options on ARM-based Macs 
-     (M1, etc), but will be much slower on X86 (AMD/Intel) based machine. If you're on an X86 machine, pass the architecture in
-     an environment variable when building the api gateway: `ARCH=X86 nx build api-gateway`.
+   * On Linux systems, the DOCKER_HOST value should be `unix:///var/run/docker.sock` instead of what's in the sample.
  * To run all services:
    * `nx run-many -t serve`
-   * Note: the first time this runs, the gateway will take quite awhile to start. It'll be faster on subsequent starts.
-   * For now, this starts up the ApiGateway and the User service
- * In `.env` in your `wri-terramatch-website` repository, set your BE connection URL correctly:
-   * `NEXT_PUBLIC_API_BASE_URL='http://localhost:4000'`
+ * In `.env` in your `wri-terramatch-website` repository, set your BE connection URL correctly by noting the config
+   in `.env.local.sample` for local development.
+   * The `NEXT_PUBLIC_API_BASE_URL` still points at the PHP BE directly
+   * New `NEXT_PUBLIC_<SERVICE>_URL` values are needed for each service you're running locally. This will typically match
+     the services defined in `V3_NAMESPACES` in `src/generated/v3/utils.ts`.
 
 # Deployment
-Deployment is handled via manual trigger of GitHub actions. There is one for each service, and one for the ApiGateway. The 
+Deployment is handled via manual trigger of GitHub actions. There is one for services, and one for the ApiGateway. The 
 ApiGateway only needs to be redeployed if its code changes; it does not need to be redeployed for updates to individual services
 to take effect.
 
 Once this project is live in production, we can explore continuous deployment to at least staging and prod envs on the staging
 and main branches.
 
+# Creating a new service
+ * In the root directory: `nx g @nx/nest:app apps/foo-service`
+ * Set up the new `main.ts` similarly to existing services.
+   * Make sure swagger docs and the `/health` endpoint are implemented
+   * Pick a default local port that is unique from other services
+ * In your `.env` and `.env.local.sample`, add `_PORT` for the new service
+ * In `api-gateway-stack.ts`, add the new service and namespace to `V3_SERVICES`
+ * In your local web repo, follow directions in `README.md` for setting up a new service.
+ * For deployment to AWS:
+   * Add a Dockerfile in the new app directory. A simple copy and modify from user-service is sufficient
+   * In AWS:
+     * Add ECR repositories for each env (follow the naming scheme from user-service, e.g. `terramatch-microservices/foo-service-staging`, etc)
+       * Set the repo to Immutable
+       * After creation, set a Lifecycle Policy. In lower envs, we retain the most recent 2 images, and in prod it's set to 5
+     * In CloudWatch, create a log group for each env (follow the naming scheme from user-service, e.g. `ecs/foo-service-staging`, etc).
+       * TODO: the log groups could be created as part of the stack. The ECR repository is needed before the stack runs, so that will
+         need to remain a manual process.
+
 # Database work
 For now, Laravel is the source of truth for all things related to the DB schema. As such, TypeORM is not allowed to modify the 
 schema, and is expected to interface with exactly the schema that is managed by Laravel. This note is included in user.entity.ts,