1646 lambda (#401)

* fix(infra): change destination for public scores & config of cloudfront * fix(api,infra): fix bug in and removed acl from task exporting scores * feat(infra,api): add dockerfile for AWS lambda * feat(load-test): adjustments for running submit-passport as lambda * feat(infra): forgot to add lambda function * feat(infra,api): update readme in the load_test folder * feat(api): adding lambda to ci and building lambda * chore(infra): wip lambda build * fix(infra): succesfully building image * feat(lambdas): wip submit passport lambda auth and rate limiting with tests * feat(api): moves submit passport lambda adds logger * chore(infra): add image from ci * fix(ci): lambda ecr * fix(ci): aws_lambda * fix(infra): fix docker file build * fix(infra): adjust docker file to point at module, add requirements.txt * fix(api): fix build for lambda image * chore(api): test lambda rate limiting and auth * chore(api): rename submit-passport module and cleanup * chore(api): add lambda testing readme * fix(api): fix module resolution for pytest * fix(ci): update ECR name * feat(infra): move lambda definition to service and add to prod * chore(ci): add comments and correct ECR name alias * chore(ci): add lambda to prod ci * fix(infra): build image uri using pulumi * fix(infra): use private ECR image for lambda * fix(infra): login to private ECR * fix(ci): update ECR to private * fix(ci): region * fix(ci): use env for image url * fix(ci): name image using env variable * fix(ci): update region * fix(ci): hard code us-west-2 * fix(api): update how x-api-key is parsed and validated * fix(api): fix parsing of body request * chore(api, ci): fix test and prod ci * fix(api): remove rate limiting test due to conflict * chore(infra): add v2 endpoint * feat(api): fix tests, remove unused request handler --------- Co-authored-by: schultztimothy <[email protected]> Co-authored-by: Aminah Burch <[email protected]>
passportxyz · Oct 10, 2023 · b0ed430 · b0ed430
1 parent 72859f0
commit b0ed430
Show file tree

Hide file tree

Showing 18 changed files with 1,774 additions and 20 deletions.
diff --git a/.github/workflows/api-promote-prod.yml b/.github/workflows/api-promote-prod.yml
@@ -196,9 +196,56 @@ jobs:
     outputs:
       dockerTag: ${{ steps.vars.outputs.sha_short }}
 
+  build-submit-passport-lambda:
+    environment: Staging
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ inputs.commit }}
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+
+      - name: Declare some variables
+        id: vars
+        shell: bash
+        run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_STAGING }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_STAGING }}
+          aws-region: us-west-2
+
+      - name: Login to Amazon ECR Private
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v1
+
+      - name: Build, tag, and push image to Amazon ECR
+        env:
+          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+          ECR_REPOSITORY: submit-passport-lambdas
+          IMAGE_TAG: ${{ steps.vars.outputs.sha_short }}
+        run: |
+          docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG -f api/aws_lambdas/submit_passport/Dockerfile ./api/
+          docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
+
+    outputs:
+      dockerTag: ${{ steps.vars.outputs.sha_short }}
+
   deploy-production:
     environment: Production
-    needs: [test, build-api, build-verifier, build-indexer]
+    needs:
+      [
+        test,
+        build-api,
+        build-verifier,
+        build-indexer,
+        build-submit-passport-lambda,
+      ]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
@@ -246,7 +293,8 @@ jobs:
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           DOCKER_GTC_PASSPORT_SCORER_IMAGE: public.ecr.aws/c8n6v8e9/passport-scorer:${{ needs.build-api.outputs.dockerTag }}
           DOCKER_GTC_PASSPORT_VERIFIER_IMAGE: public.ecr.aws/c8n6v8e9/passport-verifier:${{ needs.build-verifier.outputs.dockerTag }}
-          DOCKER_GTC_PASSPORT_INDEXER_IMAGE: public.ecr.aws/t1g3k9q8/passport-indexer:${{ needs.build-indexer.outputs.dockerTag }}
+          DOCKER_GTC_PASSPORT_INDEXER_IMAGE: public.ecr.aws/c8n6v8e9/passport-indexer:${{ needs.build-indexer.outputs.dockerTag }}
+          DOCKER_GTC_SUBMIT_PASSPORT_LAMBDA_IMAGE: ${{secrets.PRIVATE_ECR_URL}}:${{ needs.build-submit-passport-lambda.outputs.dockerTag }}
           ROUTE_53_ZONE: ${{ secrets.ROUTE53_ZONE_ID }}
           ROUTE_53_ZONE_FOR_PUBLIC_DATA: ${{ secrets.ROUTE_53_ZONE_FOR_PUBLIC_DATA }}
           DOMAIN: ${{ secrets.DOMAIN }}

diff --git a/.github/workflows/api-promote-staging.yml b/.github/workflows/api-promote-staging.yml
@@ -121,6 +121,46 @@ jobs:
     outputs:
       dockerTag: ${{ steps.vars.outputs.sha_short }}
 
+  build-submit-passport-lambda:
+    environment: Staging
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ inputs.commit }}
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+
+      - name: Declare some variables
+        id: vars
+        shell: bash
+        run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_STAGING }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_STAGING }}
+          aws-region: us-west-2
+
+      - name: Login to Amazon ECR Private
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v1
+
+      - name: Build, tag, and push image to Amazon ECR
+        env:
+          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+          ECR_REPOSITORY: submit-passport-lambdas
+          IMAGE_TAG: ${{ steps.vars.outputs.sha_short }}
+        run: |
+          docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG -f api/aws_lambdas/submit_passport/Dockerfile ./api/
+          docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
+
+    outputs:
+      dockerTag: ${{ steps.vars.outputs.sha_short }}
+
   build-indexer:
     environment: Staging
     runs-on: ubuntu-latest
@@ -204,7 +244,14 @@ jobs:
 
   deploy-staging:
     environment: Staging
-    needs: [test, build-api, build-verifier, build-indexer]
+    needs:
+      [
+        test,
+        build-api,
+        build-verifier,
+        build-indexer,
+        build-submit-passport-lambda,
+      ]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
@@ -256,6 +303,7 @@ jobs:
           DOCKER_GTC_PASSPORT_SCORER_IMAGE: public.ecr.aws/t1g3k9q8/passport-scorer:${{ needs.build-api.outputs.dockerTag }}
           DOCKER_GTC_PASSPORT_VERIFIER_IMAGE: public.ecr.aws/t1g3k9q8/passport-verifier:${{ needs.build-verifier.outputs.dockerTag }}
           DOCKER_GTC_PASSPORT_INDEXER_IMAGE: public.ecr.aws/t1g3k9q8/passport-indexer:${{ needs.build-indexer.outputs.dockerTag }}
+          DOCKER_GTC_SUBMIT_PASSPORT_LAMBDA_IMAGE: ${{secrets.PRIVATE_ECR_URL}}:${{ needs.build-submit-passport-lambda.outputs.dockerTag }}
           ROUTE_53_ZONE: ${{ secrets.ROUTE53_ZONE_ID }}
           ROUTE_53_ZONE_FOR_PUBLIC_DATA: ${{ secrets.ROUTE_53_ZONE_FOR_PUBLIC_DATA }}
           DOMAIN: ${{ secrets.DOMAIN_STAGING }}

diff --git a/api/Dockerfile.python3.11 b/api/Dockerfile.python3.11
@@ -0,0 +1,53 @@
+
+# https://pipenv.pypa.io/en/latest/basics/#pipenv-and-docker-containers
+FROM public.ecr.aws/lambda/python:3.11 AS base
+
+
+#########################################################
+# Builder
+#########################################################
+FROM base AS builder
+
+RUN pip install pipenv
+
+# Tell pipenv to create venv in the current directory
+ENV PIPENV_VENV_IN_PROJECT=1
+
+ADD Pipfile.lock /usr/src/
+
+WORKDIR /usr/src
+
+
+# NOTE: If you install binary packages required for a python module, you need
+# to install them again in the runtime. For example, if you need to install pycurl
+# you need to have pycurl build dependencies libcurl4-gnutls-dev and libcurl3-gnutls
+# In the runtime container you need only libcurl3-gnutls
+
+# RUN apt install -y libcurl3-gnutls libcurl4-gnutls-dev
+# RUN /root/.local/bin/pipenv lock
+
+RUN pipenv requirements > requirements.txt
+# RUN /usr/src/.venv/bin/python -c "import django; print(django.__version__)"
+# RUN ls /usr/src/.venv/lib/
+
+#########################################################
+# Runtime
+#########################################################
+FROM base AS runtime
+
+# RUN mkdir -v /usr/src/venv
+
+COPY --from=builder /usr/src/requirements.txt ${LAMBDA_TASK_ROOT}
+COPY . ${LAMBDA_TASK_ROOT}
+
+
+WORKDIR ${LAMBDA_TASK_ROOT}
+
+RUN pip install -r requirements.txt
+RUN python -c "import django; print(f\"\n{'*' * 40}\n* DJANGO VERSION: {django.__version__ : <21}*\n{'*' * 40}\n\")"
+# ENV PATH="/usr/src/.venv/bin/:${PATH}"
+
+
+RUN STATIC_ROOT=${LAMBDA_TASK_ROOT}/static SECRET_KEY=secret_is_irelevent_here CERAMIC_CACHE_SCORER_ID=1 DATABASE_URL=sqlite:////dunmmy_db.sqlite3 python manage.py collectstatic --noinput
+
+CMD [ "lambda_function.handler" ]
diff --git a/api/aws_lambdas/Readme.md b/api/aws_lambdas/Readme.md
@@ -0,0 +1,47 @@
+# Testing the Application Locally with Docker
+
+## 1. Building the Docker Image
+
+First, navigate to the passport-scorer root directory. Then, execute the command below to build the Docker image:
+
+```bash
+docker build --platform linux/amd64 -t docker-image:test -f ./api/aws_lambdas/submit_passport/Dockerfile ./api
+```
+
+## 2. Running the Docker Image
+
+Once you've successfully built the image, you can run it. Ensure you point to the location of your running PostgreSQL instance. In this context, the database is running in another Docker container:
+
+```bash
+docker run -e DATABASE_URL=postgres://passport_scorer:[email protected]:5432/passport_scorer -p 8080:8080 docker-image:test
+```
+
+### 3. Making Curl Requests
+
+After you have your application up and running, you can test its endpoints using `curl`. Use the command below to make a request to the instance:
+
+````markdown
+```bash
+curl -X 'POST' \
+  'http://localhost:8080/2015-03-31/functions/function/invocations' \
+  -H 'accept: application/json' \
+  -H 'Content-Type: application/json' \
+  -d '{
+  "requestContext": {
+    "elb": {
+      "targetGroupArn": "arn"
+    }
+  },
+  "httpMethod": "POST",
+  "path": "/registry/submit-passport",
+  "queryStringParameters": {},
+  "headers": {add necessary headers here},
+  "body": "{\"address\":\"0x868asAe3B27asdF475e41FAdDF9F0cf97fDB71fC\",\"community\":\"24\"}",
+  "isBase64Encoded": false
+}'
+```
+````
+
+```
+
+```
diff --git a/api/aws_lambdas/__init__.py b/api/aws_lambdas/__init__.py
diff --git a/api/aws_lambdas/docker-compose-lambda.yml b/api/aws_lambdas/docker-compose-lambda.yml
@@ -0,0 +1,74 @@
+version: "3.9" # optional since v1.27.0
+services:
+  # redis:
+  #   image: redis
+  #   restart: unless-stopped
+  #   ports:
+  #     - 6379:6379
+
+  api:
+    build:
+      context: ./api/aws_lambdas/submit-passport
+      dockerfile: Dockerfile
+
+    ports:
+      - "127.0.0.1:8000:8000"
+      - 8080:8080
+
+    volumes:
+      - ./api:/var/task
+
+    environment:
+      - CELERY_BROKER_URL=redis://redis:6379/0
+      - CERAMIC_CACHE_CACAO_VALIDATION_URL=http://verifier:8001/verify
+      - CERAMIC_CACHE_SCORER_ID=1
+      - DATABASE_URL=postgres://passport_scorer:passport_scorer_pwd@postgres:5432/passport_scorer
+
+    # command: uvicorn scorer.asgi:application --reload --host 0.0.0.0 --port 8002
+    # command: uvicorn scorer.asgi:application --host 0.0.0.0 --port 8002 --backlog 5000 --workers 6 --log-level debug
+    # command: gunicorn -w 8 -t 60 -k uvicorn.workers.UvicornWorker --worker-connections 5000 scorer.asgi:application -b 0.0.0.0:8002
+    # command: gunicorn -w 4 -t 60 -k uvicorn.workers.UvicornWorker scorer.asgi:application -b 0.0.0.0:8002
+
+  # worker:
+  #   build: api
+  #   volumes:
+  #     - ./api:/app
+  #   environment:
+  #     - CELERY_BROKER_URL=redis://redis:6379/0
+  #     - CERAMIC_CACHE_SCORER_ID=1
+  #   command: celery -A scorer worker -Q score_passport_passport,score_registry_passport -l DEBUG
+
+  # interface:
+  #   build:
+  #     context: ./interface
+  #     dockerfile: Dockerfile
+
+  #   ports:
+  #     - 3001:3001
+
+  #   volumes:
+  #     - ./interface:/app
+  #     - /app/node_modules
+  #     - /app/.next
+
+  #   command: yarn dev
+
+  verifier:
+    build: verifier
+
+  postgres:
+    image: postgres:12.3-alpine
+    restart: unless-stopped
+
+    environment:
+      POSTGRES_USER: passport_scorer
+      POSTGRES_PASSWORD: passport_scorer_pwd
+      POSTGRES_DB: passport_scorer
+      PGDATA: /db_data
+
+    command: postgres -c 'max_connections=2000'
+    volumes:
+      - ./postgres_db_data:/db_data
+
+    ports:
+      - 5432:5432
diff --git a/api/aws_lambdas/submit_passport/Dockerfile b/api/aws_lambdas/submit_passport/Dockerfile
@@ -0,0 +1,40 @@
+
+# https://pipenv.pypa.io/en/latest/basics/#pipenv-and-docker-containers
+FROM public.ecr.aws/lambda/python:3.11 AS base
+
+
+#########################################################
+# Builder
+#########################################################
+FROM base AS builder
+
+RUN pip install pipenv
+
+# Tell pipenv to create venv in the current directory
+ENV PIPENV_VENV_IN_PROJECT=1
+
+ADD Pipfile.lock /usr/src/
+
+WORKDIR /usr/src
+
+RUN pipenv requirements > requirements.txt
+
+#########################################################
+# Runtime
+#########################################################
+FROM base AS runtime
+
+COPY --from=builder /usr/src/requirements.txt ${LAMBDA_TASK_ROOT}
+COPY ./ ${LAMBDA_TASK_ROOT}
+
+
+WORKDIR ${LAMBDA_TASK_ROOT}
+
+RUN pip install -r requirements.txt
+
+RUN python -c "import django; print(f\"\n{'*' * 40}\n* DJANGO VERSION: {django.__version__ : <21}*\n{'*' * 40}\n\")"
+
+
+RUN STATIC_ROOT=${LAMBDA_TASK_ROOT}/static SECRET_KEY=secret_is_irelevent_here CERAMIC_CACHE_SCORER_ID=1 DATABASE_URL=sqlite:////dunmmy_db.sqlite3 python manage.py collectstatic --noinput
+
+CMD [ "aws_lambdas.submit_passport.submit_passport.handler" ]
diff --git a/api/aws_lambdas/submit_passport/__init__.py b/api/aws_lambdas/submit_passport/__init__.py