Dataflow Initialization triggered by @mt7180 #21
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Create Infrastructure and Deploy Prefect to AWS ECS Push Work Pool | |
| run-name: Dataflow Initialization triggered by @${{ github.actor }} | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| aws_region: | |
| description: Desired region for AWS resources | |
| required: true | |
| default: 'eu-central-1' | |
| type: string | |
| aws_credential_block_id: | |
| description: The ID of your AWS Credentials Block, use "prefect blocks ls" command in your terminal to find it. | |
| required: false | |
| default: '' | |
| type: string | |
| jobs: | |
| create-aws-infrastructure: | |
| name: Create AWS Infrastructure | |
| runs-on: ubuntu-latest | |
| outputs: | |
| aws_ecs_cluster_arn: ${{ steps.pulumi.outputs.ecs_cluster_arn }} | |
| aws_vpc_id: ${{ steps.pulumi.outputs.vpc_id }} | |
| aws_task_role_arn: ${{ steps.pulumi.outputs.iam_task_role_arn }} | |
| aws_execution_role_arn: ${{ steps.pulumi.outputs.iam_execution_role_arn }} | |
| aws_ecr_url : ${{ steps.pulumi.outputs.ecr_repo_url}} | |
| aws_ecr_name: ${{ steps.pulumi.outputs.ecr_repo_name}} | |
| defaults: | |
| run: | |
| working-directory: ./infrastructure | |
| steps: | |
| - name: Code Checkout | |
| uses: actions/checkout@v3 | |
| - name: Send Information to Summary | |
| run: echo "### Workflow $GITHUB_WORKFLOW with GITHUB_SHA $GITHUB_SHA" >> $GITHUB_STEP_SUMMARY | |
| - name: Set up Python | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: '3.10' | |
| - name: Install Pulumi Dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirements.txt | |
| - name: Set AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: ${{ github.event.inputs.aws_region}} | |
| - name: Create Infrastructure with Pulumi | |
| id: pulumi | |
| uses: pulumi/actions@v3 | |
| with: | |
| command: up | |
| stack-name: ${{ github.actor }}/ecs-flow-infrastructure/dev | |
| work-dir: ./infrastructure | |
| env: | |
| PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_ACCESS_TOKEN }} | |
| AWS_IAM_ROLE_TO_ASSUME: ${{ secrets.AWS_IAM_ROLE_TO_ASSUME }} | |
| AWS_IAM_USER_NAME: ${{ secrets.AWS_IAM_USER_NAME }} | |
| - name: Create AWS Infrastructure Summary | |
| run: | | |
| echo "#### use the following ECR repo name as input to the delete_infra github action: ${{ steps.pulumi.outputs.ecr_repo_name }}" >> $GITHUB_STEP_SUMMARY | |
| echo "### AWS Infrastructure build at $(date +'%Y-%m-%dT%H:%M:%S')" >> $GITHUB_STEP_SUMMARY | |
| deploy-prefect-dataflow: | |
| name: Deploy Flow to Prefect Push Workpool and push image to AWS ECR Repo | |
| runs-on: ubuntu-latest | |
| needs: create-aws-infrastructure | |
| steps: | |
| - name: Code Checkout | |
| uses: actions/checkout@v3 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.10' | |
| - name: Install Dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r etl/requirements.txt | |
| - name: Set AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: ${{ github.event.inputs.aws_region }} | |
| - name: Login to Amazon ECR | |
| id: login-ecr | |
| uses: aws-actions/amazon-ecr-login@v1 | |
| - name: Prefect Auth | |
| uses: PrefectHQ/actions-prefect-auth@v1 | |
| with: | |
| prefect-api-key: ${{ secrets.PREFECT_API_KEY }} | |
| prefect-workspace: ${{ secrets.PREFECT_WORKSPACE }} | |
| - name: Create Prefect AWS ECS Push Workpool | |
| run: | | |
| prefect work-pool create --type ecs:push air-to-air_push | |
| continue-on-error: true # if work pool already exists | |
| - name: Create Prefect Webhook | |
| run: | | |
| prefect cloud webhook create entsoe-msg-webhook \ | |
| --template '{ "event": "entsoe-event", "resource": { "prefect.resource.id": "entsoe-msg-webhook-id" } }' | |
| - name: Deploy Flow | |
| run: | | |
| python -m etl.dataflow | |
| env: | |
| ECR_REPO_URL: ${{ needs.create-aws-infrastructure.outputs.aws_ecr_url}} | |
| ECR_REPO_NAME: ${{ needs.create-aws-infrastructure.outputs.aws_ecr_name}} | |
| IMAGE_TAG: ${{ github.sha }} | |
| EXECUTION_ROLE: ${{ needs.create-aws-infrastructure.outputs.aws_execution_role_arn }} | |
| TASK_ROLE: ${{ needs.create-aws-infrastructure.outputs.aws_task_role_arn }} | |
| ECS_CLUSTER: ${{ needs.create-aws-infrastructure.outputs.aws_ecs_cluster_arn }} | |
| VPC_ID: ${{ needs.create-aws-infrastructure.outputs.aws_vpc_id }} | |
| AWS_CREDENTIAL_BLOCK_ID: ${{ github.event.inputs.aws_credential_block_id }} | |
| - name: Let's test our automated dataflow by sending a test event | |
| run: python -m etl.emit_event | |
| - name: Summary | |
| run: echo '### AWS infrastructure and flow successfully deployed! :rocket:' >> $GITHUB_STEP_SUMMARY |