Dataflow Initialization triggered by @mt7180 #6
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Create Infrastructure and Deploy Prefect to AWS ECS Push Work Pool | |
| run-name: Dataflow Initialization triggered by @${{ github.actor }} | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| aws_region: | |
| description: Desired region for AWS resources | |
| required: true | |
| default: 'eu-central-1' | |
| type: string | |
| aws_credential_block_id: | |
| description: The ID of your AWS Credentials Block, use "prefect blocks ls" command in your terminal to find it. | |
| required: false | |
| default: '' | |
| type: string | |
| jobs: | |
| create-aws-infrastructure: | |
| name: Create AWS Infrastructure | |
| runs-on: ubuntu-latest | |
| outputs: | |
| aws_ecs_cluster_arn: ${{ steps.pulumi.outputs.ecs_cluster_arn }} | |
| aws_vpc_id: ${{ steps.pulumi.outputs.vpc_id }} | |
| aws_task_role_arn: ${{ steps.pulumi.outputs.iam_task_role_arn }} | |
| aws_execution_role_arn: ${{ steps.pulumi.outputs.iam_execution_role_arn }} | |
| aws_ecr_url : ${{ steps.pulumi.outputs.ecr_repo_url}} | |
| aws_ecr_name: ${{ steps.pulumi.outputs.ecr_repo_name}} | |
| defaults: | |
| run: | |
| working-directory: ./infrastructure | |
| steps: | |
| - name: Code Checkout | |
| uses: actions/checkout@v3 | |
| - name: Send Information to Summary | |
| run: echo "### Workflow $GITHUB_WORKFLOW with GITHUB_SHA $GITHUB_SHA" >> $GITHUB_STEP_SUMMARY | |
| - name: Set up Python | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: '3.10' | |
| - name: Install Pulumi Dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirements.txt | |
| - name: Set AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: ${{ github.event.inputs.aws_region}} | |
| - name: Create Infrastructure with Pulumi | |
| id: pulumi | |
| uses: pulumi/actions@v3 | |
| with: | |
| command: up | |
| stack-name: ${{ github.actor }}/ecs-flow-infrastructure/dev | |
| work-dir: ./infrastructure | |
| env: | |
| PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_ACCESS_TOKEN }} | |
| - run: | | |
| echo "Current AWS ECR Repository: ${{ steps.pulumi.outputs.ecr_repo_url }}" >> $GITHUB_STEP_SUMMARY | |
| echo "Current AWS ECS Cluster arn: ${{ steps.pulumi.outputs.ecs_cluster_arn }}" >> $GITHUB_STEP_SUMMARY | |
| echo "Current AWS VPC id: ${{ steps.pulumi.outputs.vpc_id }}" >> $GITHUB_STEP_SUMMARY | |
| echo "Current AWS iam task role arn: ${{ steps.pulumi.outputs.iam_task_role_arn }}" >> $GITHUB_STEP_SUMMARY | |
| echo "Current AWS iam execution role arn ${{ steps.pulumi.outputs.iam_execution_role_arn }}" >> $GITHUB_STEP_SUMMARY | |
| - name: Create AWS Infrastructure Summary | |
| run: echo "AWS Infrastructure build at $(date +'%Y-%m-%dT%H:%M:%S')" >> $GITHUB_STEP_SUMMARY | |
| deploy-prefect-dataflow: | |
| name: Deploy Flow to Prefect Push Workpool and push image to AWS ECR Repo | |
| runs-on: ubuntu-latest | |
| needs: create-aws-infrastructure | |
| steps: | |
| - name: Code Checkout | |
| uses: actions/checkout@v3 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.10' | |
| - name: Install Dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r etl/requirements.txt | |
| - name: Set AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: ${{ github.event.inputs.aws_region }} | |
| - name: Login to Amazon ECR | |
| id: login-ecr | |
| uses: aws-actions/amazon-ecr-login@v1 | |
| - name: Prefect Auth | |
| uses: PrefectHQ/actions-prefect-auth@v1 | |
| with: | |
| prefect-api-key: ${{ secrets.PREFECT_API_KEY }} | |
| prefect-workspace: ${{ secrets.PREFECT_WORKSPACE }} | |
| - name: Create Prefect AWS ECS Push Workpool | |
| run: | | |
| prefect work-pool create --type ecs:push air-to-air_push | |
| continue-on-error: true # if work pool already exists | |
| - name: Create Prefect Webhook | |
| run: | | |
| prefect cloud webhook create entsoe-msg-webhook \ | |
| --template '{ "event": "entsoe-event", "resource": { "prefect.resource.id": "entsoe-msg-webhook-id" } }' | |
| - name: Deploy Flow | |
| run: | | |
| python -m etl.dataflow | |
| env: | |
| ECR_REPO_URL: ${{ needs.create-aws-infrastructure.outputs.aws_ecr_url}} | |
| ECR_REPO_NAME: ${{ needs.create-aws-infrastructure.outputs.aws_ecr_name}} | |
| IMAGE_TAG: ${{ github.sha }} | |
| EXECUTION_ROLE: ${{ needs.create-aws-infrastructure.outputs.aws_execution_role_arn }} | |
| TASK_ROLE: ${{ needs.create-aws-infrastructure.outputs.aws_task_role_arn }} | |
| ECS_CLUSTER: ${{ needs.create-aws-infrastructure.outputs.aws_ecs_cluster_arn }} | |
| VPC_ID: ${{ needs.create-aws-infrastructure.outputs.aws_vpc_id }} | |
| AWS_CREDENTIAL_BLOCK_ID: ${{ github.event.inputs.aws_credential_block_id }} | |
| - name: Let's test our automated dataflow by sending a test event | |
| run: python -m etl.emit_event | |
| - name: Summary | |
| run: | | |
| echo '### AWS infrastructure and flow successfully deployed! :rocket:' >> $GITHUB_STEP_SUMMARY | |
| echo "#### use the following ECR repo name as input to the delete_infra github action: ${{ steps.pulumi.outputs.ecr_repo_name }}" >> $GITHUB_STEP_SUMMARY |