From 954436c5df674e8d291e5edf5ca3290347670da4 Mon Sep 17 00:00:00 2001 From: Tom Romano Date: Mon, 19 Aug 2024 08:51:32 -0400 Subject: [PATCH 1/6] Retriece API from a CFN stack --- .../serverless-testing-workshop/demo-app/urs-ui.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/workshops/serverless-testing-workshop/demo-app/urs-ui.py b/workshops/serverless-testing-workshop/demo-app/urs-ui.py index 98bd26cb..c20c3079 100644 --- a/workshops/serverless-testing-workshop/demo-app/urs-ui.py +++ b/workshops/serverless-testing-workshop/demo-app/urs-ui.py @@ -12,12 +12,22 @@ import uuid import time import requests +import boto3 import streamlit as st from streamlit_js_eval import streamlit_js_eval # Initialize Contexts if 'api_endpoint_url' not in st.session_state: - if os.path.isfile("config.json"): + try: + cfn_client = boto3.client('cloudformation') + response = cfn_client.describe_stacks(StackName=os.environ.get('BACKEND_STACK_NAME','urs-backend')) + for output in response['Stacks'][0]['Outputs']: + if output['OutputKey'] == 'ApiEndpoint': + st.session_state['api_endpoint_url'] = output['OutputValue'] + except: + print("Failed to get API Endpoint from CloudFormation Stack") + + if os.path.isfile("config.json") and 'api_endpoint_url' not in st.session_state: with open("config.json","r",encoding="utf-8") as f: app_config = json.load(f) st.session_state['api_endpoint_url'] = app_config["api_endpoint"].strip() From 080690ced594ea2956289ca5c2abf15e228e7bf9 Mon Sep 17 00:00:00 2001 From: Tom Romano Date: Mon, 19 Aug 2024 16:32:02 -0400 Subject: [PATCH 2/6] Adding permissions for ECS task --- .../serverless-testing-workshop/template.yaml | 73 ++++++++++++++----- 1 file changed, 56 insertions(+), 17 deletions(-) diff --git a/workshops/serverless-testing-workshop/template.yaml b/workshops/serverless-testing-workshop/template.yaml index 50d89b78..0903355d 100644 --- a/workshops/serverless-testing-workshop/template.yaml +++ b/workshops/serverless-testing-workshop/template.yaml @@ -296,8 +296,8 @@ Resources: Description: Allow outbound access SecurityGroupIngress: - IpProtocol: tcp - FromPort: !Ref iECRStreamlitPort - ToPort: !Ref iECRStreamlitPort + FromPort: 8501 + ToPort: 8501 CidrIp: 0.0.0.0/0 Description: Inbound only on Streamlit port VpcId: !Ref StreamlitVPC @@ -406,7 +406,7 @@ Resources: Type: AWS::ElasticLoadBalancingV2::Listener Properties: LoadBalancerArn: !Ref LoadBalancer - Port: !Ref iECRStreamlitPort + Port: 8501 Protocol: HTTP DefaultActions: - Type: forward @@ -417,13 +417,13 @@ Resources: Properties: Name: !Sub "${AWS::StackName}-tg-http" VpcId: !Ref StreamlitVPC - Port: !Ref iECRStreamlitPort + Port: 8501 Protocol: HTTP TargetType: ip HealthCheckEnabled: true HealthCheckIntervalSeconds: 60 HealthCheckPath: "/_stcore/health" - HealthCheckPort: !Ref iECRStreamlitPort + HealthCheckPort: 8501 HealthCheckProtocol: HTTP TargetGroupAttributes: - Key: stickiness.enabled @@ -449,13 +449,16 @@ Resources: TaskRoleArn: !Ref TaskRole ContainerDefinitions: - Name: "streamlit" - Image: !Sub "${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/docsearch-ecr" + Image: !Sub "${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/urs-ui" MemoryReservation: 2048 Cpu: 2048 Memory: 4096 Essential: true PortMappings: - - ContainerPort: !Ref iECRStreamlitPort + - ContainerPort: 8501 + Environment: + - Name: BACKEND_STACK_NAME + Value: !Sub "{AWS::StackName}" LogConfiguration: LogDriver: awslogs Options: @@ -500,24 +503,57 @@ Resources: Statement: - Effect: Allow Principal: - Service: ecs-tasks.amazonaws.com + Service: + - ecs.amazonaws.com + - ecs-tasks.amazonaws.com Action: 'sts:AssumeRole' Policies: - - PolicyName: root + - PolicyName: cw PolicyDocument: Version: "2012-10-17" Statement: - Effect: Allow Action: - - "ecr:GetAuthorizationToken" - - "ecr:BatchCheckLayerAvailability" - - "ecr:GetDownloadUrlForLayer" - - "ecr:BatchGetImage" - - "logs:CreateLogStream" - - "logs:PutLogEvents" - - "logs:CreateLogGroup" + - logs:CreateLogGroup + - logs:CreateLogStream + - logs:PutLogEvents Resource: '*' - + - PolicyName: s3-read-access-policy + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Action: + - s3:GetObject + - s3:ListBucket + - s3:GetBucketLocation + - s3:GetObjectVersion + - s3:GetLifecycleConfiguration + - s3:PutObject + Resource: + - !Sub "arn:aws:s3:::unicorn-inv-${AWS::StackName}-${AWS::AccountId}" + - !Sub "arn:aws:s3:::unicorn-inv-${AWS::StackName}-${AWS::AccountId}/*" + - PolicyName: ecr_access_policy + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Action: + - ecr:GetAuthorizationToken + - ecr:BatchCheckLayerAvailability + - ecr:GetDownloadUrlForLayer + - ecr:BatchGetImage + Resource: "*" + - PolicyName: stack_describe_for_config + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Action: + - cloudformation:DescribeStacks + Resource: + - !Sub "arn:aws:cloudformation:${AWS::Region}:${AWS::AccountId}:stack/${AWS::StackName}" + - !Sub "arn:aws:cloudformation:${AWS::Region}:${AWS::AccountId}:stack/${AWS::StackName}" TaskRole: Type: AWS::IAM::Role Properties: @@ -529,6 +565,9 @@ Resources: Service: ecs-tasks.amazonaws.com Action: 'sts:AssumeRole' + + + Outputs: # ServerlessRestApi is an implicit API created out of Events key under Serverless::Function # Find out more about other implicit resources you can reference within SAM From 7b402e74b8dcf8fb27756590053a2005ce1f2b4b Mon Sep 17 00:00:00 2001 From: Tom Romano Date: Tue, 20 Aug 2024 07:51:36 -0400 Subject: [PATCH 3/6] remove UI components --- .../serverless-testing-workshop/template.yaml | 329 ------------------ 1 file changed, 329 deletions(-) diff --git a/workshops/serverless-testing-workshop/template.yaml b/workshops/serverless-testing-workshop/template.yaml index 0903355d..d97a6b0f 100644 --- a/workshops/serverless-testing-workshop/template.yaml +++ b/workshops/serverless-testing-workshop/template.yaml @@ -242,332 +242,6 @@ Resources: name: - !Ref UnicornInventoryBucket -################################################## -################################################## -##### demo-app (begin) -################################################## -################################################## - - # - # Networking - # VPC, 2 Public Subnets, S3/ECR/Cloudwatch Service Endpoints, Internet Gateway - # - - StreamlitVPC: - Type: AWS::EC2::VPC - Properties: - CidrBlock: 192.168.0.0/24 - EnableDnsHostnames : true - EnableDnsSupport : true - - StreamlitSubnet1: - Type: AWS::EC2::Subnet - Properties: - VpcId: !Ref StreamlitVPC - CidrBlock: 192.168.0.0/25 - AvailabilityZone: !Sub "${AWS::Region}b" - - StreamlitSubnet2: - Type: AWS::EC2::Subnet - Properties: - VpcId: !Ref StreamlitVPC - CidrBlock: 192.168.0.128/25 - AvailabilityZone: !Sub "${AWS::Region}c" - - StreamlitInternetGateway: - Type: AWS::EC2::InternetGateway - - StreamlitGatewayAttachment: - Type: AWS::EC2::VPCGatewayAttachment - Properties: - InternetGatewayId: !Ref StreamlitInternetGateway - VpcId: !Ref StreamlitVPC - - StreamlitSecurityGroup: - Type: AWS::EC2::SecurityGroup - Properties: - GroupDescription: "Streamlit UI Security Group" - GroupName: !Sub "${AWS::StackName}-ds-sg" - SecurityGroupEgress: - - IpProtocol: tcp - FromPort: 0 - ToPort: 65535 - CidrIp: 0.0.0.0/0 - Description: Allow outbound access - SecurityGroupIngress: - - IpProtocol: tcp - FromPort: 8501 - ToPort: 8501 - CidrIp: 0.0.0.0/0 - Description: Inbound only on Streamlit port - VpcId: !Ref StreamlitVPC - - EndpointSecurityGroup: - Type: AWS::EC2::SecurityGroup - Properties: - GroupDescription: "Streamlit UI Endpoint Security Group" - GroupName: !Sub "${AWS::StackName}-ep-sg" - SecurityGroupEgress: - - IpProtocol: tcp - FromPort: 0 - ToPort: 65535 - CidrIp: 0.0.0.0/0 - Description: Allow outbound access - SecurityGroupIngress: - - IpProtocol: tcp - FromPort: 0 - ToPort: 65535 - SourceSecurityGroupId: !Ref StreamlitSecurityGroup - Description: Allow inbound from Streamlit sg only - VpcId: !Ref StreamlitVPC - - StreamlitRouteTable: - Type: 'AWS::EC2::RouteTable' - Properties: - VpcId: !Ref StreamlitVPC - - InternetGatewayRoute: - Type: AWS::EC2::Route - Properties: - GatewayId: !Ref StreamlitInternetGateway - RouteTableId: !Ref StreamlitRouteTable - DestinationCidrBlock: 0.0.0.0/0 - - SubnetRouteTableAssociation1: - Type: 'AWS::EC2::SubnetRouteTableAssociation' - Properties: - SubnetId: !Ref StreamlitSubnet1 - RouteTableId: !Ref StreamlitRouteTable - - SubnetRouteTableAssociation2: - Type: 'AWS::EC2::SubnetRouteTableAssociation' - Properties: - SubnetId: !Ref StreamlitSubnet2 - RouteTableId: !Ref StreamlitRouteTable - - StreamlitVPCEndpointECRApi: - Type: AWS::EC2::VPCEndpoint - Properties: - SecurityGroupIds: - - !Ref EndpointSecurityGroup - ServiceName: !Sub 'com.amazonaws.${AWS::Region}.ecr.api' - SubnetIds: - - !Ref StreamlitSubnet1 - - !Ref StreamlitSubnet2 - VpcEndpointType: Interface - VpcId: !Ref StreamlitVPC - PrivateDnsEnabled: true - - StreamlitVPCEndpointDocker: - Type: AWS::EC2::VPCEndpoint - Properties: - SecurityGroupIds: - - !Ref EndpointSecurityGroup - ServiceName: !Sub 'com.amazonaws.${AWS::Region}.ecr.dkr' - SubnetIds: - - !Ref StreamlitSubnet1 - - !Ref StreamlitSubnet2 - VpcEndpointType: Interface - VpcId: !Ref StreamlitVPC - PrivateDnsEnabled: true - - StreamlitVPCEndpointLogs: - Type: AWS::EC2::VPCEndpoint - Properties: - SecurityGroupIds: - - !Ref EndpointSecurityGroup - ServiceName: !Sub 'com.amazonaws.${AWS::Region}.logs' - SubnetIds: - - !Ref StreamlitSubnet1 - - !Ref StreamlitSubnet2 - VpcEndpointType: Interface - VpcId: !Ref StreamlitVPC - PrivateDnsEnabled: true - - StreamlitVPCEndpointS3: - Type: AWS::EC2::VPCEndpoint - Properties: - ServiceName: !Sub 'com.amazonaws.${AWS::Region}.s3' - VpcEndpointType: Gateway - VpcId: !Ref StreamlitVPC - RouteTableIds: - - !Ref StreamlitRouteTable - - LoadBalancer: - Type: AWS::ElasticLoadBalancingV2::LoadBalancer - Properties: - Subnets: - - !Ref StreamlitSubnet1 - - !Ref StreamlitSubnet2 - SecurityGroups: - - !Ref StreamlitSecurityGroup - - LoadBalancerListener: - Type: AWS::ElasticLoadBalancingV2::Listener - Properties: - LoadBalancerArn: !Ref LoadBalancer - Port: 8501 - Protocol: HTTP - DefaultActions: - - Type: forward - TargetGroupArn: !Ref TargetGroup - - TargetGroup: - Type: AWS::ElasticLoadBalancingV2::TargetGroup - Properties: - Name: !Sub "${AWS::StackName}-tg-http" - VpcId: !Ref StreamlitVPC - Port: 8501 - Protocol: HTTP - TargetType: ip - HealthCheckEnabled: true - HealthCheckIntervalSeconds: 60 - HealthCheckPath: "/_stcore/health" - HealthCheckPort: 8501 - HealthCheckProtocol: HTTP - TargetGroupAttributes: - - Key: stickiness.enabled - Value: "true" - - Key: stickiness.type - Value: lb_cookie - - Key: stickiness.lb_cookie.duration_seconds - Value: "86500" - - ECSTask: - Type: AWS::ECS::TaskDefinition - DependsOn: LoadBalancerListener - Properties: - RequiresCompatibilities: - - FARGATE - Cpu: '2048' - Memory: '4096' - NetworkMode: awsvpc - RuntimePlatform: - CpuArchitecture: "X86_64" - OperatingSystemFamily: "LINUX" - ExecutionRoleArn: !Ref ExecutionRole - TaskRoleArn: !Ref TaskRole - ContainerDefinitions: - - Name: "streamlit" - Image: !Sub "${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/urs-ui" - MemoryReservation: 2048 - Cpu: 2048 - Memory: 4096 - Essential: true - PortMappings: - - ContainerPort: 8501 - Environment: - - Name: BACKEND_STACK_NAME - Value: !Sub "{AWS::StackName}" - LogConfiguration: - LogDriver: awslogs - Options: - awslogs-create-group: "true" - awslogs-group: !Sub "/ecs/${AWS::StackName}-ECSTask" - awslogs-region: !Sub "${AWS::Region}" - awslogs-stream-prefix: "ecs" - - ECSCluster: - Type: 'AWS::ECS::Cluster' - Properties: - ClusterName: !Sub "${AWS::StackName}-cluster" - - ECSService: - Type: 'AWS::ECS::Service' - Properties: - Cluster: !Ref ECSCluster - TaskDefinition: !Ref ECSTask - DesiredCount: 1 - LaunchType: FARGATE - ServiceName: !Sub "${AWS::StackName}-svc" - SchedulingStrategy: "REPLICA" - LoadBalancers: - - ContainerName: "streamlit" - ContainerPort: !Ref iECRStreamlitPort - TargetGroupArn: !Ref TargetGroup - HealthCheckGracePeriodSeconds: 50 - NetworkConfiguration: - AwsvpcConfiguration: - AssignPublicIp: ENABLED - SecurityGroups: - - !Ref StreamlitSecurityGroup - Subnets: - - !Ref StreamlitSubnet1 - - !Ref StreamlitSubnet2 - - ExecutionRole: - Type: AWS::IAM::Role - Properties: - RoleName: !Sub "${AWS::StackName}-execution-role" - AssumeRolePolicyDocument: - Statement: - - Effect: Allow - Principal: - Service: - - ecs.amazonaws.com - - ecs-tasks.amazonaws.com - Action: 'sts:AssumeRole' - Policies: - - PolicyName: cw - PolicyDocument: - Version: "2012-10-17" - Statement: - - Effect: Allow - Action: - - logs:CreateLogGroup - - logs:CreateLogStream - - logs:PutLogEvents - Resource: '*' - - PolicyName: s3-read-access-policy - PolicyDocument: - Version: "2012-10-17" - Statement: - - Effect: Allow - Action: - - s3:GetObject - - s3:ListBucket - - s3:GetBucketLocation - - s3:GetObjectVersion - - s3:GetLifecycleConfiguration - - s3:PutObject - Resource: - - !Sub "arn:aws:s3:::unicorn-inv-${AWS::StackName}-${AWS::AccountId}" - - !Sub "arn:aws:s3:::unicorn-inv-${AWS::StackName}-${AWS::AccountId}/*" - - PolicyName: ecr_access_policy - PolicyDocument: - Version: "2012-10-17" - Statement: - - Effect: Allow - Action: - - ecr:GetAuthorizationToken - - ecr:BatchCheckLayerAvailability - - ecr:GetDownloadUrlForLayer - - ecr:BatchGetImage - Resource: "*" - - PolicyName: stack_describe_for_config - PolicyDocument: - Version: "2012-10-17" - Statement: - - Effect: Allow - Action: - - cloudformation:DescribeStacks - Resource: - - !Sub "arn:aws:cloudformation:${AWS::Region}:${AWS::AccountId}:stack/${AWS::StackName}" - - !Sub "arn:aws:cloudformation:${AWS::Region}:${AWS::AccountId}:stack/${AWS::StackName}" - TaskRole: - Type: AWS::IAM::Role - Properties: - RoleName: !Sub "${AWS::StackName}-task-role" - AssumeRolePolicyDocument: - Statement: - - Effect: Allow - Principal: - Service: ecs-tasks.amazonaws.com - Action: 'sts:AssumeRole' - - - - Outputs: # ServerlessRestApi is an implicit API created out of Events key under Serverless::Function # Find out more about other implicit resources you can reference within SAM @@ -587,6 +261,3 @@ Outputs: GetFileValidatorARN: Description: "ARN of the Lambda function required in the 'OPTIONAL: Invoke a Lambda function in the cloud' section." Value: !GetAtt FileValidator.Arn - oUiDnsName: - Description: Host UI web link name - Value: !Sub "http://${LoadBalancer.DNSName}:${iECRStreamlitPort}" From 25c405f910a1d92e02a53ffc71ed427e5bddb1a2 Mon Sep 17 00:00:00 2001 From: Tom Romano Date: Tue, 20 Aug 2024 09:56:47 -0400 Subject: [PATCH 4/6] Stack config retrieval test --- workshops/serverless-testing-workshop/demo-app/urs-ui.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/workshops/serverless-testing-workshop/demo-app/urs-ui.py b/workshops/serverless-testing-workshop/demo-app/urs-ui.py index c20c3079..b3b49894 100644 --- a/workshops/serverless-testing-workshop/demo-app/urs-ui.py +++ b/workshops/serverless-testing-workshop/demo-app/urs-ui.py @@ -225,6 +225,13 @@ def update_unicorn_reserve_list(): key="api_endpoint_url", on_change=update_api_endpoint ) + if st.button("Retrieve API Endpoint"): + cfn_client = boto3.client('cloudformation') + response = cfn_client.describe_stacks(StackName=os.environ.get('BACKEND_STACK_NAME','urs-backend')) + for output in response['Stacks'][0]['Outputs']: + if output['OutputKey'] == 'ApiEndpoint': + st.session_state['api_endpoint_url'] = output['OutputValue'] + # File picker for uploading to the unicorn inventory uploaded_file = st.file_uploader("Choose a CSV file for the Unicorn Inventory.", type=["csv"]) From d7b321a1dca952ea335cf0796a4107e20bb38718 Mon Sep 17 00:00:00 2001 From: Tom Romano Date: Tue, 20 Aug 2024 10:56:17 -0400 Subject: [PATCH 5/6] API fetch changes --- .../serverless-testing-workshop/demo-app/urs-ui.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/workshops/serverless-testing-workshop/demo-app/urs-ui.py b/workshops/serverless-testing-workshop/demo-app/urs-ui.py index b3b49894..94c56b0a 100644 --- a/workshops/serverless-testing-workshop/demo-app/urs-ui.py +++ b/workshops/serverless-testing-workshop/demo-app/urs-ui.py @@ -25,13 +25,6 @@ if output['OutputKey'] == 'ApiEndpoint': st.session_state['api_endpoint_url'] = output['OutputValue'] except: - print("Failed to get API Endpoint from CloudFormation Stack") - - if os.path.isfile("config.json") and 'api_endpoint_url' not in st.session_state: - with open("config.json","r",encoding="utf-8") as f: - app_config = json.load(f) - st.session_state['api_endpoint_url'] = app_config["api_endpoint"].strip() - else: st.session_state['api_endpoint_url'] = "https://{APIGATEWAYID}.execute-api.{REGION}.amazonaws.com/Prod/" if 'unicorn_art' not in st.session_state: @@ -225,14 +218,14 @@ def update_unicorn_reserve_list(): key="api_endpoint_url", on_change=update_api_endpoint ) - if st.button("Retrieve API Endpoint"): + if st.button(f"Reset API Endpoint from Stack"): cfn_client = boto3.client('cloudformation') response = cfn_client.describe_stacks(StackName=os.environ.get('BACKEND_STACK_NAME','urs-backend')) for output in response['Stacks'][0]['Outputs']: if output['OutputKey'] == 'ApiEndpoint': - st.session_state['api_endpoint_url'] = output['OutputValue'] + st.write(f"Resetting API Endpoint to: {output['OutputValue']}") + - # File picker for uploading to the unicorn inventory uploaded_file = st.file_uploader("Choose a CSV file for the Unicorn Inventory.", type=["csv"]) if uploaded_file is not None: From 8a534fd579fe0d66c569e62c04facfbb12ab7c6b Mon Sep 17 00:00:00 2001 From: Tom Romano Date: Tue, 20 Aug 2024 11:23:30 -0400 Subject: [PATCH 6/6] Auto-set the backend API based on the stack name in env BACKEND_STACK_NAME --- workshops/serverless-testing-workshop/demo-app/urs-ui.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/workshops/serverless-testing-workshop/demo-app/urs-ui.py b/workshops/serverless-testing-workshop/demo-app/urs-ui.py index 94c56b0a..40f230b5 100644 --- a/workshops/serverless-testing-workshop/demo-app/urs-ui.py +++ b/workshops/serverless-testing-workshop/demo-app/urs-ui.py @@ -218,13 +218,6 @@ def update_unicorn_reserve_list(): key="api_endpoint_url", on_change=update_api_endpoint ) - if st.button(f"Reset API Endpoint from Stack"): - cfn_client = boto3.client('cloudformation') - response = cfn_client.describe_stacks(StackName=os.environ.get('BACKEND_STACK_NAME','urs-backend')) - for output in response['Stacks'][0]['Outputs']: - if output['OutputKey'] == 'ApiEndpoint': - st.write(f"Resetting API Endpoint to: {output['OutputValue']}") - # File picker for uploading to the unicorn inventory uploaded_file = st.file_uploader("Choose a CSV file for the Unicorn Inventory.", type=["csv"])