diff --git a/examples/README.md b/examples/README.md index 5ea89bc2..0f9f2ec5 100644 --- a/examples/README.md +++ b/examples/README.md @@ -29,7 +29,7 @@ Run the example: ./examples/agent/bin/run.sh ``` -## FireLens on ECS +## FireLens on ECS EC2 You can deploy the example by running the following: @@ -45,7 +45,7 @@ aws s3api create-bucket --bucket --region # create ECS service # deploy -./examples/ecs-firelens/publish.sh \ +./examples/ecs-firelens/bin/publish.sh \ \ \ \ @@ -54,3 +54,48 @@ aws s3api create-bucket --bucket --region \ ``` + +## FireLens on ECS Fargate + +For running on Fargate, s3 file option is not supported for Fluent-bit config. Hence, we need to build the fluent-bit custom config image and then use its reference in our Firelens container definition. + +For building the custom fluent-bit image, clone the [amazon-ecs-firelens-examples](https://github.com/aws-samples/amazon-ecs-firelens-examples) and modify the contents of [extra.conf](https://github.com/aws-samples/amazon-ecs-firelens-examples/blob/mainline/examples/fluent-bit/config-file-type-file/extra.conf) in the amazon-ecs-firelens-examples repository. Post this run the following commands to build the custom fluent-bit image:- + +```sh +# create an ECR repository for the Fluentbit-config image +aws ecr create-repository --repository-name --region + +# Navigate to the config file directory +cd examples/fluent-bit/config-file-type-file + +# Build the docker image from Dockerfile. Replace config-image-name with your image name +docker build -t . + +# Tag the recently built docker image . Replace the config-image-name, account-id and region with your values. +docker tag :latest .dkr.ecr..amazonaws.com/:latest + +# Push the docker image to ECR +docker push .dkr.ecr..amazonaws.com/:latest +``` + +For executing EMF application on Fargate, you need to execute the following commands :- + +```sh +# create an ECR repository for the example image +aws ecr create-repository --repository-name --region + +# create ECS cluster +# create ECS task definition +# create ECS service + +# deploy +./examples/ecs-firelens/bin/publish-fargate.sh + \ + \ + \ + \ + \ + \ + + +``` diff --git a/examples/ecs-firelens/Dockerfile b/examples/ecs-firelens/Dockerfile index aa1455eb..3bb7eba2 100644 --- a/examples/ecs-firelens/Dockerfile +++ b/examples/ecs-firelens/Dockerfile @@ -1,8 +1,9 @@ -FROM openjdk:8-jdk-slim +#Make sure the jdk pulled in dockerfile matches your IDE compile version used for compiling Jar file +FROM openjdk:11 RUN mkdir -p /app # copy the source files over COPY build/libs/*.jar /app/app.jar ENV JAVA_OPTS="" -ENTRYPOINT [ "sh", "-c", "java $JAVA_OPTS -Djava.security.egd=file:/dev/./urandom -jar /app/app.jar" ] \ No newline at end of file +ENTRYPOINT [ "sh", "-c", "java $JAVA_OPTS -Djava.security.egd=file:/dev/./urandom -jar /app/app.jar" ] diff --git a/examples/ecs-firelens/bin/publish-fargate.sh b/examples/ecs-firelens/bin/publish-fargate.sh new file mode 100644 index 00000000..83267ea9 --- /dev/null +++ b/examples/ecs-firelens/bin/publish-fargate.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +# Usage: +# ./examples/ecs-firelens/bin/publish-fargate.sh \ +# \ +# \ +# \ +# \ +# \ +# \ +# + +rootdir=$(git rev-parse --show-toplevel) + +ACCOUNT_ID=$1 +REGION=$2 +IMAGE_NAME=$3 # emf-ecs-firelens +FLUENT_BIT_CONFIG=$4 +CLUSTER_NAME=$5 # emf-example +ECS_TASK_FAMILY=$6 # aws-emf-ecs-app-example +ECS_SERVICE_NAME=$7 # aws-emf-ecs-firelens-ec2 + +LIB_PATH=$rootdir +EXAMPLE_DIR=$rootdir/examples/ecs-firelens +ECR_REMOTE=$ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com/$IMAGE_NAME + +function check_exit() { + last_exit_code=$? + if [ $last_exit_code -ne 0 ]; + then + echo "Last command failed with exit code: $last_exit_code." + echo "Exiting." + exit $last_exit_code; + fi +} + +echo 'BUILDING THE LOCAL PROJECT' +pushd $rootdir +./gradlew :examples:ecs-firelens:build +check_exit +popd + +pushd $EXAMPLE_DIR +pwd + + +echo 'UPDATING CONTAINER DEFINITIONS' +sed "s//$ACCOUNT_ID/g" $EXAMPLE_DIR/container-definitions-fargate.template.json \ +| sed "s//$REGION/g" \ +| sed "s//$FLUENT_BIT_CONFIG/g" \ +| sed "s//$IMAGE_NAME/g" \ +> $EXAMPLE_DIR/container-definitions.json +check_exit + +echo 'BUILDING THE EXAMPLE DOCKER IMAGE' +`aws ecr get-login --no-include-email --region $REGION` +docker build . -t $IMAGE_NAME:latest +check_exit + +echo 'PUSHING THE EXAMPLE DOCKER IMAGE TO ECR' +imageid=$(docker images -q $IMAGE_NAME:latest) +docker tag $imageid $ECR_REMOTE +docker push $ECR_REMOTE +check_exit + +echo 'UPDATING THE ECS SERVICE' +aws ecs update-service \ + --region $REGION \ + --cluster $CLUSTER_NAME \ + --service $ECS_SERVICE_NAME \ + --force-new-deployment \ + --task-definition $(aws ecs register-task-definition \ + --network-mode awsvpc \ + --task-role arn:aws:iam::$ACCOUNT_ID:role/ecsTaskExecutionRole \ + --execution-role-arn "arn:aws:iam::$ACCOUNT_ID:role/ecsTaskExecutionRole" \ + --region $REGION \ + --memory 512 \ + --cpu 256 \ + --family $ECS_TASK_FAMILY \ + --container-definitions "$(cat container-definitions.json)" \ + | jq --raw-output '.taskDefinition.taskDefinitionArn' | awk -F '/' '{ print $2 }') + +popd diff --git a/examples/ecs-firelens/container-definitions-fargate.template.json b/examples/ecs-firelens/container-definitions-fargate.template.json new file mode 100644 index 00000000..737c5e9c --- /dev/null +++ b/examples/ecs-firelens/container-definitions-fargate.template.json @@ -0,0 +1,42 @@ +[ + { + "name": "example", + "image": ".dkr.ecr..amazonaws.com/:latest", + "essential": true, + "logConfiguration": { + "logDriver": "awsfirelens", + "options": { + "Name": "cloudwatch", + "region": "", + "log_key": "log", + "log_group_name": "aws-emf-ecs-firelens-example-metrics", + "auto_create_group": "true", + "log_stream_prefix": "emf-", + "retry_limit": "2", + "log_format": "json/emf" + } + } + }, + { + "name": "fluent-bit", + "image": ".dkr.ecr..amazonaws.com/:latest", + "essential": true, + "firelensConfiguration": { + "type": "fluentbit", + "options": { + "config-file-type": "file", + "config-file-value": "/extra.conf", + "enable-ecs-log-metadata": "false" + } + }, + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-group": "firelens-container", + "awslogs-region": "", + "awslogs-create-group": "true", + "awslogs-stream-prefix": "firelens" + } + } + } +] diff --git a/examples/ecs-firelens/extra.conf b/examples/ecs-firelens/extra.conf new file mode 100644 index 00000000..396301e6 --- /dev/null +++ b/examples/ecs-firelens/extra.conf @@ -0,0 +1,26 @@ +# TCP input used for EMF payloads +[INPUT] + Name tcp + Listen 0.0.0.0 + Port 25888 + Chunk_Size 32 + Buffer_Size 64 + Format none + Tag emf-${HOSTNAME} + # This tag is used by the output plugin to determine the LogStream + # including the HOSTNAME is a way to increase the number of LogStreams. + # The maximum throughput on a + # single LogStream is 5 MB/s (max 1 MB at max 5 TPS). + # In AWSVPC mode, the HOSTNAME is the ENI private IP + # in bridge mode, the HOSTNAME is the Docker container ID + +# Output for EMF over TCP -> CloudWatch +[OUTPUT] + Name cloudwatch + Match emf-* + region us-east-1 + log_key log + log_group_name aws-emf-ecs-firelens-example-metrics + log_stream_prefix from-fluent-bit- + auto_create_group true + log_format json/emf diff --git a/examples/ecs-firelens/src/main/java/App.java b/examples/ecs-firelens/src/main/java/App.java index ee72a451..c71f5368 100644 --- a/examples/ecs-firelens/src/main/java/App.java +++ b/examples/ecs-firelens/src/main/java/App.java @@ -24,7 +24,6 @@ import software.amazon.cloudwatchlogs.emf.logger.MetricsLogger; import software.amazon.cloudwatchlogs.emf.model.Unit; import sun.misc.Signal; - import java.io.IOException; import java.io.OutputStream; import java.net.InetSocketAddress; @@ -36,9 +35,12 @@ public class App { public static void main(String[] args) throws Exception { registerShutdownHook(); - - int portNumber = 8000; + MetricsLogger logger = new MetricsLogger(); + logger.setNamespace("FargateEMF"); + logger.putMetric("Latency", 63, Unit.MILLISECONDS); + logger.flush(); HttpServer server = HttpServer.create(new InetSocketAddress(8000), 0); + int portNumber = 8000; System.out.println("Server started. Listening on " + portNumber); server.createContext("/", new SimpleHandler()); server.setExecutor(null);