Merge pull request #102 from simran029/master

Added example to use EMF Firelens for ECS Fargate

examples/README.md

@@ -29,7 +29,7 @@ Run the example:
 ./examples/agent/bin/run.sh
 ```
 
-## FireLens on ECS
+## FireLens on ECS EC2
 
 You can deploy the example by running the following:
 
@@ -45,7 +45,7 @@ aws s3api create-bucket --bucket <bucket-name> --region <region>
 # create ECS service
 
 # deploy
-./examples/ecs-firelens/publish.sh \
+./examples/ecs-firelens/bin/publish.sh \
   <account-id> \
   <region> \
   <image-name> \
@@ -54,3 +54,48 @@ aws s3api create-bucket --bucket <bucket-name> --region <region>
   <ecs-task-family> \
   <ecs-service-name>
 ```  
+
+## FireLens on ECS Fargate
+
+For running on Fargate, s3 file option is not supported for Fluent-bit config. Hence, we need to build the fluent-bit custom config image and then use its reference in our Firelens container definition.
+
+For building the custom fluent-bit image, clone the [amazon-ecs-firelens-examples](https://github.com/aws-samples/amazon-ecs-firelens-examples) and modify the contents of [extra.conf](https://github.com/aws-samples/amazon-ecs-firelens-examples/blob/mainline/examples/fluent-bit/config-file-type-file/extra.conf) in the amazon-ecs-firelens-examples repository. Post this run the following commands to build the custom fluent-bit image:-
+
+```sh 
+# create an ECR repository for the Fluentbit-config image
+aws ecr create-repository --repository-name <config-image-name> --region <region>
+
+# Navigate to the config file directory
+cd examples/fluent-bit/config-file-type-file
+
+# Build the docker image from Dockerfile. Replace config-image-name with your image name
+docker build -t <config-image-name> .
+
+# Tag the recently built docker image . Replace the config-image-name, account-id and region with your values.
+docker tag <config-image-name>:latest <account-id>.dkr.ecr.<region>.amazonaws.com/<config-image-name>:latest
+
+# Push the docker image to ECR
+docker push <account-id>.dkr.ecr.<region>.amazonaws.com/<config-image-name>:latest
+```
+
+For executing EMF application on Fargate, you need to execute the following commands :-
+
+```sh 
+# create an ECR repository for the example image
+aws ecr create-repository --repository-name <image-name> --region <region>
+
+# create ECS cluster
+# create ECS task definition
+# create ECS service
+
+# deploy
+./examples/ecs-firelens/bin/publish-fargate.sh 
+<account-id> \
+<region> \
+<example-image> \
+<fargate-config-image> \
+<ecs-cluster> \
+<ecs-task> \
+<ecs-service>
+
+```

examples/ecs-firelens/Dockerfile

@@ -1,8 +1,9 @@
-FROM openjdk:8-jdk-slim
+#Make sure the jdk pulled in dockerfile matches your IDE compile version used for compiling Jar file
+FROM openjdk:11
 RUN mkdir -p /app
 
 # copy the source files over
 COPY build/libs/*.jar /app/app.jar
 
 ENV JAVA_OPTS=""
-ENTRYPOINT [ "sh", "-c", "java $JAVA_OPTS -Djava.security.egd=file:/dev/./urandom -jar /app/app.jar" ]
+ENTRYPOINT [ "sh", "-c", "java $JAVA_OPTS -Djava.security.egd=file:/dev/./urandom -jar /app/app.jar" ]

examples/ecs-firelens/bin/publish-fargate.sh

@@ -0,0 +1,82 @@
+#!/usr/bin/env bash
+# Usage:
+#   ./examples/ecs-firelens/bin/publish-fargate.sh \
+#     <account-id> \
+#     <region> \
+#     <image-name> \
+#     <config-image-name> \
+#     <ecs-cluster-name> \
+#     <ecs-task-family> \
+#     <ecs-service-name>
+
+rootdir=$(git rev-parse --show-toplevel)
+
+ACCOUNT_ID=$1
+REGION=$2
+IMAGE_NAME=$3 # emf-ecs-firelens
+FLUENT_BIT_CONFIG=$4
+CLUSTER_NAME=$5 # emf-example
+ECS_TASK_FAMILY=$6 # aws-emf-ecs-app-example
+ECS_SERVICE_NAME=$7 # aws-emf-ecs-firelens-ec2
+
+LIB_PATH=$rootdir
+EXAMPLE_DIR=$rootdir/examples/ecs-firelens
+ECR_REMOTE=$ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com/$IMAGE_NAME
+
+function check_exit() {
+    last_exit_code=$?
+    if [ $last_exit_code -ne 0 ];
+    then
+        echo "Last command failed with exit code: $last_exit_code."
+        echo "Exiting."
+        exit $last_exit_code;
+    fi
+}
+
+echo 'BUILDING THE LOCAL PROJECT'
+pushd $rootdir
+./gradlew :examples:ecs-firelens:build
+check_exit
+popd
+
+pushd $EXAMPLE_DIR
+pwd
+
+
+echo 'UPDATING CONTAINER DEFINITIONS'
+sed "s/<account-id>/$ACCOUNT_ID/g" $EXAMPLE_DIR/container-definitions-fargate.template.json \
+| sed "s/<region>/$REGION/g" \
+| sed "s/<Firelens-custom-conf-image>/$FLUENT_BIT_CONFIG/g" \
+| sed "s/<image-name>/$IMAGE_NAME/g" \
+> $EXAMPLE_DIR/container-definitions.json
+check_exit
+
+echo 'BUILDING THE EXAMPLE DOCKER IMAGE'
+`aws ecr get-login --no-include-email --region $REGION`
+docker build . -t $IMAGE_NAME:latest
+check_exit
+
+echo 'PUSHING THE EXAMPLE DOCKER IMAGE TO ECR'
+imageid=$(docker images -q $IMAGE_NAME:latest)
+docker tag $imageid $ECR_REMOTE
+docker push $ECR_REMOTE
+check_exit
+
+echo 'UPDATING THE ECS SERVICE'
+aws ecs update-service \
+  --region $REGION \
+  --cluster $CLUSTER_NAME \
+  --service $ECS_SERVICE_NAME \
+  --force-new-deployment \
+  --task-definition $(aws ecs register-task-definition \
+                        --network-mode awsvpc \
+                        --task-role arn:aws:iam::$ACCOUNT_ID:role/ecsTaskExecutionRole \
+                        --execution-role-arn "arn:aws:iam::$ACCOUNT_ID:role/ecsTaskExecutionRole" \
+                        --region $REGION \
+                        --memory 512 \
+                        --cpu 256 \
+                        --family $ECS_TASK_FAMILY \
+                        --container-definitions "$(cat container-definitions.json)" \
+                    | jq --raw-output '.taskDefinition.taskDefinitionArn' | awk -F '/' '{ print $2 }')
+
+popd

examples/ecs-firelens/container-definitions-fargate.template.json

@@ -0,0 +1,42 @@
+[
+  {
+    "name": "example",
+    "image": "<account-id>.dkr.ecr.<region>.amazonaws.com/<image-name>:latest",
+    "essential": true,
+    "logConfiguration": {
+      "logDriver": "awsfirelens",
+      "options": {
+        "Name": "cloudwatch",
+        "region": "<region>",
+        "log_key": "log",
+        "log_group_name": "aws-emf-ecs-firelens-example-metrics",
+        "auto_create_group": "true",
+        "log_stream_prefix": "emf-",
+        "retry_limit": "2",
+        "log_format": "json/emf"
+      }
+    }
+  },
+  {
+    "name": "fluent-bit",
+    "image": "<account-id>.dkr.ecr.<region>.amazonaws.com/<Firelens-custom-conf-image>:latest",
+    "essential": true,
+    "firelensConfiguration": {
+      "type": "fluentbit",
+      "options": {
+        "config-file-type": "file",
+        "config-file-value": "/extra.conf",
+        "enable-ecs-log-metadata": "false"
+      }
+    },
+    "logConfiguration": {
+      "logDriver": "awslogs",
+      "options": {
+        "awslogs-group": "firelens-container",
+        "awslogs-region": "<region>",
+        "awslogs-create-group": "true",
+        "awslogs-stream-prefix": "firelens"
+      }
+    }
+  }
+]

examples/ecs-firelens/extra.conf

@@ -0,0 +1,26 @@
+# TCP input used for EMF payloads
+[INPUT]
+    Name        tcp
+    Listen      0.0.0.0
+    Port        25888
+    Chunk_Size  32
+    Buffer_Size 64
+    Format      none
+    Tag         emf-${HOSTNAME}
+    # This tag is used by the output plugin to determine the LogStream
+    # including the HOSTNAME is a way to increase the number of LogStreams.
+    # The maximum throughput on a
+    # single LogStream is 5 MB/s (max 1 MB at max 5 TPS).
+    # In AWSVPC mode, the HOSTNAME is the ENI private IP
+    # in bridge mode, the HOSTNAME is the Docker container ID
+
+# Output for EMF over TCP -> CloudWatch
+[OUTPUT]
+    Name                cloudwatch
+    Match               emf-*
+    region              us-east-1
+    log_key             log
+    log_group_name      aws-emf-ecs-firelens-example-metrics
+    log_stream_prefix   from-fluent-bit-
+    auto_create_group   true
+    log_format          json/emf

examples/ecs-firelens/src/main/java/App.java

@@ -24,7 +24,6 @@
 import software.amazon.cloudwatchlogs.emf.logger.MetricsLogger;
 import software.amazon.cloudwatchlogs.emf.model.Unit;
 import sun.misc.Signal;
-
 import java.io.IOException;
 import java.io.OutputStream;
 import java.net.InetSocketAddress;
@@ -36,9 +35,12 @@ public class App {
 
     public static void main(String[] args) throws Exception {
         registerShutdownHook();
-
-        int portNumber = 8000;
+        MetricsLogger logger = new MetricsLogger();
+        logger.setNamespace("FargateEMF");
+        logger.putMetric("Latency", 63, Unit.MILLISECONDS);
+        logger.flush();
         HttpServer server = HttpServer.create(new InetSocketAddress(8000), 0);
+        int portNumber = 8000;
         System.out.println("Server started. Listening on " + portNumber);
         server.createContext("/", new SimpleHandler());
         server.setExecutor(null);