bug fixes, performance, usability and docs improvements

.travis.yml

@@ -1,7 +1,7 @@
 language: java
 jdk:
-  - oraclejdk8
-  - oraclejdk7
+  - openjdk8
+  - openjdk11
 cache:
   directories:
   - $HOME/.m2

README.md

@@ -1,6 +1,6 @@
 Gruffalo
 ========
-[![Build Status](https://travis-ci.org/outbrain/gruffalo.svg?branch=master)](https://travis-ci.org/outbrain/gruffalo)
+[![Build Status](https://travis-ci.org/eranharel/gruffalo.svg?branch=master)](https://travis-ci.org/eranharel/gruffalo)
 
 Gruffalo is an asynchronous Netty based graphite proxy.
 It protects Graphite from the herds of clients by minimizing context switches and interrupts; by batching and aggregating metrics.
@@ -9,7 +9,68 @@ Gruffalo also allows you to replicate metrics between Graphite installations for
 
 Gruffalo can easily handle a massive amount of traffic, and thus increase your metrics delivery system availability.
 
-At Outbrain, we currently handle over 1700 concurrent connections, and over 2M metrics per minute per instance.
+In practice, Gruffalo handles thousands of concurrent connections, and over several million metrics per minute per instance at our production environment.
+
+## Usage
+```
+$ ./gruffalo-standalone.sh -h
+usage: com.outbrain.gruffalo.StandaloneGruffaloServer
+ -c,--clusters <arg>          Graphite relay clusters (required)
+ -flushOnIdleTime <arg>       Max amount of time (seconds) in which a
+                              channel can be idle before the current batch
+                              is flushed (default 10). 0 means disabled.
+ -h,--help                    print this message
+ -maxBatchSize <arg>          Max size of metrics batch in characters.
+                              This setting affects the memory footprint of
+                              the server - each channel holds a buffer of
+                              the specified size for batching (default
+                              8192)
+ -maxInflightBatches <arg>    Max number of allowed in-flight batches
+                              before we start pushing back clients
+                              (default 1500)
+ -maxMetricLength <arg>       Maximum length of a frame we're willing to
+                              decode (default 4096)
+ -p,--port <arg>              TCP listen port (required)
+ -reconnectOnIdleTime <arg>   Max time in seconds before we reconnect an
+                              idle downstream channel (default 120)
+```
+
+The `gruffalo-standalone.sh` script allows you to configure the inbound port, multiple replication targets, 
+and to optionally control the behavior of the proxy.
+
+**Example execution**
+Start Gruffalo on TCP port 3003, and proxy to 2 clusters: 
+[localhost:2003,localhost:2004] and [localhost:2006,localhost:2006]
+(traffic will be replicated to both clusters)
+```
+$ ./gruffalo-standalone.sh -p 3003 -c localhost:2003,localhost:2004 -c localhost:2005,localhost:2006
+{"timestamp":"2019-05-25T16:11:07.074+03:00","message":"Creating a client pool for [localhost:2003,localhost:2004]","logger_name":"c.o.g.netty.GraphiteClientPool","thread_name":"main","level":"INFO","facility":"Gruffalo","environment":"DEV","hostName":"my.host"}
+{"timestamp":"2019-05-25T16:11:07.084+03:00","message":"Client for [localhost:2003] initialized","logger_name":"c.o.g.netty.NettyGraphiteClient","thread_name":"main","level":"INFO","facility":"Gruffalo","environment":"DEV","hostName":"my.host"}
+{"timestamp":"2019-05-25T16:11:07.089+03:00","message":"Client for [localhost:2004] initialized","logger_name":"c.o.g.netty.NettyGraphiteClient","thread_name":"main","level":"INFO","facility":"Gruffalo","environment":"DEV","hostName":"my.host"}
+{"timestamp":"2019-05-25T16:11:07.089+03:00","message":"Client for [localhost:2003] is reconnecting","logger_name":"c.o.g.netty.NettyGraphiteClient","thread_name":"main","level":"INFO","facility":"Gruffalo","environment":"DEV","hostName":"my.host"}
+{"timestamp":"2019-05-25T16:11:07.149+03:00","message":"Client for [localhost:2004] is reconnecting","logger_name":"c.o.g.netty.NettyGraphiteClient","thread_name":"main","level":"INFO","facility":"Gruffalo","environment":"DEV","hostName":"my.host"}
+{"timestamp":"2019-05-25T16:11:07.153+03:00","message":"Creating a client pool for [localhost:2005,localhost:2006]","logger_name":"c.o.g.netty.GraphiteClientPool","thread_name":"main","level":"INFO","facility":"Gruffalo","environment":"DEV","hostName":"my.host"}
+{"timestamp":"2019-05-25T16:11:07.153+03:00","message":"Client for [localhost:2005] initialized","logger_name":"c.o.g.netty.NettyGraphiteClient","thread_name":"main","level":"INFO","facility":"Gruffalo","environment":"DEV","hostName":"my.host"}
+{"timestamp":"2019-05-25T16:11:07.154+03:00","message":"Client for [localhost:2006] initialized","logger_name":"c.o.g.netty.NettyGraphiteClient","thread_name":"main","level":"INFO","facility":"Gruffalo","environment":"DEV","hostName":"my.host"}
+{"timestamp":"2019-05-25T16:11:07.154+03:00","message":"Client for [localhost:2005] is reconnecting","logger_name":"c.o.g.netty.NettyGraphiteClient","thread_name":"main","level":"INFO","facility":"Gruffalo","environment":"DEV","hostName":"my.host"}
+{"timestamp":"2019-05-25T16:11:07.155+03:00","message":"Client for [localhost:2006] is reconnecting","logger_name":"c.o.g.netty.NettyGraphiteClient","thread_name":"main","level":"INFO","facility":"Gruffalo","environment":"DEV","hostName":"my.host"}
+{"timestamp":"2019-05-25T16:11:07.166+03:00","message":"Connected to: localhost/127.0.0.1:2004","logger_name":"c.o.g.n.GraphiteChannelInboundHandler","thread_name":"nioEventLoopGroup-2-2","level":"INFO","facility":"Gruffalo","environment":"DEV","hostName":"my.host"}
+{"timestamp":"2019-05-25T16:11:07.168+03:00","message":"Initializing TCP...","logger_name":"c.o.gruffalo.netty.GruffaloProxy","thread_name":"main","level":"INFO","facility":"Gruffalo","environment":"DEV","hostName":"my.host"}
+{"timestamp":"2019-05-25T16:11:07.168+03:00","message":"Connected to: localhost/127.0.0.1:2003","logger_name":"c.o.g.n.GraphiteChannelInboundHandler","thread_name":"nioEventLoopGroup-2-1","level":"INFO","facility":"Gruffalo","environment":"DEV","hostName":"my.host"}
+{"timestamp":"2019-05-25T16:11:07.168+03:00","message":"Connected to: localhost/127.0.0.1:2006","logger_name":"c.o.g.n.GraphiteChannelInboundHandler","thread_name":"nioEventLoopGroup-2-4","level":"INFO","facility":"Gruffalo","environment":"DEV","hostName":"my.host"}
+{"timestamp":"2019-05-25T16:11:07.172+03:00","message":"Connected to: localhost/127.0.0.1:2005","logger_name":"c.o.g.n.GraphiteChannelInboundHandler","thread_name":"nioEventLoopGroup-2-3","level":"INFO","facility":"Gruffalo","environment":"DEV","hostName":"my.host"}
+{"timestamp":"2019-05-25T16:11:07.176+03:00","message":"Binding to TCP port 3003","logger_name":"c.o.gruffalo.netty.GruffaloProxy","thread_name":"main","level":"INFO","facility":"Gruffalo","environment":"DEV","hostName":"my.host"}
+{"timestamp":"2019-05-25T16:11:07.176+03:00","message":"Initialization completed","logger_name":"c.o.gruffalo.netty.GruffaloProxy","thread_name":"main","level":"INFO","facility":"Gruffalo","environment":"DEV","hostName":"my.host"}
+{"timestamp":"2019-05-25T16:11:07.177+03:00","message":"******** Gruffalo started ********","logger_name":"c.o.g.StandaloneGruffaloServer","thread_name":"main","level":"INFO","facility":"Gruffalo","environment":"DEV","hostName":"my.host"}
+```
+
+### Logging
+Gruffalo supports logstash compatible output, and logs to standard output, where logs can easily be collected.
+
+The logging can be slightly modified using the following flags:
+* `GRUFFALO_LOG_APPENDER` - if present and equals to `STDOUT` logs will be in text format.
+If absent - logs will be in the logstash JSON format by default.
+* `GRUFFLAO_ENVIRONMENT` - controls the value of the environment field, defaults to `DEV`
 
 Reference Docs
 --------------

gruffalo-standalone.sh

@@ -1,7 +1,5 @@
 #!/bin/sh
 
-mvn dependency:build-classpath -Dmdep.outputFile=classpath
-export CLASSPATH=target/classes:`cat classpath`
-export log4jfile=$(pwd)/src/main/resources/stand-alone-log4j.xml
-echo $log4jfile
-java -cp $CLASSPATH -Dlog4j.configuration=file://$log4jfile -Xmx2024m -Xms2024m com.outbrain.gruffalo.StandaloneGruffaloServer
+mvn dependency:build-classpath -Dmdep.outputFile=classpath -q
+CLASSPATH=target/classes:`cat classpath`
+java -cp ${CLASSPATH} -Xmx2024m -Xms2024m com.outbrain.gruffalo.StandaloneGruffaloServer "$@"

pom.xml

@@ -5,7 +5,7 @@
   <groupId>com.outbrain.swinfra</groupId>  
   <packaging>jar</packaging>
   <name>Gruffalo</name>
-  <version>0.23-SNAPSHOT</version>
+  <version>1.0.0-SNAPSHOT</version>
 
   <licenses>
     <license>
@@ -16,17 +16,12 @@
   </licenses>
 
   <scm>
-    <connection>scm:git:https://github.com/outbrain/gruffalo.git</connection>
-    <developerConnection>scm:git:https://github.com/outbrain/gruffalo.git</developerConnection>
-    <url>https://github.com/outbrain/gruffalo/</url>
+    <connection>scm:git:https://github.com/eranahrel/gruffalo.git</connection>
+    <developerConnection>scm:git:https://github.com/eranahrel/gruffalo.git</developerConnection>
+    <url>https://github.com/eranahrel/gruffalo/</url>
     <tag>HEAD</tag>
   </scm>
 
-  <organization>
-    <name>outbrain</name>
-    <url>http://www.outbrain.com</url>
-  </organization>
-
   <developers>
     <developer>
       <name>Eran Harel</name>
@@ -37,209 +32,130 @@
 
   <properties>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-    <java.compiler.source>1.7</java.compiler.source>
-    <java.compiler.target>1.7</java.compiler.target>
-
-    <org.springframework.version>4.0.7.RELEASE</org.springframework.version>
+    <java.compiler.source>1.8</java.compiler.source>
+    <java.compiler.target>1.8</java.compiler.target>
   </properties>
 
   <distributionManagement>
-    <!--
-    <repository>
-      <id>nexus-3rdparty</id>
-      <name>3rd party repository</name>
-      <url>http://mavenrepo:8081/nexus/content/repositories/thirdparty</url>
-    </repository>
-    -->
-    <!-- Temp Bintray repo
-    <repository>
-      <id>bintray-harel-eran-Gruffalo-com.outbrain.gruffalo</id>
-      <name>harel-eran-Gruffalo-com.outbrain.gruffalo</name>
-      <url>https://api.bintray.com/maven/harel-eran/Gruffalo/com.outbrain.gruffalo</url>
-    </repository>
-    -->
   </distributionManagement>
 
   <repositories>
-    <!--
-    <repository>
-      <id>nexus</id>
-      <name>Local nexus</name>
-      <url>http://nexus:8081/nexus/content/groups/public</url>
-    </repository>
-    -->
     <repository>
       <id>jcenter</id>
       <name>jcenter</name>
-      <url>http://jcenter.bintray.com</url>
+      <url>https://jcenter.bintray.com</url>
       <snapshots><enabled>false</enabled></snapshots>
     </repository>
   </repositories>
 
   <build>
     <plugins>
 
-<!--
-      <plugin>
-        <groupId>org.codehaus.mojo</groupId>
-        <artifactId>versions-maven-plugin</artifactId>
-        <version>2.1</version>
-      </plugin>
-
-      <plugin>
-        <artifactId>maven-scm-plugin</artifactId>
-        <version>1.9.2</version>
-        <configuration>
-          <tag>${project.artifactId}-${project.version}</tag>
-        </configuration>
-      </plugin>
--->
-
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-release-plugin</artifactId>
-        <version>2.5.1</version>
+        <version>2.5.3</version>
       </plugin>
 
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-compiler-plugin</artifactId>
-        <version>3.1</version>
+        <version>3.8.1</version>
         <configuration>
           <source>${java.compiler.source}</source>
           <target>${java.compiler.target}</target>
         </configuration>
       </plugin>
 
       <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-surefire-plugin</artifactId>
-        <version>2.17</version>
-        <configuration>
-          <useFile>false</useFile>
-          <useSystemClassLoader>false</useSystemClassLoader>
-          <forkMode>once</forkMode>
-          <argLine>-Xms512m -Xmx2048m -XX:MaxPermSize=1024m</argLine>
-        </configuration>
+        <version>2.22.2</version>
       </plugin>
-
-      <!-- TODO reconfigure for OS projects -->
       <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-jar-plugin</artifactId>
-        <version>2.4</version>
-        <configuration>
-          <archive>
-            <manifestEntries>
-              <timestamp>${maven.build.timestamp}</timestamp>
-            </manifestEntries>
-          </archive>
-        </configuration>
-
-        <executions>
-          <execution>
-            <id>make-a-jar</id>
-            <phase>compile</phase>
-            <goals>
-              <goal>jar</goal>
-            </goals>
-          </execution>
-          <execution>
-            <id>make-a-test-jar</id>
-            <goals>
-              <goal>test-jar</goal>
-            </goals>
-          </execution>
-        </executions>
-
+        <artifactId>maven-failsafe-plugin</artifactId>
+        <version>2.22.2</version>
       </plugin>
 
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-source-plugin</artifactId>
-        <version>2.3</version>
-        <executions>
-          <execution>
-            <id>attach-sources</id>
-            <goals>
-              <goal>jar</goal>
-            </goals>
-          </execution>
-        </executions>
-      </plugin>
 
     </plugins>
   </build>
 
   <dependencies>
 
     <dependency>
-      <groupId>com.outbrain.swinfra</groupId>
-      <artifactId>util-metrics</artifactId>
-      <version>0.122</version>
+      <groupId>io.netty</groupId>
+      <artifactId>netty-all</artifactId>
+      <version>4.1.36.Final</version>
     </dependency>
 
     <dependency>
-      <groupId>com.outbrain.swinfra</groupId>
-      <artifactId>util-config</artifactId>
-      <version>0.122</version>
+      <groupId>io.dropwizard.metrics</groupId>
+      <artifactId>metrics-core</artifactId>
+      <version>4.1.0</version>
     </dependency>
-
     <dependency>
-      <groupId>io.netty</groupId>
-      <artifactId>netty-all</artifactId>
-      <version>4.0.23.Final</version>
+      <groupId>io.dropwizard.metrics</groupId>
+      <artifactId>metrics-jvm</artifactId>
+      <version>4.1.0</version>
     </dependency>
     <dependency>
-      <groupId>joda-time</groupId>
-      <artifactId>joda-time</artifactId>
-      <version>2.2</version>
+      <groupId>io.dropwizard.metrics</groupId>
+      <artifactId>metrics-jmx</artifactId>
+      <version>4.1.0</version>
     </dependency>
 
+
     <dependency>
-      <groupId>org.springframework</groupId>
-      <artifactId>spring-core</artifactId>
-      <version>${org.springframework.version}</version>
+      <groupId>commons-cli</groupId>
+      <artifactId>commons-cli</artifactId>
+      <version>1.4</version>
     </dependency>
+
     <dependency>
-      <groupId>org.springframework</groupId>
-      <artifactId>spring-context</artifactId>
-      <version>${org.springframework.version}</version>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+      <version>1.7.25</version>
     </dependency>
     <dependency>
-      <groupId>com.google.guava</groupId>
-      <artifactId>guava</artifactId>
-      <version>16.0.1</version>
+      <groupId>ch.qos.logback</groupId>
+      <artifactId>logback-classic</artifactId>
+      <version>1.2.3</version>
     </dependency>
     <dependency>
-      <groupId>org.apache.commons</groupId>
-      <artifactId>commons-lang3</artifactId>
-      <version>3.3.2</version>
+      <groupId>ch.qos.logback</groupId>
+      <artifactId>logback-core</artifactId>
+      <version>1.2.3</version>
     </dependency>
     <dependency>
-      <groupId>com.codahale.metrics</groupId>
-      <artifactId>metrics-core</artifactId>
-      <version>3.0.2</version>
+      <groupId>net.logstash.logback</groupId>
+      <artifactId>logstash-logback-encoder</artifactId>
+      <version>5.3</version>
     </dependency>
 
+
     <dependency>
-      <groupId>org.slf4j</groupId>
-      <artifactId>slf4j-log4j12</artifactId>
-      <version>1.7.5</version>
-      <scope>provided</scope>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-lang3</artifactId>
+      <version>3.9</version>
+      <scope>test</scope>
     </dependency>
     <dependency>
-      <groupId>log4j</groupId>
-      <artifactId>log4j</artifactId>
-      <version>1.2.17</version>
-      <scope>provided</scope>
+      <groupId>com.google.guava</groupId>
+      <artifactId>guava</artifactId>
+      <version>28.0-jre</version>
     </dependency>
 
     <dependency>
-      <groupId>org.mockito</groupId>
-      <artifactId>mockito-all</artifactId>
-      <scope>test</scope>
-      <version>1.9.5</version>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-api</artifactId>
+      <version>5.5.0</version>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-engine</artifactId>
+      <version>5.5.0</version>
     </dependency>
+
+
   </dependencies>
 </project>