Skip to content

Commit

Permalink
RD-12747 - secret scrubbing (#66)
Browse files Browse the repository at this point in the history
  • Loading branch information
harelmo-lumigo authored May 22, 2024
1 parent 5adc9c4 commit 7ebb712
Show file tree
Hide file tree
Showing 16 changed files with 546 additions and 122 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -221,3 +221,4 @@ _meta
.serverless
package-lock.json
src/main/resources/lumigo-agent.jar
.vscode/
44 changes: 16 additions & 28 deletions pom.xml
Original file line number Diff line number Diff line change
@@ -1,25 +1,19 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>

<groupId>io.lumigo</groupId>
<artifactId>java-tracer</artifactId>
<version>1.0.42</version>
<packaging>jar</packaging>

<name>Lumigo java tracer</name>
<description>The Lumigo java tracer for serverless functions</description>
<url>https://lumigo.io/</url>

<scm>
<url>https://github.com/lumigo-io/java-tracer</url>
<connection>scm:git:https://github.com:lumigo-io/java-tracer.git</connection>
<developerConnection>scm:git:https://github.com:lumigo-io/java-tracer.git</developerConnection>
<tag>1.0.42</tag>
</scm>

<developers>
<developer>
<name>Lumigo Dev Team</name>
Expand All @@ -28,21 +22,17 @@
<organizationUrl>https://lumigo.io/</organizationUrl>
</developer>
</developers>

<organization>
<name>Lumigo</name>
<url>https://lumigo.io/</url>
</organization>

<licenses>
<license>
<name>Apache License, Version 2.0</name>
<url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
<distribution>repo</distribution>
</license>
</licenses>


<distributionManagement>
<snapshotRepository>
<id>ossrh</id>
Expand All @@ -53,8 +43,6 @@
<url>https://oss.sonatype.org/service/local/staging/deploy/maven2/</url>
</repository>
</distributionManagement>


<properties>
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
Expand Down Expand Up @@ -159,7 +147,7 @@
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>1.18.22</version>
<version>1.18.32</version>
<scope>provided</scope>
</dependency>
<dependency>
Expand All @@ -182,7 +170,11 @@
<artifactId>byte-buddy-agent</artifactId>
<version>1.14.14</version>
</dependency>

<dependency>
<groupId>org.json</groupId>
<artifactId>json</artifactId>
<version>20210307</version>
</dependency>
<!-- Testing dependencies -->
<dependency>
<groupId>org.junit.jupiter</groupId>
Expand Down Expand Up @@ -233,7 +225,7 @@
<includes>
<include>**/lumigo-version.txt</include>
</includes>
</resource>
</resource>
</resources>
<plugins>
<!-- Used to make it so the JavaDoc and sources are included. -->
Expand Down Expand Up @@ -267,8 +259,7 @@
</executions>
</plugin>
<!-- End of sources/javadoc inclusion. -->

<!-- Use for running tests-->
<!-- Use for running tests -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
Expand Down Expand Up @@ -297,12 +288,11 @@
<version>3.0.5</version>
<configuration>
<!--
Enables analysis which takes more memory but finds more bugs.
If you run out of memory, changes the value of the effort element
to 'Low'.
-->
Enables analysis which takes more memory but finds more bugs.
If you run out of memory, changes the value of the effort element
to 'Low'.
-->
<effort>Max</effort>

<!-- Reports all bugs (other values are medium and max) -->
<threshold>Low</threshold>
<!-- Produces XML report -->
Expand All @@ -312,9 +302,7 @@
<excludeFilterFile>findbugs/findbugs-exclude.xml</excludeFilterFile>
</configuration>
<executions>
<!--
Ensures that FindBugs inspects source code when project is compiled.
-->
<!-- Ensures that FindBugs inspects source code when project is compiled. -->
<execution>
<id>analyze-compile</id>
<phase>compile</phase>
Expand All @@ -327,7 +315,7 @@
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.3</version>
<version>0.8.11</version>
<executions>
<execution>
<goals>
Expand Down Expand Up @@ -377,4 +365,4 @@
</plugin>
</plugins>
</build>
</project>
</project>
4 changes: 2 additions & 2 deletions scripts/checks.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,5 @@
set -eo pipefail

java -jar libs/google-java-format-1.7-all-deps.jar --set-exit-if-changed -i -a $(find . -type f -name "*.java" | grep ".*/src/.*java")
mvn -f agent/pom.xml clean package
mvn clean package
mvn -Djava.security.manager=allow -f agent/pom.xml clean package
mvn -Djava.security.manager=allow clean package
65 changes: 14 additions & 51 deletions src/main/java/io/lumigo/core/SpansContainer.java
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,12 @@
import io.lumigo.core.parsers.v1.AwsSdkV1ParserFactory;
import io.lumigo.core.parsers.v2.AwsSdkV2ParserFactory;
import io.lumigo.core.utils.AwsUtils;
import io.lumigo.core.utils.EnvUtil;
import io.lumigo.core.utils.JsonUtils;
import io.lumigo.core.utils.SecretScrubber;
import io.lumigo.core.utils.StringUtils;
import io.lumigo.models.HttpSpan;
import io.lumigo.models.Reportable;
import io.lumigo.models.Span;
import java.io.*;
import java.util.*;
Expand All @@ -38,14 +41,14 @@ public class SpansContainer {
private static final String AMZN_TRACE_ID = "_X_AMZN_TRACE_ID";
private static final String FUNCTION_SPAN_TYPE = "function";
private static final String HTTP_SPAN_TYPE = "http";
private static final SecretScrubber secretScrubber = new SecretScrubber(new EnvUtil().getEnv());

private Span baseSpan;
private Span startFunctionSpan;
private Long rttDuration;
private Span endFunctionSpan;
private Reporter reporter;
private List<HttpSpan> httpSpans = new LinkedList<>();

private static final SpansContainer ourInstance = new SpansContainer();

public static SpansContainer getInstance() {
Expand All @@ -68,6 +71,7 @@ private SpansContainer() {}
public void init(Map<String, String> env, Reporter reporter, Context context, Object event) {
this.clear();
this.reporter = reporter;

int javaVersion = AwsUtils.parseJavaVersion(System.getProperty("java.version"));
if (javaVersion > 11) {
awsTracerId = System.getProperty("com.amazonaws.xray.traceHeader");
Expand Down Expand Up @@ -214,8 +218,8 @@ public Span getStartFunctionSpan() {
return startFunctionSpan;
}

public List<Object> getAllCollectedSpans() {
List<Object> spans = new LinkedList<>();
public List<Reportable> getAllCollectedSpans() {
List<Reportable> spans = new LinkedList<>();
spans.add(endFunctionSpan);
spans.addAll(httpSpans);
return spans;
Expand Down Expand Up @@ -518,63 +522,22 @@ protected static <T> T callIfVerbose(Callable<T> method) {
}
}

private Object prepareToSend(Object span, boolean hasError) {
return reduceSpanSize(span, hasError);
private Reportable prepareToSend(Reportable span, boolean hasError) {
return reduceSpanSize(span.scrub(secretScrubber), hasError);
}

private List<Object> prepareToSend(List<Object> spans, boolean hasError) {
for (Object span : spans) {
reduceSpanSize(span, hasError);
private List<Reportable> prepareToSend(List<Reportable> spans, boolean hasError) {
for (Reportable span : spans) {
reduceSpanSize(span.scrub(secretScrubber), hasError);
}
return spans;
}

public Object reduceSpanSize(Object span, boolean hasError) {
public Reportable reduceSpanSize(Reportable span, boolean hasError) {
int maxFieldSize =
hasError
? Configuration.getInstance().maxSpanFieldSizeWhenError()
: Configuration.getInstance().maxSpanFieldSize();
if (span instanceof Span) {
Span functionSpan = (Span) span;
functionSpan.setEnvs(
StringUtils.getMaxSizeString(
functionSpan.getEnvs(),
Configuration.getInstance().maxSpanFieldSize()));
functionSpan.setReturn_value(
StringUtils.getMaxSizeString(functionSpan.getReturn_value(), maxFieldSize));
functionSpan.setEvent(
StringUtils.getMaxSizeString(functionSpan.getEvent(), maxFieldSize));
} else if (span instanceof HttpSpan) {
HttpSpan httpSpan = (HttpSpan) span;
httpSpan.getInfo()
.getHttpInfo()
.getRequest()
.setHeaders(
StringUtils.getMaxSizeString(
httpSpan.getInfo().getHttpInfo().getRequest().getHeaders(),
maxFieldSize));
httpSpan.getInfo()
.getHttpInfo()
.getRequest()
.setBody(
StringUtils.getMaxSizeString(
httpSpan.getInfo().getHttpInfo().getRequest().getBody(),
maxFieldSize));
httpSpan.getInfo()
.getHttpInfo()
.getResponse()
.setHeaders(
StringUtils.getMaxSizeString(
httpSpan.getInfo().getHttpInfo().getResponse().getHeaders(),
maxFieldSize));
httpSpan.getInfo()
.getHttpInfo()
.getResponse()
.setBody(
StringUtils.getMaxSizeString(
httpSpan.getInfo().getHttpInfo().getResponse().getBody(),
maxFieldSize));
}
return span;
return span.reduceSize(maxFieldSize);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ public class Configuration {
public static final String LUMIGO_MAX_RESPONSE_SIZE = "LUMIGO_MAX_RESPONSE_SIZE";
public static final String LUMIGO_MAX_SIZE_FOR_REQUEST = "LUMIGO_MAX_SIZE_FOR_REQUEST";
public static final String LUMIGO_INSTRUMENTATION = "LUMIGO_INSTRUMENTATION";
public static final String LUMIGO_SECRET_MASKING_REGEX = "LUMIGO_SECRET_MASKING_REGEX";

private static Configuration instance;
private LumigoConfiguration inlineConf;
Expand Down
5 changes: 3 additions & 2 deletions src/main/java/io/lumigo/core/network/Reporter.java
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import io.lumigo.core.configuration.Configuration;
import io.lumigo.core.utils.JsonUtils;
import io.lumigo.core.utils.StringUtils;
import io.lumigo.models.Reportable;
import java.io.IOException;
import java.util.Collections;
import java.util.LinkedList;
Expand All @@ -21,11 +22,11 @@ public Reporter() {
.build();
}

public long reportSpans(Object span, int maxSize) throws IOException {
public long reportSpans(Reportable span, int maxSize) throws IOException {
return reportSpans(Collections.singletonList(span), maxSize);
}

public long reportSpans(List<Object> spans, int maxSize) throws IOException {
public long reportSpans(List<Reportable> spans, int maxSize) throws IOException {
long time = System.currentTimeMillis();
List<String> spansAsStringList = new LinkedList<>();
int sizeCount = 0;
Expand Down
66 changes: 66 additions & 0 deletions src/main/java/io/lumigo/core/utils/SecretScrubber.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
package io.lumigo.core.utils;

import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;
import org.json.JSONArray;
import org.json.JSONObject;

public class SecretScrubber {
private List<Pattern> scrubbingPatterns;

private static final String SECRET_PLACEHOLDER = "****";

public SecretScrubber(Map<String, String> env) {
this.scrubbingPatterns = new SecretScrubbingPatternProvider(env).getScrubbingPatterns();
}

public String scrubStringifiedObject(String stringifiedObject) {
try {
JSONObject jsonObject = new JSONObject(stringifiedObject);
return scrubJsonObject(jsonObject, this.scrubbingPatterns).toString();
} catch (Exception e) {
return stringifiedObject;
}
}

private JSONObject scrubJsonObject(JSONObject jsonObject, List<Pattern> patterns) {
for (String key : jsonObject.keySet()) {
Object value = jsonObject.get(key);

if (value instanceof String && isSecret(key, patterns)) {
jsonObject.put(key, SECRET_PLACEHOLDER);
} else if (value instanceof JSONArray) {
ArrayList<Object> scrubbedArray = new ArrayList<>();

for (Object item : (JSONArray) value) {
if (item instanceof String && isSecret(key, patterns)) {
scrubbedArray.add(SECRET_PLACEHOLDER);
} else if (item instanceof JSONObject) {
scrubbedArray.add(scrubJsonObject((JSONObject) item, patterns));
} else {
scrubbedArray.add(item);
}
}

jsonObject.put(key, scrubbedArray.toArray());

} else if (value instanceof JSONObject) {
jsonObject.put(key, scrubJsonObject((JSONObject) value, patterns));
}
}

return jsonObject;
}

private boolean isSecret(String value, List<Pattern> patterns) {
for (Pattern pattern : patterns) {
if (pattern.matcher(value).matches()) {
return true;
}
}

return false;
}
}
Loading

0 comments on commit 7ebb712

Please sign in to comment.