Skip to content

Commit

Permalink
Merge pull request #20 from rapid7/dotnet-parser
Browse files Browse the repository at this point in the history
Dotnet parser
  • Loading branch information
bholden-r7 authored Oct 29, 2021
2 parents 5446635 + 29e4df1 commit ac9a020
Show file tree
Hide file tree
Showing 9 changed files with 224 additions and 13 deletions.
2 changes: 0 additions & 2 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,6 @@
<commons.codec.version>1.12</commons.codec.version>
<commons.compress.version>1.19</commons.compress.version>
<commons.io.version>2.6</commons.io.version>
<!-- <jackson.version>[2.9.9,2.10.0)</jackson.version>-->
<jackson.version>2.11.4</jackson.version>
<slf4j.version>1.7.30</slf4j.version>
<dependency.check.version>5.3.2</dependency.check.version>
Expand All @@ -69,7 +68,6 @@
</properties>

<dependencies>

<!-- 3rd party dependencies -->
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
package com.rapid7.container.analyzer.docker.fingerprinter;

import com.rapid7.container.analyzer.docker.analyzer.LayerFileHandler;
import com.rapid7.container.analyzer.docker.model.LayerPath;
import com.rapid7.container.analyzer.docker.model.image.Image;
import com.rapid7.container.analyzer.docker.model.json.Configuration;
import com.rapid7.container.analyzer.docker.packages.DotNetParser;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Paths;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;

public class DotNetFingerprinter implements LayerFileHandler {

private final DotNetParser parser;

public DotNetFingerprinter(DotNetParser parser) {
this.parser = parser;
}

@Override
public void handle(String name, TarArchiveEntry entry, InputStream contents, Image image, Configuration configuration, LayerPath layerPath) throws IOException {
if (parser.supports(name, entry)) {
File tmpFile = Paths.get(layerPath.getPath(), name).toFile();
if (tmpFile.isFile()) {
layerPath.getLayer().addPackages(parser.parse(tmpFile, image.getOperatingSystem() == null
? layerPath.getLayer().getOperatingSystem() : image.getOperatingSystem()));
}
}
}

}
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.rapid7.container.analyzer.docker.model.json;

import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonSubTypes;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
import com.rapid7.container.analyzer.docker.model.image.ImageId;
Expand All @@ -18,15 +19,15 @@
})
public interface Manifest {

public List<LayerId> getLayers();
List<LayerId> getLayers();

public default List<LayerId> getLayerBlobIds() {
default List<LayerId> getLayerBlobIds() {
return getLayers();
}

public ImageId getImageId();
ImageId getImageId();

public long getSize();
long getSize();

public String getType();
String getType();
}
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.rapid7.container.analyzer.docker.os;

import com.rapid7.container.analyzer.docker.analyzer.LayerFileHandler;
import com.rapid7.container.analyzer.docker.model.image.OperatingSystem;
import java.io.BufferedReader;
import java.io.IOException;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
package com.rapid7.container.analyzer.docker.packages;

import com.rapid7.container.analyzer.docker.model.image.OperatingSystem;
import com.rapid7.container.analyzer.docker.model.image.Package;
import com.rapid7.container.analyzer.docker.model.image.PackageType;
import java.io.File;
import java.io.IOException;
import java.util.HashSet;
import java.util.Set;
import java.util.regex.Pattern;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;

public class DotNetParser implements PackageParser<File> {

private static final Logger LOGGER = LoggerFactory.getLogger(DotNetParser.class);
private static final Pattern DOT_NET_PATTERN = Pattern.compile(".*(?i)(\\.nuspec)$");

@Override
public boolean supports(String name, TarArchiveEntry entry) {
return !entry.isSymbolicLink() && DOT_NET_PATTERN.matcher(name).matches();
}

@Override
public Set<Package> parse(File input, OperatingSystem operatingSystem) throws IOException {

Set<Package> packages = new HashSet<>();
try {
DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
DocumentBuilder db = dbFactory.newDocumentBuilder();
Document document = db.parse(input);
document.getDocumentElement().normalize();

NodeList nodeList = document.getElementsByTagName("package");
for (int i = 0; i < nodeList.getLength(); i++) {
Node node = nodeList.item(i);

if (node.getNodeType() == Node.ELEMENT_NODE) {
Element element = (Element) node;

String source = input.getName();
String name = getValueForAttribute(element, "id");
String version = getValueForAttribute(element, "version");
String description = getValueForAttribute(element, "description");
packages.add(new Package(source, PackageType.DOTNET, operatingSystem, name, version, description, null, null, null, null));
}
}
} catch (ParserConfigurationException | SAXException exception) {
LOGGER.error("Could not parse .nuspec file", exception);
}

return packages;
}


private String getValueForAttribute(Element element, String attribute) {
NodeList nodeList = element.getElementsByTagName(attribute);
if (nodeList.getLength() > 0) {
return nodeList.item(0).getTextContent();
} else {
return null;
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
package com.rapid7.container.analyzer.docker.packages.settings;

import com.google.common.collect.ImmutableMap;
import com.rapid7.container.analyzer.docker.analyzer.LayerFileHandler;
import com.rapid7.container.analyzer.docker.fingerprinter.DotNetFingerprinter;
import com.rapid7.container.analyzer.docker.model.image.PackageType;
import com.rapid7.container.analyzer.docker.packages.DotNetParser;
import java.util.Collection;
import java.util.HashSet;
import java.util.Set;

public class CustomParserSettingsBuilder {

// Mappings for customer parsers
private static final ImmutableMap<PackageType, LayerFileHandler> FINGERPRINTER_MAPPINGS = ImmutableMap.of(
PackageType.DOTNET, new DotNetFingerprinter(new DotNetParser())
);

public static final CustomParserSettingsBuilder ALL = CustomParserSettingsBuilder.builder()
.addFingerprinters(FINGERPRINTER_MAPPINGS.keySet());

private final Set<LayerFileHandler> enabledFingerprinters = new HashSet<>();

private CustomParserSettingsBuilder() {
}

public static CustomParserSettingsBuilder builder() {
return new CustomParserSettingsBuilder();
}

public CustomParserSettingsBuilder addFingerprinter(PackageType packageType) {
LayerFileHandler handler = FINGERPRINTER_MAPPINGS.get(packageType);
if (handler != null) {
enabledFingerprinters.add(handler);
}
return this;
}

public CustomParserSettingsBuilder addFingerprinters(Collection<PackageType> packageTypes) {
for (PackageType packageType : packageTypes) {
addFingerprinter(packageType);
}
return this;
}

public Set<LayerFileHandler> getFingerprinters() {
return enabledFingerprinters;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import com.rapid7.container.analyzer.docker.analyzer.LayerExtractor;
import com.rapid7.container.analyzer.docker.analyzer.LayerFileHandler;
import com.rapid7.container.analyzer.docker.fingerprinter.ApkgFingerprinter;
import com.rapid7.container.analyzer.docker.fingerprinter.DotNetFingerprinter;
import com.rapid7.container.analyzer.docker.fingerprinter.DpkgFingerprinter;
import com.rapid7.container.analyzer.docker.fingerprinter.FileFingerprinter;
import com.rapid7.container.analyzer.docker.fingerprinter.OsReleaseFingerprinter;
Expand All @@ -31,10 +32,12 @@
import com.rapid7.container.analyzer.docker.model.json.TarManifestJson;
import com.rapid7.container.analyzer.docker.os.Fingerprinter;
import com.rapid7.container.analyzer.docker.packages.ApkgParser;
import com.rapid7.container.analyzer.docker.packages.DotNetParser;
import com.rapid7.container.analyzer.docker.packages.DpkgParser;
import com.rapid7.container.analyzer.docker.packages.OwaspDependencyParser;
import com.rapid7.container.analyzer.docker.packages.PacmanPackageParser;
import com.rapid7.container.analyzer.docker.packages.RpmPackageParser;
import com.rapid7.container.analyzer.docker.packages.settings.CustomParserSettingsBuilder;
import com.rapid7.container.analyzer.docker.packages.settings.OwaspDependencyParserSettingsBuilder;
import com.rapid7.container.analyzer.docker.util.InstantParser;
import com.rapid7.container.analyzer.docker.util.InstantParserModule;
Expand All @@ -57,8 +60,10 @@
import java.util.Objects;
import java.util.zip.GZIPInputStream;
import java.util.zip.ZipException;
import java.util.zip.ZipInputStream;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
import org.apache.commons.compress.utils.IOUtils;
import org.apache.commons.io.FilenameUtils;
import org.slf4j.Logger;
Expand All @@ -80,7 +85,15 @@ public DockerImageAnalyzerService(String rpmDockerImage) {
this(rpmDockerImage, OwaspDependencyParserSettingsBuilder.EXPERIMENTAL);
}

public DockerImageAnalyzerService(String rpmDockerImage, CustomParserSettingsBuilder customBuilder) {
this(rpmDockerImage, OwaspDependencyParserSettingsBuilder.EXPERIMENTAL, customBuilder);
}

public DockerImageAnalyzerService(String rpmDockerImage, OwaspDependencyParserSettingsBuilder builder) {
this(rpmDockerImage, builder, CustomParserSettingsBuilder.builder());
}

public DockerImageAnalyzerService(String rpmDockerImage, OwaspDependencyParserSettingsBuilder owaspBuilder, CustomParserSettingsBuilder customBuilder) {
objectMapper = new ObjectMapper();
objectMapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES);
objectMapper.registerModule(new InstantParserModule());
Expand All @@ -92,7 +105,8 @@ public DockerImageAnalyzerService(String rpmDockerImage, OwaspDependencyParserSe
layerHandlers.add(new DpkgFingerprinter(new DpkgParser()));
layerHandlers.add(new ApkgFingerprinter(new ApkgParser()));
layerHandlers.add(new PacmanFingerprinter(new PacmanPackageParser()));
layerHandlers.add(new OwaspDependencyFingerprinter(new OwaspDependencyParser(builder)));
layerHandlers.add(new OwaspDependencyFingerprinter(new OwaspDependencyParser(owaspBuilder)));
layerHandlers.addAll(customBuilder.getFingerprinters());
}

public void addFileHandler(LayerFileHandler handler) {
Expand Down Expand Up @@ -250,7 +264,12 @@ else if (previousLayer != null && previousLayer.getCreated() != null)
}

public Manifest parseManifest(File file) throws JsonParseException, JsonMappingException, IOException {
return objectMapper.readValue(file, Manifest.class); // TODO: polymorphic
// TODO: polymorphic
try (GZIPInputStream stream = new GZIPInputStream(new FileInputStream(file))) {
return objectMapper.readValue(stream, Manifest.class);
} catch (ZipException exception) {
return objectMapper.readValue(file, Manifest.class);
}
}

public Configuration parseConfiguration(File file) throws JsonParseException, JsonMappingException, IOException {
Expand Down Expand Up @@ -315,11 +334,13 @@ private void processLayer(Image image, Configuration configuration, Layer layer,
if (tar.length() < 100)
return;

try (TarArchiveInputStream tarIn = new TarArchiveInputStream(new GZIPInputStream(new FileInputStream(tar), 65536))) {
try (TarArchiveInputStream tarIn = new TarArchiveInputStream(new GzipCompressorInputStream(new BufferedInputStream(new FileInputStream(tar), 65536)))) {
processLayerTar(image, configuration, layer, tar, tarIn);
} catch (ZipException ze) {
try (TarArchiveInputStream tarIn = new TarArchiveInputStream(new BufferedInputStream(new FileInputStream(tar), 65536))) {
processLayerTar(image, configuration, layer, tar, tarIn);
} catch (IOException exception) {
if (exception.getMessage().equals("Input is not in the .gz format") || exception.getCause() instanceof ZipException) {
try (TarArchiveInputStream tarIn = new TarArchiveInputStream(new BufferedInputStream(new FileInputStream(tar), 65536))) {
processLayerTar(image, configuration, layer, tar, tarIn);
}
}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,23 @@

import com.rapid7.container.analyzer.docker.model.image.Image;
import com.rapid7.container.analyzer.docker.model.image.ImageId;
import com.rapid7.container.analyzer.docker.model.image.Layer;
import com.rapid7.container.analyzer.docker.model.image.OperatingSystem;
import com.rapid7.container.analyzer.docker.model.image.Package;
import com.rapid7.container.analyzer.docker.model.image.PackageType;
import com.rapid7.container.analyzer.docker.packages.settings.CustomParserSettingsBuilder;
import com.rapid7.container.analyzer.docker.packages.settings.OwaspDependencyParserSettingsBuilder;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import org.junit.jupiter.api.Test;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.is;
import static org.junit.jupiter.api.Assertions.assertEquals;

class DockerImageAnalyzerServiceTest {
Expand Down Expand Up @@ -57,4 +68,28 @@ public void testCentos() throws IOException {
assertEquals(expectedPackages, image.getPackages().size());
assertEquals("A set of system configuration and setup files", image.getPackages().stream().findFirst().get().getDescription());
}


@Test
public void parseDotnet() throws FileNotFoundException, IOException {
// given
File tarFile = new File(getClass().getClassLoader().getResource("containers/dotnet-packages.tar").getFile());

// when
DockerImageAnalyzerService analyzer = new DockerImageAnalyzerService(null, OwaspDependencyParserSettingsBuilder.builder(),
CustomParserSettingsBuilder.builder().addFingerprinter(PackageType.DOTNET));
Path tmpdir = Files.createTempDirectory("r7dia");
Image image = analyzer.analyze(tarFile, tmpdir.toString());

// then
List<Layer> layersWithDotnetPackages = image.getLayers().stream()
.filter(layer -> layer.getPackages().stream().anyMatch(pkg -> pkg.getType() == PackageType.DOTNET))
.collect(Collectors.toList());
Set<Package> dotnetPackages = image.getPackages().stream()
.filter(pkg -> pkg.getType() == PackageType.DOTNET)
.collect(Collectors.toSet());

assertThat(layersWithDotnetPackages.size(), is(1));
assertThat(dotnetPackages.size(), is(5));
}
}
Binary file added src/test/resources/containers/dotnet-packages.tar
Binary file not shown.

0 comments on commit ac9a020

Please sign in to comment.