Skip to content

Commit

Permalink
Merge pull request #4359 from LaVibeX/lastModifiedIndicator
Browse files Browse the repository at this point in the history
  • Loading branch information
@nscuro
nscuro authored Nov 12, 2024
2 parents ba4400e + f08804f commit f7e2fa4
Show file tree
Hide file tree
Showing 13 changed files with 281 additions and 135 deletions.
1 change: 1 addition & 0 deletions src/main/java/org/dependencytrack/parser/snyk/SnykParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,7 @@ public Vulnerability parse(JSONArray data, QueryManager qm, String purl, int cou
}
final List<VulnerableSoftware> vsListOld = qm.detach(qm.getVulnerableSoftwareByVulnId(vulnerability.getSource(), vulnerability.getVulnId()));
synchronizedVulnerability = qm.synchronizeVulnerability(vulnerability, false);
if (synchronizedVulnerability == null) return vulnerability;
qm.persist(vsList);
qm.updateAffectedVersionAttributions(synchronizedVulnerability, vsList, Vulnerability.Source.SNYK);
vsList = qm.reconcileVulnerableSoftware(synchronizedVulnerability, vsListOld, vsList, Vulnerability.Source.SNYK);
Expand Down
8 changes: 4 additions & 4 deletions src/main/java/org/dependencytrack/parser/vulndb/ModelConverter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,12 +94,12 @@ public static Vulnerability convert(final QueryManager qm, final org.dependencyt
final OffsetDateTime odt = OffsetDateTime.parse(vulnDbVuln.disclosureDate());
vuln.setPublished(Date.from(odt.toInstant()));
}
/*
if (StringUtils.isNotBlank(vulnDbVuln.getUpdatedAt())) {
final OffsetDateTime odt = OffsetDateTime.parse(vulnDbVuln.getUpdatedAt());

if (StringUtils.isNotBlank(vulnDbVuln.lastModified())) {
final OffsetDateTime odt = OffsetDateTime.parse(vulnDbVuln.lastModified());
vuln.setUpdated(Date.from(odt.toInstant()));
}
*/



/* References */
Expand Down
19 changes: 10 additions & 9 deletions src/main/java/org/dependencytrack/parser/vulndb/VulnDbParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,15 @@
*/
package org.dependencytrack.parser.vulndb;

import java.io.File;
import java.io.IOException;
import java.math.BigDecimal;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;

import org.apache.commons.lang3.StringUtils;
import org.dependencytrack.parser.vulndb.model.ApiObject;
import org.dependencytrack.parser.vulndb.model.Author;
Expand All @@ -40,15 +49,6 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.IOException;
import java.math.BigDecimal;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;

/*
* Model class needed by VulnDBAnalysis task. Class brought over from the vulndb-data-mirror repo:
* <a href="https://github.com/stevespringett/vulndb-data-mirror">...</a>
Expand Down Expand Up @@ -329,6 +329,7 @@ private List<Vulnerability> parseVulnerabilities(JSONArray rso) {
StringUtils.trimToNull(object.optString("short_description", (String) null)),
StringUtils.trimToNull(object.optString("description", (String) null)),
StringUtils.trimToNull(object.optString("solution", (String) null)),
StringUtils.trimToNull(object.optString("vulndb_last_modified", (String) null)),
StringUtils.trimToNull(object.optString("manual_notes", (String) null)),
StringUtils.trimToNull(object.optString("t_description", (String) null)),
StringUtils.trimToNull(object.optString("solution_date", (String) null)),
Expand Down
1 change: 1 addition & 0 deletions src/main/java/org/dependencytrack/parser/vulndb/model/Vulnerability.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ public record Vulnerability(int id,
String shortDescription,
String description,
String solution,
String lastModified,
String manualNotes,
String technicalDescription,
String solutionDate,
Expand Down
121 changes: 72 additions & 49 deletions src/main/java/org/dependencytrack/persistence/VulnerabilityQueryManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
*/
package org.dependencytrack.persistence;

import alpine.common.logging.Logger;
import alpine.event.framework.Event;
import alpine.persistence.PaginatedResult;
import alpine.resources.AlpineRequest;
Expand All @@ -33,7 +34,9 @@
import org.dependencytrack.model.VulnerabilityAlias;
import org.dependencytrack.model.VulnerableSoftware;
import org.dependencytrack.resources.v1.vo.AffectedProject;
import org.dependencytrack.tasks.VulnDbSyncTask;
import org.dependencytrack.tasks.scanners.AnalyzerIdentity;
import org.dependencytrack.util.PersistenceUtil;

import javax.jdo.PersistenceManager;
import javax.jdo.Query;
Expand All @@ -50,7 +53,7 @@
import java.util.stream.Collectors;

final class VulnerabilityQueryManager extends QueryManager implements IQueryManager {

private static final Logger LOGGER = Logger.getLogger(VulnDbSyncTask.class);
/**
* Constructs a new QueryManager.
* @param pm a PersistenceManager object
Expand Down Expand Up @@ -81,57 +84,69 @@ public Vulnerability createVulnerability(Vulnerability vulnerability, boolean co
return result;
}

private boolean hasChanges(Vulnerability vulnerability, Vulnerability transientVulnerability) {
return vulnerability.getUpdated() == null || transientVulnerability.getUpdated() == null || !vulnerability.getUpdated().equals(transientVulnerability.getUpdated());
}

private Vulnerability getExistingVulnerability(Vulnerability transientVulnerability){
if (transientVulnerability == null) {
return null;
}
if (transientVulnerability.getId() > 0) {
return getObjectById(Vulnerability.class, transientVulnerability.getId());
}
return getVulnerabilityByVulnId(transientVulnerability.getSource(), transientVulnerability.getVulnId());
}

/**
* Updates a vulnerability.
* @param transientVulnerability the vulnerability to update
* @param commitIndex specifies if the search index should be committed (an expensive operation)
* @return a Vulnerability object
*/
@Override
public Vulnerability updateVulnerability(Vulnerability transientVulnerability, boolean commitIndex) {
final Vulnerability vulnerability;
if (transientVulnerability.getId() > 0) {
vulnerability = getObjectById(Vulnerability.class, transientVulnerability.getId());
} else {
vulnerability = getVulnerabilityByVulnId(transientVulnerability.getSource(), transientVulnerability.getVulnId());
}
if (vulnerability != null) {
vulnerability.setCreated(transientVulnerability.getCreated());
vulnerability.setPublished(transientVulnerability.getPublished());
vulnerability.setUpdated(transientVulnerability.getUpdated());
vulnerability.setVulnId(transientVulnerability.getVulnId());
vulnerability.setSource(transientVulnerability.getSource());
vulnerability.setCredits(transientVulnerability.getCredits());
vulnerability.setVulnerableVersions(transientVulnerability.getVulnerableVersions());
vulnerability.setPatchedVersions(transientVulnerability.getPatchedVersions());
vulnerability.setDescription(transientVulnerability.getDescription());
vulnerability.setDetail(transientVulnerability.getDetail());
vulnerability.setTitle(transientVulnerability.getTitle());
vulnerability.setSubTitle(transientVulnerability.getSubTitle());
vulnerability.setReferences(transientVulnerability.getReferences());
vulnerability.setRecommendation(transientVulnerability.getRecommendation());
vulnerability.setSeverity(transientVulnerability.getSeverity());
vulnerability.setCvssV2Vector(transientVulnerability.getCvssV2Vector());
vulnerability.setCvssV2BaseScore(transientVulnerability.getCvssV2BaseScore());
vulnerability.setCvssV2ImpactSubScore(transientVulnerability.getCvssV2ImpactSubScore());
vulnerability.setCvssV2ExploitabilitySubScore(transientVulnerability.getCvssV2ExploitabilitySubScore());
vulnerability.setCvssV3Vector(transientVulnerability.getCvssV3Vector());
vulnerability.setCvssV3BaseScore(transientVulnerability.getCvssV3BaseScore());
vulnerability.setCvssV3ImpactSubScore(transientVulnerability.getCvssV3ImpactSubScore());
vulnerability.setCvssV3ExploitabilitySubScore(transientVulnerability.getCvssV3ExploitabilitySubScore());
vulnerability.setOwaspRRLikelihoodScore(transientVulnerability.getOwaspRRLikelihoodScore());
vulnerability.setOwaspRRBusinessImpactScore(transientVulnerability.getOwaspRRBusinessImpactScore());
vulnerability.setOwaspRRTechnicalImpactScore(transientVulnerability.getOwaspRRTechnicalImpactScore());
vulnerability.setOwaspRRVector(transientVulnerability.getOwaspRRVector());
vulnerability.setCwes(transientVulnerability.getCwes());
if (transientVulnerability.getVulnerableSoftware() != null) {
vulnerability.setVulnerableSoftware(transientVulnerability.getVulnerableSoftware());
return callInTransaction(() -> {
Vulnerability existingVulnerability = getExistingVulnerability(transientVulnerability);
if (existingVulnerability != null) {
final PersistenceUtil.Differ<Vulnerability> differ = new PersistenceUtil.Differ<>(existingVulnerability, transientVulnerability);
differ.applyIfChanged("created", Vulnerability::getCreated, existingVulnerability::setCreated);
differ.applyIfChanged("published", Vulnerability::getPublished, existingVulnerability::setPublished);
differ.applyIfChanged("updated", Vulnerability::getUpdated, existingVulnerability::setUpdated);
differ.applyIfChanged("vulnId", Vulnerability::getVulnId, existingVulnerability::setVulnId);
differ.applyIfChanged("source", Vulnerability::getSource, existingVulnerability::setSource);
differ.applyIfChanged("credits", Vulnerability::getCredits, existingVulnerability::setCredits);
differ.applyIfChanged("vulnerableVersions", Vulnerability::getVulnerableVersions, existingVulnerability::setVulnerableVersions);
differ.applyIfChanged("patchedVersions", Vulnerability::getPatchedVersions, existingVulnerability::setPatchedVersions);
differ.applyIfChanged("description", Vulnerability::getDescription, existingVulnerability::setDescription);
differ.applyIfChanged("detail", Vulnerability::getDetail, existingVulnerability::setDetail);
differ.applyIfChanged("title", Vulnerability::getTitle, existingVulnerability::setTitle);
differ.applyIfChanged("subTitle", Vulnerability::getSubTitle, existingVulnerability::setSubTitle);
differ.applyIfChanged("references", Vulnerability::getReferences, existingVulnerability::setReferences);
differ.applyIfChanged("recommendation", Vulnerability::getRecommendation, existingVulnerability::setRecommendation);
differ.applyIfChanged("severity", Vulnerability::getSeverity, existingVulnerability::setSeverity);
differ.applyIfChanged("cvssV2Vector", Vulnerability::getCvssV2Vector, existingVulnerability::setCvssV2Vector);
differ.applyIfChanged("cvssV2BaseScore", Vulnerability::getCvssV2BaseScore, existingVulnerability::setCvssV2BaseScore);
differ.applyIfChanged("cvssV2ImpactSubScore", Vulnerability::getCvssV2ImpactSubScore, existingVulnerability::setCvssV2ImpactSubScore);
differ.applyIfChanged("cvssV2ExploitabilitySubScore", Vulnerability::getCvssV2ExploitabilitySubScore, existingVulnerability::setCvssV2ExploitabilitySubScore);
differ.applyIfChanged("cvssV3Vector", Vulnerability::getCvssV3Vector, existingVulnerability::setCvssV3Vector);
differ.applyIfChanged("cvssV3BaseScore", Vulnerability::getCvssV3BaseScore, existingVulnerability::setCvssV3BaseScore);
differ.applyIfChanged("cvssV3ImpactSubScore", Vulnerability::getCvssV3ImpactSubScore, existingVulnerability::setCvssV3ImpactSubScore);
differ.applyIfChanged("cvssV3ExploitabilitySubScore", Vulnerability::getCvssV3ExploitabilitySubScore, existingVulnerability::setCvssV3ExploitabilitySubScore);
differ.applyIfChanged("owaspRRLikelihoodScore", Vulnerability::getOwaspRRLikelihoodScore, existingVulnerability::setOwaspRRLikelihoodScore);
differ.applyIfChanged("owaspRRBusinessImpactScore", Vulnerability::getOwaspRRBusinessImpactScore, existingVulnerability::setOwaspRRBusinessImpactScore);
differ.applyIfChanged("owaspRRTechnicalImpactScore", Vulnerability::getOwaspRRTechnicalImpactScore, existingVulnerability::setOwaspRRTechnicalImpactScore);
differ.applyIfChanged("owaspRRVector", Vulnerability::getOwaspRRVector, existingVulnerability::setOwaspRRVector);
differ.applyIfChanged("cwes", Vulnerability::getCwes, existingVulnerability::setCwes);
differ.applyIfNonNullAndChanged("vulnerableSoftware", Vulnerability::getVulnerableSoftware, existingVulnerability::setVulnerableSoftware);
}
final Vulnerability result = persist(vulnerability);
Event.dispatch(new IndexEvent(IndexEvent.Action.UPDATE, result));
commitSearchIndex(commitIndex, Vulnerability.class);
return result;
}
return null;
else {
// Handle cases where no existing vulnerability is found if needed (e.g., log an error)
LOGGER.warn("No existing vulnerability found for update operation.");
}
//return updated Vulnerability
return existingVulnerability;
});
}

/**
Expand All @@ -142,12 +157,20 @@ public Vulnerability updateVulnerability(Vulnerability transientVulnerability, b
* @param commitIndex specifies if the search index should be committed (an expensive operation)
* @return a Vulnerability object
*/
@Override
public Vulnerability synchronizeVulnerability(Vulnerability vulnerability, boolean commitIndex) {
Vulnerability result = updateVulnerability(vulnerability, commitIndex);
if (result == null) {
result = createVulnerability(vulnerability, commitIndex);
}
return result;
return callInTransaction(() -> {
Vulnerability existingVulnerability = getExistingVulnerability(vulnerability);
if (existingVulnerability == null) {
// Create new vulnerability if it doesnt exist
return createVulnerability(vulnerability, commitIndex);
} else {
// Update only if changes are detected
return hasChanges(existingVulnerability, vulnerability)
? updateVulnerability(vulnerability, commitIndex)
: null;
}
});
}

/**
Expand Down
50 changes: 26 additions & 24 deletions src/main/java/org/dependencytrack/tasks/GitHubAdvisoryMirrorTask.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,17 +18,17 @@
*/
package org.dependencytrack.tasks;

import alpine.common.logging.Logger;
import alpine.event.framework.Event;
import alpine.event.framework.LoggableSubscriber;
import alpine.model.ConfigProperty;
import alpine.notification.Notification;
import alpine.notification.NotificationLevel;
import com.github.packageurl.MalformedPackageURLException;
import com.github.packageurl.PackageURL;
import com.github.packageurl.PackageURLBuilder;
import io.pebbletemplates.pebble.PebbleEngine;
import io.pebbletemplates.pebble.template.PebbleTemplate;
import java.io.IOException;
import java.io.StringWriter;
import java.io.Writer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.apache.commons.lang3.tuple.Pair;
import org.apache.http.HttpStatus;
import org.apache.http.client.methods.CloseableHttpResponse;
Expand All @@ -38,6 +38,9 @@
import org.dependencytrack.common.HttpClientPool;
import org.dependencytrack.event.GitHubAdvisoryMirrorEvent;
import org.dependencytrack.event.IndexEvent;
import static org.dependencytrack.model.ConfigPropertyConstants.VULNERABILITY_SOURCE_GITHUB_ADVISORIES_ACCESS_TOKEN;
import static org.dependencytrack.model.ConfigPropertyConstants.VULNERABILITY_SOURCE_GITHUB_ADVISORIES_ALIAS_SYNC_ENABLED;
import static org.dependencytrack.model.ConfigPropertyConstants.VULNERABILITY_SOURCE_GITHUB_ADVISORIES_ENABLED;
import org.dependencytrack.model.Cwe;
import org.dependencytrack.model.Severity;
import org.dependencytrack.model.Vulnerability;
Expand All @@ -54,20 +57,18 @@
import org.dependencytrack.persistence.QueryManager;
import org.json.JSONObject;

import java.io.IOException;
import java.io.StringWriter;
import java.io.Writer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import com.github.packageurl.MalformedPackageURLException;
import com.github.packageurl.PackageURL;
import com.github.packageurl.PackageURLBuilder;

import static org.dependencytrack.model.ConfigPropertyConstants.VULNERABILITY_SOURCE_GITHUB_ADVISORIES_ACCESS_TOKEN;
import static org.dependencytrack.model.ConfigPropertyConstants.VULNERABILITY_SOURCE_GITHUB_ADVISORIES_ALIAS_SYNC_ENABLED;
import static org.dependencytrack.model.ConfigPropertyConstants.VULNERABILITY_SOURCE_GITHUB_ADVISORIES_ENABLED;
import alpine.common.logging.Logger;
import alpine.event.framework.Event;
import alpine.event.framework.LoggableSubscriber;
import alpine.model.ConfigProperty;
import alpine.notification.Notification;
import alpine.notification.NotificationLevel;
import io.pebbletemplates.pebble.PebbleEngine;
import io.pebbletemplates.pebble.template.PebbleTemplate;

public class GitHubAdvisoryMirrorTask implements LoggableSubscriber {

Expand Down Expand Up @@ -193,6 +194,7 @@ void updateDatasource(final List<GitHubSecurityAdvisory> advisories) {
final Vulnerability mappedVulnerability = mapAdvisoryToVulnerability(qm, advisory);
final List<VulnerableSoftware> vsListOld = qm.detach(qm.getVulnerableSoftwareByVulnId(mappedVulnerability.getSource(), mappedVulnerability.getVulnId()));
final Vulnerability synchronizedVulnerability = qm.synchronizeVulnerability(mappedVulnerability, false);
if (synchronizedVulnerability == null) continue;
List<VulnerableSoftware> vsList = new ArrayList<>();
for (GitHubVulnerability ghvuln : advisory.getVulnerabilities()) {
final VulnerableSoftware vs = mapVulnerabilityToVulnerableSoftware(qm, ghvuln, advisory);
Expand Down
Loading

0 comments on commit f7e2fa4

Please sign in to comment.