Optimized synchronizeVulnerability

src/main/java/org/dependencytrack/parser/snyk/SnykParser.java

@@ -102,6 +102,7 @@ public Vulnerability parse(JSONArray data, QueryManager qm, String purl, int cou
             }
             final List<VulnerableSoftware> vsListOld = qm.detach(qm.getVulnerableSoftwareByVulnId(vulnerability.getSource(), vulnerability.getVulnId()));
             synchronizedVulnerability = qm.synchronizeVulnerability(vulnerability, false);
+            if (synchronizedVulnerability == null) return vulnerability;
             qm.persist(vsList);
             qm.updateAffectedVersionAttributions(synchronizedVulnerability, vsList, Vulnerability.Source.SNYK);
             vsList = qm.reconcileVulnerableSoftware(synchronizedVulnerability, vsListOld, vsList, Vulnerability.Source.SNYK);

src/main/java/org/dependencytrack/parser/vulndb/ModelConverter.java

@@ -94,12 +94,12 @@ public static Vulnerability convert(final QueryManager qm, final org.dependencyt
             final OffsetDateTime odt = OffsetDateTime.parse(vulnDbVuln.disclosureDate());
             vuln.setPublished(Date.from(odt.toInstant()));
         }
-        /*
-        if (StringUtils.isNotBlank(vulnDbVuln.getUpdatedAt())) {
-            final OffsetDateTime odt = OffsetDateTime.parse(vulnDbVuln.getUpdatedAt());
+
+        if (StringUtils.isNotBlank(vulnDbVuln.lastModified())) {
+            final OffsetDateTime odt = OffsetDateTime.parse(vulnDbVuln.lastModified());
             vuln.setUpdated(Date.from(odt.toInstant()));
         }
-        */
+
 
 
         /* References */

src/main/java/org/dependencytrack/parser/vulndb/VulnDbParser.java

@@ -18,6 +18,15 @@
  */
 package org.dependencytrack.parser.vulndb;
 
+import java.io.File;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.nio.charset.Charset;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+
 import org.apache.commons.lang3.StringUtils;
 import org.dependencytrack.parser.vulndb.model.ApiObject;
 import org.dependencytrack.parser.vulndb.model.Author;
@@ -40,15 +49,6 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.File;
-import java.io.IOException;
-import java.math.BigDecimal;
-import java.nio.charset.Charset;
-import java.nio.file.Files;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.List;
-
 /*
  * Model class needed by VulnDBAnalysis task. Class brought over from the vulndb-data-mirror repo:
  * <a href="https://github.com/stevespringett/vulndb-data-mirror">...</a>
@@ -329,6 +329,7 @@ private List<Vulnerability> parseVulnerabilities(JSONArray rso) {
                         StringUtils.trimToNull(object.optString("short_description", (String) null)),
                         StringUtils.trimToNull(object.optString("description", (String) null)),
                         StringUtils.trimToNull(object.optString("solution", (String) null)),
+                        StringUtils.trimToNull(object.optString("vulndb_last_modified", (String) null)),
                         StringUtils.trimToNull(object.optString("manual_notes", (String) null)),
                         StringUtils.trimToNull(object.optString("t_description", (String) null)),
                         StringUtils.trimToNull(object.optString("solution_date", (String) null)),

src/main/java/org/dependencytrack/parser/vulndb/model/Vulnerability.java

@@ -34,6 +34,7 @@ public record Vulnerability(int id,
                             String shortDescription,
                             String description,
                             String solution,
+                            String lastModified,
                             String manualNotes,
                             String technicalDescription,
                             String solutionDate,

src/main/java/org/dependencytrack/persistence/VulnerabilityQueryManager.java

@@ -18,6 +18,7 @@
  */
 package org.dependencytrack.persistence;
 
+import alpine.common.logging.Logger;
 import alpine.event.framework.Event;
 import alpine.persistence.PaginatedResult;
 import alpine.resources.AlpineRequest;
@@ -33,7 +34,9 @@
 import org.dependencytrack.model.VulnerabilityAlias;
 import org.dependencytrack.model.VulnerableSoftware;
 import org.dependencytrack.resources.v1.vo.AffectedProject;
+import org.dependencytrack.tasks.VulnDbSyncTask;
 import org.dependencytrack.tasks.scanners.AnalyzerIdentity;
+import org.dependencytrack.util.PersistenceUtil;
 
 import javax.jdo.PersistenceManager;
 import javax.jdo.Query;
@@ -50,7 +53,7 @@
 import java.util.stream.Collectors;
 
 final class VulnerabilityQueryManager extends QueryManager implements IQueryManager {
-
+    private static final Logger LOGGER = Logger.getLogger(VulnDbSyncTask.class);
     /**
      * Constructs a new QueryManager.
      * @param pm a PersistenceManager object
@@ -81,57 +84,69 @@ public Vulnerability createVulnerability(Vulnerability vulnerability, boolean co
         return result;
     }
 
+    private boolean hasChanges(Vulnerability vulnerability, Vulnerability transientVulnerability) {
+        return vulnerability.getUpdated() == null || transientVulnerability.getUpdated() == null || !vulnerability.getUpdated().equals(transientVulnerability.getUpdated());
+    }
+
+    private Vulnerability getExistingVulnerability(Vulnerability transientVulnerability){
+        if (transientVulnerability == null) {
+            return null;
+        }
+        if (transientVulnerability.getId() > 0) {
+            return getObjectById(Vulnerability.class, transientVulnerability.getId());
+        }
+        return getVulnerabilityByVulnId(transientVulnerability.getSource(), transientVulnerability.getVulnId());
+    }
+
     /**
      * Updates a vulnerability.
      * @param transientVulnerability the vulnerability to update
      * @param commitIndex specifies if the search index should be committed (an expensive operation)
      * @return a Vulnerability object
      */
+    @Override
     public Vulnerability updateVulnerability(Vulnerability transientVulnerability, boolean commitIndex) {
-        final Vulnerability vulnerability;
-        if (transientVulnerability.getId() > 0) {
-            vulnerability = getObjectById(Vulnerability.class, transientVulnerability.getId());
-        } else {
-            vulnerability = getVulnerabilityByVulnId(transientVulnerability.getSource(), transientVulnerability.getVulnId());
-        }
-        if (vulnerability != null) {
-            vulnerability.setCreated(transientVulnerability.getCreated());
-            vulnerability.setPublished(transientVulnerability.getPublished());
-            vulnerability.setUpdated(transientVulnerability.getUpdated());
-            vulnerability.setVulnId(transientVulnerability.getVulnId());
-            vulnerability.setSource(transientVulnerability.getSource());
-            vulnerability.setCredits(transientVulnerability.getCredits());
-            vulnerability.setVulnerableVersions(transientVulnerability.getVulnerableVersions());
-            vulnerability.setPatchedVersions(transientVulnerability.getPatchedVersions());
-            vulnerability.setDescription(transientVulnerability.getDescription());
-            vulnerability.setDetail(transientVulnerability.getDetail());
-            vulnerability.setTitle(transientVulnerability.getTitle());
-            vulnerability.setSubTitle(transientVulnerability.getSubTitle());
-            vulnerability.setReferences(transientVulnerability.getReferences());
-            vulnerability.setRecommendation(transientVulnerability.getRecommendation());
-            vulnerability.setSeverity(transientVulnerability.getSeverity());
-            vulnerability.setCvssV2Vector(transientVulnerability.getCvssV2Vector());
-            vulnerability.setCvssV2BaseScore(transientVulnerability.getCvssV2BaseScore());
-            vulnerability.setCvssV2ImpactSubScore(transientVulnerability.getCvssV2ImpactSubScore());
-            vulnerability.setCvssV2ExploitabilitySubScore(transientVulnerability.getCvssV2ExploitabilitySubScore());
-            vulnerability.setCvssV3Vector(transientVulnerability.getCvssV3Vector());
-            vulnerability.setCvssV3BaseScore(transientVulnerability.getCvssV3BaseScore());
-            vulnerability.setCvssV3ImpactSubScore(transientVulnerability.getCvssV3ImpactSubScore());
-            vulnerability.setCvssV3ExploitabilitySubScore(transientVulnerability.getCvssV3ExploitabilitySubScore());
-            vulnerability.setOwaspRRLikelihoodScore(transientVulnerability.getOwaspRRLikelihoodScore());
-            vulnerability.setOwaspRRBusinessImpactScore(transientVulnerability.getOwaspRRBusinessImpactScore());
-            vulnerability.setOwaspRRTechnicalImpactScore(transientVulnerability.getOwaspRRTechnicalImpactScore());
-            vulnerability.setOwaspRRVector(transientVulnerability.getOwaspRRVector());
-            vulnerability.setCwes(transientVulnerability.getCwes());
-            if (transientVulnerability.getVulnerableSoftware() != null) {
-                vulnerability.setVulnerableSoftware(transientVulnerability.getVulnerableSoftware());
+        return callInTransaction(() -> {
+            Vulnerability existingVulnerability = getExistingVulnerability(transientVulnerability);
+            if (existingVulnerability != null) {
+                final PersistenceUtil.Differ<Vulnerability> differ = new PersistenceUtil.Differ<>(existingVulnerability, transientVulnerability);
+                differ.applyIfChanged("created", Vulnerability::getCreated, existingVulnerability::setCreated);
+                differ.applyIfChanged("published", Vulnerability::getPublished, existingVulnerability::setPublished);
+                differ.applyIfChanged("updated", Vulnerability::getUpdated, existingVulnerability::setUpdated);
+                differ.applyIfChanged("vulnId", Vulnerability::getVulnId, existingVulnerability::setVulnId);
+                differ.applyIfChanged("source", Vulnerability::getSource, existingVulnerability::setSource);
+                differ.applyIfChanged("credits", Vulnerability::getCredits, existingVulnerability::setCredits);
+                differ.applyIfChanged("vulnerableVersions", Vulnerability::getVulnerableVersions, existingVulnerability::setVulnerableVersions);
+                differ.applyIfChanged("patchedVersions", Vulnerability::getPatchedVersions, existingVulnerability::setPatchedVersions);
+                differ.applyIfChanged("description", Vulnerability::getDescription, existingVulnerability::setDescription);
+                differ.applyIfChanged("detail", Vulnerability::getDetail, existingVulnerability::setDetail);
+                differ.applyIfChanged("title", Vulnerability::getTitle, existingVulnerability::setTitle);
+                differ.applyIfChanged("subTitle", Vulnerability::getSubTitle, existingVulnerability::setSubTitle);
+                differ.applyIfChanged("references", Vulnerability::getReferences, existingVulnerability::setReferences);
+                differ.applyIfChanged("recommendation", Vulnerability::getRecommendation, existingVulnerability::setRecommendation);
+                differ.applyIfChanged("severity", Vulnerability::getSeverity, existingVulnerability::setSeverity);
+                differ.applyIfChanged("cvssV2Vector", Vulnerability::getCvssV2Vector, existingVulnerability::setCvssV2Vector);
+                differ.applyIfChanged("cvssV2BaseScore", Vulnerability::getCvssV2BaseScore, existingVulnerability::setCvssV2BaseScore);
+                differ.applyIfChanged("cvssV2ImpactSubScore", Vulnerability::getCvssV2ImpactSubScore, existingVulnerability::setCvssV2ImpactSubScore);
+                differ.applyIfChanged("cvssV2ExploitabilitySubScore", Vulnerability::getCvssV2ExploitabilitySubScore, existingVulnerability::setCvssV2ExploitabilitySubScore);
+                differ.applyIfChanged("cvssV3Vector", Vulnerability::getCvssV3Vector, existingVulnerability::setCvssV3Vector);
+                differ.applyIfChanged("cvssV3BaseScore", Vulnerability::getCvssV3BaseScore, existingVulnerability::setCvssV3BaseScore);
+                differ.applyIfChanged("cvssV3ImpactSubScore", Vulnerability::getCvssV3ImpactSubScore, existingVulnerability::setCvssV3ImpactSubScore);
+                differ.applyIfChanged("cvssV3ExploitabilitySubScore", Vulnerability::getCvssV3ExploitabilitySubScore, existingVulnerability::setCvssV3ExploitabilitySubScore);
+                differ.applyIfChanged("owaspRRLikelihoodScore", Vulnerability::getOwaspRRLikelihoodScore, existingVulnerability::setOwaspRRLikelihoodScore);
+                differ.applyIfChanged("owaspRRBusinessImpactScore", Vulnerability::getOwaspRRBusinessImpactScore, existingVulnerability::setOwaspRRBusinessImpactScore);
+                differ.applyIfChanged("owaspRRTechnicalImpactScore", Vulnerability::getOwaspRRTechnicalImpactScore, existingVulnerability::setOwaspRRTechnicalImpactScore);
+                differ.applyIfChanged("owaspRRVector", Vulnerability::getOwaspRRVector, existingVulnerability::setOwaspRRVector);
+                differ.applyIfChanged("cwes", Vulnerability::getCwes, existingVulnerability::setCwes);
+                differ.applyIfNonNullAndChanged("vulnerableSoftware", Vulnerability::getVulnerableSoftware, existingVulnerability::setVulnerableSoftware);
             }
-            final Vulnerability result = persist(vulnerability);
-            Event.dispatch(new IndexEvent(IndexEvent.Action.UPDATE, result));
-            commitSearchIndex(commitIndex, Vulnerability.class);
-            return result;
-        }
-        return null;
+            else {
+                // Handle cases where no existing vulnerability is found if needed (e.g., log an error)
+                LOGGER.warn("No existing vulnerability found for update operation.");
+            }
+            //return updated Vulnerability
+            return existingVulnerability;
+        });
     }
 
     /**
@@ -142,12 +157,20 @@ public Vulnerability updateVulnerability(Vulnerability transientVulnerability, b
      * @param commitIndex specifies if the search index should be committed (an expensive operation)
      * @return a Vulnerability object
      */
+    @Override
     public Vulnerability synchronizeVulnerability(Vulnerability vulnerability, boolean commitIndex) {
-        Vulnerability result = updateVulnerability(vulnerability, commitIndex);
-        if (result == null) {
-            result = createVulnerability(vulnerability, commitIndex);
-        }
-        return result;
+        return callInTransaction(() -> {
+            Vulnerability existingVulnerability  = getExistingVulnerability(vulnerability);
+            if (existingVulnerability == null) {
+                // Create new vulnerability if it doesnt exist
+                return  createVulnerability(vulnerability, commitIndex);
+            } else {
+                // Update only if changes are detected
+                return hasChanges(existingVulnerability, vulnerability)
+                ? updateVulnerability(vulnerability, commitIndex)
+                : null;
+            }
+        });
     }
 
     /**

src/main/java/org/dependencytrack/tasks/GitHubAdvisoryMirrorTask.java

@@ -18,17 +18,17 @@
  */
 package org.dependencytrack.tasks;
 
-import alpine.common.logging.Logger;
-import alpine.event.framework.Event;
-import alpine.event.framework.LoggableSubscriber;
-import alpine.model.ConfigProperty;
-import alpine.notification.Notification;
-import alpine.notification.NotificationLevel;
-import com.github.packageurl.MalformedPackageURLException;
-import com.github.packageurl.PackageURL;
-import com.github.packageurl.PackageURLBuilder;
-import io.pebbletemplates.pebble.PebbleEngine;
-import io.pebbletemplates.pebble.template.PebbleTemplate;
+import java.io.IOException;
+import java.io.StringWriter;
+import java.io.Writer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
 import org.apache.commons.lang3.tuple.Pair;
 import org.apache.http.HttpStatus;
 import org.apache.http.client.methods.CloseableHttpResponse;
@@ -38,6 +38,9 @@
 import org.dependencytrack.common.HttpClientPool;
 import org.dependencytrack.event.GitHubAdvisoryMirrorEvent;
 import org.dependencytrack.event.IndexEvent;
+import static org.dependencytrack.model.ConfigPropertyConstants.VULNERABILITY_SOURCE_GITHUB_ADVISORIES_ACCESS_TOKEN;
+import static org.dependencytrack.model.ConfigPropertyConstants.VULNERABILITY_SOURCE_GITHUB_ADVISORIES_ALIAS_SYNC_ENABLED;
+import static org.dependencytrack.model.ConfigPropertyConstants.VULNERABILITY_SOURCE_GITHUB_ADVISORIES_ENABLED;
 import org.dependencytrack.model.Cwe;
 import org.dependencytrack.model.Severity;
 import org.dependencytrack.model.Vulnerability;
@@ -54,20 +57,18 @@
 import org.dependencytrack.persistence.QueryManager;
 import org.json.JSONObject;
 
-import java.io.IOException;
-import java.io.StringWriter;
-import java.io.Writer;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import com.github.packageurl.MalformedPackageURLException;
+import com.github.packageurl.PackageURL;
+import com.github.packageurl.PackageURLBuilder;
 
-import static org.dependencytrack.model.ConfigPropertyConstants.VULNERABILITY_SOURCE_GITHUB_ADVISORIES_ACCESS_TOKEN;
-import static org.dependencytrack.model.ConfigPropertyConstants.VULNERABILITY_SOURCE_GITHUB_ADVISORIES_ALIAS_SYNC_ENABLED;
-import static org.dependencytrack.model.ConfigPropertyConstants.VULNERABILITY_SOURCE_GITHUB_ADVISORIES_ENABLED;
+import alpine.common.logging.Logger;
+import alpine.event.framework.Event;
+import alpine.event.framework.LoggableSubscriber;
+import alpine.model.ConfigProperty;
+import alpine.notification.Notification;
+import alpine.notification.NotificationLevel;
+import io.pebbletemplates.pebble.PebbleEngine;
+import io.pebbletemplates.pebble.template.PebbleTemplate;
 
 public class GitHubAdvisoryMirrorTask implements LoggableSubscriber {
 
@@ -193,6 +194,7 @@ void updateDatasource(final List<GitHubSecurityAdvisory> advisories) {
                 final Vulnerability mappedVulnerability = mapAdvisoryToVulnerability(qm, advisory);
                 final List<VulnerableSoftware> vsListOld = qm.detach(qm.getVulnerableSoftwareByVulnId(mappedVulnerability.getSource(), mappedVulnerability.getVulnId()));
                 final Vulnerability synchronizedVulnerability = qm.synchronizeVulnerability(mappedVulnerability, false);
+                if (synchronizedVulnerability == null) continue;
                 List<VulnerableSoftware> vsList = new ArrayList<>();
                 for (GitHubVulnerability ghvuln : advisory.getVulnerabilities()) {
                     final VulnerableSoftware vs = mapVulnerabilityToVulnerableSoftware(qm, ghvuln, advisory);