NVSHAS-8481: Merge CPE and FixedIn for feed entries of same vuln

common/debug.go

@@ -49,6 +49,7 @@ func DEBUG_VULN(x interface{}, msg string) {
 			if Debugs.CVEs.Contains(v.Name) {
 				log.WithFields(log.Fields{
 					"name": v.Name, "distro": v.Namespace, "severity": v.Severity, "v2": v.CVSSv2, "v3": v.CVSSv3, "rate": v.FeedRating,
+					"fix": v.FixedIn, "cpes": v.CPEs,
 					"pub": v.IssuedDate.Format(time.RFC3339), "lastMod": v.LastModDate.Format(time.RFC3339),
 					"description": firstN(v.Description, 64),
 				}).Debug(msg)

common/types.go

@@ -14,6 +14,7 @@ const RHELCpeMapFile = "rhel-cpe.map"
 
 type NVDMetadata struct {
 	Description      string `json:"description,omitempty"`
+	Severity         Priority
 	CVSSv2           CVSS
 	CVSSv3           CVSS
 	VulnVersions     []NVDvulnerableVersion
@@ -62,19 +63,18 @@ type VulShort struct {
 }
 
 type FeaFull struct {
-	Name      string `json:"N"`
-	Namespace string `json:"NS"`
-	Version   string `json:"V"`
-	MinVer    string `json:"MV"`
-	AddedBy   string `json:"A"`
+	Name    string `json:"N"`
+	Version string `json:"V"`
+	MinVer  string `json:"MV"`
+	AddedBy string `json:"A"`
 }
 
 type VulFull struct {
 	Name        string    `json:"N"`
 	Namespace   string    `json:"NS"`
 	Description string    `json:"D"`
 	Link        string    `json:"L"`
-	Severity    string    `json:"S"`
+	Severity    Priority  `json:"S"`
 	CVSSv2      CVSS      `json:"C2"`
 	CVSSv3      CVSS      `json:"C3"`
 	FixedBy     string    `json:"FB"`

dbgen.go

@@ -13,6 +13,7 @@ import (
 	"github.com/vul-dbgen/common"
 	utils "github.com/vul-dbgen/share"
 	"github.com/vul-dbgen/updater"
+
 	_ "github.com/vul-dbgen/updater/fetchers/alpine"
 	_ "github.com/vul-dbgen/updater/fetchers/amazon"
 	_ "github.com/vul-dbgen/updater/fetchers/apps"
@@ -37,7 +38,7 @@ func main() {
 
 	version := flag.String("v", "0.90", "cve database version")
 	dbPath := flag.String("d", "", "cve database path")
-	debug := flag.String("debug", "", "debug filters")
+	debug := flag.String("debug", "", "debug filters. -debug v=CVE-2023-1000")
 	flag.Usage = usage
 	flag.Parse()
 

memdb.go

@@ -55,7 +55,7 @@ func modVulToVulFull(v *common.Vulnerability) *common.VulFull {
 	vv1.Namespace = v.Namespace
 	vv1.Description = v.Description
 	vv1.Link = v.Link
-	vv1.Severity = string(v.Severity)
+	vv1.Severity = v.Severity
 	vv1.FeedRating = v.FeedRating
 	vv1.CPEs = v.CPEs
 	vv1.CVEs = make([]string, len(v.CVEs))
@@ -72,10 +72,9 @@ func modVulToVulFull(v *common.Vulnerability) *common.VulFull {
 
 func modFeaToFeaFull(fx common.FeatureVersion) common.FeaFull {
 	var v1fx = common.FeaFull{
-		Name:      fx.Feature.Name,
-		Namespace: fx.Feature.Namespace,
-		Version:   fx.Version.String(),
-		MinVer:    fx.MinVer.String(),
+		Name:    fx.Feature.Name,
+		Version: fx.Version.String(),
+		MinVer:  fx.MinVer.String(),
 	}
 	return v1fx
 }
@@ -288,7 +287,22 @@ func (db *memDB) InsertVulnerabilities(osVuls []*common.Vulnerability, appVuls [
 			vv1.FixedIn = append(vv1.FixedIn, v1fx)
 		}
 		cveName := fmt.Sprintf("%s:%s", vv1.Namespace, vv1.Name)
-		db.osVuls[cveName] = vv1
+		if vf, ok := db.osVuls[cveName]; ok {
+			fixes := utils.NewSetFromSliceKind(vf.FixedIn)
+			cpes := utils.NewSetFromSliceKind(vf.CPEs)
+			for _, f := range vv1.FixedIn {
+				if !fixes.Contains(f) {
+					vf.FixedIn = append(vf.FixedIn, f)
+				}
+			}
+			for _, c := range vv1.CPEs {
+				if !cpes.Contains(c) {
+					vf.CPEs = append(vf.CPEs, c)
+				}
+			}
+		} else {
+			db.osVuls[cveName] = vv1
+		}
 	}
 	db.appVuls = appVuls
 

updater/fetchers/rhel2/rhel.go

@@ -141,7 +141,7 @@ func (f *RHELFetcher) fetchPreDownload(rhelFolder string) ([]common.Vulnerabilit
 		}
 		for _, f := range files {
 			if strings.HasSuffix(f.Name(), ".xml.bz2") {
-				log.WithFields(log.Fields{"file": f.Name()}).Debug("Read redhat feed")
+				log.WithFields(log.Fields{"os": ros, "file": f.Name()}).Debug("Read redhat feed")
 
 				rfp, err := os.Open(fmt.Sprintf("%s/%s", folder, f.Name()))
 				cr := bzip2.NewReader(rfp)
@@ -450,7 +450,7 @@ func parseRHSA(ros int, rhsa string, ovalReader io.Reader) (vulnerabilities []co
 
 		pkgs := toFeatureVersions(ros, rhsa, nameId, definition.Criteria)
 		if len(pkgs) > 0 {
-			vulnerability := common.Vulnerability{
+			v := common.Vulnerability{
 				Name:        nameId,
 				Namespace:   "centos" + ":" + strconv.Itoa(ros),
 				Link:        link(definition),
@@ -461,14 +461,12 @@ func parseRHSA(ros int, rhsa string, ovalReader io.Reader) (vulnerabilities []co
 				CPEs:        definition.CpeList.CPEs,
 				FeedRating:  definition.Severity,
 			}
-			if vulnerability.Link == "" {
-				vulnerability.Link = cveLink(definition)
+			if v.Link == "" {
+				v.Link = cveLink(definition)
 			}
-			// if vulnerability.Severity == common.Unknown {
-			// 	log.WithFields(log.Fields{"nameId": nameId, "rhsa": rhsa}).Error("\"Unknown\" severity")
-			// }
+
 			for _, p := range pkgs {
-				vulnerability.FixedIn = append(vulnerability.FixedIn, p)
+				v.FixedIn = append(v.FixedIn, p)
 			}
 			for _, r := range definition.Cves {
 				var v2, v3 string
@@ -485,19 +483,29 @@ func parseRHSA(ros int, rhsa string, ovalReader io.Reader) (vulnerabilities []co
 						v3 = r.Cvss3[s+1:]
 					}
 				}
-				vulnerability.CVEs = append(vulnerability.CVEs, common.CVE{
+				cve := common.CVE{
 					Name:   r.ID,
 					CVSSv2: common.CVSS{Vectors: v2, Score: s2},
 					CVSSv3: common.CVSS{Vectors: v3, Score: s3},
-				})
+				}
+				if s2 > v.CVSSv2.Score {
+					v.CVSSv2 = cve.CVSSv2
+				}
+				if s3 > v.CVSSv3.Score {
+					v.CVSSv3 = cve.CVSSv3
+				}
+				v.CVEs = append(v.CVEs, cve)
 			}
-			if vulnerability.IssuedDate.IsZero() {
-				vulnerability.IssuedDate = vulnerability.LastModDate
+			if v.IssuedDate.IsZero() {
+				v.IssuedDate = v.LastModDate
 			}
-			if vulnerability.LastModDate.IsZero() {
-				vulnerability.LastModDate = vulnerability.IssuedDate
+			if v.LastModDate.IsZero() {
+				v.LastModDate = v.IssuedDate
 			}
-			vulnerabilities = append(vulnerabilities, vulnerability)
+
+			common.DEBUG_VULN(&v, "redhat")
+
+			vulnerabilities = append(vulnerabilities, v)
 		}
 	}
 

updater/fetchers/ubuntu/ubuntu.go

@@ -304,7 +304,7 @@ func parseUbuntuCVE(fileContent io.Reader) (vulnerability common.Vulnerability,
 				priority = priority[:strings.Index(priority, " ")]
 			}
 
-			vulnerability.Severity = ubuntuPriorityToSeverity(priority)
+			vulnerability.Severity = toSeverity(priority)
 			vulnerability.FeedRating = priority
 			continue
 		}
@@ -409,7 +409,7 @@ func parseUbuntuCVE(fileContent io.Reader) (vulnerability common.Vulnerability,
 	return
 }
 
-func ubuntuPriorityToSeverity(priority string) common.Priority {
+func toSeverity(priority string) common.Priority {
 	switch priority {
 	case "untriaged":
 		return common.Unknown

updater/nvd/nvd.go

@@ -75,7 +75,7 @@ type NvdCve struct {
 				Source                  string   `json:"source"`
 				Type                    string   `json:"type"`
 				CvssData                CvssData `json:"cvssData"`
-				Severity                string   `json:"severity"`
+				Severity                string   `json:"baseSeverity"`
 				ExploitabilityScore     float64  `json:"exploitabilityScore"`
 				ImpactScore             float64  `json:"impactScore"`
 				ObtainAllPrivilege      bool     `json:"obtainAllPrivilege"`
@@ -124,6 +124,7 @@ type CvssData struct {
 	IntegrityImpact       string  `json:"integrityImpact"`
 	AvailabilityImpact    string  `json:"availabilityImpact"`
 	BaseScore             float64 `json:"baseScore"`
+	BaseSeverity          string  `json:"baseSeverity"`
 }
 
 var NVD NVDMetadataFetcher
@@ -247,17 +248,22 @@ func (fetcher *NVDMetadataFetcher) Load() error {
 			meta.Description = cve.Cve.Description[0].Value
 		}
 		if cve.Cve.ID != "" {
-			//Prefer CVSS31 over CVSS30 if it exists.
+			// Prefer CVSS31 over CVSS30 if it exists.
 			if len(cve.Cve.Metrics.BaseMetricV31) > 0 && cve.Cve.Metrics.BaseMetricV31[0].CvssData.BaseScore != 0 {
 				meta.CVSSv3.Vectors = cve.Cve.Metrics.BaseMetricV31[0].CvssData.VectorString
 				meta.CVSSv3.Score = cve.Cve.Metrics.BaseMetricV31[0].CvssData.BaseScore
+				meta.Severity = fetcher.toSeverity(cve.Cve.Metrics.BaseMetricV31[0].CvssData.BaseSeverity)
 			} else if len(cve.Cve.Metrics.BaseMetricV3) > 0 && cve.Cve.Metrics.BaseMetricV3[0].CvssData.BaseScore != 0 {
 				meta.CVSSv3.Vectors = cve.Cve.Metrics.BaseMetricV3[0].CvssData.VectorString
 				meta.CVSSv3.Score = cve.Cve.Metrics.BaseMetricV3[0].CvssData.BaseScore
+				meta.Severity = fetcher.toSeverity(cve.Cve.Metrics.BaseMetricV3[0].CvssData.BaseSeverity)
 			}
 			if len(cve.Cve.Metrics.BaseMetricV2) > 0 && cve.Cve.Metrics.BaseMetricV2[0].CvssData.BaseScore != 0 {
 				meta.CVSSv2.Vectors = cve.Cve.Metrics.BaseMetricV2[0].CvssData.VectorString
 				meta.CVSSv2.Score = cve.Cve.Metrics.BaseMetricV2[0].CvssData.BaseScore
+				if meta.Severity == "" {
+					meta.Severity = fetcher.toSeverity(cve.Cve.Metrics.BaseMetricV2[0].Severity)
+				}
 			}
 			if cve.Cve.PublishedDate != "" {
 				// Use new format, try old format if parse fails.
@@ -314,6 +320,22 @@ func (fetcher *NVDMetadataFetcher) Load() error {
 	return nil
 }
 
+func (fetcher *NVDMetadataFetcher) toSeverity(s string) common.Priority {
+	switch s {
+	case "LOW":
+		return common.Low
+	case "MEDIUM":
+		return common.Medium
+	case "HIGH":
+		return common.High
+	case "CRITICAL":
+		return common.Critical
+	}
+
+	// return empty instead of Unknown
+	return ""
+}
+
 func (fetcher *NVDMetadataFetcher) GetMetadata(cve string) (*common.NVDMetadata, bool) {
 	if nvd, ok := fetcher.metadata[cve]; ok {
 		var description string
@@ -324,6 +346,7 @@ func (fetcher *NVDMetadataFetcher) GetMetadata(cve string) (*common.NVDMetadata,
 		}
 		return &common.NVDMetadata{
 			Description:      description,
+			Severity:         nvd.Severity,
 			CVSSv3:           nvd.CVSSv3,
 			CVSSv2:           nvd.CVSSv2,
 			PublishedDate:    nvd.PublishedDate,

updater/updater.go

@@ -210,6 +210,9 @@ func enrichAppMeta(meta *common.NVDMetadata, v *common.AppModuleVul) {
 		meta.CVSSv2.Score = v.Score
 		meta.CVSSv2.Vectors = v.Vectors
 	}
+	if meta.Severity == "" || meta.Severity == common.Unknown {
+		meta.Severity = v.Severity
+	}
 	if meta.PublishedDate.IsZero() {
 		meta.PublishedDate = v.IssuedDate
 	}
@@ -228,6 +231,9 @@ func enrichDistroMeta(meta *common.NVDMetadata, v *common.Vulnerability, cve *co
 	if meta.CVSSv2.Score == 0 {
 		meta.CVSSv2 = cve.CVSSv2
 	}
+	if meta.Severity == "" || meta.Severity == common.Unknown {
+		meta.Severity = v.Severity
+	}
 	if meta.PublishedDate.IsZero() {
 		meta.PublishedDate = v.IssuedDate
 	}
@@ -302,6 +308,7 @@ func assignMetadata(vuls []*common.Vulnerability, apps []*common.AppModuleVul) (
 					meta = &common.NVDMetadata{
 						CVSSv3:           cve.CVSSv3,
 						CVSSv2:           cve.CVSSv2,
+						Severity:         v.Severity,
 						PublishedDate:    v.IssuedDate,
 						LastModifiedDate: v.LastModDate,
 					}
@@ -333,6 +340,7 @@ func assignMetadata(vuls []*common.Vulnerability, apps []*common.AppModuleVul) (
 					meta = &common.NVDMetadata{
 						CVSSv3:           common.CVSS{Score: app.ScoreV3, Vectors: app.VectorsV3},
 						CVSSv2:           common.CVSS{Score: app.Score, Vectors: app.Vectors},
+						Severity:         app.Severity,
 						PublishedDate:    app.IssuedDate,
 						LastModifiedDate: app.LastModDate,
 					}
@@ -372,6 +380,9 @@ func assignMetadata(vuls []*common.Vulnerability, apps []*common.AppModuleVul) (
 				if cvss2.Score == 0 {
 					cvss2 = meta.CVSSv2
 				}
+				if v.Severity == "" || v.Severity == common.Unknown {
+					v.Severity = meta.Severity
+				}
 			}
 		}