From ca2ca4f73bd46aed93704876c0075b953b0ca5c5 Mon Sep 17 00:00:00 2001 From: acmarr Date: Mon, 7 Oct 2024 14:23:35 -0700 Subject: [PATCH 1/2] 9394 rewrite rocky fetcher to use new API source --- updater/fetchers/rocky/rocky.go | 303 ++++++++++++++++---------------- 1 file changed, 150 insertions(+), 153 deletions(-) diff --git a/updater/fetchers/rocky/rocky.go b/updater/fetchers/rocky/rocky.go index 41112e4..a9d594e 100644 --- a/updater/fetchers/rocky/rocky.go +++ b/updater/fetchers/rocky/rocky.go @@ -6,6 +6,7 @@ import ( "net/http" "strconv" "strings" + "time" "github.com/vul-dbgen/common" "github.com/vul-dbgen/updater" @@ -13,68 +14,42 @@ import ( log "github.com/sirupsen/logrus" ) -var endpoint = "https://apollo.build.resf.org/api/v3/advisories/" +var endpoint = "https://errata.rockylinux.org/api/v2/advisories?filters.product=&filters.fetchRelated=true&" +var limit = int64(100) type RockyFetcher struct{} type apiResponse struct { Advisories []advisory `json:"advisories"` + Total int64 `json:"total"` } type advisory struct { - Id int `json:"id"` - Created_at string `json:"created_at"` - Updated_at string `json:"updated_at"` - Published_at string `json:"published_at"` - Name string `json:"name"` - Synopsis string `json:"synopsis"` - Description string `json:"description"` - Kind string `json:"kind"` - Severity string `json:"severity"` - Topic string `json:"topic"` - Red_hat_advisory_id int `json:"red_hat_advisory_id"` - AffectedProducts []Product `json:"affected_products"` - Cves []cve `json:"cves"` - Packages []ApiPackage `json:"packages"` - Fixes []Fix `json:"fixes"` + AffectedProducts []string `json:"affectedProducts"` + Cves []cve `json:"cves"` + Description string `json:"description"` + Fixes []fix `json:"fixes"` + Name string `json:"name"` + PublishedAt string `json:"publishedAt"` + RPMs map[string]map[string][]string `json:"rpms"` + Severity string `json:"severity"` } -type cve struct { - Id int `json:"id"` - Cve string `json:"cve"` - CvssVector string `json:"cvss3_scoring_vector"` - CvssScore string `json:"cvss3_base_score"` - Cwe string `json:"cwe"` -} - -type ApiPackage struct { - Id int `json"id"` - Nevra string `json:"nevra"` - Checksum string `json:"checksum"` - ChecksumType string `json:"checksum_type"` - ModuleContext string `json:"module_context"` - ModuleName string `json:"module_name"` - ModuleStream string `json:"module_stream"` - ModuleVersion string `json:"module_version"` - RepoName string `json:"repo_name"` - PackageName string `json:"package_name"` - ProductName string `json:"product_name"` -} - -type Fix struct { - Id int `json:"id"` - TicketId string `json:"ticket_id"` - Source string `json:"source"` +type fix struct { Description string `json:"description"` + SourceBy string `json:"sourceBy"` + SourceLink string `json:"sourceLink"` + Ticket string `json:"ticket"` } -type Product struct { - Id int `json:"id"` - Variant string `json:"variant"` - Name string `json:"name"` - MajorVersion int `json:"major_version"` - MinorVersion int `json:"minor_version"` - Arch string `json:"arch"` +// cve cvss score are currently unfilled by API +type cve struct { + CvssVector string `json:"cvss3ScoringVector"` + Cvss3Score string `json:"cvss3BaseScore"` + Cwe string `json:"cwe"` + Name string `json:"name"` + SourceBy string `json:"sourceBy"` + SourceLink string `json:"sourceLink"` } func init() { @@ -89,146 +64,168 @@ func (f *RockyFetcher) FetchUpdate() (resp updater.FetcherResponse, err error) { log.WithFields(log.Fields{"err": err}).Error("Error fetching rocky remote") } //process data into standard format - resp.Vulnerabilities = parseRockyJSON(remoteResponse) - + resp.Vulnerabilities = translateRockyJSON(remoteResponse) log.WithFields(log.Fields{"Vulnerabilities": len(resp.Vulnerabilities)}).Info("fetching rocky done") return resp, nil } // fetchRemote retrieves and stores the api json response. func (f *RockyFetcher) fetchRemote() (*apiResponse, error) { - req, err := http.NewRequest("GET", endpoint, nil) - if err != nil { - log.WithFields(log.Fields{"err": err}).Error("Error creating rocky linux request") - return nil, err - } - client := http.Client{} - r, err := client.Do(req) - if err != nil { - log.WithFields(log.Fields{"err": err}).Error("Error retrieving rocky linux response") - return nil, err - } - body, err := ioutil.ReadAll(r.Body) + response, err := fetchRockyLinuxErrata() if err != nil { - log.WithFields(log.Fields{"err": err}).Error("Error reading rocky linux response") return nil, err } - var jsonResponse apiResponse - err = json.Unmarshal(body, &jsonResponse) - if err != nil { - log.WithFields(log.Fields{"err": err}).Error("Error unmarshalling rocky linux response") - } - log.WithFields(log.Fields{"number of advisories": len(jsonResponse.Advisories)}).Debug("Rocky reponse") - return &jsonResponse, nil + return response, nil } -// pruneDuplicates returns a slice with all duplicate entries removed from the input slice. -func pruneDuplicates(slice []common.FeatureVersion) []common.FeatureVersion { - prunedList := []common.FeatureVersion{} - set := map[string]common.FeatureVersion{} - for _, entry := range slice { - if _, ok := set[entry.Feature.Name+":"+entry.Version.String()]; !ok { - set[entry.Feature.Name+":"+entry.Version.String()] = entry - } - } - - for _, val := range set { - prunedList = append(prunedList, val) - } - - return prunedList -} - -func makeFV(fixPackage ApiPackage, majorVersion int) common.FeatureVersion { - //Split the nevra for the version - start := strings.Index(fixPackage.Nevra, ":") - last := strings.LastIndex(fixPackage.Nevra, ".") - last = strings.LastIndex(fixPackage.Nevra[:last], ".") - verString := fixPackage.Nevra[start-1 : last] - version, err := common.NewVersion(verString) - if err != nil { - log.WithFields(log.Fields{"err": err}).Error("Error making rocky linux feature version") - } +// translateRockyJSON translates the apiResponse struct to an array of common.Vulnerability to be used in fetcher response later. +func translateRockyJSON(response *apiResponse) []common.Vulnerability { + vulns := []common.Vulnerability{} - fixed := common.FeatureVersion{ - Feature: common.Feature{ - Name: fixPackage.PackageName, - Namespace: "rocky:" + strconv.Itoa(majorVersion), - }, - Version: version, - } - return fixed -} - -// parseRockyJSON takes the data and formats it into the format []common.Vulnerability. -func parseRockyJSON(data *apiResponse) []common.Vulnerability { - var vulns []common.Vulnerability - vulMap := map[string]common.Vulnerability{} - - //Iterate over advisory and make the corresponding vulnerability. - for _, advisory := range data.Advisories { - Namespaces := getNamespaces(advisory.AffectedProducts) - vuln := common.Vulnerability{ + for _, advisory := range response.Advisories { + fixedIns := map[string][]common.FeatureVersion{} + for key, rpms := range advisory.RPMs { + nvras := rpms["nvras"] + namespace := productToNamespace(key) + fixedIns[namespace] = nvraToFeatureVersion(nvras, namespace) + } + entry := common.Vulnerability{ Name: advisory.Name, Description: advisory.Description, + Link: "", Severity: translateSeverity(advisory.Severity), + IssuedDate: issuedDate(advisory.PublishedAt), + CVEs: []common.CVE{}, } - fixedIns := []common.FeatureVersion{} - for _, fix := range advisory.Packages { - fv := makeFV(fix, advisory.AffectedProducts[0].MajorVersion) - fixedIns = append(fixedIns, fv) + //populate CVEs + for _, cve := range advisory.Cves { + commonCVE := common.CVE{ + Name: cve.Name, + } + entry.CVEs = append(entry.CVEs, commonCVE) } - fixedIns = pruneDuplicates(fixedIns) - vuln.FixedIn = fixedIns - - //Add entry for each unique namespace - for _, ns := range Namespaces { - //Check if the advisory already exists in the namespace - vuln.Namespace = ns - if _, ok := vulMap[ns+":"+advisory.Name]; !ok { - vulMap[ns+":"+advisory.Name] = vuln - } else { - log.WithFields(log.Fields{"Name": advisory.Name}).Debug("Duplicate rocky advisory entry") - continue + //For each potential affected product we need to consider namespace changing + for _, productName := range advisory.AffectedProducts { + entry.Namespace = productToNamespace(productName) + if fi, ok := fixedIns[entry.Namespace]; ok { + entry.FixedIn = fi } + vulns = append(vulns, entry) } + } + return vulns +} + +func productToNamespace(product string) string { + lastSpace := strings.LastIndex(product, " ") + majorVersion := strings.Trim(product[lastSpace:], " ") + return "rocky:" + majorVersion +} +func issuedDate(dateString string) time.Time { + defTime := strings.Split(dateString, "T")[0] + if t, err := time.Parse("2006-01-02", defTime); err == nil { + return t + } else { + return time.Time{} } +} - //Make slice from map - for _, val := range vulMap { - vulns = append(vulns, val) +func nvraToFeatureVersion(nvras []string, namespace string) []common.FeatureVersion { + results := []common.FeatureVersion{} + //map with key modulename:moduleversion to prevent duplicates + set := map[string]struct{}{} + + for _, nvraString := range nvras { + //Remove rpm and arch sections + lastPeriod := strings.LastIndex(nvraString, ".") + nvraString = nvraString[:lastPeriod] + lastPeriod = strings.LastIndex(nvraString, ".") + nvraString = nvraString[:lastPeriod] + //Get module name from section before epoch + epochIndex := strings.Index(nvraString, ":") + moduleName := nvraString[:epochIndex-2] + //Remaining section is version + moduleVersion := nvraString[epochIndex-1:] + key := moduleName + ":" + moduleVersion + if _, ok := set[key]; ok { + continue + } else { + set[key] = struct{}{} + } + + fvVer, err := common.NewVersion(moduleVersion) + if err != nil { + log.WithFields(log.Fields{"err": err, "nvra": nvraString, "ftVer": fvVer}).Debug("Error converting nvra to FeatureVersion") + } + entry := common.FeatureVersion{ + Feature: common.Feature{ + Name: moduleName, + Namespace: namespace, + }, + Version: fvVer, + } + results = append(results, entry) } - return vulns + return results } -func getNamespaces(affectedProducts []Product) []string { - nsMap := map[string]string{} - for _, product := range affectedProducts { - majorVersion := strconv.Itoa(product.MajorVersion) - if _, ok := nsMap["rocky:"+majorVersion]; !ok { - nsMap["rocky:"+majorVersion] = "rocky:" + majorVersion +func fetchRockyLinuxErrata() (*apiResponse, error) { + results := &apiResponse{} + page := 0 + count := int64(0) + total := int64(1) + count2 := 0 + for count < total { + req, err := http.NewRequest("GET", endpoint+"page="+strconv.Itoa(page)+"&limit="+strconv.FormatInt(limit, 10), nil) + if err != nil { + log.WithFields(log.Fields{"err": err}).Error("Error creating rocky linux request") + return nil, err + } + client := http.Client{} + r, err := client.Do(req) + if err != nil { + log.WithFields(log.Fields{"err": err}).Error("Error retrieving rocky linux response") + return nil, err + } + body, err := ioutil.ReadAll(r.Body) + if err != nil { + log.WithFields(log.Fields{"err": err}).Error("Error reading rocky linux response") + return nil, err + } + var jsonResponse apiResponse + err = json.Unmarshal(body, &jsonResponse) + if err != nil { + log.WithFields(log.Fields{"err": err}).Error("Error unmarshalling rocky linux response") + } + results.Advisories = append(results.Advisories, jsonResponse.Advisories...) + total = jsonResponse.Total + count += limit + page++ + for _, adv := range jsonResponse.Advisories { + if len(adv.Cves) > 0 || len(adv.Fixes) > 0 { + count2++ + } } } - results := []string{} - for _, val := range nsMap { - results = append(results, val) - } - return results + log.WithFields(log.Fields{"number of advisories": len(results.Advisories)}).Debug("Rocky reponse") + return results, nil } func translateSeverity(incSev string) common.Priority { var severity common.Priority - switch strings.ToLower(incSev) { - case "important": + switch incSev { + case "SEVERITY_CRITICAL": + severity = common.Critical + case "SEVERITY_IMPORTANT": severity = common.High - case "moderate": + case "SEVERITY_MODERATE": severity = common.Medium - case "low": + case "SEVERITY_LOW": severity = common.Low - case "none": + case "SEVERITY_UNKNOWN": severity = common.Low default: log.WithFields(log.Fields{"sev": incSev}).Debug("unhandled severity") From 3f8a00bcfbb64c861ff951cc20e177cf93cc51c1 Mon Sep 17 00:00:00 2001 From: acmarr Date: Fri, 8 Nov 2024 10:18:46 -0800 Subject: [PATCH 2/2] NVSHAS-9620 change go version to 1.22.7 remove overly verbose suse fetcher logging NVSHAS-9620 --- .github/workflows/unitest.yaml | 2 +- go.mod | 7 +++++-- go.sum | 4 ---- updater/fetchers/suse/suse.go | 7 ------- 4 files changed, 6 insertions(+), 14 deletions(-) diff --git a/.github/workflows/unitest.yaml b/.github/workflows/unitest.yaml index 0f3441f..e7c0211 100644 --- a/.github/workflows/unitest.yaml +++ b/.github/workflows/unitest.yaml @@ -11,6 +11,6 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-go@v2 with: - go-version: '1.14.15' + go-version: '1.22.7' - run: | ./unitest.sh diff --git a/go.mod b/go.mod index b852756..0515b72 100644 --- a/go.mod +++ b/go.mod @@ -1,11 +1,14 @@ module github.com/vul-dbgen -go 1.14 +go 1.22.7 require ( github.com/k3a/html2text v1.0.8 github.com/sirupsen/logrus v1.8.1 + gopkg.in/yaml.v2 v2.4.0 +) + +require ( github.com/stretchr/testify v1.7.0 // indirect golang.org/x/sys v0.0.0-20211019181941-9d821ace8654 // indirect - gopkg.in/yaml.v2 v2.4.0 ) diff --git a/go.sum b/go.sum index ba6b1d3..853f9d7 100644 --- a/go.sum +++ b/go.sum @@ -15,17 +15,13 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykE github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/streadway/simpleuuid v0.0.0-20130420165545-6617b501e485 h1:tvEO2/Btzw9L4N2VlAHD7AXjk1g1yFTwbGEm8dz7QWY= -github.com/streadway/simpleuuid v0.0.0-20130420165545-6617b501e485/go.mod h1:fMlyZAyOBbIsA9SgKX9V3X8DvF+5ImkZ+Z1HZcmo8Ec= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654 h1:id054HUawV2/6IGm2IV8KZQjqtwAOo2CYlOToYqa0d0= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/updater/fetchers/suse/suse.go b/updater/fetchers/suse/suse.go index 1bb15db..4851274 100644 --- a/updater/fetchers/suse/suse.go +++ b/updater/fetchers/suse/suse.go @@ -237,8 +237,6 @@ func parseOVAL(o *ovalInfo, ovalReader io.Reader) ([]common.Vulnerability, error vulnerability.CVEs = append(vulnerability.CVEs, common.CVE{ Name: cve, }) - } else { - log.WithFields(log.Fields{"definition": definition.Title, "id": r.ID}).Debug("defintion entry missing cve ID") } } if vulnerability.IssuedDate.IsZero() { @@ -254,7 +252,6 @@ func parseOVAL(o *ovalInfo, ovalReader io.Reader) ([]common.Vulnerability, error // } } } - return vulnerabilities, nil } @@ -371,8 +368,6 @@ func parsePackageVersions(o *ovalInfo, cvename string, criteria criteria, testMa } else { fv.Feature.Namespace = fmt.Sprintf("%s%s", o.nsPrefix, ti.version) } - } else { - log.WithFields(log.Fields{"cve": cvename, "test": c.TestRef}).Warn("Failed locate test record") } } else if !strings.HasPrefix(c.Comment, "SUSE") && (strings.Contains(c.Comment, " is installed") || strings.Contains(c.Comment, " is not affected")) { // This is the package line @@ -384,8 +379,6 @@ func parsePackageVersions(o *ovalInfo, cvename string, criteria criteria, testMa fv.Version = ti.version fv.Feature.Name = ti.name - } else { - log.WithFields(log.Fields{"cve": cvename, "test": c.TestRef}).Warn("Failed locate test record") } } }