From 61fee7def22ea1f0bdafdf5ef6cd1913d7d0a78b Mon Sep 17 00:00:00 2001 From: MaineK00n Date: Sat, 4 May 2024 06:23:00 +0900 Subject: [PATCH] feat(fetch/redhat): fetch including-unpatched --- commands/fetch-redhat.go | 2 +- db/db.go | 4 +- db/db_test.go | 29 ++++++++++++ db/rdb.go | 10 ++-- fetcher/redhat/redhat.go | 8 ++-- models/models.go | 20 +++++++- models/redhat/redhat.go | 89 +++++++++++++++++++++++------------- models/redhat/redhat_test.go | 42 +++++++++++++++++ models/redhat/types.go | 2 +- 9 files changed, 163 insertions(+), 43 deletions(-) diff --git a/commands/fetch-redhat.go b/commands/fetch-redhat.go index 5dc401f6..53cd878b 100644 --- a/commands/fetch-redhat.go +++ b/commands/fetch-redhat.go @@ -85,7 +85,7 @@ func fetchRedHat(_ *cobra.Command, args []string) (err error) { } roots := make([]redhat.Root, 0, len(m)) - for _, k := range []string{fmt.Sprintf("rhel-%s.oval.xml.bz2", v), fmt.Sprintf("rhel-%s-extras.oval.xml.bz2", v), fmt.Sprintf("rhel-%s-supplementary.oval.xml.bz2", v), fmt.Sprintf("rhel-%s-els.oval.xml.bz2", v), fmt.Sprintf("com.redhat.rhsa-RHEL%s.xml", v), fmt.Sprintf("com.redhat.rhsa-RHEL%s-ELS.xml", v)} { + for _, k := range []string{fmt.Sprintf("rhel-%s-including-unpatched.oval.xml.bz2", v), fmt.Sprintf("rhel-%s-extras-including-unpatched.oval.xml.bz2", v), fmt.Sprintf("rhel-%s-supplementary.oval.xml.bz2", v), fmt.Sprintf("rhel-%s-els.oval.xml.bz2", v), fmt.Sprintf("com.redhat.rhsa-RHEL%s.xml", v), fmt.Sprintf("com.redhat.rhsa-RHEL%s-ELS.xml", v)} { roots = append(roots, m[k]) } diff --git a/db/db.go b/db/db.go index 547a03ce..898d6805 100644 --- a/db/db.go +++ b/db/db.go @@ -154,8 +154,8 @@ func chunkSlice(length int, chunkSize int) <-chan IndexChunk { func filterByRedHatMajor(packs []models.Package, majorVer string) (filtered []models.Package) { for _, p := range packs { - if strings.Contains(p.Version, ".el"+majorVer) || - strings.Contains(p.Version, ".module+el"+majorVer) { + if p.NotFixedYet || + strings.Contains(p.Version, ".el"+majorVer) || strings.Contains(p.Version, ".module+el"+majorVer) { filtered = append(filtered, p) } } diff --git a/db/db_test.go b/db/db_test.go index 641b3521..9c0675a0 100644 --- a/db/db_test.go +++ b/db/db_test.go @@ -373,6 +373,35 @@ func Test_filterByRedHatMajor(t *testing.T) { }, }, }, + { + in: args{ + packs: []models.Package{ + { + Name: "name-el7", + Version: "0:0.0.1-0.0.1.el7", + }, + { + Name: "name-el8", + Version: "0:0.0.1-0.0.1.el8", + }, + { + Name: "notfixedyet", + NotFixedYet: true, + }, + }, + majorVer: "8", + }, + expected: []models.Package{ + { + Name: "name-el8", + Version: "0:0.0.1-0.0.1.el8", + }, + { + Name: "notfixedyet", + NotFixedYet: true, + }, + }, + }, } for i, tt := range tests { diff --git a/db/rdb.go b/db/rdb.go index 043bb867..8e7156c4 100644 --- a/db/rdb.go +++ b/db/rdb.go @@ -132,6 +132,8 @@ func (r *RDBDriver) MigrateDB() error { &models.Advisory{}, &models.Cve{}, &models.Bugzilla{}, + &models.Resolution{}, + &models.Component{}, &models.Cpe{}, &models.Debian{}, ); err != nil { @@ -156,9 +158,7 @@ func (r *RDBDriver) MigrateDB() error { } } case dialectMysql, dialectPostgreSQL: - if err != nil { - return xerrors.Errorf("Failed to migrate. err: %w", err) - } + return xerrors.Errorf("Failed to migrate. err: %w", err) default: return xerrors.Errorf("Not Supported DB dialects. r.name: %s", r.name) } @@ -196,6 +196,8 @@ func (r *RDBDriver) GetByPackName(family, osVer, packName, arch string) ([]model Preload("Advisory"). Preload("Advisory.Cves"). Preload("Advisory.Bugzillas"). + Preload("Advisory.AffectedResolution"). + Preload("Advisory.AffectedResolution.Components"). Preload("Advisory.AffectedCPEList"). Preload("References") @@ -253,6 +255,8 @@ func (r *RDBDriver) GetByCveID(family, osVer, cveID, arch string) ([]models.Defi Preload("Advisory"). Preload("Advisory.Cves"). Preload("Advisory.Bugzillas"). + Preload("Advisory.AffectedResolution"). + Preload("Advisory.AffectedResolution.Components"). Preload("Advisory.AffectedCPEList"). Preload("References") diff --git a/fetcher/redhat/redhat.go b/fetcher/redhat/redhat.go index 90563f27..70b3a8a9 100644 --- a/fetcher/redhat/redhat.go +++ b/fetcher/redhat/redhat.go @@ -42,7 +42,7 @@ func FetchFiles(versions []string) (map[string][]util.FetchResult, error) { } results[v] = rs - rs, err = fetchOVALv2([]string{v, fmt.Sprintf("%s-extras", v), fmt.Sprintf("%s-supplementary", v), fmt.Sprintf("%s-els", v)}) + rs, err = fetchOVALv2([]string{fmt.Sprintf("%s-including-unpatched", v), fmt.Sprintf("%s-extras-including-unpatched", v), fmt.Sprintf("%s-supplementary", v), fmt.Sprintf("%s-els", v)}) if err != nil { return nil, xerrors.Errorf("Failed to fetch OVALv2. err: %w", err) } @@ -54,7 +54,7 @@ func FetchFiles(versions []string) (map[string][]util.FetchResult, error) { } results[v] = rs - rs, err = fetchOVALv2([]string{v, fmt.Sprintf("%s-extras", v), fmt.Sprintf("%s-supplementary", v)}) + rs, err = fetchOVALv2([]string{fmt.Sprintf("%s-including-unpatched", v), fmt.Sprintf("%s-extras-including-unpatched", v), fmt.Sprintf("%s-supplementary", v)}) if err != nil { return nil, xerrors.Errorf("Failed to fetch OVALv2. err: %w", err) } @@ -66,7 +66,7 @@ func FetchFiles(versions []string) (map[string][]util.FetchResult, error) { } results[v] = rs - rs, err = fetchOVALv2([]string{v}) + rs, err = fetchOVALv2([]string{fmt.Sprintf("%s-including-unpatched", v)}) if err != nil { return nil, xerrors.Errorf("Failed to fetch OVALv2. err: %w", err) } @@ -77,7 +77,7 @@ func FetchFiles(versions []string) (map[string][]util.FetchResult, error) { break } - rs, err := fetchOVALv2([]string{v}) + rs, err := fetchOVALv2([]string{fmt.Sprintf("%s-including-unpatched", v)}) if err != nil { return nil, xerrors.Errorf("Failed to fetch OVALv2. err: %w", err) } diff --git a/models/models.go b/models/models.go index 694b124c..28243967 100644 --- a/models/models.go +++ b/models/models.go @@ -53,7 +53,7 @@ type Package struct { Name string `gorm:"index:idx_packages_name"` // If the type:text, varchar(255) is specified, MySQL overflows and gives an error. No problem in GORMv2. (https://github.com/go-gorm/mysql/tree/15e2cbc6fd072be99215a82292e025dab25e2e16#configuration) Version string `gorm:"type:varchar(255)"` // affected earlier than this version Arch string `gorm:"type:varchar(255)"` // Used for Amazon Linux, Oracle Linux and Fedora - NotFixedYet bool // Ubuntu Only + NotFixedYet bool // Used for RedHat, Ubuntu ModularityLabel string `gorm:"type:varchar(255)"` // RHEL 8 or later only } @@ -75,6 +75,7 @@ type Advisory struct { Severity string `gorm:"type:varchar(255)"` Cves []Cve Bugzillas []Bugzilla + AffectedResolution []Resolution AffectedCPEList []Cpe AffectedRepository string `gorm:"type:varchar(255)"` // Amazon Linux 2 Only Issued time.Time @@ -105,6 +106,23 @@ type Bugzilla struct { Title string `gorm:"type:varchar(255)"` } +// Resolution : >definitions>definition>metadata>advisory>affected>resolution +type Resolution struct { + ID uint `gorm:"primary_key" json:"-"` + AdvisoryID uint `gorm:"index:idx_resolution_advisory_id" json:"-" xml:"-"` + + State string `gorm:"type:varchar(255)"` + Components []Component +} + +// Component : >definitions>definition>metadata>advisory>affected>resolution>component +type Component struct { + ID uint `gorm:"primary_key" json:"-"` + ResolutionID uint `gorm:"index:idx_component_resolution_id" json:"-" xml:"-"` + + Component string `gorm:"type:varchar(255)"` +} + // Cpe : >definitions>definition>metadata>advisory>affected_cpe_list type Cpe struct { ID uint `gorm:"primary_key" json:"-"` diff --git a/models/redhat/redhat.go b/models/redhat/redhat.go index 3a186544..a61a30b1 100644 --- a/models/redhat/redhat.go +++ b/models/redhat/redhat.go @@ -18,11 +18,11 @@ func ConvertToModel(v string, roots []Root) []models.Definition { defs := map[string]models.Definition{} for _, root := range roots { for _, d := range root.Definitions.Definitions { - if strings.Contains(d.Description, "** REJECT **") { + if strings.HasPrefix(d.ID, "oval:com.redhat.unaffected:def:") || strings.Contains(d.Description, "** REJECT **") { continue } - cves := []models.Cve{} + var cves = make([]models.Cve, 0, len(d.Advisory.Cves)) for _, c := range d.Advisory.Cves { cves = append(cves, models.Cve{ CveID: c.CveID, @@ -35,7 +35,7 @@ func ConvertToModel(v string, roots []Root) []models.Definition { }) } - rs := []models.Reference{} + var rs = make([]models.Reference, 0, len(d.References)) for _, r := range d.References { rs = append(rs, models.Reference{ Source: r.Source, @@ -44,14 +44,14 @@ func ConvertToModel(v string, roots []Root) []models.Definition { }) } - cl := []models.Cpe{} + var cpes = make([]models.Cpe, 0, len(d.Advisory.AffectedCPEList)) for _, cpe := range d.Advisory.AffectedCPEList { - cl = append(cl, models.Cpe{ + cpes = append(cpes, models.Cpe{ Cpe: cpe, }) } - bs := []models.Bugzilla{} + var bs = make([]models.Bugzilla, 0, len(d.Advisory.Bugzillas)) for _, b := range d.Advisory.Bugzillas { bs = append(bs, models.Bugzilla{ BugzillaID: b.ID, @@ -60,6 +60,22 @@ func ConvertToModel(v string, roots []Root) []models.Definition { }) } + var ress = make([]models.Resolution, 0, len(d.Advisory.Affected.Resolution)) + for _, r := range d.Advisory.Affected.Resolution { + ress = append(ress, models.Resolution{ + State: r.State, + Components: func() []models.Component { + var comps = make([]models.Component, 0, len(r.Component)) + for _, c := range r.Component { + comps = append(comps, models.Component{ + Component: c, + }) + } + return comps + }(), + }) + } + issued := util.ParsedOrDefaultTime([]string{"2006-01-02"}, d.Advisory.Issued.Date) updated := util.ParsedOrDefaultTime([]string{"2006-01-02"}, d.Advisory.Updated.Date) @@ -68,14 +84,14 @@ func ConvertToModel(v string, roots []Root) []models.Definition { Title: d.Title, Description: d.Description, Advisory: models.Advisory{ - Severity: d.Advisory.Severity, - Cves: cves, - Bugzillas: bs, - AffectedCPEList: cl, - Issued: issued, - Updated: updated, + Severity: d.Advisory.Severity, + Cves: cves, + Bugzillas: bs, + AffectedResolution: ress, + AffectedCPEList: cpes, + Issued: issued, + Updated: updated, }, - Debian: nil, AffectedPacks: collectRedHatPacks(v, d.Criteria), References: rs, } @@ -100,19 +116,23 @@ func ConvertToModel(v string, roots []Root) []models.Definition { } func collectRedHatPacks(v string, cri Criteria) []models.Package { - ps := walkRedHat(cri, []models.Package{}, "") pkgs := map[string]models.Package{} - for _, p := range ps { - // OVALv1 includes definitions other than the target RHEL version - if !strings.Contains(p.Version, ".el"+v) && !strings.Contains(p.Version, ".module+el"+v) { - continue - } - + for _, p := range walkRedHat(cri, []models.Package{}, "") { n := p.Name if p.ModularityLabel != "" { n = fmt.Sprintf("%s::%s", p.ModularityLabel, p.Name) } + if p.NotFixedYet { + pkgs[n] = p + continue + } + + // OVALv1 includes definitions other than the target RHEL version + if !strings.Contains(p.Version, ".el"+v) && !strings.Contains(p.Version, ".module+el"+v) { + continue + } + // since different versions are defined for the same package, the newer version is adopted // example: OVALv2: oval:com.redhat.rhsa:def:20111349, oval:com.redhat.rhsa:def:20120451 if base, ok := pkgs[n]; ok { @@ -130,19 +150,26 @@ func collectRedHatPacks(v string, cri Criteria) []models.Package { func walkRedHat(cri Criteria, acc []models.Package, label string) []models.Package { for _, c := range cri.Criterions { - if strings.HasPrefix(c.Comment, "Module ") && strings.HasSuffix(c.Comment, " is enabled") { + switch { + case strings.HasPrefix(c.Comment, "Module ") && strings.HasSuffix(c.Comment, " is enabled"): label = strings.TrimSuffix(strings.TrimPrefix(c.Comment, "Module "), " is enabled") + case strings.Contains(c.Comment, " is earlier than "): + ss := strings.Split(c.Comment, " is earlier than ") + if len(ss) != 2 { + continue + } + acc = append(acc, models.Package{ + Name: ss[0], + Version: strings.Split(ss[1], " ")[0], + ModularityLabel: label, + }) + case !strings.HasPrefix(c.Comment, "Red Hat Enterprise Linux") && !strings.HasPrefix(c.Comment, "Red Hat CoreOS") && strings.HasSuffix(c.Comment, " is installed"): + acc = append(acc, models.Package{ + Name: strings.TrimSuffix(c.Comment, " is installed"), + ModularityLabel: label, + NotFixedYet: true, + }) } - - ss := strings.Split(c.Comment, " is earlier than ") - if len(ss) != 2 { - continue - } - acc = append(acc, models.Package{ - Name: ss[0], - Version: strings.Split(ss[1], " ")[0], - ModularityLabel: label, - }) } if len(cri.Criterias) == 0 { diff --git a/models/redhat/redhat_test.go b/models/redhat/redhat_test.go index 7b88dd7c..59045405 100644 --- a/models/redhat/redhat_test.go +++ b/models/redhat/redhat_test.go @@ -253,6 +253,48 @@ func TestWalkRedHat(t *testing.T) { }, }, }, + { + version: "8", + cri: Criteria{ + Criterias: []Criteria{ + { + Criterias: []Criteria{ + { + Criterias: []Criteria{ + { + Criterias: []Criteria{ + { + Criterions: []Criterion{ + {Comment: "python2 is installed"}, + {Comment: "python2 is signed with Red Hat redhatrelease2 key"}, + }, + }, + }, + }, + }, + Criterions: []Criterion{ + {Comment: "Module inkscape:flatpak is enabled"}, + }, + }, + }, + Criterions: []Criterion{ + {Comment: "Red Hat Enterprise Linux 8 is installed"}, + {Comment: "Red Hat CoreOS 4 is installed"}, + }, + }, + }, + Criterions: []Criterion{ + {Comment: "Red Hat Enterprise Linux must be installed"}, + }, + }, + expected: []models.Package{ + { + Name: "python2", + ModularityLabel: "inkscape:flatpak", + NotFixedYet: true, + }, + }, + }, } for i, tt := range tests { diff --git a/models/redhat/types.go b/models/redhat/types.go index 1b22c1e1..0f89f832 100644 --- a/models/redhat/types.go +++ b/models/redhat/types.go @@ -110,7 +110,7 @@ type Bugzilla struct { // AffectedPkgs : >definitions>definition>metadata>advisory>affected type AffectedPkgs struct { - Resolution struct { + Resolution []struct { State string `xml:"state,attr"` Component []string `xml:"component"` } `xml:"resolution"`