feat(fetch/redhat): fetch including-unpatched

commands/fetch-redhat.go

@@ -85,7 +85,7 @@ func fetchRedHat(_ *cobra.Command, args []string) (err error) {
  }
 
  roots := make([]redhat.Root, 0, len(m))
- for _, k := range []string{fmt.Sprintf("rhel-%s.oval.xml.bz2", v), fmt.Sprintf("rhel-%s-extras.oval.xml.bz2", v), fmt.Sprintf("rhel-%s-supplementary.oval.xml.bz2", v), fmt.Sprintf("rhel-%s-els.oval.xml.bz2", v), fmt.Sprintf("com.redhat.rhsa-RHEL%s.xml", v), fmt.Sprintf("com.redhat.rhsa-RHEL%s-ELS.xml", v)} {
+ for _, k := range []string{fmt.Sprintf("rhel-%s-including-unpatched.oval.xml.bz2", v), fmt.Sprintf("rhel-%s-extras-including-unpatched.oval.xml.bz2", v), fmt.Sprintf("rhel-%s-supplementary.oval.xml.bz2", v), fmt.Sprintf("rhel-%s-els.oval.xml.bz2", v), fmt.Sprintf("com.redhat.rhsa-RHEL%s.xml", v), fmt.Sprintf("com.redhat.rhsa-RHEL%s-ELS.xml", v)} {
  roots = append(roots, m[k])
  }
 

db/db.go

@@ -154,8 +154,8 @@ func chunkSlice(length int, chunkSize int) <-chan IndexChunk {
 
 func filterByRedHatMajor(packs []models.Package, majorVer string) (filtered []models.Package) {
  for _, p := range packs {
- if strings.Contains(p.Version, ".el"+majorVer) ||
- strings.Contains(p.Version, ".module+el"+majorVer) {
+ if p.NotFixedYet ||
+ strings.Contains(p.Version, ".el"+majorVer) || strings.Contains(p.Version, ".module+el"+majorVer) {
  filtered = append(filtered, p)
  }
  }

db/db_test.go

@@ -373,6 +373,35 @@ func Test_filterByRedHatMajor(t *testing.T) {
  },
  },
  },
+ {
+ in: args{
+ packs: []models.Package{
+ {
+ Name: "name-el7",
+ Version: "0:0.0.1-0.0.1.el7",
+ },
+ {
+ Name: "name-el8",
+ Version: "0:0.0.1-0.0.1.el8",
+ },
+ {
+ Name: "notfixedyet",
+ NotFixedYet: true,
+ },
+ },
+ majorVer: "8",
+ },
+ expected: []models.Package{
+ {
+ Name: "name-el8",
+ Version: "0:0.0.1-0.0.1.el8",
+ },
+ {
+ Name: "notfixedyet",
+ NotFixedYet: true,
+ },
+ },
+ },
  }
 
  for i, tt := range tests {

db/rdb.go

@@ -132,6 +132,8 @@ func (r *RDBDriver) MigrateDB() error {
  &models.Advisory{},
  &models.Cve{},
  &models.Bugzilla{},
+ &models.Resolution{},
+ &models.Component{},
  &models.Cpe{},
  &models.Debian{},
  ); err != nil {
@@ -156,9 +158,7 @@ func (r *RDBDriver) MigrateDB() error {
  }
  }
  case dialectMysql, dialectPostgreSQL:
- if err != nil {
- return xerrors.Errorf("Failed to migrate. err: %w", err)
- }
+ return xerrors.Errorf("Failed to migrate. err: %w", err)
  default:
  return xerrors.Errorf("Not Supported DB dialects. r.name: %s", r.name)
  }
@@ -196,6 +196,8 @@ func (r *RDBDriver) GetByPackName(family, osVer, packName, arch string) ([]model
  Preload("Advisory").
  Preload("Advisory.Cves").
  Preload("Advisory.Bugzillas").
+ Preload("Advisory.AffectedResolution").
+ Preload("Advisory.AffectedResolution.Components").
  Preload("Advisory.AffectedCPEList").
  Preload("References")
 
@@ -253,6 +255,8 @@ func (r *RDBDriver) GetByCveID(family, osVer, cveID, arch string) ([]models.Defi
  Preload("Advisory").
  Preload("Advisory.Cves").
  Preload("Advisory.Bugzillas").
+ Preload("Advisory.AffectedResolution").
+ Preload("Advisory.AffectedResolution.Components").
  Preload("Advisory.AffectedCPEList").
  Preload("References")
 

fetcher/redhat/redhat.go

@@ -42,7 +42,7 @@ func FetchFiles(versions []string) (map[string][]util.FetchResult, error) {
  }
  results[v] = rs
 
- rs, err = fetchOVALv2([]string{v, fmt.Sprintf("%s-extras", v), fmt.Sprintf("%s-supplementary", v), fmt.Sprintf("%s-els", v)})
+ rs, err = fetchOVALv2([]string{fmt.Sprintf("%s-including-unpatched", v), fmt.Sprintf("%s-extras-including-unpatched", v), fmt.Sprintf("%s-supplementary", v), fmt.Sprintf("%s-els", v)})
  if err != nil {
  return nil, xerrors.Errorf("Failed to fetch OVALv2. err: %w", err)
  }
@@ -54,7 +54,7 @@ func FetchFiles(versions []string) (map[string][]util.FetchResult, error) {
  }
  results[v] = rs
 
- rs, err = fetchOVALv2([]string{v, fmt.Sprintf("%s-extras", v), fmt.Sprintf("%s-supplementary", v)})
+ rs, err = fetchOVALv2([]string{fmt.Sprintf("%s-including-unpatched", v), fmt.Sprintf("%s-extras-including-unpatched", v), fmt.Sprintf("%s-supplementary", v)})
  if err != nil {
  return nil, xerrors.Errorf("Failed to fetch OVALv2. err: %w", err)
  }
@@ -66,7 +66,7 @@ func FetchFiles(versions []string) (map[string][]util.FetchResult, error) {
  }
  results[v] = rs
 
- rs, err = fetchOVALv2([]string{v})
+ rs, err = fetchOVALv2([]string{fmt.Sprintf("%s-including-unpatched", v)})
  if err != nil {
  return nil, xerrors.Errorf("Failed to fetch OVALv2. err: %w", err)
  }
@@ -77,7 +77,7 @@ func FetchFiles(versions []string) (map[string][]util.FetchResult, error) {
  break
  }
 
- rs, err := fetchOVALv2([]string{v})
+ rs, err := fetchOVALv2([]string{fmt.Sprintf("%s-including-unpatched", v)})
  if err != nil {
  return nil, xerrors.Errorf("Failed to fetch OVALv2. err: %w", err)
  }

models/models.go

@@ -53,7 +53,7 @@ type Package struct {
  Name string `gorm:"index:idx_packages_name"` // If the type:text, varchar(255) is specified, MySQL overflows and gives an error. No problem in GORMv2. (https://github.com/go-gorm/mysql/tree/15e2cbc6fd072be99215a82292e025dab25e2e16#configuration)
  Version string `gorm:"type:varchar(255)"` // affected earlier than this version
  Arch string `gorm:"type:varchar(255)"` // Used for Amazon Linux, Oracle Linux and Fedora
- NotFixedYet bool // Ubuntu Only
+ NotFixedYet bool // Used for RedHat, Ubuntu
  ModularityLabel string `gorm:"type:varchar(255)"` // RHEL 8 or later only
 }
 
@@ -75,6 +75,7 @@ type Advisory struct {
  Severity string `gorm:"type:varchar(255)"`
  Cves []Cve
  Bugzillas []Bugzilla
+ AffectedResolution []Resolution
  AffectedCPEList []Cpe
  AffectedRepository string `gorm:"type:varchar(255)"` // Amazon Linux 2 Only
  Issued time.Time
@@ -105,6 +106,23 @@ type Bugzilla struct {
  Title string `gorm:"type:varchar(255)"`
 }
 
+// Resolution : >definitions>definition>metadata>advisory>affected>resolution
+type Resolution struct {
+ ID uint `gorm:"primary_key" json:"-"`
+ AdvisoryID uint `gorm:"index:idx_resolution_advisory_id" json:"-" xml:"-"`
+
+ State string `gorm:"type:varchar(255)"`
+ Components []Component
+}
+
+// Component : >definitions>definition>metadata>advisory>affected>resolution>component
+type Component struct {
+ ID uint `gorm:"primary_key" json:"-"`
+ ResolutionID uint `gorm:"index:idx_component_resolution_id" json:"-" xml:"-"`
+
+ Component string `gorm:"type:varchar(255)"`
+}
+
 // Cpe : >definitions>definition>metadata>advisory>affected_cpe_list
 type Cpe struct {
  ID uint `gorm:"primary_key" json:"-"`

models/redhat/redhat.go

@@ -18,11 +18,11 @@ func ConvertToModel(v string, roots []Root) []models.Definition {
  defs := map[string]models.Definition{}
  for _, root := range roots {
  for _, d := range root.Definitions.Definitions {
- if strings.Contains(d.Description, "** REJECT **") {
+ if strings.HasPrefix(d.ID, "oval:com.redhat.unaffected:def:") || strings.Contains(d.Description, "** REJECT **") {
  continue
  }
 
- cves := []models.Cve{}
+ var cves = make([]models.Cve, 0, len(d.Advisory.Cves))
  for _, c := range d.Advisory.Cves {
  cves = append(cves, models.Cve{
  CveID: c.CveID,
@@ -35,7 +35,7 @@ func ConvertToModel(v string, roots []Root) []models.Definition {
  })
  }
 
- rs := []models.Reference{}
+ var rs = make([]models.Reference, 0, len(d.References))
  for _, r := range d.References {
  rs = append(rs, models.Reference{
  Source: r.Source,
@@ -44,14 +44,14 @@ func ConvertToModel(v string, roots []Root) []models.Definition {
  })
  }
 
- cl := []models.Cpe{}
+ var cpes = make([]models.Cpe, 0, len(d.Advisory.AffectedCPEList))
  for _, cpe := range d.Advisory.AffectedCPEList {
- cl = append(cl, models.Cpe{
+ cpes = append(cpes, models.Cpe{
  Cpe: cpe,
  })
  }
 
- bs := []models.Bugzilla{}
+ var bs = make([]models.Bugzilla, 0, len(d.Advisory.Bugzillas))
  for _, b := range d.Advisory.Bugzillas {
  bs = append(bs, models.Bugzilla{
  BugzillaID: b.ID,
@@ -60,6 +60,22 @@ func ConvertToModel(v string, roots []Root) []models.Definition {
  })
  }
 
+ var ress = make([]models.Resolution, 0, len(d.Advisory.Affected.Resolution))
+ for _, r := range d.Advisory.Affected.Resolution {
+ ress = append(ress, models.Resolution{
+ State: r.State,
+ Components: func() []models.Component {
+ var comps = make([]models.Component, 0, len(r.Component))
+ for _, c := range r.Component {
+ comps = append(comps, models.Component{
+ Component: c,
+ })
+ }
+ return comps
+ }(),
+ })
+ }
+
  issued := util.ParsedOrDefaultTime([]string{"2006-01-02"}, d.Advisory.Issued.Date)
  updated := util.ParsedOrDefaultTime([]string{"2006-01-02"}, d.Advisory.Updated.Date)
 
@@ -68,14 +84,14 @@ func ConvertToModel(v string, roots []Root) []models.Definition {
  Title: d.Title,
  Description: d.Description,
  Advisory: models.Advisory{
- Severity: d.Advisory.Severity,
- Cves: cves,
- Bugzillas: bs,
- AffectedCPEList: cl,
- Issued: issued,
- Updated: updated,
+ Severity: d.Advisory.Severity,
+ Cves: cves,
+ Bugzillas: bs,
+ AffectedResolution: ress,
+ AffectedCPEList: cpes,
+ Issued: issued,
+ Updated: updated,
  },
- Debian: nil,
  AffectedPacks: collectRedHatPacks(v, d.Criteria),
  References: rs,
  }
@@ -100,19 +116,23 @@ func ConvertToModel(v string, roots []Root) []models.Definition {
 }
 
 func collectRedHatPacks(v string, cri Criteria) []models.Package {
- ps := walkRedHat(cri, []models.Package{}, "")
  pkgs := map[string]models.Package{}
- for _, p := range ps {
- // OVALv1 includes definitions other than the target RHEL version
- if !strings.Contains(p.Version, ".el"+v) && !strings.Contains(p.Version, ".module+el"+v) {
- continue
- }
-
+ for _, p := range walkRedHat(cri, []models.Package{}, "") {
  n := p.Name
  if p.ModularityLabel != "" {
  n = fmt.Sprintf("%s::%s", p.ModularityLabel, p.Name)
  }
 
+ if p.NotFixedYet {
+ pkgs[n] = p
+ continue
+ }
+
+ // OVALv1 includes definitions other than the target RHEL version
+ if !strings.Contains(p.Version, ".el"+v) && !strings.Contains(p.Version, ".module+el"+v) {
+ continue
+ }
+
  // since different versions are defined for the same package, the newer version is adopted
  // example: OVALv2: oval:com.redhat.rhsa:def:20111349, oval:com.redhat.rhsa:def:20120451
  if base, ok := pkgs[n]; ok {
@@ -130,19 +150,26 @@ func collectRedHatPacks(v string, cri Criteria) []models.Package {
 
 func walkRedHat(cri Criteria, acc []models.Package, label string) []models.Package {
  for _, c := range cri.Criterions {
- if strings.HasPrefix(c.Comment, "Module ") && strings.HasSuffix(c.Comment, " is enabled") {
+ switch {
+ case strings.HasPrefix(c.Comment, "Module ") && strings.HasSuffix(c.Comment, " is enabled"):
  label = strings.TrimSuffix(strings.TrimPrefix(c.Comment, "Module "), " is enabled")
+ case strings.Contains(c.Comment, " is earlier than "):
+ ss := strings.Split(c.Comment, " is earlier than ")
+ if len(ss) != 2 {
+ continue
+ }
+ acc = append(acc, models.Package{
+ Name: ss[0],
+ Version: strings.Split(ss[1], " ")[0],
+ ModularityLabel: label,
+ })
+ case !strings.HasPrefix(c.Comment, "Red Hat Enterprise Linux") && !strings.HasPrefix(c.Comment, "Red Hat CoreOS") && strings.HasSuffix(c.Comment, " is installed"):
+ acc = append(acc, models.Package{
+ Name: strings.TrimSuffix(c.Comment, " is installed"),
+ ModularityLabel: label,
+ NotFixedYet: true,
+ })
  }
-
- ss := strings.Split(c.Comment, " is earlier than ")
- if len(ss) != 2 {
- continue
- }
- acc = append(acc, models.Package{
- Name: ss[0],
- Version: strings.Split(ss[1], " ")[0],
- ModularityLabel: label,
- })
  }
 
  if len(cri.Criterias) == 0 {

models/redhat/redhat_test.go

@@ -253,6 +253,48 @@ func TestWalkRedHat(t *testing.T) {
  },
  },
  },
+ {
+ version: "8",
+ cri: Criteria{
+ Criterias: []Criteria{
+ {
+ Criterias: []Criteria{
+ {
+ Criterias: []Criteria{
+ {
+ Criterias: []Criteria{
+ {
+ Criterions: []Criterion{
+ {Comment: "python2 is installed"},
+ {Comment: "python2 is signed with Red Hat redhatrelease2 key"},
+ },
+ },
+ },
+ },
+ },
+ Criterions: []Criterion{
+ {Comment: "Module inkscape:flatpak is enabled"},
+ },
+ },
+ },
+ Criterions: []Criterion{
+ {Comment: "Red Hat Enterprise Linux 8 is installed"},
+ {Comment: "Red Hat CoreOS 4 is installed"},
+ },
+ },
+ },
+ Criterions: []Criterion{
+ {Comment: "Red Hat Enterprise Linux must be installed"},
+ },
+ },
+ expected: []models.Package{
+ {
+ Name: "python2",
+ ModularityLabel: "inkscape:flatpak",
+ NotFixedYet: true,
+ },
+ },
+ },
  }
 
  for i, tt := range tests {

models/redhat/types.go

@@ -110,7 +110,7 @@ type Bugzilla struct {
 
 // AffectedPkgs : >definitions>definition>metadata>advisory>affected
 type AffectedPkgs struct {
- Resolution struct {
+ Resolution []struct {
  State string `xml:"state,attr"`
  Component []string `xml:"component"`
  } `xml:"resolution"`