Filter between mirrors

AUTHORS

@@ -63,3 +63,4 @@ List of contributors, in chronological order:
 * Ramón N.Rodriguez (https://github.com/runitonmetal)
 * Golf Hu (https://github.com/hudeng-go)
 * Cookie Fei (https://github.com/wuhuang26)
+* Malte Swart (https://github.com/mswart)

api/api.go

@@ -228,7 +228,7 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList, collectionFactory
  list.PrepareIndex()
 
  list, err = list.Filter([]deb.PackageQuery{q}, withDeps,
- nil, context.DependencyOptions(), architecturesList)
+ nil, nil, context.DependencyOptions(), architecturesList)
  if err != nil {
  AbortWithJSONError(c, 500, fmt.Errorf("unable to search: %s", err))
  return
@@ -244,7 +244,7 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList, collectionFactory
  fmt.Println("filter packages by version, query string parse err: ", err)
  c.AbortWithError(500, fmt.Errorf("unable to parse %s maximum version query string: %s", p.Name, err))
  } else {
- tmpList, err := list.Filter([]deb.PackageQuery{versionQ}, false,
+ tmpList, err := list.Filter([]deb.PackageQuery{versionQ}, false, nil,
  nil, 0, []string{})
 
  if err == nil {

api/mirror.go

@@ -306,7 +306,7 @@ func apiMirrorsPackages(c *gin.Context) {
 
  list.PrepareIndex()
 
- list, err = list.Filter([]deb.PackageQuery{q}, withDeps,
+ list, err = list.Filter([]deb.PackageQuery{q}, withDeps, nil,
  nil, context.DependencyOptions(), architecturesList)
  if err != nil {
  AbortWithJSONError(c, 500, fmt.Errorf("unable to search: %s", err))
@@ -468,7 +468,7 @@ func apiMirrorsUpdate(c *gin.Context) {
  return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
  }
 
- _, _, err = remote.ApplyFilter(context.DependencyOptions(), filterQuery, out)
+ _, _, err = remote.ApplyFilter(context.DependencyOptions(), filterQuery, nil, out)
  if err != nil {
  return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
  }

api/repos.go

@@ -558,7 +558,7 @@ func apiReposCopyPackage(c *gin.Context) {
  return &task.ProcessReturnValue{Code: http.StatusUnprocessableEntity, Value: nil}, fmt.Errorf("unable to parse query '%s': %s", fileName, err)
  }
 
- toProcess, err := srcList.FilterWithProgress(queries, jsonBody.WithDeps, dstList, context.DependencyOptions(), architecturesList, context.Progress())
+ toProcess, err := srcList.FilterWithProgress(queries, jsonBody.WithDeps, dstList, nil, context.DependencyOptions(), architecturesList, context.Progress())
  if err != nil {
  return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("filter error: %s", err)
  }

api/snapshot.go

@@ -633,7 +633,7 @@ func apiSnapshotsPull(c *gin.Context) {
  }
 
  // Filter with dependencies as requested
- destinationPackageList, err := sourcePackageList.FilterWithProgress(queries, !noDeps, toPackageList, context.DependencyOptions(), architecturesList, context.Progress())
+ destinationPackageList, err := sourcePackageList.FilterWithProgress(queries, !noDeps, toPackageList, nil, context.DependencyOptions(), architecturesList, context.Progress())
  if err != nil {
  return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
  }

cmd/mirror_create.go

@@ -51,6 +51,15 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
  repo.SkipComponentCheck = context.Flags().Lookup("force-components").Value.Get().(bool)
  repo.SkipArchitectureCheck = context.Flags().Lookup("force-architectures").Value.Get().(bool)
 
+ extraDepsFromMirrors := context.Flags().Lookup("deps-from-mirrors").Value.String()
+ if extraDepsFromMirrors != "" {
+ repo.DepsFromMirrors = strings.Split(extraDepsFromMirrors, ",")
+ }
+ extraDepsFromRepos := context.Flags().Lookup("deps-from-repos").Value.String()
+ if extraDepsFromRepos != "" {
+ repo.DepsFromRepos = strings.Split(extraDepsFromRepos, ",")
+ }
+
  if repo.Filter != "" {
  _, err = query.Parse(repo.Filter)
  if err != nil {
@@ -109,6 +118,8 @@ Example:
  cmd.Flag.Bool("force-architectures", false, "(only with architecture list) skip check that requested architectures are listed in Release file")
  cmd.Flag.Int("max-tries", 1, "max download tries till process fails with download error")
  cmd.Flag.Var(&keyRingsFlag{}, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")
+ cmd.Flag.String("deps-from-mirrors", "", "list of mirrors thats packages's dependencies should be fulfilled (comma-separated), default to none")
+ cmd.Flag.String("deps-from-repos", "", "list of repositories thats packages's dependencies should be fulfilled (comma-separated), default to none")
 
  return cmd
 }

cmd/mirror_edit.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
  "fmt"
+ "strings"
 
  "github.com/aptly-dev/aptly/pgp"
  "github.com/aptly-dev/aptly/query"
@@ -46,6 +47,20 @@ func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
  fetchMirror = true
  case "ignore-signatures":
  ignoreSignatures = true
+ case "deps-from-repos":
+ repoNames := flag.Value.String()
+ if repoNames != "" {
+ repo.DepsFromRepos = strings.Split(repoNames, ",")
+ } else {
+ repo.DepsFromRepos = make([]string, 0)
+ }
+ case "deps-from-mirrors":
+ mirrorNames := flag.Value.String()
+ if mirrorNames != "" {
+ repo.DepsFromMirrors = strings.Split(mirrorNames, ",")
+ } else {
+ repo.DepsFromMirrors = make([]string, 0)
+ }
  }
  })
 
@@ -111,6 +126,8 @@ Example:
  cmd.Flag.Bool("with-sources", false, "download source packages in addition to binary packages")
  cmd.Flag.Bool("with-udebs", false, "download .udeb packages (Debian installer support)")
  cmd.Flag.Var(&keyRingsFlag{}, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")
+ cmd.Flag.String("deps-from-mirrors", "", "list of mirrors thats packages's dependencies should be fulfilled (comma-separated), default to none")
+ cmd.Flag.String("deps-from-repos", "", "list of repositories thats packages's dependencies should be fulfilled (comma-separated), default to none")
 
  return cmd
 }

cmd/mirror_show.go

@@ -68,6 +68,14 @@ func aptlyMirrorShowTxt(_ *commander.Command, args []string) error {
  filterWithDeps = Yes
  }
  fmt.Printf("Filter With Deps: %s\n", filterWithDeps)
+ if repo.FilterWithDeps {
+ if len(repo.DepsFromMirrors) > 0 {
+ fmt.Printf("Include Dependencies Of Mirrors: %s\n", strings.Join(repo.DepsFromMirrors, ", "))
+ }
+ if len(repo.DepsFromRepos) > 0 {
+ fmt.Printf("Include Dependencies Of Repositories: %s\n", strings.Join(repo.DepsFromRepos, ", "))
+ }
+ }
  }
  if repo.LastDownloadDate.IsZero() {
  fmt.Printf("Last update: never\n")

cmd/mirror_update.go

@@ -67,14 +67,51 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
  if repo.Filter != "" {
  context.Progress().Printf("Applying filter...\n")
  var filterQuery deb.PackageQuery
+ var packagesWithExtraDeps *deb.PackageList
 
  filterQuery, err = query.Parse(repo.Filter)
  if err != nil {
  return fmt.Errorf("unable to update: %s", err)
  }
 
+ collectionFactory := context.NewCollectionFactory()
+ if len(repo.DepsFromMirrors)+len(repo.DepsFromRepos) > 0 {
+ packagesWithExtraDeps = deb.NewPackageList()
+ for _, mirrorName := range repo.DepsFromMirrors {
+ extraMirror, lerr := collectionFactory.RemoteRepoCollection().ByName(mirrorName)
+ if lerr != nil {
+ return fmt.Errorf("unable to update: %s", lerr)
+ }
+ lerr = collectionFactory.RemoteRepoCollection().LoadComplete(extraMirror)
+ if lerr != nil {
+ return fmt.Errorf("unable to update: %s", lerr)
+ }
+ packageList, lerr := deb.NewPackageListFromRefList(extraMirror.RefList(), collectionFactory.PackageCollection(), context.Progress())
+ if lerr != nil {
+ return fmt.Errorf("unable to update: %s", lerr)
+ }
+ packagesWithExtraDeps.Append(packageList)
+ }
+ for _, repoName := range repo.DepsFromRepos {
+ extraRepo, lerr := collectionFactory.LocalRepoCollection().ByName(repoName)
+ if lerr != nil {
+ return fmt.Errorf("unable to update: %s", lerr)
+ }
+ lerr = collectionFactory.LocalRepoCollection().LoadComplete(extraRepo)
+ if lerr != nil {
+ return fmt.Errorf("unable to update: %s", lerr)
+ }
+ packageList, lerr := deb.NewPackageListFromRefList(extraRepo.RefList(), collectionFactory.PackageCollection(), context.Progress())
+ if lerr != nil {
+ return fmt.Errorf("unable to update: %s", lerr)
+ }
+ packagesWithExtraDeps.Append(packageList)
+ }
+ context.Progress().Printf("Extra packages with deps: %d\n", packagesWithExtraDeps.Len())
+ }
+
  var oldLen, newLen int
- oldLen, newLen, err = repo.ApplyFilter(context.DependencyOptions(), filterQuery, context.Progress())
+ oldLen, newLen, err = repo.ApplyFilter(context.DependencyOptions(), filterQuery, packagesWithExtraDeps, context.Progress())
  if err != nil {
  return fmt.Errorf("unable to update: %s", err)
  }

cmd/repo_move.go

@@ -116,7 +116,7 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
  }
  }
 
- toProcess, err := srcList.FilterWithProgress(queries, withDeps, dstList, context.DependencyOptions(), architecturesList, context.Progress())
+ toProcess, err := srcList.FilterWithProgress(queries, withDeps, dstList, nil, context.DependencyOptions(), architecturesList, context.Progress())
  if err != nil {
  return fmt.Errorf("unable to %s: %s", command, err)
  }

cmd/repo_remove.go

@@ -45,7 +45,7 @@ func aptlyRepoRemove(cmd *commander.Command, args []string) error {
  }
 
  list.PrepareIndex()
- toRemove, err := list.Filter(queries, false, nil, 0, nil)
+ toRemove, err := list.Filter(queries, false, nil, nil, 0, nil)
  if err != nil {
  return fmt.Errorf("unable to remove: %s", err)
  }

cmd/snapshot_filter.go

@@ -67,7 +67,7 @@ func aptlySnapshotFilter(cmd *commander.Command, args []string) error {
  }
 
  // Filter with dependencies as requested
- result, err := packageList.FilterWithProgress(queries, withDeps, nil, context.DependencyOptions(), architecturesList, context.Progress())
+ result, err := packageList.FilterWithProgress(queries, withDeps, nil, nil, context.DependencyOptions(), architecturesList, context.Progress())
  if err != nil {
  return fmt.Errorf("unable to filter: %s", err)
  }

cmd/snapshot_pull.go

@@ -97,7 +97,7 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
  }
 
  // Filter with dependencies as requested
- result, err := sourcePackageList.FilterWithProgress(queries, !noDeps, packageList, context.DependencyOptions(), architecturesList, context.Progress())
+ result, err := sourcePackageList.FilterWithProgress(queries, !noDeps, packageList, nil, context.DependencyOptions(), architecturesList, context.Progress())
  if err != nil {
  return fmt.Errorf("unable to pull: %s", err)
  }

cmd/snapshot_search.go

@@ -104,7 +104,7 @@ func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error
  }
 
  result, err := list.FilterWithProgress([]deb.PackageQuery{q}, withDeps,
- nil, context.DependencyOptions(), architecturesList, context.Progress())
+ nil, nil, context.DependencyOptions(), architecturesList, context.Progress())
  if err != nil {
  return fmt.Errorf("unable to search: %s", err)
  }

completion.d/aptly

@@ -154,15 +154,15 @@ _aptly()
  "create")
  if [[ $numargs -eq 0 ]]; then
  if [[ "$cur" == -* ]]; then
- COMPREPLY=($(compgen -W "-filter= -filter-with-deps -force-components -ignore-signatures -keyring= -with-installer -with-sources -with-udebs" -- ${cur}))
+ COMPREPLY=($(compgen -W "-filter= -filter-with-deps -force-components -ignore-signatures -keyring= -with-installer -with-sources -with-udebs -deps-from-mirrors= -deps-from-repos=" -- ${cur}))
  return 0
  fi
  fi
  ;;
  "edit")
  if [[ $numargs -eq 0 ]]; then
  if [[ "$cur" == -* ]]; then
- COMPREPLY=($(compgen -W "-archive-url= -filter= -filter-with-deps -ignore-signatures -keyring= -with-installer -with-sources -with-udebs" -- ${cur}))
+ COMPREPLY=($(compgen -W "-archive-url= -filter= -filter-with-deps -ignore-signatures -keyring= -with-installer -with-sources -with-udebs -deps-from-mirrors= -deps-from-repos=" -- ${cur}))
  else
  COMPREPLY=($(compgen -W "$(__aptly_mirror_list)" -- ${cur}))
  fi

deb/list.go

@@ -499,12 +499,12 @@ func (l *PackageList) Search(dep Dependency, allMatches bool) (searchResults []*
 }
 
 // Filter filters package index by specified queries (ORed together), possibly pulling dependencies
-func (l *PackageList) Filter(queries []PackageQuery, withDependencies bool, source *PackageList, dependencyOptions int, architecturesList []string) (*PackageList, error) {
- return l.FilterWithProgress(queries, withDependencies, source, dependencyOptions, architecturesList, nil)
+func (l *PackageList) Filter(queries []PackageQuery, withDependencies bool, source, extraDeps *PackageList, dependencyOptions int, architecturesList []string) (*PackageList, error) {
+ return l.FilterWithProgress(queries, withDependencies, source, extraDeps, dependencyOptions, architecturesList, nil)
 }
 
 // FilterWithProgress filters package index by specified queries (ORed together), possibly pulling dependencies and displays progress
-func (l *PackageList) FilterWithProgress(queries []PackageQuery, withDependencies bool, source *PackageList, dependencyOptions int, architecturesList []string, progress aptly.Progress) (*PackageList, error) {
+func (l *PackageList) FilterWithProgress(queries []PackageQuery, withDependencies bool, source, extraDeps *PackageList, dependencyOptions int, architecturesList []string, progress aptly.Progress) (*PackageList, error) {
  if !l.indexed {
  panic("list not indexed, can't filter")
  }
@@ -520,18 +520,43 @@ func (l *PackageList) FilterWithProgress(queries []PackageQuery, withDependencie
  result.PrepareIndex()
 
  dependencySource := NewPackageList()
+ validationSet := NewPackageList()
  if source != nil {
  dependencySource.Append(source)
  }
+ if extraDeps != nil {
+ added += extraDeps.Len()
+ dependencySource.Append(extraDeps)
+ validationSet.Append(extraDeps)
+ if dependencyOptions&DepFollowAllVariants > 0 {
+ // keep all packages that are also present in extraDeps
+ // --> they are updates
+ extraDeps.ForEachIndexed(func(p *Package) error {
+ var err error
+ searchResults := l.Search(Dependency{Pkg: p.Name, Relation: VersionDontCare}, true)
+ if searchResults != nil {
+ for _, p := range searchResults {
+ if dependencyOptions&DepVerboseResolve == DepVerboseResolve && progress != nil {
+ progress.ColoredPrintf("@{g}Include package as update@|: %s", p)
+ }
+ result.Add(p)
+ added++
+ }
+ }
+ return err
+ })
+ }
+ }
  dependencySource.Append(result)
+ validationSet.Append(result)
  dependencySource.PrepareIndex()
+ validationSet.PrepareIndex()
 
  // while some new dependencies were discovered
  for added > 0 {
  added = 0
 
- // find missing dependencies
- missing, err := result.VerifyDependencies(dependencyOptions, architecturesList, dependencySource, progress)
+ missing, err := validationSet.VerifyDependencies(dependencyOptions, architecturesList, dependencySource, progress)
  if err != nil {
  return nil, err
  }
@@ -564,6 +589,7 @@ func (l *PackageList) FilterWithProgress(queries []PackageQuery, withDependencie
  }
  result.Add(p)
  dependencySource.Add(p)
+ validationSet.Add(p)
  added++
  if dependencyOptions&DepFollowAllVariants == 0 {
  break
@@ -573,7 +599,27 @@ func (l *PackageList) FilterWithProgress(queries []PackageQuery, withDependencie
  if dependencyOptions&DepVerboseResolve == DepVerboseResolve && progress != nil {
  progress.ColoredPrintf("@{r}Unsatisfied dependency@|: %s", dep.String())
  }
+ }
 
+ if extraDeps == nil {
+ continue
+ }
+ // the missing dependency might come from the extraDeps package list
+ // if we can fulfil the dependency from extraDeps, include these packages
+ // within the validationSet -> we want to satisfy their dependencies as well
+ extraSearchResults := extraDeps.Search(dep, true)
+ if extraSearchResults != nil {
+ for _, p := range extraSearchResults {
+ if dependencyOptions&DepVerboseResolve == DepVerboseResolve && progress != nil {
+ progress.ColoredPrintf("@{g}Injecting extra package@|: %s", p)
+ }
+ dependencySource.Add(p)
+ validationSet.Add(p)
+ added++
+ if dependencyOptions&DepFollowAllVariants == 0 {
+ break
+ }
+ }
  }
  }
  }