Enable upgrading only specified repos

Reimplement support for `newt upgrade <repo1> <repo2>...`. Not
specifying repos upgrades everything; specifying repos update only the ones
given, without upgrading/installing its dependencies, etc.

This should allow advanced users to install only what is required, eg.
if developing an app for nrf52x one could simply run:

`newt upgrade --depth=1 apache-mynewt-core apache-mynewt-nimble nordic-nrfx`

This is also useful for CI, eg, for MCUboot it's possible to use MCUboot as
the root project and run:

`newt upgrade --depth=1 apache-mynewt-core nordic-nrfx mbedtls`

And this brings down the complete download from 1GB to just around 250MB.

Signed-off-by: Fabio Utzig <utzig@apache.org>

newt/cli/project_cmds.go

@@ -121,14 +121,14 @@ func makeRepoPredicate(repoNames []string) func(r *repo.Repo) bool {
 }
 
 func upgradeRunCmd(cmd *cobra.Command, args []string) {
-	proj := TryGetOrDownloadProject()
+	proj := TryGetOrDownloadProject(args)
 	interfaces.SetProject(proj)
 
-	proj.GetPkgRepos()
+	proj.GetPkgRepos(args)
 
 	pred := makeRepoPredicate(args)
 	if err := proj.UpgradeIf(
-		newtutil.NewtForce, newtutil.NewtAsk, pred); err != nil {
+		newtutil.NewtForce, newtutil.NewtAsk, args, pred); err != nil {
 
 		NewtUsage(nil, err)
 	}

newt/cli/util.go

@@ -210,11 +210,11 @@ func TryGetProject() *project.Project {
 	return p
 }
 
-func TryGetOrDownloadProject() *project.Project {
+func TryGetOrDownloadProject(okRepos []string) *project.Project {
 	var p *project.Project
 	var err error
 
-	if p, err = project.TryGetOrDownloadProject(); err != nil {
+	if p, err = project.TryGetOrDownloadProject(okRepos); err != nil {
 		NewtUsage(nil, err)
 	}
 

newt/deprepo/deprepo.go

@@ -107,14 +107,26 @@ func (vm VersionMap) String() string {
 	return s
 }
 
+func isInReqs(repo string, reqs RequirementMap) bool {
+	for r, _ := range reqs {
+		if repo == r {
+			return true
+		}
+	}
+	return false
+}
+
 // Builds a repo dependency graph from the repo requirements expressed in the
 // `project.yml` file.
 func BuildDepGraph(repos RepoMap, rootReqs RequirementMap) (DepGraph, error) {
 	dg := DepGraph{}
 
 	// First, add the hard dependencies expressed in `project.yml`.
 	for repoName, verReq := range rootReqs {
-		repo := repos[repoName]
+		repo, ok := repos[repoName]
+		if !ok {
+			continue
+		}
 		normalizedReq, err := repo.NormalizeVerReq(verReq)
 		if err != nil {
 			return nil, err
@@ -128,13 +140,16 @@ func BuildDepGraph(repos RepoMap, rootReqs RequirementMap) (DepGraph, error) {
 	// Add inter-repo dependencies to the graph.
 	for _, r := range repos.Sorted() {
 		nvers, err := r.NormalizedVersions()
-		if err != nil {
+		if err != nil && isInReqs(r.Name(), rootReqs) {
 			return nil, err
 		}
 		for _, v := range nvers {
 			deps := r.DepsForVersion(v)
 			reqMap := RequirementMap{}
 			for _, d := range deps {
+				if !isInReqs(d.Name, rootReqs) {
+					continue
+				}
 				depRepo := repos[d.Name]
 				verReqs, err := depRepo.NormalizeVerReq(d.VerReqs)
 				if err != nil {

newt/install/install.go

@@ -198,8 +198,10 @@ func (inst *Installer) ensureDepsInList(repos []*repo.Repo,
 			deps = r.DepsForVersion(vm[r.Name()])
 		}
 		for _, d := range deps {
-			depRepo := inst.repos[d.Name]
-			result = append(result, recurse(depRepo)...)
+			depRepo, ok := inst.repos[d.Name]
+			if ok {
+				result = append(result, recurse(depRepo)...)
+			}
 		}
 
 		return result
@@ -706,12 +708,14 @@ func (inst *Installer) remoteRepoInfo(r *repo.Repo, vm *deprepo.VersionMap) {
 	ri := inst.gatherInfo(r, vm)
 	s := fmt.Sprintf("    * %s:", r.Name())
 
-	s += fmt.Sprintf(" %s", ri.commitHash)
 	if ri.installedVer == nil {
+		s += fmt.Sprintf(" ?")
 		s += ", (not installed)"
 	} else if ri.errorText != "" {
+		s += fmt.Sprintf(" %s", ri.commitHash)
 		s += fmt.Sprintf(", (unknown: %s)", ri.errorText)
 	} else {
+		s += fmt.Sprintf(" %s", ri.commitHash)
 		if ri.installedVer.Commit == "" {
 			s += fmt.Sprintf(", %s", ri.installedVer.String())
 		}

newt/project/project.go

@@ -85,10 +85,10 @@ type Project struct {
 	yc ycfg.YCfg
 }
 
-func initProject(dir string, download bool) error {
+func initProject(dir string, download bool, okRepos []string) error {
 	var err error
 
-	globalProject, err = LoadProject(dir, download)
+	globalProject, err = loadProject(dir, download, okRepos)
 	if err != nil {
 		return err
 	}
@@ -99,29 +99,29 @@ func initProject(dir string, download bool) error {
 	return nil
 }
 
-func initialize(download bool) error {
+func initialize(download bool, okRepos []string) error {
 	if globalProject == nil {
 		wd, err := os.Getwd()
 		wd = filepath.ToSlash(wd)
 		if err != nil {
 			return util.NewNewtError(err.Error())
 		}
-		if err := initProject(wd, download); err != nil {
+		if err := initProject(wd, download, okRepos); err != nil {
 			return err
 		}
 	}
 	return nil
 }
 
 func TryGetProject() (*Project, error) {
-	if err := initialize(false); err != nil {
+	if err := initialize(false, nil); err != nil {
 		return nil, err
 	}
 	return globalProject, nil
 }
 
-func TryGetOrDownloadProject() (*Project, error) {
-	if err := initialize(true); err != nil {
+func TryGetOrDownloadProject(okRepos []string) (*Project, error) {
+	if err := initialize(true, okRepos); err != nil {
 		return nil, err
 	}
 	return globalProject, nil
@@ -153,24 +153,36 @@ func ResetDeps(newList interfaces.PackageList) interfaces.PackageList {
 	return oldList
 }
 
-func NewProject(dir string, download bool) (*Project, error) {
+func newProject(dir string, download bool, okRepos []string) (*Project, error) {
 	proj := &Project{}
 
-	if err := proj.Init(dir, download); err != nil {
+	if err := proj.Init(dir, download, okRepos); err != nil {
 		return nil, err
 	}
 
 	return proj, nil
 }
 
-func (proj *Project) GetPkgRepos() error {
+func isOkRepo(repo string, okRepos []string) bool {
+	if okRepos == nil || len(okRepos) == 0 {
+		return true
+	}
+	for _, v := range okRepos {
+		if repo == v {
+			return true
+		}
+	}
+	return false
+}
+
+func (proj *Project) GetPkgRepos(okRepos []string) error {
 
 	for _, pkgList := range proj.packages {
 		for _, pkg := range *pkgList {
 			if pkg.PkgConfig().HasKey("repository") {
 				for k, _ := range pkg.PkgConfig().AllSettings() {
 					repoName := strings.TrimPrefix(k, "repository.")
-					if repoName != k {
+					if repoName != k && isOkRepo(repoName, okRepos) {
 						fields, err := pkg.PkgConfig().GetValStringMapString(k, nil)
 						util.OneTimeWarningError(err)
 
@@ -190,7 +202,7 @@ func (proj *Project) GetPkgRepos() error {
 								repoName, fields["vers"], err.Error())
 						}
 						r.SetPkgName(pkg.Name())
-						if err := proj.addRepo(r, true); err != nil {
+						if err := proj.addRepo(r, true, false); err != nil {
 							return err
 						}
 						proj.rootRepoReqs[repoName] = verReq
@@ -302,10 +314,10 @@ func (proj *Project) SelectRepos(pred func(r *repo.Repo) bool) []*repo.Repo {
 
 // Installs or upgrades repos matching the specified predicate.
 func (proj *Project) UpgradeIf(
-	force bool, ask bool, predicate func(r *repo.Repo) bool) error {
+	force bool, ask bool, okRepos []string, predicate func(r *repo.Repo) bool) error {
 
 	// Make sure we have an up to date copy of all `repository.yml` files.
-	if err := proj.downloadRepositoryYmlFiles(); err != nil {
+	if err := proj.downloadRepositoryYmlFiles(okRepos); err != nil {
 		return err
 	}
 
@@ -328,7 +340,7 @@ func (proj *Project) InfoIf(predicate func(r *repo.Repo) bool,
 
 	if remote {
 		// Make sure we have an up to date copy of all `repository.yml` files.
-		if err := proj.downloadRepositoryYmlFiles(); err != nil {
+		if err := proj.downloadRepositoryYmlFiles(nil); err != nil {
 			return err
 		}
 	}
@@ -425,7 +437,7 @@ func (proj *Project) checkNewtVer() error {
 }
 
 // Loads the `repository.yml` file for each depended-on repo.  This
-func (proj *Project) loadRepoDeps(download bool) error {
+func (proj *Project) loadRepoDeps(download bool, okRepos []string) error {
 	seen := map[string]struct{}{}
 
 	loadDeps := func(r *repo.Repo) ([]*repo.Repo, error) {
@@ -434,6 +446,11 @@ func (proj *Project) loadRepoDeps(download bool) error {
 		depMap := r.CommitDepMap()
 		for _, depSlice := range depMap {
 			for _, dep := range depSlice {
+				if !isOkRepo(dep.Name, okRepos) {
+					log.Debugf("Skipping repo %s", dep.Name)
+					continue
+				}
+
 				if _, ok := seen[dep.Name]; !ok {
 					seen[r.Name()] = struct{}{}
 
@@ -448,7 +465,7 @@ func (proj *Project) loadRepoDeps(download bool) error {
 								return nil, err
 							}
 						}
-						if err := proj.addRepo(depRepo, download); err != nil {
+						if err := proj.addRepo(depRepo, download, true); err != nil {
 							return nil, err
 						}
 					}
@@ -485,7 +502,7 @@ func (proj *Project) loadRepoDeps(download bool) error {
 	return nil
 }
 
-func (proj *Project) downloadRepositoryYmlFiles() error {
+func (proj *Project) downloadRepositoryYmlFiles(okRepos []string) error {
 	// Download the `repository.yml` file for each root-level repo (those
 	// specified in the `project.yml` file).
 	for _, r := range proj.repos.Sorted() {
@@ -500,7 +517,7 @@ func (proj *Project) downloadRepositoryYmlFiles() error {
 	}
 
 	// Download the `repository.yml` file for each depended-on repo.
-	if err := proj.loadRepoDeps(true); err != nil {
+	if err := proj.loadRepoDeps(true, okRepos); err != nil {
 		return err
 	}
 
@@ -548,13 +565,13 @@ func (proj *Project) verifyNewtCompat() error {
 
 // addRepo Adds an entry to the project's repo map.  It clones the repo if it
 // does not exist locally.
-func (proj *Project) addRepo(r *repo.Repo, download bool) error {
+func (proj *Project) addRepo(r *repo.Repo, download bool, isDep bool) error {
 	if download {
 		if err := r.EnsureExists(); err != nil {
 			return err
 		}
 	} else {
-		if !r.CheckExists() {
+		if !r.CheckExists() && !isDep {
 			return util.NewNewtError(
 				fmt.Sprintf(
 					"Repo \"%s\" is not installed, please run `newt upgrade`!",
@@ -566,7 +583,7 @@ func (proj *Project) addRepo(r *repo.Repo, download bool) error {
 	return nil
 }
 
-func (proj *Project) loadConfig(download bool) error {
+func (proj *Project) loadConfig(download bool, okRepos []string) error {
 	yc, err := config.ReadFile(proj.BasePath + "/" + PROJECT_FILE_NAME)
 	if err != nil {
 		return util.NewNewtError(err.Error())
@@ -615,7 +632,7 @@ func (proj *Project) loadConfig(download bool) error {
 					repoName, fields["vers"], err.Error())
 			}
 
-			if err := proj.addRepo(r, download); err != nil {
+			if err := proj.addRepo(r, download, false); err != nil {
 				return err
 			}
 			proj.rootRepoReqs[repoName] = verReq
@@ -625,7 +642,7 @@ func (proj *Project) loadConfig(download bool) error {
 	// Read `repository.yml` files belonging to dependee repos from disk.
 	// These repos might not be specified in the `project.yml` file, but they
 	// are still part of the project.
-	if err := proj.loadRepoDeps(download); err != nil {
+	if err := proj.loadRepoDeps(download, okRepos); err != nil {
 		return err
 	}
 
@@ -662,7 +679,7 @@ func (proj *Project) loadConfig(download bool) error {
 	return nil
 }
 
-func (proj *Project) Init(dir string, download bool) error {
+func (proj *Project) Init(dir string, download bool, okRepos []string) error {
 	proj.BasePath = filepath.ToSlash(filepath.Clean(dir))
 
 	// Only one project per system, when created, set it as the global project
@@ -672,7 +689,7 @@ func (proj *Project) Init(dir string, download bool) error {
 	proj.rootRepoReqs = map[string]newtutil.RepoVersion{}
 
 	// Load Project configuration
-	if err := proj.loadConfig(download); err != nil {
+	if err := proj.loadConfig(download, okRepos); err != nil {
 		return err
 	}
 
@@ -813,13 +830,13 @@ func (proj *Project) PackagesOfType(pkgType interfaces.PackageType) []interfaces
 	return matches
 }
 
-func LoadProject(dir string, download bool) (*Project, error) {
+func loadProject(dir string, download bool, okRepos []string) (*Project, error) {
 	projDir, err := findProjectDir(dir)
 	if err != nil {
 		return nil, err
 	}
 
-	proj, err := NewProject(projDir, download)
+	proj, err := newProject(projDir, download, okRepos)
 
 	return proj, err
 }