From 24a208ae34e3631ebfd7a07bdd4f04f75c26791b Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Wed, 27 Jul 2016 20:48:56 -0400
Subject: [PATCH 01/71] Rename errors.go to make space for sm errors

---
 errors.go => solve_failures.go | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename errors.go => solve_failures.go (100%)

diff --git a/errors.go b/solve_failures.go
similarity index 100%
rename from errors.go
rename to solve_failures.go

From 51077e10d737ab57d6fdb4ae1f52f98a89ecb615 Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Wed, 27 Jul 2016 21:46:52 -0400
Subject: [PATCH 02/71] Convert SourceManager to use on ProjectIdentifier

I pulled this out a while back, but going back to it's been a long time
coming. Not all the SourceManager methods strictly need the information
in a ProjectIdentifier, but it's much easier to be consistent and just
always require it.

This does not actually convert function/method bodies - just signatures.
In no way does this come even close to compiling.
---
 solve_basic_test.go   | 10 +++---
 solve_bimodal_test.go |  4 +--
 source_manager.go     | 75 ++++++++++++++++++++++---------------------
 3 files changed, 45 insertions(+), 44 deletions(-)

diff --git a/solve_basic_test.go b/solve_basic_test.go
index ac833e3..1083368 100644
--- a/solve_basic_test.go
+++ b/solve_basic_test.go
@@ -1202,7 +1202,7 @@ func newdepspecSM(ds []depspec, ignore []string) *depspecSourceManager {
 	}
 }
 
-func (sm *depspecSourceManager) GetManifestAndLock(n ProjectRoot, v Version) (Manifest, Lock, error) {
+func (sm *depspecSourceManager) GetManifestAndLock(id ProjectIdentifier, v Version) (Manifest, Lock, error) {
 	for _, ds := range sm.specs {
 		if n == ds.n && v.Matches(ds.v) {
 			return ds, dummyLock{}, nil
@@ -1217,7 +1217,7 @@ func (sm *depspecSourceManager) AnalyzerInfo() (string, *semver.Version) {
 	return "depspec-sm-builtin", sv("v1.0.0")
 }
 
-func (sm *depspecSourceManager) ExternalReach(n ProjectRoot, v Version) (map[string][]string, error) {
+func (sm *depspecSourceManager) ExternalReach(id ProjectIdentifier, v Version) (map[string][]string, error) {
 	id := pident{n: n, v: v}
 	if m, exists := sm.rm[id]; exists {
 		return m, nil
@@ -1225,7 +1225,7 @@ func (sm *depspecSourceManager) ExternalReach(n ProjectRoot, v Version) (map[str
 	return nil, fmt.Errorf("No reach data for %s at version %s", n, v)
 }
 
-func (sm *depspecSourceManager) ListExternal(n ProjectRoot, v Version) ([]string, error) {
+func (sm *depspecSourceManager) ListExternal(id ProjectIdentifier, v Version) ([]string, error) {
 	// This should only be called for the root
 	id := pident{n: n, v: v}
 	if r, exists := sm.rm[id]; exists {
@@ -1234,7 +1234,7 @@ func (sm *depspecSourceManager) ListExternal(n ProjectRoot, v Version) ([]string
 	return nil, fmt.Errorf("No reach data for %s at version %s", n, v)
 }
 
-func (sm *depspecSourceManager) ListPackages(n ProjectRoot, v Version) (PackageTree, error) {
+func (sm *depspecSourceManager) ListPackages(id ProjectIdentifier, v Version) (PackageTree, error) {
 	id := pident{n: n, v: v}
 	if r, exists := sm.rm[id]; exists {
 		ptree := PackageTree{
@@ -1297,7 +1297,7 @@ func (sm *depspecSourceManager) VendorCodeExists(name ProjectRoot) (bool, error)
 
 func (sm *depspecSourceManager) Release() {}
 
-func (sm *depspecSourceManager) ExportProject(n ProjectRoot, v Version, to string) error {
+func (sm *depspecSourceManager) ExportProject(id ProjectIdentifier, v Version, to string) error {
 	return fmt.Errorf("dummy sm doesn't support exporting")
 }
 
diff --git a/solve_bimodal_test.go b/solve_bimodal_test.go
index 530d6e1..aa97294 100644
--- a/solve_bimodal_test.go
+++ b/solve_bimodal_test.go
@@ -649,7 +649,7 @@ func newbmSM(bmf bimodalFixture) *bmSourceManager {
 	return sm
 }
 
-func (sm *bmSourceManager) ListPackages(n ProjectRoot, v Version) (PackageTree, error) {
+func (sm *bmSourceManager) ListPackages(id ProjectIdentifier, v Version) (PackageTree, error) {
 	for k, ds := range sm.specs {
 		// Cheat for root, otherwise we blow up b/c version is empty
 		if n == ds.n && (k == 0 || ds.v.Matches(v)) {
@@ -674,7 +674,7 @@ func (sm *bmSourceManager) ListPackages(n ProjectRoot, v Version) (PackageTree,
 	return PackageTree{}, fmt.Errorf("Project %s at version %s could not be found", n, v)
 }
 
-func (sm *bmSourceManager) GetManifestAndLock(n ProjectRoot, v Version) (Manifest, Lock, error) {
+func (sm *bmSourceManager) GetManifestAndLock(id ProjectIdentifier, v Version) (Manifest, Lock, error) {
 	for _, ds := range sm.specs {
 		if n == ds.n && v.Matches(ds.v) {
 			if l, exists := sm.lm[string(n)+" "+v.String()]; exists {
diff --git a/source_manager.go b/source_manager.go
index 7403025..8ce89b4 100644
--- a/source_manager.go
+++ b/source_manager.go
@@ -15,42 +15,42 @@ import (
 // source repositories. Its primary purpose is to serve the needs of a Solver,
 // but it is handy for other purposes, as well.
 //
-// gps's built-in SourceManager, accessible via NewSourceManager(), is
-// intended to be generic and sufficient for any purpose. It provides some
-// additional semantics around the methods defined here.
+// gps's built-in SourceManager, SourceMgr, is intended to be generic and
+// sufficient for any purpose. It provides some additional semantics around the
+// methods defined here.
 type SourceManager interface {
 	// RepoExists checks if a repository exists, either upstream or in the
 	// SourceManager's central repository cache.
-	RepoExists(ProjectRoot) (bool, error)
+	RepoExists(ProjectIdentifier) (bool, error)
 
 	// ListVersions retrieves a list of the available versions for a given
 	// repository name.
-	ListVersions(ProjectRoot) ([]Version, error)
+	ListVersions(ProjectIdentifier) ([]Version, error)
 
 	// RevisionPresentIn indicates whether the provided Version is present in
 	// the given repository.
-	RevisionPresentIn(ProjectRoot, Revision) (bool, error)
+	RevisionPresentIn(ProjectIdentifier, Revision) (bool, error)
 
-	// ListPackages retrieves a tree of the Go packages at or below the provided
-	// import path, at the provided version.
-	ListPackages(ProjectRoot, Version) (PackageTree, error)
+	// ListPackages parses the tree of the Go packages at or below root of the
+	// provided ProjectIdentifier, at the provided version.
+	ListPackages(ProjectIdentifier, Version) (PackageTree, error)
 
 	// GetManifestAndLock returns manifest and lock information for the provided
 	// root import path.
 	//
-	// gps currently requires that projects be rooted at their
-	// repository root, necessitating that this ProjectRoot must also be a
+	// gps currently requires that projects be rooted at their repository root,
+	// necessitating that the ProjectIdentifier's ProjectRoot must also be a
 	// repository root.
-	GetManifestAndLock(ProjectRoot, Version) (Manifest, Lock, error)
+	GetManifestAndLock(ProjectIdentifier, Version) (Manifest, Lock, error)
+
+	// ExportProject writes out the tree of the provided import path, at the
+	// provided version, to the provided directory.
+	ExportProject(ProjectIdentifier, Version, string) error
 
 	// AnalyzerInfo reports the name and version of the logic used to service
 	// GetManifestAndLock().
 	AnalyzerInfo() (name string, version *semver.Version)
 
-	// ExportProject writes out the tree of the provided import path, at the
-	// provided version, to the provided directory.
-	ExportProject(ProjectRoot, Version, string) error
-
 	// Release lets go of any locks held by the SourceManager.
 	Release()
 }
@@ -72,10 +72,9 @@ type ProjectAnalyzer interface {
 // tools; control via dependency injection is intended to be sufficient.
 type SourceMgr struct {
 	cachedir string
-	pms      map[ProjectRoot]*pmState
+	pms      map[ProjectIdentifier]*pmState
 	an       ProjectAnalyzer
 	ctx      build.Context
-	//pme               map[ProjectRoot]error
 }
 
 var _ SourceManager = &SourceMgr{}
@@ -148,13 +147,14 @@ func (sm *SourceMgr) AnalyzerInfo() (name string, version *semver.Version) {
 	return sm.an.Info()
 }
 
-// GetManifestAndLock returns manifest and lock information for the provided import
-// path. gps currently requires that projects be rooted at their repository
-// root, which means that this ProjectRoot must also be a repository root.
+// GetManifestAndLock returns manifest and lock information for the provided
+// import path. gps currently requires that projects be rooted at their
+// repository root, necessitating that the ProjectIdentifier's ProjectRoot must
+// also be a repository root.
 //
 // The work of producing the manifest and lock is delegated to the injected
 // ProjectAnalyzer's DeriveManifestAndLock() method.
-func (sm *SourceMgr) GetManifestAndLock(n ProjectRoot, v Version) (Manifest, Lock, error) {
+func (sm *SourceMgr) GetManifestAndLock(id ProjectIdentifier, v Version) (Manifest, Lock, error) {
 	pmc, err := sm.getProjectManager(n)
 	if err != nil {
 		return nil, nil, err
@@ -163,9 +163,9 @@ func (sm *SourceMgr) GetManifestAndLock(n ProjectRoot, v Version) (Manifest, Loc
 	return pmc.pm.GetInfoAt(v)
 }
 
-// ListPackages retrieves a tree of the Go packages at or below the provided
-// import path, at the provided version.
-func (sm *SourceMgr) ListPackages(n ProjectRoot, v Version) (PackageTree, error) {
+// ListPackages parses the tree of the Go packages at and below the ProjectRoot
+// of the given ProjectIdentifier, at the given version.
+func (sm *SourceMgr) ListPackages(id ProjectIdentifier, v Version) (PackageTree, error) {
 	pmc, err := sm.getProjectManager(n)
 	if err != nil {
 		return PackageTree{}, err
@@ -182,10 +182,11 @@ func (sm *SourceMgr) ListPackages(n ProjectRoot, v Version) (PackageTree, error)
 // expected that the caller either not care about order, or sort the result
 // themselves.
 //
-// This list is always retrieved from upstream; if upstream is not accessible
-// (network outage, access issues, or the resource actually went away), an error
-// will be returned.
-func (sm *SourceMgr) ListVersions(n ProjectRoot) ([]Version, error) {
+// This list is always retrieved from upstream on the first call. Subsequent
+// calls will return a cached version of the first call's results. if upstream
+// is not accessible (network outage, access issues, or the resource actually
+// went away), an error will be returned.
+func (sm *SourceMgr) ListVersions(id ProjectIdentifier) ([]Version, error) {
 	pmc, err := sm.getProjectManager(n)
 	if err != nil {
 		// TODO(sdboyer) More-er proper-er errors
@@ -197,7 +198,7 @@ func (sm *SourceMgr) ListVersions(n ProjectRoot) ([]Version, error) {
 
 // RevisionPresentIn indicates whether the provided Revision is present in the given
 // repository.
-func (sm *SourceMgr) RevisionPresentIn(n ProjectRoot, r Revision) (bool, error) {
+func (sm *SourceMgr) RevisionPresentIn(id ProjectIdentifier, r Revision) (bool, error) {
 	pmc, err := sm.getProjectManager(n)
 	if err != nil {
 		// TODO(sdboyer) More-er proper-er errors
@@ -208,8 +209,8 @@ func (sm *SourceMgr) RevisionPresentIn(n ProjectRoot, r Revision) (bool, error)
 }
 
 // RepoExists checks if a repository exists, either upstream or in the cache,
-// for the provided ProjectRoot.
-func (sm *SourceMgr) RepoExists(n ProjectRoot) (bool, error) {
+// for the provided ProjectIdentifier.
+func (sm *SourceMgr) RepoExists(id ProjectIdentifier) (bool, error) {
 	pms, err := sm.getProjectManager(n)
 	if err != nil {
 		return false, err
@@ -218,9 +219,9 @@ func (sm *SourceMgr) RepoExists(n ProjectRoot) (bool, error) {
 	return pms.pm.CheckExistence(existsInCache) || pms.pm.CheckExistence(existsUpstream), nil
 }
 
-// ExportProject writes out the tree of the provided import path, at the
-// provided version, to the provided directory.
-func (sm *SourceMgr) ExportProject(n ProjectRoot, v Version, to string) error {
+// ExportProject writes out the tree of the provided ProjectIdentifier's
+// ProjectRoot, at the provided version, to the provided directory.
+func (sm *SourceMgr) ExportProject(id ProjectIdentifier, v Version, to string) error {
 	pms, err := sm.getProjectManager(n)
 	if err != nil {
 		return err
@@ -229,10 +230,10 @@ func (sm *SourceMgr) ExportProject(n ProjectRoot, v Version, to string) error {
 	return pms.pm.ExportVersionTo(v, to)
 }
 
-// getProjectManager gets the project manager for the given ProjectRoot.
+// getProjectManager gets the project manager for the given ProjectIdentifier.
 //
 // If no such manager yet exists, it attempts to create one.
-func (sm *SourceMgr) getProjectManager(n ProjectRoot) (*pmState, error) {
+func (sm *SourceMgr) getProjectManager(id ProjectIdentifier) (*pmState, error) {
 	// Check pm cache and errcache first
 	if pm, exists := sm.pms[n]; exists {
 		return pm, nil

From c5573215c7f36e0a2cc9816b24efdaf53dc4fa2c Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Wed, 27 Jul 2016 22:37:43 -0400
Subject: [PATCH 03/71] Have sourceBridge compose SourceManager

---
 bridge.go           | 25 ++++++++++---------------
 hash.go             |  4 ++--
 manager_test.go     |  2 +-
 satisfy.go          |  6 +++---
 solve_basic_test.go |  2 +-
 solver.go           | 18 +++++++++---------
 source_manager.go   |  1 +
 version_queue.go    |  4 ++--
 8 files changed, 29 insertions(+), 33 deletions(-)

diff --git a/bridge.go b/bridge.go
index d09a35a..5ff8d7f 100644
--- a/bridge.go
+++ b/bridge.go
@@ -12,20 +12,15 @@ import (
 // sourceBridges provide an adapter to SourceManagers that tailor operations
 // for a single solve run.
 type sourceBridge interface {
-	getManifestAndLock(pa atom) (Manifest, Lock, error)
-	listVersions(id ProjectIdentifier) ([]Version, error)
-	listPackages(id ProjectIdentifier, v Version) (PackageTree, error)
+	SourceManager // composes SourceManager
+	verifyRootDir(path string) error
 	computeRootReach() ([]string, error)
-	revisionPresentIn(id ProjectIdentifier, r Revision) (bool, error)
 	pairRevision(id ProjectIdentifier, r Revision) []Version
 	pairVersion(id ProjectIdentifier, v UnpairedVersion) PairedVersion
-	repoExists(id ProjectIdentifier) (bool, error)
 	vendorCodeExists(id ProjectIdentifier) (bool, error)
 	matches(id ProjectIdentifier, c Constraint, v Version) bool
 	matchesAny(id ProjectIdentifier, c1, c2 Constraint) bool
 	intersect(id ProjectIdentifier, c1, c2 Constraint) Constraint
-	verifyRootDir(path string) error
-	analyzerInfo() (string, *semver.Version)
 	deduceRemoteRepo(path string) (*remoteRepo, error)
 }
 
@@ -76,14 +71,14 @@ var mkBridge func(*solver, SourceManager) sourceBridge = func(s *solver, sm Sour
 	}
 }
 
-func (b *bridge) getManifestAndLock(pa atom) (Manifest, Lock, error) {
+func (b *bridge) GetManifestAndLock(pa atom) (Manifest, Lock, error) {
 	if pa.id.ProjectRoot == b.s.params.ImportRoot {
 		return b.s.rm, b.s.rl, nil
 	}
 	return b.sm.GetManifestAndLock(ProjectRoot(pa.id.netName()), pa.v)
 }
 
-func (b *bridge) analyzerInfo() (string, *semver.Version) {
+func (b *bridge) AnalyzerInfo() (string, *semver.Version) {
 	return b.sm.AnalyzerInfo()
 }
 
@@ -96,7 +91,7 @@ func (b *bridge) key(id ProjectIdentifier) ProjectRoot {
 	return k
 }
 
-func (b *bridge) listVersions(id ProjectIdentifier) ([]Version, error) {
+func (b *bridge) ListVersions(id ProjectIdentifier) ([]Version, error) {
 	k := b.key(id)
 
 	if vl, exists := b.vlists[k]; exists {
@@ -119,12 +114,12 @@ func (b *bridge) listVersions(id ProjectIdentifier) ([]Version, error) {
 	return vl, nil
 }
 
-func (b *bridge) revisionPresentIn(id ProjectIdentifier, r Revision) (bool, error) {
+func (b *bridge) RevisionPresentIn(id ProjectIdentifier, r Revision) (bool, error) {
 	k := b.key(id)
 	return b.sm.RevisionPresentIn(k, r)
 }
 
-func (b *bridge) repoExists(id ProjectIdentifier) (bool, error) {
+func (b *bridge) RepoExists(id ProjectIdentifier) (bool, error) {
 	k := b.key(id)
 	return b.sm.RepoExists(k)
 }
@@ -141,7 +136,7 @@ func (b *bridge) vendorCodeExists(id ProjectIdentifier) (bool, error) {
 }
 
 func (b *bridge) pairVersion(id ProjectIdentifier, v UnpairedVersion) PairedVersion {
-	vl, err := b.listVersions(id)
+	vl, err := b.ListVersions(id)
 	if err != nil {
 		return nil
 	}
@@ -159,7 +154,7 @@ func (b *bridge) pairVersion(id ProjectIdentifier, v UnpairedVersion) PairedVers
 }
 
 func (b *bridge) pairRevision(id ProjectIdentifier, r Revision) []Version {
-	vl, err := b.listVersions(id)
+	vl, err := b.ListVersions(id)
 	if err != nil {
 		return nil
 	}
@@ -409,7 +404,7 @@ func (b *bridge) listRootPackages() (PackageTree, error) {
 //
 // The root project is handled separately, as the source manager isn't
 // responsible for that code.
-func (b *bridge) listPackages(id ProjectIdentifier, v Version) (PackageTree, error) {
+func (b *bridge) ListPackages(id ProjectIdentifier, v Version) (PackageTree, error) {
 	if id.ProjectRoot == b.s.params.ImportRoot {
 		return b.listRootPackages()
 	}
diff --git a/hash.go b/hash.go
index e336aaf..893c34e 100644
--- a/hash.go
+++ b/hash.go
@@ -20,7 +20,7 @@ func (s *solver) HashInputs() ([]byte, error) {
 	// Do these checks up front before any other work is needed, as they're the
 	// only things that can cause errors
 	// Pass in magic root values, and the bridge will analyze the right thing
-	ptree, err := s.b.listPackages(ProjectIdentifier{ProjectRoot: s.params.ImportRoot}, nil)
+	ptree, err := s.b.ListPackages(ProjectIdentifier{ProjectRoot: s.params.ImportRoot}, nil)
 	if err != nil {
 		return nil, badOptsFailure(fmt.Sprintf("Error while parsing packages under %s: %s", s.params.RootDir, err.Error()))
 	}
@@ -93,7 +93,7 @@ func (s *solver) HashInputs() ([]byte, error) {
 		}
 	}
 
-	an, av := s.b.analyzerInfo()
+	an, av := s.b.AnalyzerInfo()
 	h.Write([]byte(an))
 	h.Write([]byte(av.String()))
 
diff --git a/manager_test.go b/manager_test.go
index ae65ef4..02ae908 100644
--- a/manager_test.go
+++ b/manager_test.go
@@ -134,7 +134,7 @@ func TestProjectManagerInit(t *testing.T) {
 		s:      &solver{},
 	}
 
-	v, err = smc.listVersions(ProjectIdentifier{ProjectRoot: pn})
+	v, err = smc.ListVersions(ProjectIdentifier{ProjectRoot: pn})
 	if err != nil {
 		t.Errorf("Unexpected error during initial project setup/fetching %s", err)
 	}
diff --git a/satisfy.go b/satisfy.go
index 686676d..ef9e688 100644
--- a/satisfy.go
+++ b/satisfy.go
@@ -99,7 +99,7 @@ func (s *solver) checkAtomAllowable(pa atom) error {
 // checkRequiredPackagesExist ensures that all required packages enumerated by
 // existing dependencies on this atom are actually present in the atom.
 func (s *solver) checkRequiredPackagesExist(a atomWithPackages) error {
-	ptree, err := s.b.listPackages(a.a.id, a.a.v)
+	ptree, err := s.b.ListPackages(a.a.id, a.a.v)
 	if err != nil {
 		// TODO(sdboyer) handle this more gracefully
 		return err
@@ -225,7 +225,7 @@ func (s *solver) checkPackageImportsFromDepExist(a atomWithPackages, cdep comple
 		return nil
 	}
 
-	ptree, err := s.b.listPackages(sel.a.id, sel.a.v)
+	ptree, err := s.b.ListPackages(sel.a.id, sel.a.v)
 	if err != nil {
 		// TODO(sdboyer) handle this more gracefully
 		return err
@@ -266,7 +266,7 @@ func (s *solver) checkRevisionExists(a atomWithPackages, cdep completeDep) error
 		return nil
 	}
 
-	present, _ := s.b.revisionPresentIn(cdep.Ident, r)
+	present, _ := s.b.RevisionPresentIn(cdep.Ident, r)
 	if present {
 		return nil
 	}
diff --git a/solve_basic_test.go b/solve_basic_test.go
index 1083368..b02e7af 100644
--- a/solve_basic_test.go
+++ b/solve_basic_test.go
@@ -1342,7 +1342,7 @@ func (b *depspecBridge) verifyRootDir(path string) error {
 	return nil
 }
 
-func (b *depspecBridge) listPackages(id ProjectIdentifier, v Version) (PackageTree, error) {
+func (b *depspecBridge) ListPackages(id ProjectIdentifier, v Version) (PackageTree, error) {
 	return b.sm.(fixSM).ListPackages(b.key(id), v)
 }
 
diff --git a/solver.go b/solver.go
index f6efd96..92cc242 100644
--- a/solver.go
+++ b/solver.go
@@ -437,7 +437,7 @@ func (s *solver) selectRoot() error {
 		v: rootRev,
 	}
 
-	ptree, err := s.b.listPackages(pa.id, nil)
+	ptree, err := s.b.ListPackages(pa.id, nil)
 	if err != nil {
 		return err
 	}
@@ -493,12 +493,12 @@ func (s *solver) getImportsAndConstraintsOf(a atomWithPackages) ([]completeDep,
 
 	// Work through the source manager to get project info and static analysis
 	// information.
-	m, _, err := s.b.getManifestAndLock(a.a)
+	m, _, err := s.b.GetManifestAndLock(a.a)
 	if err != nil {
 		return nil, err
 	}
 
-	ptree, err := s.b.listPackages(a.a.id, a.a.v)
+	ptree, err := s.b.ListPackages(a.a.id, a.a.v)
 	if err != nil {
 		return nil, err
 	}
@@ -639,7 +639,7 @@ func (s *solver) createVersionQueue(bmi bimodalIdentifier) (*versionQueue, error
 		return newVersionQueue(id, nil, nil, s.b)
 	}
 
-	exists, err := s.b.repoExists(id)
+	exists, err := s.b.RepoExists(id)
 	if err != nil {
 		return nil, err
 	}
@@ -679,7 +679,7 @@ func (s *solver) createVersionQueue(bmi bimodalIdentifier) (*versionQueue, error
 				continue
 			}
 
-			_, l, err := s.b.getManifestAndLock(dep.depender)
+			_, l, err := s.b.GetManifestAndLock(dep.depender)
 			if err != nil || l == nil {
 				// err being non-nil really shouldn't be possible, but the lock
 				// being nil is quite likely
@@ -816,7 +816,7 @@ func (s *solver) getLockVersionIfValid(id ProjectIdentifier) (Version, error) {
 		// to be found and attempted in the repository. If it's only in vendor,
 		// though, then we have to try to use what's in the lock, because that's
 		// the only version we'll be able to get.
-		if exist, _ := s.b.repoExists(id); exist {
+		if exist, _ := s.b.RepoExists(id); exist {
 			return nil, nil
 		}
 
@@ -1001,8 +1001,8 @@ func (s *solver) unselectedComparator(i, j int) bool {
 	// We can safely ignore an err from ListVersions here because, if there is
 	// an actual problem, it'll be noted and handled somewhere else saner in the
 	// solving algorithm.
-	ivl, _ := s.b.listVersions(iname)
-	jvl, _ := s.b.listVersions(jname)
+	ivl, _ := s.b.ListVersions(iname)
+	jvl, _ := s.b.ListVersions(jname)
 	iv, jv := len(ivl), len(jvl)
 
 	// Packages with fewer versions to pick from are less likely to benefit from
@@ -1060,7 +1060,7 @@ func (s *solver) selectAtom(a atomWithPackages, pkgonly bool) {
 
 	// If this atom has a lock, pull it out so that we can potentially inject
 	// preferred versions into any bmis we enqueue
-	_, l, _ := s.b.getManifestAndLock(a.a)
+	_, l, _ := s.b.GetManifestAndLock(a.a)
 	var lmap map[ProjectIdentifier]Version
 	if l != nil {
 		lmap = make(map[ProjectIdentifier]Version)
diff --git a/source_manager.go b/source_manager.go
index 8ce89b4..ef79806 100644
--- a/source_manager.go
+++ b/source_manager.go
@@ -21,6 +21,7 @@ import (
 type SourceManager interface {
 	// RepoExists checks if a repository exists, either upstream or in the
 	// SourceManager's central repository cache.
+	// TODO rename to SourceExists
 	RepoExists(ProjectIdentifier) (bool, error)
 
 	// ListVersions retrieves a list of the available versions for a given
diff --git a/version_queue.go b/version_queue.go
index e74a1da..7c92253 100644
--- a/version_queue.go
+++ b/version_queue.go
@@ -40,7 +40,7 @@ func newVersionQueue(id ProjectIdentifier, lockv, prefv Version, b sourceBridge)
 
 	if len(vq.pi) == 0 {
 		var err error
-		vq.pi, err = vq.b.listVersions(vq.id)
+		vq.pi, err = vq.b.ListVersions(vq.id)
 		if err != nil {
 			// TODO(sdboyer) pushing this error this early entails that we
 			// unconditionally deep scan (e.g. vendor), as well as hitting the
@@ -86,7 +86,7 @@ func (vq *versionQueue) advance(fail error) (err error) {
 		}
 
 		vq.allLoaded = true
-		vq.pi, err = vq.b.listVersions(vq.id)
+		vq.pi, err = vq.b.ListVersions(vq.id)
 		if err != nil {
 			return err
 		}

From 8c226cd18b25dd8c71c2fed5d04fb102999160ac Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Fri, 29 Jul 2016 00:47:32 -0400
Subject: [PATCH 04/71] Buncha type renames; short tests now passing again

---
 bridge.go             | 42 +++++++++++++++------------------------
 manager_test.go       | 26 ++++++++++++------------
 result.go             |  2 +-
 result_test.go        |  2 +-
 solve_basic_test.go   | 46 ++++++++++++++++++++++---------------------
 solve_bimodal_test.go | 12 +++++------
 solve_test.go         |  4 ++--
 solver.go             |  6 +++---
 source_manager.go     | 25 +++++++++++------------
 9 files changed, 77 insertions(+), 88 deletions(-)

diff --git a/bridge.go b/bridge.go
index 5ff8d7f..00fb839 100644
--- a/bridge.go
+++ b/bridge.go
@@ -58,7 +58,7 @@ type bridge struct {
 	// layered on top of the proper SourceManager's cache; the only difference
 	// is that this keeps the versions sorted in the direction required by the
 	// current solve run
-	vlists map[ProjectRoot][]Version
+	vlists map[ProjectIdentifier][]Version
 }
 
 // Global factory func to create a bridge. This exists solely to allow tests to
@@ -67,38 +67,27 @@ var mkBridge func(*solver, SourceManager) sourceBridge = func(s *solver, sm Sour
 	return &bridge{
 		sm:     sm,
 		s:      s,
-		vlists: make(map[ProjectRoot][]Version),
+		vlists: make(map[ProjectIdentifier][]Version),
 	}
 }
 
-func (b *bridge) GetManifestAndLock(pa atom) (Manifest, Lock, error) {
-	if pa.id.ProjectRoot == b.s.params.ImportRoot {
+func (b *bridge) GetManifestAndLock(id ProjectIdentifier, v Version) (Manifest, Lock, error) {
+	if id.ProjectRoot == b.s.params.ImportRoot {
 		return b.s.rm, b.s.rl, nil
 	}
-	return b.sm.GetManifestAndLock(ProjectRoot(pa.id.netName()), pa.v)
+	return b.sm.GetManifestAndLock(id, v)
 }
 
 func (b *bridge) AnalyzerInfo() (string, *semver.Version) {
 	return b.sm.AnalyzerInfo()
 }
 
-func (b *bridge) key(id ProjectIdentifier) ProjectRoot {
-	k := ProjectRoot(id.NetworkName)
-	if k == "" {
-		k = id.ProjectRoot
-	}
-
-	return k
-}
-
 func (b *bridge) ListVersions(id ProjectIdentifier) ([]Version, error) {
-	k := b.key(id)
-
-	if vl, exists := b.vlists[k]; exists {
+	if vl, exists := b.vlists[id]; exists {
 		return vl, nil
 	}
 
-	vl, err := b.sm.ListVersions(k)
+	vl, err := b.sm.ListVersions(id)
 	// TODO(sdboyer) cache errors, too?
 	if err != nil {
 		return nil, err
@@ -110,18 +99,16 @@ func (b *bridge) ListVersions(id ProjectIdentifier) ([]Version, error) {
 		sort.Sort(upgradeVersionSorter(vl))
 	}
 
-	b.vlists[k] = vl
+	b.vlists[id] = vl
 	return vl, nil
 }
 
 func (b *bridge) RevisionPresentIn(id ProjectIdentifier, r Revision) (bool, error) {
-	k := b.key(id)
-	return b.sm.RevisionPresentIn(k, r)
+	return b.sm.RevisionPresentIn(id, r)
 }
 
 func (b *bridge) RepoExists(id ProjectIdentifier) (bool, error) {
-	k := b.key(id)
-	return b.sm.RepoExists(k)
+	return b.sm.RepoExists(id)
 }
 
 func (b *bridge) vendorCodeExists(id ProjectIdentifier) (bool, error) {
@@ -409,9 +396,12 @@ func (b *bridge) ListPackages(id ProjectIdentifier, v Version) (PackageTree, err
 		return b.listRootPackages()
 	}
 
-	// FIXME if we're aliasing here, the returned PackageTree will have
-	// unaliased import paths, which is super not correct
-	return b.sm.ListPackages(b.key(id), v)
+	return b.sm.ListPackages(id, v)
+}
+
+func (b *bridge) ExportProject(id ProjectIdentifier, v Version, path string) error {
+	//return b.sm.ExportProject(id, v, path)
+	panic("bridge should never be used to ExportProject")
 }
 
 // verifyRoot ensures that the provided path to the project root is in good
diff --git a/manager_test.go b/manager_test.go
index 02ae908..0164664 100644
--- a/manager_test.go
+++ b/manager_test.go
@@ -98,8 +98,8 @@ func TestProjectManagerInit(t *testing.T) {
 	}()
 	defer sm.Release()
 
-	pn := ProjectRoot("github.com/Masterminds/VCSTestRepo")
-	v, err := sm.ListVersions(pn)
+	id := mkPI("github.com/Masterminds/VCSTestRepo")
+	v, err := sm.ListVersions(id)
 	if err != nil {
 		t.Errorf("Unexpected error during initial project setup/fetching %s", err)
 	}
@@ -130,11 +130,11 @@ func TestProjectManagerInit(t *testing.T) {
 	// ensure its sorting works, as well.
 	smc := &bridge{
 		sm:     sm,
-		vlists: make(map[ProjectRoot][]Version),
+		vlists: make(map[ProjectIdentifier][]Version),
 		s:      &solver{},
 	}
 
-	v, err = smc.ListVersions(ProjectIdentifier{ProjectRoot: pn})
+	v, err = smc.ListVersions(id)
 	if err != nil {
 		t.Errorf("Unexpected error during initial project setup/fetching %s", err)
 	}
@@ -170,7 +170,7 @@ func TestProjectManagerInit(t *testing.T) {
 
 	// Ensure project existence values are what we expect
 	var exists bool
-	exists, err = sm.RepoExists(pn)
+	exists, err = sm.RepoExists(id)
 	if err != nil {
 		t.Errorf("Error on checking RepoExists: %s", err)
 	}
@@ -179,7 +179,7 @@ func TestProjectManagerInit(t *testing.T) {
 	}
 
 	// Now reach inside the black box
-	pms, err := sm.getProjectManager(pn)
+	pms, err := sm.getProjectManager(id)
 	if err != nil {
 		t.Errorf("Error on grabbing project manager obj: %s", err)
 	}
@@ -207,10 +207,10 @@ func TestRepoVersionFetching(t *testing.T) {
 		t.FailNow()
 	}
 
-	upstreams := []ProjectRoot{
-		"github.com/Masterminds/VCSTestRepo",
-		"bitbucket.org/mattfarina/testhgrepo",
-		"launchpad.net/govcstestbzrrepo",
+	upstreams := []ProjectIdentifier{
+		mkPI("github.com/Masterminds/VCSTestRepo"),
+		mkPI("bitbucket.org/mattfarina/testhgrepo"),
+		mkPI("launchpad.net/govcstestbzrrepo"),
 	}
 
 	pms := make([]*projectManager, len(upstreams))
@@ -328,14 +328,14 @@ func TestGetInfoListVersionsOrdering(t *testing.T) {
 
 	// setup done, now do the test
 
-	pn := ProjectRoot("github.com/Masterminds/VCSTestRepo")
+	id := mkPI("github.com/Masterminds/VCSTestRepo")
 
-	_, _, err = sm.GetManifestAndLock(pn, NewVersion("1.0.0"))
+	_, _, err = sm.GetManifestAndLock(id, NewVersion("1.0.0"))
 	if err != nil {
 		t.Errorf("Unexpected error from GetInfoAt %s", err)
 	}
 
-	v, err := sm.ListVersions(pn)
+	v, err := sm.ListVersions(id)
 	if err != nil {
 		t.Errorf("Unexpected error from ListVersions %s", err)
 	}
diff --git a/result.go b/result.go
index e601de9..7b13f23 100644
--- a/result.go
+++ b/result.go
@@ -46,7 +46,7 @@ func CreateVendorTree(basedir string, l Lock, sm SourceManager, sv bool) error {
 			return err
 		}
 
-		err = sm.ExportProject(p.Ident().ProjectRoot, p.Version(), to)
+		err = sm.ExportProject(p.Ident(), p.Version(), to)
 		if err != nil {
 			removeAll(basedir)
 			return fmt.Errorf("Error while exporting %s: %s", p.Ident().ProjectRoot, err)
diff --git a/result_test.go b/result_test.go
index f1544c6..1a2a8ad 100644
--- a/result_test.go
+++ b/result_test.go
@@ -77,7 +77,7 @@ func BenchmarkCreateVendorTree(b *testing.B) {
 
 	// Prefetch the projects before timer starts
 	for _, lp := range r.p {
-		_, _, err := sm.GetManifestAndLock(lp.Ident().ProjectRoot, lp.Version())
+		_, _, err := sm.GetManifestAndLock(lp.Ident(), lp.Version())
 		if err != nil {
 			b.Errorf("failed getting project info during prefetch: %s", err)
 			clean = false
diff --git a/solve_basic_test.go b/solve_basic_test.go
index b02e7af..c493b19 100644
--- a/solve_basic_test.go
+++ b/solve_basic_test.go
@@ -1204,13 +1204,13 @@ func newdepspecSM(ds []depspec, ignore []string) *depspecSourceManager {
 
 func (sm *depspecSourceManager) GetManifestAndLock(id ProjectIdentifier, v Version) (Manifest, Lock, error) {
 	for _, ds := range sm.specs {
-		if n == ds.n && v.Matches(ds.v) {
+		if id.ProjectRoot == ds.n && v.Matches(ds.v) {
 			return ds, dummyLock{}, nil
 		}
 	}
 
 	// TODO(sdboyer) proper solver-type errors
-	return nil, nil, fmt.Errorf("Project %s at version %s could not be found", n, v)
+	return nil, nil, fmt.Errorf("Project %s at version %s could not be found", id.errString(), v)
 }
 
 func (sm *depspecSourceManager) AnalyzerInfo() (string, *semver.Version) {
@@ -1218,25 +1218,27 @@ func (sm *depspecSourceManager) AnalyzerInfo() (string, *semver.Version) {
 }
 
 func (sm *depspecSourceManager) ExternalReach(id ProjectIdentifier, v Version) (map[string][]string, error) {
-	id := pident{n: n, v: v}
-	if m, exists := sm.rm[id]; exists {
+	pid := pident{n: id.ProjectRoot, v: v}
+	if m, exists := sm.rm[pid]; exists {
 		return m, nil
 	}
-	return nil, fmt.Errorf("No reach data for %s at version %s", n, v)
+	return nil, fmt.Errorf("No reach data for %s at version %s", id.errString(), v)
 }
 
 func (sm *depspecSourceManager) ListExternal(id ProjectIdentifier, v Version) ([]string, error) {
 	// This should only be called for the root
-	id := pident{n: n, v: v}
-	if r, exists := sm.rm[id]; exists {
-		return r[string(n)], nil
+	pid := pident{n: id.ProjectRoot, v: v}
+	if r, exists := sm.rm[pid]; exists {
+		return r[string(id.ProjectRoot)], nil
 	}
-	return nil, fmt.Errorf("No reach data for %s at version %s", n, v)
+	return nil, fmt.Errorf("No reach data for %s at version %s", id.errString(), v)
 }
 
 func (sm *depspecSourceManager) ListPackages(id ProjectIdentifier, v Version) (PackageTree, error) {
-	id := pident{n: n, v: v}
-	if r, exists := sm.rm[id]; exists {
+	pid := pident{n: id.ProjectRoot, v: v}
+	n := id.ProjectRoot
+
+	if r, exists := sm.rm[pid]; exists {
 		ptree := PackageTree{
 			ImportRoot: string(n),
 			Packages: map[string]PackageOrErr{
@@ -1255,35 +1257,35 @@ func (sm *depspecSourceManager) ListPackages(id ProjectIdentifier, v Version) (P
 	return PackageTree{}, fmt.Errorf("Project %s at version %s could not be found", n, v)
 }
 
-func (sm *depspecSourceManager) ListVersions(name ProjectRoot) (pi []Version, err error) {
+func (sm *depspecSourceManager) ListVersions(id ProjectIdentifier) (pi []Version, err error) {
 	for _, ds := range sm.specs {
 		// To simulate the behavior of the real SourceManager, we do not return
 		// revisions from ListVersions().
-		if _, isrev := ds.v.(Revision); !isrev && name == ds.n {
+		if _, isrev := ds.v.(Revision); !isrev && id.ProjectRoot == ds.n {
 			pi = append(pi, ds.v)
 		}
 	}
 
 	if len(pi) == 0 {
-		err = fmt.Errorf("Project %s could not be found", name)
+		err = fmt.Errorf("Project %s could not be found", id.errString())
 	}
 
 	return
 }
 
-func (sm *depspecSourceManager) RevisionPresentIn(name ProjectRoot, r Revision) (bool, error) {
+func (sm *depspecSourceManager) RevisionPresentIn(id ProjectIdentifier, r Revision) (bool, error) {
 	for _, ds := range sm.specs {
-		if name == ds.n && r == ds.v {
+		if id.ProjectRoot == ds.n && r == ds.v {
 			return true, nil
 		}
 	}
 
-	return false, fmt.Errorf("Project %s has no revision %s", name, r)
+	return false, fmt.Errorf("Project %s has no revision %s", id.errString(), r)
 }
 
-func (sm *depspecSourceManager) RepoExists(name ProjectRoot) (bool, error) {
+func (sm *depspecSourceManager) RepoExists(id ProjectIdentifier) (bool, error) {
 	for _, ds := range sm.specs {
-		if name == ds.n {
+		if id.ProjectRoot == ds.n {
 			return true, nil
 		}
 	}
@@ -1291,7 +1293,7 @@ func (sm *depspecSourceManager) RepoExists(name ProjectRoot) (bool, error) {
 	return false, nil
 }
 
-func (sm *depspecSourceManager) VendorCodeExists(name ProjectRoot) (bool, error) {
+func (sm *depspecSourceManager) VendorCodeExists(id ProjectIdentifier) (bool, error) {
 	return false, nil
 }
 
@@ -1324,7 +1326,7 @@ func (b *depspecBridge) computeRootReach() ([]string, error) {
 	dsm := b.sm.(fixSM)
 	root := dsm.rootSpec()
 
-	ptree, err := dsm.ListPackages(root.n, nil)
+	ptree, err := dsm.ListPackages(mkPI(string(root.n)), nil)
 	if err != nil {
 		return nil, err
 	}
@@ -1343,7 +1345,7 @@ func (b *depspecBridge) verifyRootDir(path string) error {
 }
 
 func (b *depspecBridge) ListPackages(id ProjectIdentifier, v Version) (PackageTree, error) {
-	return b.sm.(fixSM).ListPackages(b.key(id), v)
+	return b.sm.(fixSM).ListPackages(id, v)
 }
 
 // override deduceRemoteRepo on bridge to make all our pkg/project mappings work
diff --git a/solve_bimodal_test.go b/solve_bimodal_test.go
index aa97294..f62619d 100644
--- a/solve_bimodal_test.go
+++ b/solve_bimodal_test.go
@@ -652,9 +652,9 @@ func newbmSM(bmf bimodalFixture) *bmSourceManager {
 func (sm *bmSourceManager) ListPackages(id ProjectIdentifier, v Version) (PackageTree, error) {
 	for k, ds := range sm.specs {
 		// Cheat for root, otherwise we blow up b/c version is empty
-		if n == ds.n && (k == 0 || ds.v.Matches(v)) {
+		if id.ProjectRoot == ds.n && (k == 0 || ds.v.Matches(v)) {
 			ptree := PackageTree{
-				ImportRoot: string(n),
+				ImportRoot: string(id.ProjectRoot),
 				Packages:   make(map[string]PackageOrErr),
 			}
 			for _, pkg := range ds.pkgs {
@@ -671,13 +671,13 @@ func (sm *bmSourceManager) ListPackages(id ProjectIdentifier, v Version) (Packag
 		}
 	}
 
-	return PackageTree{}, fmt.Errorf("Project %s at version %s could not be found", n, v)
+	return PackageTree{}, fmt.Errorf("Project %s at version %s could not be found", id.errString(), v)
 }
 
 func (sm *bmSourceManager) GetManifestAndLock(id ProjectIdentifier, v Version) (Manifest, Lock, error) {
 	for _, ds := range sm.specs {
-		if n == ds.n && v.Matches(ds.v) {
-			if l, exists := sm.lm[string(n)+" "+v.String()]; exists {
+		if id.ProjectRoot == ds.n && v.Matches(ds.v) {
+			if l, exists := sm.lm[string(id.ProjectRoot)+" "+v.String()]; exists {
 				return ds, l, nil
 			}
 			return ds, dummyLock{}, nil
@@ -685,7 +685,7 @@ func (sm *bmSourceManager) GetManifestAndLock(id ProjectIdentifier, v Version) (
 	}
 
 	// TODO(sdboyer) proper solver-type errors
-	return nil, nil, fmt.Errorf("Project %s at version %s could not be found", n, v)
+	return nil, nil, fmt.Errorf("Project %s at version %s could not be found", id.errString(), v)
 }
 
 // computeBimodalExternalMap takes a set of depspecs and computes an
diff --git a/solve_test.go b/solve_test.go
index 67d0b04..94ed8ba 100644
--- a/solve_test.go
+++ b/solve_test.go
@@ -30,7 +30,7 @@ func overrideMkBridge() {
 			&bridge{
 				sm:     sm,
 				s:      s,
-				vlists: make(map[ProjectRoot][]Version),
+				vlists: make(map[ProjectIdentifier][]Version),
 			},
 		}
 	}
@@ -322,7 +322,7 @@ func TestBadSolveOpts(t *testing.T) {
 		return &bridge{
 			sm:     sm,
 			s:      s,
-			vlists: make(map[ProjectRoot][]Version),
+			vlists: make(map[ProjectIdentifier][]Version),
 		}
 	}
 
diff --git a/solver.go b/solver.go
index 92cc242..eab3b42 100644
--- a/solver.go
+++ b/solver.go
@@ -493,7 +493,7 @@ func (s *solver) getImportsAndConstraintsOf(a atomWithPackages) ([]completeDep,
 
 	// Work through the source manager to get project info and static analysis
 	// information.
-	m, _, err := s.b.GetManifestAndLock(a.a)
+	m, _, err := s.b.GetManifestAndLock(a.a.id, a.a.v)
 	if err != nil {
 		return nil, err
 	}
@@ -679,7 +679,7 @@ func (s *solver) createVersionQueue(bmi bimodalIdentifier) (*versionQueue, error
 				continue
 			}
 
-			_, l, err := s.b.GetManifestAndLock(dep.depender)
+			_, l, err := s.b.GetManifestAndLock(dep.depender.id, dep.depender.v)
 			if err != nil || l == nil {
 				// err being non-nil really shouldn't be possible, but the lock
 				// being nil is quite likely
@@ -1060,7 +1060,7 @@ func (s *solver) selectAtom(a atomWithPackages, pkgonly bool) {
 
 	// If this atom has a lock, pull it out so that we can potentially inject
 	// preferred versions into any bmis we enqueue
-	_, l, _ := s.b.GetManifestAndLock(a.a)
+	_, l, _ := s.b.GetManifestAndLock(a.a.id, a.a.v)
 	var lmap map[ProjectIdentifier]Version
 	if l != nil {
 		lmap = make(map[ProjectIdentifier]Version)
diff --git a/source_manager.go b/source_manager.go
index ef79806..4447683 100644
--- a/source_manager.go
+++ b/source_manager.go
@@ -21,7 +21,7 @@ import (
 type SourceManager interface {
 	// RepoExists checks if a repository exists, either upstream or in the
 	// SourceManager's central repository cache.
-	// TODO rename to SourceExists
+	// TODO(sdboyer) rename to SourceExists
 	RepoExists(ProjectIdentifier) (bool, error)
 
 	// ListVersions retrieves a list of the available versions for a given
@@ -51,9 +51,6 @@ type SourceManager interface {
 	// AnalyzerInfo reports the name and version of the logic used to service
 	// GetManifestAndLock().
 	AnalyzerInfo() (name string, version *semver.Version)
-
-	// Release lets go of any locks held by the SourceManager.
-	Release()
 }
 
 // A ProjectAnalyzer is responsible for analyzing a given path for Manifest and
@@ -73,7 +70,7 @@ type ProjectAnalyzer interface {
 // tools; control via dependency injection is intended to be sufficient.
 type SourceMgr struct {
 	cachedir string
-	pms      map[ProjectIdentifier]*pmState
+	pms      map[string]*pmState
 	an       ProjectAnalyzer
 	ctx      build.Context
 }
@@ -132,7 +129,7 @@ func NewSourceManager(an ProjectAnalyzer, cachedir string, force bool) (*SourceM
 
 	return &SourceMgr{
 		cachedir: cachedir,
-		pms:      make(map[ProjectRoot]*pmState),
+		pms:      make(map[string]*pmState),
 		ctx:      ctx,
 		an:       an,
 	}, nil
@@ -156,7 +153,7 @@ func (sm *SourceMgr) AnalyzerInfo() (name string, version *semver.Version) {
 // The work of producing the manifest and lock is delegated to the injected
 // ProjectAnalyzer's DeriveManifestAndLock() method.
 func (sm *SourceMgr) GetManifestAndLock(id ProjectIdentifier, v Version) (Manifest, Lock, error) {
-	pmc, err := sm.getProjectManager(n)
+	pmc, err := sm.getProjectManager(id)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -167,7 +164,7 @@ func (sm *SourceMgr) GetManifestAndLock(id ProjectIdentifier, v Version) (Manife
 // ListPackages parses the tree of the Go packages at and below the ProjectRoot
 // of the given ProjectIdentifier, at the given version.
 func (sm *SourceMgr) ListPackages(id ProjectIdentifier, v Version) (PackageTree, error) {
-	pmc, err := sm.getProjectManager(n)
+	pmc, err := sm.getProjectManager(id)
 	if err != nil {
 		return PackageTree{}, err
 	}
@@ -188,7 +185,7 @@ func (sm *SourceMgr) ListPackages(id ProjectIdentifier, v Version) (PackageTree,
 // is not accessible (network outage, access issues, or the resource actually
 // went away), an error will be returned.
 func (sm *SourceMgr) ListVersions(id ProjectIdentifier) ([]Version, error) {
-	pmc, err := sm.getProjectManager(n)
+	pmc, err := sm.getProjectManager(id)
 	if err != nil {
 		// TODO(sdboyer) More-er proper-er errors
 		return nil, err
@@ -200,7 +197,7 @@ func (sm *SourceMgr) ListVersions(id ProjectIdentifier) ([]Version, error) {
 // RevisionPresentIn indicates whether the provided Revision is present in the given
 // repository.
 func (sm *SourceMgr) RevisionPresentIn(id ProjectIdentifier, r Revision) (bool, error) {
-	pmc, err := sm.getProjectManager(n)
+	pmc, err := sm.getProjectManager(id)
 	if err != nil {
 		// TODO(sdboyer) More-er proper-er errors
 		return false, err
@@ -212,7 +209,7 @@ func (sm *SourceMgr) RevisionPresentIn(id ProjectIdentifier, r Revision) (bool,
 // RepoExists checks if a repository exists, either upstream or in the cache,
 // for the provided ProjectIdentifier.
 func (sm *SourceMgr) RepoExists(id ProjectIdentifier) (bool, error) {
-	pms, err := sm.getProjectManager(n)
+	pms, err := sm.getProjectManager(id)
 	if err != nil {
 		return false, err
 	}
@@ -223,7 +220,7 @@ func (sm *SourceMgr) RepoExists(id ProjectIdentifier) (bool, error) {
 // ExportProject writes out the tree of the provided ProjectIdentifier's
 // ProjectRoot, at the provided version, to the provided directory.
 func (sm *SourceMgr) ExportProject(id ProjectIdentifier, v Version, to string) error {
-	pms, err := sm.getProjectManager(n)
+	pms, err := sm.getProjectManager(id)
 	if err != nil {
 		return err
 	}
@@ -242,7 +239,7 @@ func (sm *SourceMgr) getProjectManager(id ProjectIdentifier) (*pmState, error) {
 		//return nil, pme
 	}
 
-	repodir := path.Join(sm.cachedir, "src", string(n))
+	repodir := filepath.Join(sm.cachedir, "src")
 	// TODO(sdboyer) be more robust about this
 	r, err := vcs.NewRepo("https://"+string(n), repodir)
 	if err != nil {
@@ -300,7 +297,7 @@ func (sm *SourceMgr) getProjectManager(id ProjectIdentifier) (*pmState, error) {
 	}
 
 	pm := &projectManager{
-		n:   n,
+		n:   id.ProjectRoot,
 		ctx: sm.ctx,
 		an:  sm.an,
 		dc:  dc,

From d6ebd9df7e36c1b36a9f2ce37b22a25def2885e9 Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Fri, 29 Jul 2016 16:55:23 -0400
Subject: [PATCH 05/71] Refactor setup of projectManager instances

---
 source_manager.go | 107 ++++++++++++++++++++++++++++++++++++++--------
 1 file changed, 89 insertions(+), 18 deletions(-)

diff --git a/source_manager.go b/source_manager.go
index 4447683..4a9d771 100644
--- a/source_manager.go
+++ b/source_manager.go
@@ -6,11 +6,18 @@ import (
 	"go/build"
 	"os"
 	"path"
+	"path/filepath"
+	"strings"
 
 	"github.com/Masterminds/semver"
 	"github.com/Masterminds/vcs"
 )
 
+// Used to compute a friendly filepath from a URL-shaped input
+//
+// TODO(sdboyer) this is awful. Right?
+var sanitizer = strings.NewReplacer(":", "-", "/", "-", "+", "-")
+
 // A SourceManager is responsible for retrieving, managing, and interrogating
 // source repositories. Its primary purpose is to serve the needs of a Solver,
 // but it is handy for other purposes, as well.
@@ -107,7 +114,7 @@ func NewSourceManager(an ProjectAnalyzer, cachedir string, force bool) (*SourceM
 		return nil, fmt.Errorf("a ProjectAnalyzer must be provided to the SourceManager")
 	}
 
-	err := os.MkdirAll(cachedir, 0777)
+	err := os.MkdirAll(filepath.Join(cachedir, "sources"), 0777)
 	if err != nil {
 		return nil, err
 	}
@@ -232,30 +239,96 @@ func (sm *SourceMgr) ExportProject(id ProjectIdentifier, v Version, to string) e
 //
 // If no such manager yet exists, it attempts to create one.
 func (sm *SourceMgr) getProjectManager(id ProjectIdentifier) (*pmState, error) {
-	// Check pm cache and errcache first
+	// TODO(sdboyer) finish this, it's not sufficient (?)
+	n := id.netName()
+	var sn string
+
+	// Early check to see if we already have a pm in the cache for this net name
 	if pm, exists := sm.pms[n]; exists {
 		return pm, nil
-		//} else if pme, errexists := sm.pme[name]; errexists {
-		//return nil, pme
 	}
 
-	repodir := filepath.Join(sm.cachedir, "src")
-	// TODO(sdboyer) be more robust about this
-	r, err := vcs.NewRepo("https://"+string(n), repodir)
+	// Figure out the remote repo path
+	rr, err := deduceRemoteRepo(n)
 	if err != nil {
-		// TODO(sdboyer) be better
+		// Not a valid import path, must reject
+		// TODO(sdboyer) wrap error
 		return nil, err
 	}
-	if !r.CheckLocal() {
-		// TODO(sdboyer) cloning the repo here puts it on a blocking, and possibly
-		// unnecessary path. defer it
+
+	// Check the cache again, see if exact resulting clone url is in there
+	if pm, exists := sm.pms[rr.CloneURL.String()]; exists {
+		// Found it - re-register this PM at the original netname so that it
+		// doesn't need to deduce next time
+		// TODO(sdboyer) is this OK to do? are there consistency side effects?
+		sm.pms[n] = pm
+		return pm, nil
+	}
+
+	// No luck again. Now, walk through the scheme options the deducer returned,
+	// checking if each is in the cache
+	for _, scheme := range rr.Schemes {
+		rr.CloneURL.Scheme = scheme
+		// See if THIS scheme has a match, now
+		if pm, exists := sm.pms[rr.CloneURL.String()]; exists {
+			// Yep - again, re-register this PM at the original netname so that it
+			// doesn't need to deduce next time
+			// TODO(sdboyer) is this OK to do? are there consistency side effects?
+			sm.pms[n] = pm
+			return pm, nil
+		}
+	}
+
+	// Definitively no match for anything in the cache, so we know we have to
+	// create the entry. Next question is whether there's already a repo on disk
+	// for any of the schemes, or if we need to create that, too.
+
+	// TODO(sdboyer) this strategy kinda locks in the scheme to use over
+	// multiple invocations in a way that maybe isn't the best.
+	var r vcs.Repo
+	for _, scheme := range rr.Schemes {
+		rr.CloneURL.Scheme = scheme
+		url := rr.CloneURL.String()
+		sn := sanitizer.Replace(url)
+		path := filepath.Join(sm.cachedir, "sources", sn)
+
+		if fi, err := os.Stat(path); err == nil && fi.IsDir() {
+			// This one exists, so set up here
+			r, err = vcs.NewRepo(url, path)
+			if err != nil {
+				return nil, err
+			}
+			goto decided
+		}
+	}
+
+	// Nothing on disk, either. Iterate through the schemes, trying each and
+	// failing out only if none resulted in successfully setting up the local.
+	for _, scheme := range rr.Schemes {
+		rr.CloneURL.Scheme = scheme
+		url := rr.CloneURL.String()
+		sn := sanitizer.Replace(url)
+		path := filepath.Join(sm.cachedir, "sources", sn)
+
+		r, err := vcs.NewRepo(url, path)
+		if err != nil {
+			continue
+		}
+
+		// FIXME(sdboyer) cloning the repo here puts it on a blocking path. that
+		// aspect of state management needs to be deferred into the
+		// projectManager
 		err = r.Get()
 		if err != nil {
-			// TODO(sdboyer) be better
-			return nil, err
+			continue
 		}
+		goto decided
 	}
 
+	// If we've gotten this far, we got some brokeass input.
+	return nil, fmt.Errorf("Could not reach source repository for %s", n)
+
+decided:
 	// Ensure cache dir exists
 	metadir := path.Join(sm.cachedir, "metadata", string(n))
 	err = os.MkdirAll(metadir, 0777)
@@ -297,12 +370,10 @@ func (sm *SourceMgr) getProjectManager(id ProjectIdentifier) (*pmState, error) {
 	}
 
 	pm := &projectManager{
-		n:   id.ProjectRoot,
-		ctx: sm.ctx,
-		an:  sm.an,
-		dc:  dc,
+		an: sm.an,
+		dc: dc,
 		crepo: &repo{
-			rpath: repodir,
+			rpath: sn,
 			r:     r,
 		},
 	}

From 4bbb0e4801a400da5a6c0a547282a6a8765df347 Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Fri, 29 Jul 2016 23:26:06 -0400
Subject: [PATCH 06/71] Revamp project managers for new type inputs

---
 project_manager.go | 18 ++++++++----------
 source_manager.go  |  7 ++++---
 2 files changed, 12 insertions(+), 13 deletions(-)

diff --git a/project_manager.go b/project_manager.go
index 6587a0c..1befade 100644
--- a/project_manager.go
+++ b/project_manager.go
@@ -7,7 +7,6 @@ import (
 	"os"
 	"os/exec"
 	"path"
-	"path/filepath"
 	"strings"
 	"sync"
 
@@ -16,9 +15,8 @@ import (
 )
 
 type projectManager struct {
-	// The identifier of the project. At this level, corresponds to the
-	// '$GOPATH/src'-relative path, *and* the network name.
-	n ProjectRoot
+	// The upstream URL from which the project is sourced.
+	n string
 
 	// build.Context to use in any analysis, and to pass to the analyzer
 	ctx build.Context
@@ -80,7 +78,7 @@ type repo struct {
 	synced bool
 }
 
-func (pm *projectManager) GetInfoAt(v Version) (Manifest, Lock, error) {
+func (pm *projectManager) GetManifestAndLock(r ProjectRoot, v Version) (Manifest, Lock, error) {
 	if err := pm.ensureCacheExistence(); err != nil {
 		return nil, nil, err
 	}
@@ -114,7 +112,7 @@ func (pm *projectManager) GetInfoAt(v Version) (Manifest, Lock, error) {
 	}
 
 	pm.crepo.mut.RLock()
-	m, l, err := pm.an.DeriveManifestAndLock(filepath.Join(pm.ctx.GOPATH, "src", string(pm.n)), pm.n)
+	m, l, err := pm.an.DeriveManifestAndLock(pm.crepo.rpath, r)
 	// TODO(sdboyer) cache results
 	pm.crepo.mut.RUnlock()
 
@@ -141,7 +139,7 @@ func (pm *projectManager) GetInfoAt(v Version) (Manifest, Lock, error) {
 	return nil, nil, err
 }
 
-func (pm *projectManager) ListPackages(v Version) (ptree PackageTree, err error) {
+func (pm *projectManager) ListPackages(pr ProjectRoot, v Version) (ptree PackageTree, err error) {
 	if err = pm.ensureCacheExistence(); err != nil {
 		return
 	}
@@ -188,7 +186,7 @@ func (pm *projectManager) ListPackages(v Version) (ptree PackageTree, err error)
 		err = pm.crepo.r.UpdateVersion(v.String())
 	}
 
-	ptree, err = listPackages(filepath.Join(pm.ctx.GOPATH, "src", string(pm.n)), string(pm.n))
+	ptree, err = listPackages(pm.crepo.rpath, string(pr))
 	pm.crepo.mut.Unlock()
 
 	// TODO(sdboyer) cache errs?
@@ -266,7 +264,7 @@ func (pm *projectManager) ListVersions() (vlist []Version, err error) {
 	return
 }
 
-func (pm *projectManager) RevisionPresentIn(r Revision) (bool, error) {
+func (pm *projectManager) RevisionPresentIn(pr ProjectRoot, r Revision) (bool, error) {
 	// First and fastest path is to check the data cache to see if the rev is
 	// present. This could give us false positives, but the cases where that can
 	// occur would require a type of cache staleness that seems *exceedingly*
@@ -279,7 +277,7 @@ func (pm *projectManager) RevisionPresentIn(r Revision) (bool, error) {
 
 	// For now at least, just run GetInfoAt(); it basically accomplishes the
 	// same thing.
-	if _, _, err := pm.GetInfoAt(r); err != nil {
+	if _, _, err := pm.GetManifestAndLock(pr, r); err != nil {
 		return false, err
 	}
 	return true, nil
diff --git a/source_manager.go b/source_manager.go
index 4a9d771..a7d4896 100644
--- a/source_manager.go
+++ b/source_manager.go
@@ -165,7 +165,7 @@ func (sm *SourceMgr) GetManifestAndLock(id ProjectIdentifier, v Version) (Manife
 		return nil, nil, err
 	}
 
-	return pmc.pm.GetInfoAt(v)
+	return pmc.pm.GetManifestAndLock(id.ProjectRoot, v)
 }
 
 // ListPackages parses the tree of the Go packages at and below the ProjectRoot
@@ -176,7 +176,7 @@ func (sm *SourceMgr) ListPackages(id ProjectIdentifier, v Version) (PackageTree,
 		return PackageTree{}, err
 	}
 
-	return pmc.pm.ListPackages(v)
+	return pmc.pm.ListPackages(id.ProjectRoot, v)
 }
 
 // ListVersions retrieves a list of the available versions for a given
@@ -210,7 +210,7 @@ func (sm *SourceMgr) RevisionPresentIn(id ProjectIdentifier, r Revision) (bool,
 		return false, err
 	}
 
-	return pmc.pm.RevisionPresentIn(r)
+	return pmc.pm.RevisionPresentIn(id.ProjectRoot, r)
 }
 
 // RepoExists checks if a repository exists, either upstream or in the cache,
@@ -370,6 +370,7 @@ decided:
 	}
 
 	pm := &projectManager{
+		n:  n,
 		an: sm.an,
 		dc: dc,
 		crepo: &repo{

From 680c99693c7ff7398c1f87afc0027d7380ea47b8 Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Sun, 31 Jul 2016 22:50:04 -0400
Subject: [PATCH 07/71] Add possible schemes by vcs in remote deduction

This strategy isn't perfect, but there's more refactoring needed for
this segment to really make it sane.
---
 remote.go      | 44 ++++++++++++++++++++++++++++++++++++++
 remote_test.go | 57 ++++++++++++++++++++++++++++++++------------------
 2 files changed, 81 insertions(+), 20 deletions(-)

diff --git a/remote.go b/remote.go
index c808d9a..cb41a8f 100644
--- a/remote.go
+++ b/remote.go
@@ -22,6 +22,13 @@ type remoteRepo struct {
 	VCS      []string
 }
 
+var (
+	gitSchemes = []string{"https", "ssh", "git", "http"}
+	bzrSchemes = []string{"https", "bzr+ssh", "bzr", "http"}
+	hgSchemes  = []string{"https", "ssh", "http"}
+	svnSchemes = []string{"https", "http", "svn", "svn+ssh"}
+)
+
 //type remoteResult struct {
 //r   remoteRepo
 //err error
@@ -105,6 +112,10 @@ func deduceRemoteRepo(path string) (rr *remoteRepo, err error) {
 		rr.Base = v[1]
 		rr.RelPkg = strings.TrimPrefix(v[3], "/")
 		rr.VCS = []string{"git"}
+		// If no scheme was already recorded, then add the possible schemes for github
+		if rr.Schemes == nil {
+			rr.Schemes = gitSchemes
+		}
 
 		return
 
@@ -129,6 +140,10 @@ func deduceRemoteRepo(path string) (rr *remoteRepo, err error) {
 		rr.Base = v[1]
 		rr.RelPkg = strings.TrimPrefix(v[6], "/")
 		rr.VCS = []string{"git"}
+		// If no scheme was already recorded, then add the possible schemes for github
+		if rr.Schemes == nil {
+			rr.Schemes = gitSchemes
+		}
 
 		return
 	//case gpinOldRegex.MatchString(path):
@@ -141,6 +156,12 @@ func deduceRemoteRepo(path string) (rr *remoteRepo, err error) {
 		rr.Base = v[1]
 		rr.RelPkg = strings.TrimPrefix(v[3], "/")
 		rr.VCS = []string{"git", "hg"}
+		// FIXME(sdboyer) this ambiguity of vcs kills us on schemes, as schemes
+		// are inherently vcs-specific. Fixing this requires a wider refactor.
+		// For now, we only allow the intersection, which is just the hg schemes
+		if rr.Schemes == nil {
+			rr.Schemes = hgSchemes
+		}
 
 		return
 
@@ -165,6 +186,9 @@ func deduceRemoteRepo(path string) (rr *remoteRepo, err error) {
 		rr.Base = v[1]
 		rr.RelPkg = strings.TrimPrefix(v[3], "/")
 		rr.VCS = []string{"bzr"}
+		if rr.Schemes == nil {
+			rr.Schemes = bzrSchemes
+		}
 
 		return
 
@@ -177,6 +201,9 @@ func deduceRemoteRepo(path string) (rr *remoteRepo, err error) {
 		rr.Base = v[1]
 		rr.RelPkg = strings.TrimPrefix(v[3], "/")
 		rr.VCS = []string{"git"}
+		if rr.Schemes == nil {
+			rr.Schemes = gitSchemes
+		}
 
 		return
 
@@ -188,6 +215,9 @@ func deduceRemoteRepo(path string) (rr *remoteRepo, err error) {
 		rr.Base = v[1]
 		rr.RelPkg = strings.TrimPrefix(v[3], "/")
 		rr.VCS = []string{"git"}
+		if rr.Schemes == nil {
+			rr.Schemes = gitSchemes
+		}
 
 		return
 
@@ -199,6 +229,9 @@ func deduceRemoteRepo(path string) (rr *remoteRepo, err error) {
 		rr.Base = v[1]
 		rr.RelPkg = strings.TrimPrefix(v[3], "/")
 		rr.VCS = []string{"git"}
+		if rr.Schemes == nil {
+			rr.Schemes = gitSchemes
+		}
 
 		return
 
@@ -214,6 +247,17 @@ func deduceRemoteRepo(path string) (rr *remoteRepo, err error) {
 			rr.VCS = []string{v[5]}
 			rr.Base = v[1]
 			rr.RelPkg = strings.TrimPrefix(v[6], "/")
+
+			if rr.Schemes == nil {
+				if v[5] == "git" {
+					rr.Schemes = gitSchemes
+				} else if v[5] == "bzr" {
+					rr.Schemes = bzrSchemes
+				} else if v[5] == "hg" {
+					rr.Schemes = hgSchemes
+				}
+			}
+
 			return
 		default:
 			return nil, fmt.Errorf("unknown repository type: %q", v[5])
diff --git a/remote_test.go b/remote_test.go
index 17de00f..6f5cb62 100644
--- a/remote_test.go
+++ b/remote_test.go
@@ -25,7 +25,7 @@ func TestDeduceRemotes(t *testing.T) {
 					Host: "github.com",
 					Path: "sdboyer/gps",
 				},
-				Schemes: nil,
+				Schemes: gitSchemes,
 				VCS:     []string{"git"},
 			},
 		},
@@ -38,7 +38,7 @@ func TestDeduceRemotes(t *testing.T) {
 					Host: "github.com",
 					Path: "sdboyer/gps",
 				},
-				Schemes: nil,
+				Schemes: gitSchemes,
 				VCS:     []string{"git"},
 			},
 		},
@@ -111,7 +111,8 @@ func TestDeduceRemotes(t *testing.T) {
 					Host: "github.com",
 					Path: "sdboyer/gps",
 				},
-				VCS: []string{"git"},
+				Schemes: gitSchemes,
+				VCS:     []string{"git"},
 			},
 		},
 		{
@@ -123,7 +124,8 @@ func TestDeduceRemotes(t *testing.T) {
 					Host: "github.com",
 					Path: "sdboyer/gps",
 				},
-				VCS: []string{"git"},
+				Schemes: gitSchemes,
+				VCS:     []string{"git"},
 			},
 		},
 		{
@@ -135,7 +137,8 @@ func TestDeduceRemotes(t *testing.T) {
 					Host: "github.com",
 					Path: "sdboyer/gps",
 				},
-				VCS: []string{"git"},
+				Schemes: gitSchemes,
+				VCS:     []string{"git"},
 			},
 		},
 		{
@@ -147,7 +150,8 @@ func TestDeduceRemotes(t *testing.T) {
 					Host: "github.com",
 					Path: "go-pkg/yaml",
 				},
-				VCS: []string{"git"},
+				Schemes: gitSchemes,
+				VCS:     []string{"git"},
 			},
 		},
 		{
@@ -159,7 +163,8 @@ func TestDeduceRemotes(t *testing.T) {
 					Host: "github.com",
 					Path: "go-pkg/yaml",
 				},
-				VCS: []string{"git"},
+				Schemes: gitSchemes,
+				VCS:     []string{"git"},
 			},
 		},
 		{
@@ -177,7 +182,8 @@ func TestDeduceRemotes(t *testing.T) {
 					Host: "hub.jazz.net",
 					Path: "git/user1/pkgname",
 				},
-				VCS: []string{"git"},
+				Schemes: gitSchemes,
+				VCS:     []string{"git"},
 			},
 		},
 		{
@@ -189,7 +195,8 @@ func TestDeduceRemotes(t *testing.T) {
 					Host: "hub.jazz.net",
 					Path: "git/user1/pkgname",
 				},
-				VCS: []string{"git"},
+				Schemes: gitSchemes,
+				VCS:     []string{"git"},
 			},
 		},
 		{
@@ -231,7 +238,8 @@ func TestDeduceRemotes(t *testing.T) {
 					Host: "hub.jazz.net",
 					Path: "git/user/pkg.name",
 				},
-				VCS: []string{"git"},
+				Schemes: gitSchemes,
+				VCS:     []string{"git"},
 			},
 		},
 		// User names cannot have uppercase letters
@@ -248,7 +256,8 @@ func TestDeduceRemotes(t *testing.T) {
 					Host: "bitbucket.org",
 					Path: "sdboyer/reporoot",
 				},
-				VCS: []string{"git", "hg"},
+				Schemes: hgSchemes,
+				VCS:     []string{"git", "hg"},
 			},
 		},
 		{
@@ -260,7 +269,8 @@ func TestDeduceRemotes(t *testing.T) {
 					Host: "bitbucket.org",
 					Path: "sdboyer/reporoot",
 				},
-				VCS: []string{"git", "hg"},
+				Schemes: hgSchemes,
+				VCS:     []string{"git", "hg"},
 			},
 		},
 		{
@@ -286,7 +296,8 @@ func TestDeduceRemotes(t *testing.T) {
 					Host: "launchpad.net",
 					Path: "govcstestbzrrepo",
 				},
-				VCS: []string{"bzr"},
+				Schemes: bzrSchemes,
+				VCS:     []string{"bzr"},
 			},
 		},
 		{
@@ -298,7 +309,8 @@ func TestDeduceRemotes(t *testing.T) {
 					Host: "launchpad.net",
 					Path: "govcstestbzrrepo",
 				},
-				VCS: []string{"bzr"},
+				Schemes: bzrSchemes,
+				VCS:     []string{"bzr"},
 			},
 		},
 		{
@@ -314,7 +326,8 @@ func TestDeduceRemotes(t *testing.T) {
 					Host: "git.launchpad.net",
 					Path: "reporoot",
 				},
-				VCS: []string{"git"},
+				Schemes: gitSchemes,
+				VCS:     []string{"git"},
 			},
 		},
 		{
@@ -326,7 +339,8 @@ func TestDeduceRemotes(t *testing.T) {
 					Host: "git.launchpad.net",
 					Path: "reporoot",
 				},
-				VCS: []string{"git"},
+				Schemes: gitSchemes,
+				VCS:     []string{"git"},
 			},
 		},
 		{
@@ -338,7 +352,8 @@ func TestDeduceRemotes(t *testing.T) {
 					Host: "git.launchpad.net",
 					Path: "reporoot",
 				},
-				VCS: []string{"git"},
+				Schemes: gitSchemes,
+				VCS:     []string{"git"},
 			},
 		},
 		{
@@ -354,7 +369,8 @@ func TestDeduceRemotes(t *testing.T) {
 					Host: "git.apache.org",
 					Path: "package-name.git",
 				},
-				VCS: []string{"git"},
+				Schemes: gitSchemes,
+				VCS:     []string{"git"},
 			},
 		},
 		{
@@ -366,7 +382,8 @@ func TestDeduceRemotes(t *testing.T) {
 					Host: "git.apache.org",
 					Path: "package-name.git",
 				},
-				VCS: []string{"git"},
+				Schemes: gitSchemes,
+				VCS:     []string{"git"},
 			},
 		},
 		// Vanity imports
@@ -422,7 +439,7 @@ func TestDeduceRemotes(t *testing.T) {
 					Host: "github.com",
 					Path: "kr/pretty",
 				},
-				Schemes: nil,
+				Schemes: gitSchemes,
 				VCS:     []string{"git"},
 			},
 		},

From 4cb9bcb0ead460a0b44947540179edf97a947c20 Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Sun, 31 Jul 2016 22:51:08 -0400
Subject: [PATCH 08/71] Fix shadowed variable assignment

---
 source_manager.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source_manager.go b/source_manager.go
index a7d4896..dc24880 100644
--- a/source_manager.go
+++ b/source_manager.go
@@ -310,7 +310,7 @@ func (sm *SourceMgr) getProjectManager(id ProjectIdentifier) (*pmState, error) {
 		sn := sanitizer.Replace(url)
 		path := filepath.Join(sm.cachedir, "sources", sn)
 
-		r, err := vcs.NewRepo(url, path)
+		r, err = vcs.NewRepo(url, path)
 		if err != nil {
 			continue
 		}

From cfa2b1fe5a9a0306c267c07e0072b6c58bae7f5d Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Sun, 31 Jul 2016 22:51:35 -0400
Subject: [PATCH 09/71] Look at dashed paths, now

---
 manager_test.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/manager_test.go b/manager_test.go
index 0164664..6843675 100644
--- a/manager_test.go
+++ b/manager_test.go
@@ -157,14 +157,14 @@ func TestProjectManagerInit(t *testing.T) {
 	}
 
 	// Ensure that the appropriate cache dirs and files exist
-	_, err = os.Stat(path.Join(cpath, "src", "github.com", "Masterminds", "VCSTestRepo", ".git"))
+	_, err = os.Stat(path.Join(cpath, "sources", "https---github.com-Masterminds-VCSTestRepo", ".git"))
 	if err != nil {
 		t.Error("Cache repo does not exist in expected location")
 	}
 
 	_, err = os.Stat(path.Join(cpath, "metadata", "github.com", "Masterminds", "VCSTestRepo", "cache.json"))
 	if err != nil {
-		// TODO(sdboyer) temporarily disabled until we turn caching back on
+		// TODO(sdboyer) disabled until we get caching working
 		//t.Error("Metadata cache json file does not exist in expected location")
 	}
 

From 50171a48168a3b50d26fa5baeecf0e4f3570ff00 Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Mon, 1 Aug 2016 00:28:13 -0400
Subject: [PATCH 10/71] Windows-friendly filepath join (hopefully)

---
 manager_test.go    | 4 ++--
 project_manager.go | 4 ++--
 source_manager.go  | 9 ++++-----
 3 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/manager_test.go b/manager_test.go
index 6843675..b8e3039 100644
--- a/manager_test.go
+++ b/manager_test.go
@@ -126,8 +126,8 @@ func TestProjectManagerInit(t *testing.T) {
 	}
 
 	// Two birds, one stone - make sure the internal ProjectManager vlist cache
-	// works by asking for the versions again, and do it through smcache to
-	// ensure its sorting works, as well.
+	// works (or at least doesn't not work) by asking for the versions again,
+	// and do it through smcache to ensure its sorting works, as well.
 	smc := &bridge{
 		sm:     sm,
 		vlists: make(map[ProjectIdentifier][]Version),
diff --git a/project_manager.go b/project_manager.go
index 1befade..98b7ac6 100644
--- a/project_manager.go
+++ b/project_manager.go
@@ -6,7 +6,7 @@ import (
 	"go/build"
 	"os"
 	"os/exec"
-	"path"
+	"path/filepath"
 	"strings"
 	"sync"
 
@@ -496,7 +496,7 @@ func (r *repo) exportVersionTo(v Version, to string) error {
 	switch r.r.(type) {
 	case *vcs.GitRepo:
 		// Back up original index
-		idx, bak := path.Join(r.rpath, ".git", "index"), path.Join(r.rpath, ".git", "origindex")
+		idx, bak := filepath.Join(r.rpath, ".git", "index"), filepath.Join(r.rpath, ".git", "origindex")
 		err := os.Rename(idx, bak)
 		if err != nil {
 			return err
diff --git a/source_manager.go b/source_manager.go
index dc24880..94e2f30 100644
--- a/source_manager.go
+++ b/source_manager.go
@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"go/build"
 	"os"
-	"path"
 	"path/filepath"
 	"strings"
 
@@ -119,7 +118,7 @@ func NewSourceManager(an ProjectAnalyzer, cachedir string, force bool) (*SourceM
 		return nil, err
 	}
 
-	glpath := path.Join(cachedir, "sm.lock")
+	glpath := filepath.Join(cachedir, "sm.lock")
 	_, err = os.Stat(glpath)
 	if err == nil && !force {
 		return nil, fmt.Errorf("cache lock file %s exists - another process crashed or is still running?", glpath)
@@ -144,7 +143,7 @@ func NewSourceManager(an ProjectAnalyzer, cachedir string, force bool) (*SourceM
 
 // Release lets go of any locks held by the SourceManager.
 func (sm *SourceMgr) Release() {
-	os.Remove(path.Join(sm.cachedir, "sm.lock"))
+	os.Remove(filepath.Join(sm.cachedir, "sm.lock"))
 }
 
 // AnalyzerInfo reports the name and version of the injected ProjectAnalyzer.
@@ -330,7 +329,7 @@ func (sm *SourceMgr) getProjectManager(id ProjectIdentifier) (*pmState, error) {
 
 decided:
 	// Ensure cache dir exists
-	metadir := path.Join(sm.cachedir, "metadata", string(n))
+	metadir := filepath.Join(sm.cachedir, "metadata", string(n))
 	err = os.MkdirAll(metadir, 0777)
 	if err != nil {
 		// TODO(sdboyer) be better
@@ -338,7 +337,7 @@ decided:
 	}
 
 	pms := &pmState{}
-	cpath := path.Join(metadir, "cache.json")
+	cpath := filepath.Join(metadir, "cache.json")
 	fi, err := os.Stat(cpath)
 	var dc *projectDataCache
 	if fi != nil {

From 90a4e17da64037d373df42391a2fb31d176e1072 Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Mon, 1 Aug 2016 00:44:06 -0400
Subject: [PATCH 11/71] Store the right repo path

---
 source_manager.go | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/source_manager.go b/source_manager.go
index 94e2f30..4617bc0 100644
--- a/source_manager.go
+++ b/source_manager.go
@@ -240,7 +240,7 @@ func (sm *SourceMgr) ExportProject(id ProjectIdentifier, v Version, to string) e
 func (sm *SourceMgr) getProjectManager(id ProjectIdentifier) (*pmState, error) {
 	// TODO(sdboyer) finish this, it's not sufficient (?)
 	n := id.netName()
-	var sn string
+	var rpath string
 
 	// Early check to see if we already have a pm in the cache for this net name
 	if pm, exists := sm.pms[n]; exists {
@@ -289,11 +289,11 @@ func (sm *SourceMgr) getProjectManager(id ProjectIdentifier) (*pmState, error) {
 		rr.CloneURL.Scheme = scheme
 		url := rr.CloneURL.String()
 		sn := sanitizer.Replace(url)
-		path := filepath.Join(sm.cachedir, "sources", sn)
+		rpath = filepath.Join(sm.cachedir, "sources", sn)
 
-		if fi, err := os.Stat(path); err == nil && fi.IsDir() {
+		if fi, err := os.Stat(rpath); err == nil && fi.IsDir() {
 			// This one exists, so set up here
-			r, err = vcs.NewRepo(url, path)
+			r, err = vcs.NewRepo(url, rpath)
 			if err != nil {
 				return nil, err
 			}
@@ -307,9 +307,9 @@ func (sm *SourceMgr) getProjectManager(id ProjectIdentifier) (*pmState, error) {
 		rr.CloneURL.Scheme = scheme
 		url := rr.CloneURL.String()
 		sn := sanitizer.Replace(url)
-		path := filepath.Join(sm.cachedir, "sources", sn)
+		rpath = filepath.Join(sm.cachedir, "sources", sn)
 
-		r, err = vcs.NewRepo(url, path)
+		r, err = vcs.NewRepo(url, rpath)
 		if err != nil {
 			continue
 		}
@@ -373,7 +373,7 @@ decided:
 		an: sm.an,
 		dc: dc,
 		crepo: &repo{
-			rpath: sn,
+			rpath: rpath,
 			r:     r,
 		},
 	}

From 445948e1655312a14154c1f146df8ba1681826f1 Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Mon, 1 Aug 2016 16:21:02 -0400
Subject: [PATCH 12/71] Add whatsInAName; a cache around deduction

---
 remote.go         |  8 --------
 source_manager.go | 45 +++++++++++++++++++++++++++++++++++++++++----
 2 files changed, 41 insertions(+), 12 deletions(-)

diff --git a/remote.go b/remote.go
index cb41a8f..d28c5e9 100644
--- a/remote.go
+++ b/remote.go
@@ -29,14 +29,6 @@ var (
 	svnSchemes = []string{"https", "http", "svn", "svn+ssh"}
 )
 
-//type remoteResult struct {
-//r   remoteRepo
-//err error
-//}
-
-// TODO(sdboyer) sync access to this map
-//var remoteCache = make(map[string]remoteResult)
-
 // Regexes for the different known import path flavors
 var (
 	// This regex allowed some usernames that github currently disallows. They
diff --git a/source_manager.go b/source_manager.go
index 4617bc0..477e705 100644
--- a/source_manager.go
+++ b/source_manager.go
@@ -7,6 +7,7 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"sync"
 
 	"github.com/Masterminds/semver"
 	"github.com/Masterminds/vcs"
@@ -77,8 +78,14 @@ type ProjectAnalyzer interface {
 type SourceMgr struct {
 	cachedir string
 	pms      map[string]*pmState
-	an       ProjectAnalyzer
-	ctx      build.Context
+	pmut     sync.RWMutex
+	rr       map[string]struct {
+		rr  *remoteRepo
+		err error
+	}
+	rmut sync.RWMutex
+	an   ProjectAnalyzer
+	ctx  build.Context
 }
 
 var _ SourceManager = &SourceMgr{}
@@ -136,8 +143,12 @@ func NewSourceManager(an ProjectAnalyzer, cachedir string, force bool) (*SourceM
 	return &SourceMgr{
 		cachedir: cachedir,
 		pms:      make(map[string]*pmState),
-		ctx:      ctx,
-		an:       an,
+		rr: make(map[string]struct {
+			rr  *remoteRepo
+			err error
+		}),
+		ctx: ctx,
+		an:  an,
 	}, nil
 }
 
@@ -382,3 +393,29 @@ decided:
 	sm.pms[n] = pms
 	return pms, nil
 }
+
+func (sm *SourceMgr) whatsInAName(nn string) (*remoteRepo, error) {
+	sm.rmut.RLock()
+	tuple, exists := sm.rr[nn]
+	sm.rmut.RUnlock()
+	if exists {
+		return tuple.rr, tuple.err
+	}
+
+	// Don't lock around the deduceRemoteRepo call, because that itself can be
+	// slow. The tradeoff is that it's possible we might duplicate work if two
+	// calls for the same id were to made simultaneously, but as those results
+	// would be the same, clobbering is OK, and better than the alternative of
+	// serializing all calls.
+	rr, err := deduceRemoteRepo(nn)
+	sm.rmut.Lock()
+	sm.rr[nn] = struct {
+		rr  *remoteRepo
+		err error
+	}{
+		rr:  rr,
+		err: err,
+	}
+	sm.rmut.Unlock()
+	return rr, err
+}

From 7c516b84d14132bf5056986e087d48f0cfc647eb Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Mon, 1 Aug 2016 21:50:53 -0400
Subject: [PATCH 13/71] Incremental refactor of remoteRepo

---
 remote.go           |  82 ++++++++++++++++++-------------
 remote_test.go      | 116 ++++++++++++++++++++++----------------------
 solve_basic_test.go |   4 +-
 solver.go           |   8 +--
 source_manager.go   |   1 -
 5 files changed, 112 insertions(+), 99 deletions(-)

diff --git a/remote.go b/remote.go
index d28c5e9..2291435 100644
--- a/remote.go
+++ b/remote.go
@@ -15,11 +15,17 @@ import (
 // one is not a guarantee that the resource it identifies actually exists or is
 // accessible.
 type remoteRepo struct {
-	Base     string
-	RelPkg   string
-	CloneURL *url.URL
-	Schemes  []string
-	VCS      []string
+	repoRoot string
+	relPkg   string
+	try      []maybeRemoteSource
+}
+
+// maybeRemoteSource represents a set of instructions for accessing a possible
+// remote resource, without knowing whether that resource actually
+// works/exists/is accessible, etc.
+type maybeRemoteSource struct {
+	vcs string
+	url *url.URL
 }
 
 var (
@@ -59,11 +65,13 @@ var (
 // repositories can be bare import paths, or urls including a checkout scheme.
 func deduceRemoteRepo(path string) (rr *remoteRepo, err error) {
 	rr = &remoteRepo{}
+	var u *url.Url
+
 	if m := scpSyntaxRe.FindStringSubmatch(path); m != nil {
 		// Match SCP-like syntax and convert it to a URL.
 		// Eg, "git@github.com:user/repo" becomes
 		// "ssh://git@github.com/user/repo".
-		rr.CloneURL = &url.URL{
+		u = &url.URL{
 			Scheme: "ssh",
 			User:   url.User(m[1]),
 			Host:   m[2],
@@ -72,26 +80,22 @@ func deduceRemoteRepo(path string) (rr *remoteRepo, err error) {
 			//RawPath: m[3],
 		}
 	} else {
-		rr.CloneURL, err = url.Parse(path)
+		u, err = url.Parse(path)
 		if err != nil {
 			return nil, fmt.Errorf("%q is not a valid import path", path)
 		}
 	}
 
-	if rr.CloneURL.Host != "" {
-		path = rr.CloneURL.Host + "/" + strings.TrimPrefix(rr.CloneURL.Path, "/")
+	if u.Host != "" {
+		path = u.Host + "/" + strings.TrimPrefix(u.Path, "/")
 	} else {
-		path = rr.CloneURL.Path
+		path = u.Path
 	}
 
 	if !pathvld.MatchString(path) {
 		return nil, fmt.Errorf("%q is not a valid import path", path)
 	}
 
-	if rr.CloneURL.Scheme != "" {
-		rr.Schemes = []string{rr.CloneURL.Scheme}
-	}
-
 	// TODO(sdboyer) instead of a switch, encode base domain in radix tree and pick
 	// detector from there; if failure, then fall back on metadata work
 
@@ -99,10 +103,19 @@ func deduceRemoteRepo(path string) (rr *remoteRepo, err error) {
 	case ghRegex.MatchString(path):
 		v := ghRegex.FindStringSubmatch(path)
 
-		rr.CloneURL.Host = "github.com"
-		rr.CloneURL.Path = v[2]
-		rr.Base = v[1]
-		rr.RelPkg = strings.TrimPrefix(v[3], "/")
+		rr.repoRoot = v[1]
+		rr.relPkg = strings.TrimPrefix(v[3], "/")
+
+		//rr.CloneURL.User = url.User("git")
+		u.CloneURL.Host = "github.com"
+		u.CloneURL.Path = v[2]
+		if u.Scheme == "" {
+			for _, scheme := range gitSchemes {
+				u2 := *u
+				u2.Scheme = scheme
+				rr.try = append(rr.try, &u2)
+			}
+		}
 		rr.VCS = []string{"git"}
 		// If no scheme was already recorded, then add the possible schemes for github
 		if rr.Schemes == nil {
@@ -121,6 +134,7 @@ func deduceRemoteRepo(path string) (rr *remoteRepo, err error) {
 		}
 
 		// gopkg.in is always backed by github
+		//rr.CloneURL.User = url.User("git")
 		rr.CloneURL.Host = "github.com"
 		// If the third position is empty, it's the shortened form that expands
 		// to the go-pkg github user
@@ -129,8 +143,8 @@ func deduceRemoteRepo(path string) (rr *remoteRepo, err error) {
 		} else {
 			rr.CloneURL.Path = v[2] + "/" + v[3]
 		}
-		rr.Base = v[1]
-		rr.RelPkg = strings.TrimPrefix(v[6], "/")
+		rr.repoRoot = v[1]
+		rr.relPkg = strings.TrimPrefix(v[6], "/")
 		rr.VCS = []string{"git"}
 		// If no scheme was already recorded, then add the possible schemes for github
 		if rr.Schemes == nil {
@@ -145,8 +159,8 @@ func deduceRemoteRepo(path string) (rr *remoteRepo, err error) {
 
 		rr.CloneURL.Host = "bitbucket.org"
 		rr.CloneURL.Path = v[2]
-		rr.Base = v[1]
-		rr.RelPkg = strings.TrimPrefix(v[3], "/")
+		rr.repoRoot = v[1]
+		rr.relPkg = strings.TrimPrefix(v[3], "/")
 		rr.VCS = []string{"git", "hg"}
 		// FIXME(sdboyer) this ambiguity of vcs kills us on schemes, as schemes
 		// are inherently vcs-specific. Fixing this requires a wider refactor.
@@ -175,8 +189,8 @@ func deduceRemoteRepo(path string) (rr *remoteRepo, err error) {
 
 		rr.CloneURL.Host = "launchpad.net"
 		rr.CloneURL.Path = v[2]
-		rr.Base = v[1]
-		rr.RelPkg = strings.TrimPrefix(v[3], "/")
+		rr.repoRoot = v[1]
+		rr.relPkg = strings.TrimPrefix(v[3], "/")
 		rr.VCS = []string{"bzr"}
 		if rr.Schemes == nil {
 			rr.Schemes = bzrSchemes
@@ -190,8 +204,8 @@ func deduceRemoteRepo(path string) (rr *remoteRepo, err error) {
 
 		rr.CloneURL.Host = "git.launchpad.net"
 		rr.CloneURL.Path = v[2]
-		rr.Base = v[1]
-		rr.RelPkg = strings.TrimPrefix(v[3], "/")
+		rr.repoRoot = v[1]
+		rr.relPkg = strings.TrimPrefix(v[3], "/")
 		rr.VCS = []string{"git"}
 		if rr.Schemes == nil {
 			rr.Schemes = gitSchemes
@@ -204,8 +218,8 @@ func deduceRemoteRepo(path string) (rr *remoteRepo, err error) {
 
 		rr.CloneURL.Host = "hub.jazz.net"
 		rr.CloneURL.Path = v[2]
-		rr.Base = v[1]
-		rr.RelPkg = strings.TrimPrefix(v[3], "/")
+		rr.repoRoot = v[1]
+		rr.relPkg = strings.TrimPrefix(v[3], "/")
 		rr.VCS = []string{"git"}
 		if rr.Schemes == nil {
 			rr.Schemes = gitSchemes
@@ -218,8 +232,8 @@ func deduceRemoteRepo(path string) (rr *remoteRepo, err error) {
 
 		rr.CloneURL.Host = "git.apache.org"
 		rr.CloneURL.Path = v[2]
-		rr.Base = v[1]
-		rr.RelPkg = strings.TrimPrefix(v[3], "/")
+		rr.repoRoot = v[1]
+		rr.relPkg = strings.TrimPrefix(v[3], "/")
 		rr.VCS = []string{"git"}
 		if rr.Schemes == nil {
 			rr.Schemes = gitSchemes
@@ -237,8 +251,8 @@ func deduceRemoteRepo(path string) (rr *remoteRepo, err error) {
 			rr.CloneURL.Host = x[0]
 			rr.CloneURL.Path = x[1]
 			rr.VCS = []string{v[5]}
-			rr.Base = v[1]
-			rr.RelPkg = strings.TrimPrefix(v[6], "/")
+			rr.repoRoot = v[1]
+			rr.relPkg = strings.TrimPrefix(v[6], "/")
 
 			if rr.Schemes == nil {
 				if v[5] == "git" {
@@ -270,8 +284,8 @@ func deduceRemoteRepo(path string) (rr *remoteRepo, err error) {
 	}
 
 	// We have a real URL. Set the other values and return.
-	rr.Base = importroot
-	rr.RelPkg = strings.TrimPrefix(path[len(importroot):], "/")
+	rr.repoRoot = importroot
+	rr.relPkg = strings.TrimPrefix(path[len(importroot):], "/")
 
 	rr.VCS = []string{vcs}
 	if rr.CloneURL.Scheme != "" {
diff --git a/remote_test.go b/remote_test.go
index 6f5cb62..e699a86 100644
--- a/remote_test.go
+++ b/remote_test.go
@@ -19,8 +19,8 @@ func TestDeduceRemotes(t *testing.T) {
 		{
 			"github.com/sdboyer/gps",
 			&remoteRepo{
-				Base:   "github.com/sdboyer/gps",
-				RelPkg: "",
+				repoRoot: "github.com/sdboyer/gps",
+				relPkg:   "",
 				CloneURL: &url.URL{
 					Host: "github.com",
 					Path: "sdboyer/gps",
@@ -32,8 +32,8 @@ func TestDeduceRemotes(t *testing.T) {
 		{
 			"github.com/sdboyer/gps/foo",
 			&remoteRepo{
-				Base:   "github.com/sdboyer/gps",
-				RelPkg: "foo",
+				repoRoot: "github.com/sdboyer/gps",
+				relPkg:   "foo",
 				CloneURL: &url.URL{
 					Host: "github.com",
 					Path: "sdboyer/gps",
@@ -45,8 +45,8 @@ func TestDeduceRemotes(t *testing.T) {
 		{
 			"git@github.com:sdboyer/gps",
 			&remoteRepo{
-				Base:   "github.com/sdboyer/gps",
-				RelPkg: "",
+				repoRoot: "github.com/sdboyer/gps",
+				relPkg:   "",
 				CloneURL: &url.URL{
 					Scheme: "ssh",
 					User:   url.User("git"),
@@ -60,8 +60,8 @@ func TestDeduceRemotes(t *testing.T) {
 		{
 			"https://github.com/sdboyer/gps/foo",
 			&remoteRepo{
-				Base:   "github.com/sdboyer/gps",
-				RelPkg: "foo",
+				repoRoot: "github.com/sdboyer/gps",
+				relPkg:   "foo",
 				CloneURL: &url.URL{
 					Scheme: "https",
 					Host:   "github.com",
@@ -74,8 +74,8 @@ func TestDeduceRemotes(t *testing.T) {
 		{
 			"https://github.com/sdboyer/gps/foo/bar",
 			&remoteRepo{
-				Base:   "github.com/sdboyer/gps",
-				RelPkg: "foo/bar",
+				repoRoot: "github.com/sdboyer/gps",
+				relPkg:   "foo/bar",
 				CloneURL: &url.URL{
 					Scheme: "https",
 					Host:   "github.com",
@@ -105,8 +105,8 @@ func TestDeduceRemotes(t *testing.T) {
 		{
 			"gopkg.in/sdboyer/gps.v0",
 			&remoteRepo{
-				Base:   "gopkg.in/sdboyer/gps.v0",
-				RelPkg: "",
+				repoRoot: "gopkg.in/sdboyer/gps.v0",
+				relPkg:   "",
 				CloneURL: &url.URL{
 					Host: "github.com",
 					Path: "sdboyer/gps",
@@ -118,8 +118,8 @@ func TestDeduceRemotes(t *testing.T) {
 		{
 			"gopkg.in/sdboyer/gps.v0/foo",
 			&remoteRepo{
-				Base:   "gopkg.in/sdboyer/gps.v0",
-				RelPkg: "foo",
+				repoRoot: "gopkg.in/sdboyer/gps.v0",
+				relPkg:   "foo",
 				CloneURL: &url.URL{
 					Host: "github.com",
 					Path: "sdboyer/gps",
@@ -131,8 +131,8 @@ func TestDeduceRemotes(t *testing.T) {
 		{
 			"gopkg.in/sdboyer/gps.v0/foo/bar",
 			&remoteRepo{
-				Base:   "gopkg.in/sdboyer/gps.v0",
-				RelPkg: "foo/bar",
+				repoRoot: "gopkg.in/sdboyer/gps.v0",
+				relPkg:   "foo/bar",
 				CloneURL: &url.URL{
 					Host: "github.com",
 					Path: "sdboyer/gps",
@@ -144,8 +144,8 @@ func TestDeduceRemotes(t *testing.T) {
 		{
 			"gopkg.in/yaml.v1",
 			&remoteRepo{
-				Base:   "gopkg.in/yaml.v1",
-				RelPkg: "",
+				repoRoot: "gopkg.in/yaml.v1",
+				relPkg:   "",
 				CloneURL: &url.URL{
 					Host: "github.com",
 					Path: "go-pkg/yaml",
@@ -157,8 +157,8 @@ func TestDeduceRemotes(t *testing.T) {
 		{
 			"gopkg.in/yaml.v1/foo/bar",
 			&remoteRepo{
-				Base:   "gopkg.in/yaml.v1",
-				RelPkg: "foo/bar",
+				repoRoot: "gopkg.in/yaml.v1",
+				relPkg:   "foo/bar",
 				CloneURL: &url.URL{
 					Host: "github.com",
 					Path: "go-pkg/yaml",
@@ -176,8 +176,8 @@ func TestDeduceRemotes(t *testing.T) {
 		{
 			"hub.jazz.net/git/user1/pkgname",
 			&remoteRepo{
-				Base:   "hub.jazz.net/git/user1/pkgname",
-				RelPkg: "",
+				repoRoot: "hub.jazz.net/git/user1/pkgname",
+				relPkg:   "",
 				CloneURL: &url.URL{
 					Host: "hub.jazz.net",
 					Path: "git/user1/pkgname",
@@ -189,8 +189,8 @@ func TestDeduceRemotes(t *testing.T) {
 		{
 			"hub.jazz.net/git/user1/pkgname/submodule/submodule/submodule",
 			&remoteRepo{
-				Base:   "hub.jazz.net/git/user1/pkgname",
-				RelPkg: "submodule/submodule/submodule",
+				repoRoot: "hub.jazz.net/git/user1/pkgname",
+				relPkg:   "submodule/submodule/submodule",
 				CloneURL: &url.URL{
 					Host: "hub.jazz.net",
 					Path: "git/user1/pkgname",
@@ -232,8 +232,8 @@ func TestDeduceRemotes(t *testing.T) {
 		{
 			"hub.jazz.net/git/user/pkg.name",
 			&remoteRepo{
-				Base:   "hub.jazz.net/git/user/pkg.name",
-				RelPkg: "",
+				repoRoot: "hub.jazz.net/git/user/pkg.name",
+				relPkg:   "",
 				CloneURL: &url.URL{
 					Host: "hub.jazz.net",
 					Path: "git/user/pkg.name",
@@ -250,8 +250,8 @@ func TestDeduceRemotes(t *testing.T) {
 		{
 			"bitbucket.org/sdboyer/reporoot",
 			&remoteRepo{
-				Base:   "bitbucket.org/sdboyer/reporoot",
-				RelPkg: "",
+				repoRoot: "bitbucket.org/sdboyer/reporoot",
+				relPkg:   "",
 				CloneURL: &url.URL{
 					Host: "bitbucket.org",
 					Path: "sdboyer/reporoot",
@@ -263,8 +263,8 @@ func TestDeduceRemotes(t *testing.T) {
 		{
 			"bitbucket.org/sdboyer/reporoot/foo/bar",
 			&remoteRepo{
-				Base:   "bitbucket.org/sdboyer/reporoot",
-				RelPkg: "foo/bar",
+				repoRoot: "bitbucket.org/sdboyer/reporoot",
+				relPkg:   "foo/bar",
 				CloneURL: &url.URL{
 					Host: "bitbucket.org",
 					Path: "sdboyer/reporoot",
@@ -276,8 +276,8 @@ func TestDeduceRemotes(t *testing.T) {
 		{
 			"https://bitbucket.org/sdboyer/reporoot/foo/bar",
 			&remoteRepo{
-				Base:   "bitbucket.org/sdboyer/reporoot",
-				RelPkg: "foo/bar",
+				repoRoot: "bitbucket.org/sdboyer/reporoot",
+				relPkg:   "foo/bar",
 				CloneURL: &url.URL{
 					Scheme: "https",
 					Host:   "bitbucket.org",
@@ -290,8 +290,8 @@ func TestDeduceRemotes(t *testing.T) {
 		{
 			"launchpad.net/govcstestbzrrepo",
 			&remoteRepo{
-				Base:   "launchpad.net/govcstestbzrrepo",
-				RelPkg: "",
+				repoRoot: "launchpad.net/govcstestbzrrepo",
+				relPkg:   "",
 				CloneURL: &url.URL{
 					Host: "launchpad.net",
 					Path: "govcstestbzrrepo",
@@ -303,8 +303,8 @@ func TestDeduceRemotes(t *testing.T) {
 		{
 			"launchpad.net/govcstestbzrrepo/foo/bar",
 			&remoteRepo{
-				Base:   "launchpad.net/govcstestbzrrepo",
-				RelPkg: "foo/bar",
+				repoRoot: "launchpad.net/govcstestbzrrepo",
+				relPkg:   "foo/bar",
 				CloneURL: &url.URL{
 					Host: "launchpad.net",
 					Path: "govcstestbzrrepo",
@@ -320,8 +320,8 @@ func TestDeduceRemotes(t *testing.T) {
 		{
 			"git.launchpad.net/reporoot",
 			&remoteRepo{
-				Base:   "git.launchpad.net/reporoot",
-				RelPkg: "",
+				repoRoot: "git.launchpad.net/reporoot",
+				relPkg:   "",
 				CloneURL: &url.URL{
 					Host: "git.launchpad.net",
 					Path: "reporoot",
@@ -333,8 +333,8 @@ func TestDeduceRemotes(t *testing.T) {
 		{
 			"git.launchpad.net/reporoot/foo/bar",
 			&remoteRepo{
-				Base:   "git.launchpad.net/reporoot",
-				RelPkg: "foo/bar",
+				repoRoot: "git.launchpad.net/reporoot",
+				relPkg:   "foo/bar",
 				CloneURL: &url.URL{
 					Host: "git.launchpad.net",
 					Path: "reporoot",
@@ -346,8 +346,8 @@ func TestDeduceRemotes(t *testing.T) {
 		{
 			"git.launchpad.net/reporoot",
 			&remoteRepo{
-				Base:   "git.launchpad.net/reporoot",
-				RelPkg: "",
+				repoRoot: "git.launchpad.net/reporoot",
+				relPkg:   "",
 				CloneURL: &url.URL{
 					Host: "git.launchpad.net",
 					Path: "reporoot",
@@ -363,8 +363,8 @@ func TestDeduceRemotes(t *testing.T) {
 		{
 			"git.apache.org/package-name.git",
 			&remoteRepo{
-				Base:   "git.apache.org/package-name.git",
-				RelPkg: "",
+				repoRoot: "git.apache.org/package-name.git",
+				relPkg:   "",
 				CloneURL: &url.URL{
 					Host: "git.apache.org",
 					Path: "package-name.git",
@@ -376,8 +376,8 @@ func TestDeduceRemotes(t *testing.T) {
 		{
 			"git.apache.org/package-name.git/foo/bar",
 			&remoteRepo{
-				Base:   "git.apache.org/package-name.git",
-				RelPkg: "foo/bar",
+				repoRoot: "git.apache.org/package-name.git",
+				relPkg:   "foo/bar",
 				CloneURL: &url.URL{
 					Host: "git.apache.org",
 					Path: "package-name.git",
@@ -390,8 +390,8 @@ func TestDeduceRemotes(t *testing.T) {
 		{
 			"golang.org/x/exp",
 			&remoteRepo{
-				Base:   "golang.org/x/exp",
-				RelPkg: "",
+				repoRoot: "golang.org/x/exp",
+				relPkg:   "",
 				CloneURL: &url.URL{
 					Scheme: "https",
 					Host:   "go.googlesource.com",
@@ -404,8 +404,8 @@ func TestDeduceRemotes(t *testing.T) {
 		{
 			"golang.org/x/exp/inotify",
 			&remoteRepo{
-				Base:   "golang.org/x/exp",
-				RelPkg: "inotify",
+				repoRoot: "golang.org/x/exp",
+				relPkg:   "inotify",
 				CloneURL: &url.URL{
 					Scheme: "https",
 					Host:   "go.googlesource.com",
@@ -418,8 +418,8 @@ func TestDeduceRemotes(t *testing.T) {
 		{
 			"rsc.io/pdf",
 			&remoteRepo{
-				Base:   "rsc.io/pdf",
-				RelPkg: "",
+				repoRoot: "rsc.io/pdf",
+				relPkg:   "",
 				CloneURL: &url.URL{
 					Scheme: "https",
 					Host:   "github.com",
@@ -433,8 +433,8 @@ func TestDeduceRemotes(t *testing.T) {
 		{
 			"github.com/kr/pretty",
 			&remoteRepo{
-				Base:   "github.com/kr/pretty",
-				RelPkg: "",
+				repoRoot: "github.com/kr/pretty",
+				relPkg:   "",
 				CloneURL: &url.URL{
 					Host: "github.com",
 					Path: "kr/pretty",
@@ -461,11 +461,11 @@ func TestDeduceRemotes(t *testing.T) {
 			continue
 		}
 
-		if got.Base != want.Base {
-			t.Errorf("deduceRemoteRepo(%q): Base was %s, wanted %s", fix.path, got.Base, want.Base)
+		if got.repoRoot != want.repoRoot {
+			t.Errorf("deduceRemoteRepo(%q): Base was %s, wanted %s", fix.path, got.repoRoot, want.repoRoot)
 		}
-		if got.RelPkg != want.RelPkg {
-			t.Errorf("deduceRemoteRepo(%q): RelPkg was %s, wanted %s", fix.path, got.RelPkg, want.RelPkg)
+		if got.relPkg != want.relPkg {
+			t.Errorf("deduceRemoteRepo(%q): RelPkg was %s, wanted %s", fix.path, got.relPkg, want.relPkg)
 		}
 		if !reflect.DeepEqual(got.CloneURL, want.CloneURL) {
 			// misspelling things is cool when it makes columns line up
diff --git a/solve_basic_test.go b/solve_basic_test.go
index c493b19..7550e10 100644
--- a/solve_basic_test.go
+++ b/solve_basic_test.go
@@ -1355,8 +1355,8 @@ func (b *depspecBridge) deduceRemoteRepo(path string) (*remoteRepo, error) {
 		n := string(ds.n)
 		if path == n || strings.HasPrefix(path, n+"/") {
 			return &remoteRepo{
-				Base:   n,
-				RelPkg: strings.TrimPrefix(path, n+"/"),
+				repoRoot: n,
+				relPkg:   strings.TrimPrefix(path, n+"/"),
 			}, nil
 		}
 	}
diff --git a/solver.go b/solver.go
index eab3b42..0597e31 100644
--- a/solver.go
+++ b/solver.go
@@ -605,17 +605,17 @@ func (s *solver) intersectConstraintsWithImports(deps []workingConstraint, reach
 		// Make a new completeDep with an open constraint, respecting overrides
 		pd := s.ovr.override(ProjectConstraint{
 			Ident: ProjectIdentifier{
-				ProjectRoot: ProjectRoot(root.Base),
-				NetworkName: root.Base,
+				ProjectRoot: ProjectRoot(root.repoRoot),
+				NetworkName: root.repoRoot,
 			},
 			Constraint: Any(),
 		})
 
 		// Insert the pd into the trie so that further deps from this
 		// project get caught by the prefix search
-		xt.Insert(root.Base, pd)
+		xt.Insert(root.repoRoot, pd)
 		// And also put the complete dep into the dmap
-		dmap[ProjectRoot(root.Base)] = completeDep{
+		dmap[ProjectRoot(root.repoRoot)] = completeDep{
 			workingConstraint: pd,
 			pl:                []string{rp},
 		}
diff --git a/source_manager.go b/source_manager.go
index 477e705..80fb0ea 100644
--- a/source_manager.go
+++ b/source_manager.go
@@ -252,7 +252,6 @@ func (sm *SourceMgr) getProjectManager(id ProjectIdentifier) (*pmState, error) {
 	// TODO(sdboyer) finish this, it's not sufficient (?)
 	n := id.netName()
 	var rpath string
-
 	// Early check to see if we already have a pm in the cache for this net name
 	if pm, exists := sm.pms[n]; exists {
 		return pm, nil

From 6a1d540a0bc7f14e8cb73591bd24be773c5c619f Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Tue, 2 Aug 2016 09:30:37 -0400
Subject: [PATCH 14/71] Incremental move towards 'source'

Yeah, we're just gonna have to go whole hog on this. The system's too
borky and complected as-is to sanely do a minor refactor.
---
 project_manager.go |  10 ---
 source.go          | 154 +++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 154 insertions(+), 10 deletions(-)
 create mode 100644 source.go

diff --git a/project_manager.go b/project_manager.go
index 98b7ac6..8631a51 100644
--- a/project_manager.go
+++ b/project_manager.go
@@ -49,15 +49,6 @@ type existence struct {
 	f projectExistence
 }
 
-// TODO(sdboyer) figure out shape of versions, then implement marshaling/unmarshaling
-type projectDataCache struct {
-	Version  string                   `json:"version"` // TODO(sdboyer) use this
-	Infos    map[Revision]projectInfo `json:"infos"`
-	Packages map[Revision]PackageTree `json:"packages"`
-	VMap     map[Version]Revision     `json:"vmap"`
-	RMap     map[Revision][]Version   `json:"rmap"`
-}
-
 // projectInfo holds manifest and lock
 type projectInfo struct {
 	Manifest
@@ -233,7 +224,6 @@ func (pm *projectManager) ListVersions() (vlist []Version, err error) {
 
 		if err != nil {
 			// TODO(sdboyer) More-er proper-er error
-			fmt.Println(err)
 			return nil, err
 		}
 
diff --git a/source.go b/source.go
new file mode 100644
index 0000000..db38e26
--- /dev/null
+++ b/source.go
@@ -0,0 +1,154 @@
+package gps
+
+import (
+	"fmt"
+	"net/url"
+	"path/filepath"
+
+	"github.com/Masterminds/vcs"
+)
+
+type source interface {
+	checkExistence(projectExistence) bool
+	exportVersionTo(Version, string) error
+	getManifestAndLock(ProjectRoot, Version) (Manifest, Lock, error)
+	listPackages(ProjectRoot, Version) (PackageTree, error)
+	listVersions() ([]Version, error)
+	revisionPresentIn(ProjectRoot, Revision) (bool, error)
+}
+
+type projectDataCache struct {
+	Version  string                   `json:"version"` // TODO(sdboyer) use this
+	Infos    map[Revision]projectInfo `json:"infos"`
+	Packages map[Revision]PackageTree `json:"packages"`
+	VMap     map[Version]Revision     `json:"vmap"`
+	RMap     map[Revision][]Version   `json:"rmap"`
+}
+
+func newDataCache() *projectDataCache {
+	return &projectDataCache{
+		Infos:    make(map[Revision]projectInfo),
+		Packages: make(map[Revision]PackageTree),
+		VMap:     make(map[Version]Revision),
+		RMap:     make(map[Revision][]Version),
+	}
+}
+
+type maybeSource interface {
+	try(cachedir string, an ProjectAnalyzer) (source, error)
+}
+
+type maybeSources []maybeSource
+
+type maybeGitSource struct {
+	n   string
+	url *url.URL
+}
+
+func (s maybeGitSource) try(cachedir string, an ProjectAnalyzer) (source, error) {
+	path := filepath.Join(cachedir, "sources", sanitizer.Replace(s.url.String()))
+	pm := &gitSource{
+		baseSource: baseSource{
+			an: an,
+			dc: newDataCache(),
+			crepo: &repo{
+				r:     vcs.NewGitRepo(path, s.url.String()),
+				rpath: path,
+			},
+		},
+	}
+
+	_, err := pm.ListVersions()
+	if err != nil {
+		return nil, err
+		//} else if pm.ex.f&existsUpstream == existsUpstream {
+		//return pm, nil
+	}
+
+	return pm, nil
+}
+
+type baseSource struct {
+	// Object for the cache repository
+	crepo *repo
+
+	// Indicates the extent to which we have searched for, and verified, the
+	// existence of the project/repo.
+	ex existence
+
+	// ProjectAnalyzer used to fulfill getManifestAndLock
+	an ProjectAnalyzer
+
+	// Whether the cache has the latest info on versions
+	cvsync bool
+
+	// The project metadata cache. This is persisted to disk, for reuse across
+	// solver runs.
+	// TODO(sdboyer) protect with mutex
+	dc *projectDataCache
+}
+
+func (bs *baseSource) getManifestAndLock(r ProjectRoot, v Version) (Manifest, Lock, error) {
+	if err := bs.ensureCacheExistence(); err != nil {
+		return nil, nil, err
+	}
+
+	if r, exists := bs.dc.VMap[v]; exists {
+		if pi, exists := bs.dc.Infos[r]; exists {
+			return pi.Manifest, pi.Lock, nil
+		}
+	}
+
+	bs.crepo.mut.Lock()
+	var err error
+	if !bs.crepo.synced {
+		err = bs.crepo.r.Update()
+		if err != nil {
+			return nil, nil, fmt.Errorf("Could not fetch latest updates into repository")
+		}
+		bs.crepo.synced = true
+	}
+
+	// Always prefer a rev, if it's available
+	if pv, ok := v.(PairedVersion); ok {
+		err = bs.crepo.r.UpdateVersion(pv.Underlying().String())
+	} else {
+		err = bs.crepo.r.UpdateVersion(v.String())
+	}
+	bs.crepo.mut.Unlock()
+	if err != nil {
+		// TODO(sdboyer) More-er proper-er error
+		panic(fmt.Sprintf("canary - why is checkout/whatever failing: %s %s %s", bs.n, v.String(), err))
+	}
+
+	bs.crepo.mut.RLock()
+	m, l, err := bs.an.DeriveManifestAndLock(bs.crepo.rpath, r)
+	// TODO(sdboyer) cache results
+	bs.crepo.mut.RUnlock()
+
+	if err == nil {
+		if l != nil {
+			l = prepLock(l)
+		}
+
+		// If m is nil, prebsanifest will provide an empty one.
+		pi := projectInfo{
+			Manifest: prebsanifest(m),
+			Lock:     l,
+		}
+
+		// TODO(sdboyer) this just clobbers all over and ignores the paired/unpaired
+		// distinction; serious fix is needed
+		if r, exists := bs.dc.VMap[v]; exists {
+			bs.dc.Infos[r] = pi
+		}
+
+		return pi.Manifest, pi.Lock, nil
+	}
+
+	return nil, nil, err
+}
+
+type gitSource struct {
+	bs baseSource
+}

From 3311597b5e3d2ccd8cc2ce03aff60607104c2c37 Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Tue, 2 Aug 2016 09:32:07 -0400
Subject: [PATCH 15/71] Revert "Incremental refactor of remoteRepo"

This reverts commit 7c516b84d14132bf5056986e087d48f0cfc647eb.
---
 remote.go           |  82 +++++++++++++------------------
 remote_test.go      | 116 ++++++++++++++++++++++----------------------
 solve_basic_test.go |   4 +-
 solver.go           |   8 +--
 source_manager.go   |   1 +
 5 files changed, 99 insertions(+), 112 deletions(-)

diff --git a/remote.go b/remote.go
index 2291435..d28c5e9 100644
--- a/remote.go
+++ b/remote.go
@@ -15,17 +15,11 @@ import (
 // one is not a guarantee that the resource it identifies actually exists or is
 // accessible.
 type remoteRepo struct {
-	repoRoot string
-	relPkg   string
-	try      []maybeRemoteSource
-}
-
-// maybeRemoteSource represents a set of instructions for accessing a possible
-// remote resource, without knowing whether that resource actually
-// works/exists/is accessible, etc.
-type maybeRemoteSource struct {
-	vcs string
-	url *url.URL
+	Base     string
+	RelPkg   string
+	CloneURL *url.URL
+	Schemes  []string
+	VCS      []string
 }
 
 var (
@@ -65,13 +59,11 @@ var (
 // repositories can be bare import paths, or urls including a checkout scheme.
 func deduceRemoteRepo(path string) (rr *remoteRepo, err error) {
 	rr = &remoteRepo{}
-	var u *url.Url
-
 	if m := scpSyntaxRe.FindStringSubmatch(path); m != nil {
 		// Match SCP-like syntax and convert it to a URL.
 		// Eg, "git@github.com:user/repo" becomes
 		// "ssh://git@github.com/user/repo".
-		u = &url.URL{
+		rr.CloneURL = &url.URL{
 			Scheme: "ssh",
 			User:   url.User(m[1]),
 			Host:   m[2],
@@ -80,22 +72,26 @@ func deduceRemoteRepo(path string) (rr *remoteRepo, err error) {
 			//RawPath: m[3],
 		}
 	} else {
-		u, err = url.Parse(path)
+		rr.CloneURL, err = url.Parse(path)
 		if err != nil {
 			return nil, fmt.Errorf("%q is not a valid import path", path)
 		}
 	}
 
-	if u.Host != "" {
-		path = u.Host + "/" + strings.TrimPrefix(u.Path, "/")
+	if rr.CloneURL.Host != "" {
+		path = rr.CloneURL.Host + "/" + strings.TrimPrefix(rr.CloneURL.Path, "/")
 	} else {
-		path = u.Path
+		path = rr.CloneURL.Path
 	}
 
 	if !pathvld.MatchString(path) {
 		return nil, fmt.Errorf("%q is not a valid import path", path)
 	}
 
+	if rr.CloneURL.Scheme != "" {
+		rr.Schemes = []string{rr.CloneURL.Scheme}
+	}
+
 	// TODO(sdboyer) instead of a switch, encode base domain in radix tree and pick
 	// detector from there; if failure, then fall back on metadata work
 
@@ -103,19 +99,10 @@ func deduceRemoteRepo(path string) (rr *remoteRepo, err error) {
 	case ghRegex.MatchString(path):
 		v := ghRegex.FindStringSubmatch(path)
 
-		rr.repoRoot = v[1]
-		rr.relPkg = strings.TrimPrefix(v[3], "/")
-
-		//rr.CloneURL.User = url.User("git")
-		u.CloneURL.Host = "github.com"
-		u.CloneURL.Path = v[2]
-		if u.Scheme == "" {
-			for _, scheme := range gitSchemes {
-				u2 := *u
-				u2.Scheme = scheme
-				rr.try = append(rr.try, &u2)
-			}
-		}
+		rr.CloneURL.Host = "github.com"
+		rr.CloneURL.Path = v[2]
+		rr.Base = v[1]
+		rr.RelPkg = strings.TrimPrefix(v[3], "/")
 		rr.VCS = []string{"git"}
 		// If no scheme was already recorded, then add the possible schemes for github
 		if rr.Schemes == nil {
@@ -134,7 +121,6 @@ func deduceRemoteRepo(path string) (rr *remoteRepo, err error) {
 		}
 
 		// gopkg.in is always backed by github
-		//rr.CloneURL.User = url.User("git")
 		rr.CloneURL.Host = "github.com"
 		// If the third position is empty, it's the shortened form that expands
 		// to the go-pkg github user
@@ -143,8 +129,8 @@ func deduceRemoteRepo(path string) (rr *remoteRepo, err error) {
 		} else {
 			rr.CloneURL.Path = v[2] + "/" + v[3]
 		}
-		rr.repoRoot = v[1]
-		rr.relPkg = strings.TrimPrefix(v[6], "/")
+		rr.Base = v[1]
+		rr.RelPkg = strings.TrimPrefix(v[6], "/")
 		rr.VCS = []string{"git"}
 		// If no scheme was already recorded, then add the possible schemes for github
 		if rr.Schemes == nil {
@@ -159,8 +145,8 @@ func deduceRemoteRepo(path string) (rr *remoteRepo, err error) {
 
 		rr.CloneURL.Host = "bitbucket.org"
 		rr.CloneURL.Path = v[2]
-		rr.repoRoot = v[1]
-		rr.relPkg = strings.TrimPrefix(v[3], "/")
+		rr.Base = v[1]
+		rr.RelPkg = strings.TrimPrefix(v[3], "/")
 		rr.VCS = []string{"git", "hg"}
 		// FIXME(sdboyer) this ambiguity of vcs kills us on schemes, as schemes
 		// are inherently vcs-specific. Fixing this requires a wider refactor.
@@ -189,8 +175,8 @@ func deduceRemoteRepo(path string) (rr *remoteRepo, err error) {
 
 		rr.CloneURL.Host = "launchpad.net"
 		rr.CloneURL.Path = v[2]
-		rr.repoRoot = v[1]
-		rr.relPkg = strings.TrimPrefix(v[3], "/")
+		rr.Base = v[1]
+		rr.RelPkg = strings.TrimPrefix(v[3], "/")
 		rr.VCS = []string{"bzr"}
 		if rr.Schemes == nil {
 			rr.Schemes = bzrSchemes
@@ -204,8 +190,8 @@ func deduceRemoteRepo(path string) (rr *remoteRepo, err error) {
 
 		rr.CloneURL.Host = "git.launchpad.net"
 		rr.CloneURL.Path = v[2]
-		rr.repoRoot = v[1]
-		rr.relPkg = strings.TrimPrefix(v[3], "/")
+		rr.Base = v[1]
+		rr.RelPkg = strings.TrimPrefix(v[3], "/")
 		rr.VCS = []string{"git"}
 		if rr.Schemes == nil {
 			rr.Schemes = gitSchemes
@@ -218,8 +204,8 @@ func deduceRemoteRepo(path string) (rr *remoteRepo, err error) {
 
 		rr.CloneURL.Host = "hub.jazz.net"
 		rr.CloneURL.Path = v[2]
-		rr.repoRoot = v[1]
-		rr.relPkg = strings.TrimPrefix(v[3], "/")
+		rr.Base = v[1]
+		rr.RelPkg = strings.TrimPrefix(v[3], "/")
 		rr.VCS = []string{"git"}
 		if rr.Schemes == nil {
 			rr.Schemes = gitSchemes
@@ -232,8 +218,8 @@ func deduceRemoteRepo(path string) (rr *remoteRepo, err error) {
 
 		rr.CloneURL.Host = "git.apache.org"
 		rr.CloneURL.Path = v[2]
-		rr.repoRoot = v[1]
-		rr.relPkg = strings.TrimPrefix(v[3], "/")
+		rr.Base = v[1]
+		rr.RelPkg = strings.TrimPrefix(v[3], "/")
 		rr.VCS = []string{"git"}
 		if rr.Schemes == nil {
 			rr.Schemes = gitSchemes
@@ -251,8 +237,8 @@ func deduceRemoteRepo(path string) (rr *remoteRepo, err error) {
 			rr.CloneURL.Host = x[0]
 			rr.CloneURL.Path = x[1]
 			rr.VCS = []string{v[5]}
-			rr.repoRoot = v[1]
-			rr.relPkg = strings.TrimPrefix(v[6], "/")
+			rr.Base = v[1]
+			rr.RelPkg = strings.TrimPrefix(v[6], "/")
 
 			if rr.Schemes == nil {
 				if v[5] == "git" {
@@ -284,8 +270,8 @@ func deduceRemoteRepo(path string) (rr *remoteRepo, err error) {
 	}
 
 	// We have a real URL. Set the other values and return.
-	rr.repoRoot = importroot
-	rr.relPkg = strings.TrimPrefix(path[len(importroot):], "/")
+	rr.Base = importroot
+	rr.RelPkg = strings.TrimPrefix(path[len(importroot):], "/")
 
 	rr.VCS = []string{vcs}
 	if rr.CloneURL.Scheme != "" {
diff --git a/remote_test.go b/remote_test.go
index e699a86..6f5cb62 100644
--- a/remote_test.go
+++ b/remote_test.go
@@ -19,8 +19,8 @@ func TestDeduceRemotes(t *testing.T) {
 		{
 			"github.com/sdboyer/gps",
 			&remoteRepo{
-				repoRoot: "github.com/sdboyer/gps",
-				relPkg:   "",
+				Base:   "github.com/sdboyer/gps",
+				RelPkg: "",
 				CloneURL: &url.URL{
 					Host: "github.com",
 					Path: "sdboyer/gps",
@@ -32,8 +32,8 @@ func TestDeduceRemotes(t *testing.T) {
 		{
 			"github.com/sdboyer/gps/foo",
 			&remoteRepo{
-				repoRoot: "github.com/sdboyer/gps",
-				relPkg:   "foo",
+				Base:   "github.com/sdboyer/gps",
+				RelPkg: "foo",
 				CloneURL: &url.URL{
 					Host: "github.com",
 					Path: "sdboyer/gps",
@@ -45,8 +45,8 @@ func TestDeduceRemotes(t *testing.T) {
 		{
 			"git@github.com:sdboyer/gps",
 			&remoteRepo{
-				repoRoot: "github.com/sdboyer/gps",
-				relPkg:   "",
+				Base:   "github.com/sdboyer/gps",
+				RelPkg: "",
 				CloneURL: &url.URL{
 					Scheme: "ssh",
 					User:   url.User("git"),
@@ -60,8 +60,8 @@ func TestDeduceRemotes(t *testing.T) {
 		{
 			"https://github.com/sdboyer/gps/foo",
 			&remoteRepo{
-				repoRoot: "github.com/sdboyer/gps",
-				relPkg:   "foo",
+				Base:   "github.com/sdboyer/gps",
+				RelPkg: "foo",
 				CloneURL: &url.URL{
 					Scheme: "https",
 					Host:   "github.com",
@@ -74,8 +74,8 @@ func TestDeduceRemotes(t *testing.T) {
 		{
 			"https://github.com/sdboyer/gps/foo/bar",
 			&remoteRepo{
-				repoRoot: "github.com/sdboyer/gps",
-				relPkg:   "foo/bar",
+				Base:   "github.com/sdboyer/gps",
+				RelPkg: "foo/bar",
 				CloneURL: &url.URL{
 					Scheme: "https",
 					Host:   "github.com",
@@ -105,8 +105,8 @@ func TestDeduceRemotes(t *testing.T) {
 		{
 			"gopkg.in/sdboyer/gps.v0",
 			&remoteRepo{
-				repoRoot: "gopkg.in/sdboyer/gps.v0",
-				relPkg:   "",
+				Base:   "gopkg.in/sdboyer/gps.v0",
+				RelPkg: "",
 				CloneURL: &url.URL{
 					Host: "github.com",
 					Path: "sdboyer/gps",
@@ -118,8 +118,8 @@ func TestDeduceRemotes(t *testing.T) {
 		{
 			"gopkg.in/sdboyer/gps.v0/foo",
 			&remoteRepo{
-				repoRoot: "gopkg.in/sdboyer/gps.v0",
-				relPkg:   "foo",
+				Base:   "gopkg.in/sdboyer/gps.v0",
+				RelPkg: "foo",
 				CloneURL: &url.URL{
 					Host: "github.com",
 					Path: "sdboyer/gps",
@@ -131,8 +131,8 @@ func TestDeduceRemotes(t *testing.T) {
 		{
 			"gopkg.in/sdboyer/gps.v0/foo/bar",
 			&remoteRepo{
-				repoRoot: "gopkg.in/sdboyer/gps.v0",
-				relPkg:   "foo/bar",
+				Base:   "gopkg.in/sdboyer/gps.v0",
+				RelPkg: "foo/bar",
 				CloneURL: &url.URL{
 					Host: "github.com",
 					Path: "sdboyer/gps",
@@ -144,8 +144,8 @@ func TestDeduceRemotes(t *testing.T) {
 		{
 			"gopkg.in/yaml.v1",
 			&remoteRepo{
-				repoRoot: "gopkg.in/yaml.v1",
-				relPkg:   "",
+				Base:   "gopkg.in/yaml.v1",
+				RelPkg: "",
 				CloneURL: &url.URL{
 					Host: "github.com",
 					Path: "go-pkg/yaml",
@@ -157,8 +157,8 @@ func TestDeduceRemotes(t *testing.T) {
 		{
 			"gopkg.in/yaml.v1/foo/bar",
 			&remoteRepo{
-				repoRoot: "gopkg.in/yaml.v1",
-				relPkg:   "foo/bar",
+				Base:   "gopkg.in/yaml.v1",
+				RelPkg: "foo/bar",
 				CloneURL: &url.URL{
 					Host: "github.com",
 					Path: "go-pkg/yaml",
@@ -176,8 +176,8 @@ func TestDeduceRemotes(t *testing.T) {
 		{
 			"hub.jazz.net/git/user1/pkgname",
 			&remoteRepo{
-				repoRoot: "hub.jazz.net/git/user1/pkgname",
-				relPkg:   "",
+				Base:   "hub.jazz.net/git/user1/pkgname",
+				RelPkg: "",
 				CloneURL: &url.URL{
 					Host: "hub.jazz.net",
 					Path: "git/user1/pkgname",
@@ -189,8 +189,8 @@ func TestDeduceRemotes(t *testing.T) {
 		{
 			"hub.jazz.net/git/user1/pkgname/submodule/submodule/submodule",
 			&remoteRepo{
-				repoRoot: "hub.jazz.net/git/user1/pkgname",
-				relPkg:   "submodule/submodule/submodule",
+				Base:   "hub.jazz.net/git/user1/pkgname",
+				RelPkg: "submodule/submodule/submodule",
 				CloneURL: &url.URL{
 					Host: "hub.jazz.net",
 					Path: "git/user1/pkgname",
@@ -232,8 +232,8 @@ func TestDeduceRemotes(t *testing.T) {
 		{
 			"hub.jazz.net/git/user/pkg.name",
 			&remoteRepo{
-				repoRoot: "hub.jazz.net/git/user/pkg.name",
-				relPkg:   "",
+				Base:   "hub.jazz.net/git/user/pkg.name",
+				RelPkg: "",
 				CloneURL: &url.URL{
 					Host: "hub.jazz.net",
 					Path: "git/user/pkg.name",
@@ -250,8 +250,8 @@ func TestDeduceRemotes(t *testing.T) {
 		{
 			"bitbucket.org/sdboyer/reporoot",
 			&remoteRepo{
-				repoRoot: "bitbucket.org/sdboyer/reporoot",
-				relPkg:   "",
+				Base:   "bitbucket.org/sdboyer/reporoot",
+				RelPkg: "",
 				CloneURL: &url.URL{
 					Host: "bitbucket.org",
 					Path: "sdboyer/reporoot",
@@ -263,8 +263,8 @@ func TestDeduceRemotes(t *testing.T) {
 		{
 			"bitbucket.org/sdboyer/reporoot/foo/bar",
 			&remoteRepo{
-				repoRoot: "bitbucket.org/sdboyer/reporoot",
-				relPkg:   "foo/bar",
+				Base:   "bitbucket.org/sdboyer/reporoot",
+				RelPkg: "foo/bar",
 				CloneURL: &url.URL{
 					Host: "bitbucket.org",
 					Path: "sdboyer/reporoot",
@@ -276,8 +276,8 @@ func TestDeduceRemotes(t *testing.T) {
 		{
 			"https://bitbucket.org/sdboyer/reporoot/foo/bar",
 			&remoteRepo{
-				repoRoot: "bitbucket.org/sdboyer/reporoot",
-				relPkg:   "foo/bar",
+				Base:   "bitbucket.org/sdboyer/reporoot",
+				RelPkg: "foo/bar",
 				CloneURL: &url.URL{
 					Scheme: "https",
 					Host:   "bitbucket.org",
@@ -290,8 +290,8 @@ func TestDeduceRemotes(t *testing.T) {
 		{
 			"launchpad.net/govcstestbzrrepo",
 			&remoteRepo{
-				repoRoot: "launchpad.net/govcstestbzrrepo",
-				relPkg:   "",
+				Base:   "launchpad.net/govcstestbzrrepo",
+				RelPkg: "",
 				CloneURL: &url.URL{
 					Host: "launchpad.net",
 					Path: "govcstestbzrrepo",
@@ -303,8 +303,8 @@ func TestDeduceRemotes(t *testing.T) {
 		{
 			"launchpad.net/govcstestbzrrepo/foo/bar",
 			&remoteRepo{
-				repoRoot: "launchpad.net/govcstestbzrrepo",
-				relPkg:   "foo/bar",
+				Base:   "launchpad.net/govcstestbzrrepo",
+				RelPkg: "foo/bar",
 				CloneURL: &url.URL{
 					Host: "launchpad.net",
 					Path: "govcstestbzrrepo",
@@ -320,8 +320,8 @@ func TestDeduceRemotes(t *testing.T) {
 		{
 			"git.launchpad.net/reporoot",
 			&remoteRepo{
-				repoRoot: "git.launchpad.net/reporoot",
-				relPkg:   "",
+				Base:   "git.launchpad.net/reporoot",
+				RelPkg: "",
 				CloneURL: &url.URL{
 					Host: "git.launchpad.net",
 					Path: "reporoot",
@@ -333,8 +333,8 @@ func TestDeduceRemotes(t *testing.T) {
 		{
 			"git.launchpad.net/reporoot/foo/bar",
 			&remoteRepo{
-				repoRoot: "git.launchpad.net/reporoot",
-				relPkg:   "foo/bar",
+				Base:   "git.launchpad.net/reporoot",
+				RelPkg: "foo/bar",
 				CloneURL: &url.URL{
 					Host: "git.launchpad.net",
 					Path: "reporoot",
@@ -346,8 +346,8 @@ func TestDeduceRemotes(t *testing.T) {
 		{
 			"git.launchpad.net/reporoot",
 			&remoteRepo{
-				repoRoot: "git.launchpad.net/reporoot",
-				relPkg:   "",
+				Base:   "git.launchpad.net/reporoot",
+				RelPkg: "",
 				CloneURL: &url.URL{
 					Host: "git.launchpad.net",
 					Path: "reporoot",
@@ -363,8 +363,8 @@ func TestDeduceRemotes(t *testing.T) {
 		{
 			"git.apache.org/package-name.git",
 			&remoteRepo{
-				repoRoot: "git.apache.org/package-name.git",
-				relPkg:   "",
+				Base:   "git.apache.org/package-name.git",
+				RelPkg: "",
 				CloneURL: &url.URL{
 					Host: "git.apache.org",
 					Path: "package-name.git",
@@ -376,8 +376,8 @@ func TestDeduceRemotes(t *testing.T) {
 		{
 			"git.apache.org/package-name.git/foo/bar",
 			&remoteRepo{
-				repoRoot: "git.apache.org/package-name.git",
-				relPkg:   "foo/bar",
+				Base:   "git.apache.org/package-name.git",
+				RelPkg: "foo/bar",
 				CloneURL: &url.URL{
 					Host: "git.apache.org",
 					Path: "package-name.git",
@@ -390,8 +390,8 @@ func TestDeduceRemotes(t *testing.T) {
 		{
 			"golang.org/x/exp",
 			&remoteRepo{
-				repoRoot: "golang.org/x/exp",
-				relPkg:   "",
+				Base:   "golang.org/x/exp",
+				RelPkg: "",
 				CloneURL: &url.URL{
 					Scheme: "https",
 					Host:   "go.googlesource.com",
@@ -404,8 +404,8 @@ func TestDeduceRemotes(t *testing.T) {
 		{
 			"golang.org/x/exp/inotify",
 			&remoteRepo{
-				repoRoot: "golang.org/x/exp",
-				relPkg:   "inotify",
+				Base:   "golang.org/x/exp",
+				RelPkg: "inotify",
 				CloneURL: &url.URL{
 					Scheme: "https",
 					Host:   "go.googlesource.com",
@@ -418,8 +418,8 @@ func TestDeduceRemotes(t *testing.T) {
 		{
 			"rsc.io/pdf",
 			&remoteRepo{
-				repoRoot: "rsc.io/pdf",
-				relPkg:   "",
+				Base:   "rsc.io/pdf",
+				RelPkg: "",
 				CloneURL: &url.URL{
 					Scheme: "https",
 					Host:   "github.com",
@@ -433,8 +433,8 @@ func TestDeduceRemotes(t *testing.T) {
 		{
 			"github.com/kr/pretty",
 			&remoteRepo{
-				repoRoot: "github.com/kr/pretty",
-				relPkg:   "",
+				Base:   "github.com/kr/pretty",
+				RelPkg: "",
 				CloneURL: &url.URL{
 					Host: "github.com",
 					Path: "kr/pretty",
@@ -461,11 +461,11 @@ func TestDeduceRemotes(t *testing.T) {
 			continue
 		}
 
-		if got.repoRoot != want.repoRoot {
-			t.Errorf("deduceRemoteRepo(%q): Base was %s, wanted %s", fix.path, got.repoRoot, want.repoRoot)
+		if got.Base != want.Base {
+			t.Errorf("deduceRemoteRepo(%q): Base was %s, wanted %s", fix.path, got.Base, want.Base)
 		}
-		if got.relPkg != want.relPkg {
-			t.Errorf("deduceRemoteRepo(%q): RelPkg was %s, wanted %s", fix.path, got.relPkg, want.relPkg)
+		if got.RelPkg != want.RelPkg {
+			t.Errorf("deduceRemoteRepo(%q): RelPkg was %s, wanted %s", fix.path, got.RelPkg, want.RelPkg)
 		}
 		if !reflect.DeepEqual(got.CloneURL, want.CloneURL) {
 			// misspelling things is cool when it makes columns line up
diff --git a/solve_basic_test.go b/solve_basic_test.go
index 7550e10..c493b19 100644
--- a/solve_basic_test.go
+++ b/solve_basic_test.go
@@ -1355,8 +1355,8 @@ func (b *depspecBridge) deduceRemoteRepo(path string) (*remoteRepo, error) {
 		n := string(ds.n)
 		if path == n || strings.HasPrefix(path, n+"/") {
 			return &remoteRepo{
-				repoRoot: n,
-				relPkg:   strings.TrimPrefix(path, n+"/"),
+				Base:   n,
+				RelPkg: strings.TrimPrefix(path, n+"/"),
 			}, nil
 		}
 	}
diff --git a/solver.go b/solver.go
index 0597e31..eab3b42 100644
--- a/solver.go
+++ b/solver.go
@@ -605,17 +605,17 @@ func (s *solver) intersectConstraintsWithImports(deps []workingConstraint, reach
 		// Make a new completeDep with an open constraint, respecting overrides
 		pd := s.ovr.override(ProjectConstraint{
 			Ident: ProjectIdentifier{
-				ProjectRoot: ProjectRoot(root.repoRoot),
-				NetworkName: root.repoRoot,
+				ProjectRoot: ProjectRoot(root.Base),
+				NetworkName: root.Base,
 			},
 			Constraint: Any(),
 		})
 
 		// Insert the pd into the trie so that further deps from this
 		// project get caught by the prefix search
-		xt.Insert(root.repoRoot, pd)
+		xt.Insert(root.Base, pd)
 		// And also put the complete dep into the dmap
-		dmap[ProjectRoot(root.repoRoot)] = completeDep{
+		dmap[ProjectRoot(root.Base)] = completeDep{
 			workingConstraint: pd,
 			pl:                []string{rp},
 		}
diff --git a/source_manager.go b/source_manager.go
index 80fb0ea..477e705 100644
--- a/source_manager.go
+++ b/source_manager.go
@@ -252,6 +252,7 @@ func (sm *SourceMgr) getProjectManager(id ProjectIdentifier) (*pmState, error) {
 	// TODO(sdboyer) finish this, it's not sufficient (?)
 	n := id.netName()
 	var rpath string
+
 	// Early check to see if we already have a pm in the cache for this net name
 	if pm, exists := sm.pms[n]; exists {
 		return pm, nil

From 6562383f5c0f77092b1f41aac1692086de00c85b Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Tue, 2 Aug 2016 10:10:01 -0400
Subject: [PATCH 16/71] Add commented mutexes on data cache

There needs to be a mutex *somewhere* around these maps. These are there
as a warning reminder that something needs to be done, even though a
per-map mutex system seems very unlikely to have sufficient performance
benefit to offset the complexity cost (and concomitant deadlock risk).
---
 source.go | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/source.go b/source.go
index db38e26..6ead645 100644
--- a/source.go
+++ b/source.go
@@ -17,12 +17,17 @@ type source interface {
 	revisionPresentIn(ProjectRoot, Revision) (bool, error)
 }
 
+// TODO(sdboyer) de-export these fields
 type projectDataCache struct {
 	Version  string                   `json:"version"` // TODO(sdboyer) use this
 	Infos    map[Revision]projectInfo `json:"infos"`
 	Packages map[Revision]PackageTree `json:"packages"`
 	VMap     map[Version]Revision     `json:"vmap"`
 	RMap     map[Revision][]Version   `json:"rmap"`
+	// granular mutexes for each map. this has major complexity costs, so we
+	// handle elsewhere - but keep these mutexes here as a TODO(sdboyer) to
+	// remind that we may want to do this eventually
+	//imut, pmut, vmut, rmut sync.RWMutex
 }
 
 func newDataCache() *projectDataCache {

From 84b91f7c18754a1afca72e4ace30fa1d10cd883e Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Tue, 2 Aug 2016 10:44:28 -0400
Subject: [PATCH 17/71] Pull most of projectManager over into baseSource

---
 source.go | 221 +++++++++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 211 insertions(+), 10 deletions(-)

diff --git a/source.go b/source.go
index 6ead645..a8b4011 100644
--- a/source.go
+++ b/source.go
@@ -3,7 +3,9 @@ package gps
 import (
 	"fmt"
 	"net/url"
+	"os"
 	"path/filepath"
+	"strings"
 
 	"github.com/Masterminds/vcs"
 )
@@ -50,20 +52,29 @@ type maybeGitSource struct {
 	url *url.URL
 }
 
-func (s maybeGitSource) try(cachedir string, an ProjectAnalyzer) (source, error) {
-	path := filepath.Join(cachedir, "sources", sanitizer.Replace(s.url.String()))
+type gitSource struct {
+	baseSource
+}
+
+func (m maybeGitSource) try(cachedir string, an ProjectAnalyzer) (source, error) {
+	path := filepath.Join(cachedir, "sources", sanitizer.Replace(m.url.String()))
+	r, err := vcs.NewGitRepo(path, m.url.String())
+	if err != nil {
+		return nil, err
+	}
+
 	pm := &gitSource{
 		baseSource: baseSource{
 			an: an,
 			dc: newDataCache(),
 			crepo: &repo{
-				r:     vcs.NewGitRepo(path, s.url.String()),
+				r:     r,
 				rpath: path,
 			},
 		},
 	}
 
-	_, err := pm.ListVersions()
+	_, err = pm.listVersions()
 	if err != nil {
 		return nil, err
 		//} else if pm.ex.f&existsUpstream == existsUpstream {
@@ -73,7 +84,7 @@ func (s maybeGitSource) try(cachedir string, an ProjectAnalyzer) (source, error)
 	return pm, nil
 }
 
-type baseSource struct {
+type baseSource struct { // TODO(sdboyer) rename to baseVCSSource
 	// Object for the cache repository
 	crepo *repo
 
@@ -123,11 +134,11 @@ func (bs *baseSource) getManifestAndLock(r ProjectRoot, v Version) (Manifest, Lo
 	bs.crepo.mut.Unlock()
 	if err != nil {
 		// TODO(sdboyer) More-er proper-er error
-		panic(fmt.Sprintf("canary - why is checkout/whatever failing: %s %s %s", bs.n, v.String(), err))
+		panic(fmt.Sprintf("canary - why is checkout/whatever failing: %s %s %s", bs.crepo.r.LocalPath(), v.String(), err))
 	}
 
 	bs.crepo.mut.RLock()
-	m, l, err := bs.an.DeriveManifestAndLock(bs.crepo.rpath, r)
+	m, l, err := bs.an.DeriveManifestAndLock(bs.crepo.r.LocalPath(), r)
 	// TODO(sdboyer) cache results
 	bs.crepo.mut.RUnlock()
 
@@ -138,7 +149,7 @@ func (bs *baseSource) getManifestAndLock(r ProjectRoot, v Version) (Manifest, Lo
 
 		// If m is nil, prebsanifest will provide an empty one.
 		pi := projectInfo{
-			Manifest: prebsanifest(m),
+			Manifest: prepManifest(m),
 			Lock:     l,
 		}
 
@@ -154,6 +165,196 @@ func (bs *baseSource) getManifestAndLock(r ProjectRoot, v Version) (Manifest, Lo
 	return nil, nil, err
 }
 
-type gitSource struct {
-	bs baseSource
+func (bs *baseSource) listVersions() (vlist []Version, err error) {
+	if !bs.cvsync {
+		// This check only guarantees that the upstream exists, not the cache
+		bs.ex.s |= existsUpstream
+		vpairs, exbits, err := bs.crepo.getCurrentVersionPairs()
+		// But it *may* also check the local existence
+		bs.ex.s |= exbits
+		bs.ex.f |= exbits
+
+		if err != nil {
+			// TODO(sdboyer) More-er proper-er error
+			return nil, err
+		}
+
+		vlist = make([]Version, len(vpairs))
+		// mark our cache as synced if we got ExistsUpstream back
+		if exbits&existsUpstream == existsUpstream {
+			bs.cvsync = true
+		}
+
+		// Process the version data into the cache
+		// TODO(sdboyer) detect out-of-sync data as we do this?
+		for k, v := range vpairs {
+			bs.dc.VMap[v] = v.Underlying()
+			bs.dc.RMap[v.Underlying()] = append(bs.dc.RMap[v.Underlying()], v)
+			vlist[k] = v
+		}
+	} else {
+		vlist = make([]Version, len(bs.dc.VMap))
+		k := 0
+		// TODO(sdboyer) key type of VMap should be string; recombine here
+		//for v, r := range bs.dc.VMap {
+		for v := range bs.dc.VMap {
+			vlist[k] = v
+			k++
+		}
+	}
+
+	return
+}
+
+func (bs *baseSource) ensureCacheExistence() error {
+	// Technically, methods could could attempt to return straight from the
+	// metadata cache even if the repo cache doesn't exist on disk. But that
+	// would allow weird state inconsistencies (cache exists, but no repo...how
+	// does that even happen?) that it'd be better to just not allow so that we
+	// don't have to think about it elsewhere
+	if !bs.checkExistence(existsInCache) {
+		if bs.checkExistence(existsUpstream) {
+			bs.crepo.mut.Lock()
+			err := bs.crepo.r.Get()
+			bs.crepo.mut.Unlock()
+
+			if err != nil {
+				return fmt.Errorf("failed to create repository cache for %s", bs.crepo.r.Remote())
+			}
+			bs.ex.s |= existsInCache
+			bs.ex.f |= existsInCache
+		} else {
+			return fmt.Errorf("project %s does not exist upstream", bs.crepo.r.Remote())
+		}
+	}
+
+	return nil
+}
+
+// checkExistence provides a direct method for querying existence levels of the
+// source. It will only perform actual searching (local fs or over the network)
+// if no previous attempt at that search has been made.
+//
+// Note that this may perform read-ish operations on the cache repo, and it
+// takes a lock accordingly. This makes it unsafe to call from a segment where
+// the cache repo mutex is already write-locked, as deadlock will occur.
+func (bs *baseSource) checkExistence(ex projectExistence) bool {
+	if bs.ex.s&ex != ex {
+		if ex&existsInVendorRoot != 0 && bs.ex.s&existsInVendorRoot == 0 {
+			panic("should now be implemented in bridge")
+		}
+		if ex&existsInCache != 0 && bs.ex.s&existsInCache == 0 {
+			bs.crepo.mut.RLock()
+			bs.ex.s |= existsInCache
+			if bs.crepo.r.CheckLocal() {
+				bs.ex.f |= existsInCache
+			}
+			bs.crepo.mut.RUnlock()
+		}
+		if ex&existsUpstream != 0 && bs.ex.s&existsUpstream == 0 {
+			bs.crepo.mut.RLock()
+			bs.ex.s |= existsUpstream
+			if bs.crepo.r.Ping() {
+				bs.ex.f |= existsUpstream
+			}
+			bs.crepo.mut.RUnlock()
+		}
+	}
+
+	return ex&bs.ex.f == ex
+}
+
+func (bs *baseSource) listPackages(pr ProjectRoot, v Version) (ptree PackageTree, err error) {
+	if err = bs.ensureCacheExistence(); err != nil {
+		return
+	}
+
+	// See if we can find it in the cache
+	var r Revision
+	switch v.(type) {
+	case Revision, PairedVersion:
+		var ok bool
+		if r, ok = v.(Revision); !ok {
+			r = v.(PairedVersion).Underlying()
+		}
+
+		if ptree, cached := bs.dc.Packages[r]; cached {
+			return ptree, nil
+		}
+	default:
+		var has bool
+		if r, has = bs.dc.VMap[v]; has {
+			if ptree, cached := bs.dc.Packages[r]; cached {
+				return ptree, nil
+			}
+		}
+	}
+
+	// TODO(sdboyer) handle the case where we have a version w/out rev, and not in cache
+
+	// Not in the cache; check out the version and do the analysis
+	bs.crepo.mut.Lock()
+	// Check out the desired version for analysis
+	if r != "" {
+		// Always prefer a rev, if it's available
+		err = bs.crepo.r.UpdateVersion(string(r))
+	} else {
+		// If we don't have a rev, ensure the repo is up to date, otherwise we
+		// could have a desync issue
+		if !bs.crepo.synced {
+			err = bs.crepo.r.Update()
+			if err != nil {
+				return PackageTree{}, fmt.Errorf("Could not fetch latest updates into repository: %s", err)
+			}
+			bs.crepo.synced = true
+		}
+		err = bs.crepo.r.UpdateVersion(v.String())
+	}
+
+	ptree, err = listPackages(bs.crepo.r.LocalPath(), string(pr))
+	bs.crepo.mut.Unlock()
+
+	// TODO(sdboyer) cache errs?
+	if err != nil {
+		bs.dc.Packages[r] = ptree
+	}
+
+	return
+}
+
+func (s *gitSource) exportVersionTo(v Version, to string) error {
+	s.crepo.mut.Lock()
+	defer s.crepo.mut.Unlock()
+
+	r := s.crepo.r
+	// Back up original index
+	idx, bak := filepath.Join(r.LocalPath(), ".git", "index"), filepath.Join(r.LocalPath(), ".git", "origindex")
+	err := os.Rename(idx, bak)
+	if err != nil {
+		return err
+	}
+
+	// TODO(sdboyer) could have an err here
+	defer os.Rename(bak, idx)
+
+	vstr := v.String()
+	if rv, ok := v.(PairedVersion); ok {
+		vstr = rv.Underlying().String()
+	}
+	_, err = r.RunFromDir("git", "read-tree", vstr)
+	if err != nil {
+		return err
+	}
+
+	// Ensure we have exactly one trailing slash
+	to = strings.TrimSuffix(to, string(os.PathSeparator)) + string(os.PathSeparator)
+	// Checkout from our temporary index to the desired target location on disk;
+	// now it's git's job to make it fast. Sadly, this approach *does* also
+	// write out vendor dirs. There doesn't appear to be a way to make
+	// checkout-index respect sparse checkout rules (-a supercedes it);
+	// the alternative is using plain checkout, though we have a bunch of
+	// housekeeping to do to set up, then tear down, the sparse checkout
+	// controls, as well as restore the original index and HEAD.
+	_, err = r.RunFromDir("git", "checkout-index", "-a", "--prefix="+to)
+	return err
 }

From fa8766545e5e21f39d1c291a835351ea57e976c0 Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Tue, 2 Aug 2016 13:20:29 -0400
Subject: [PATCH 18/71] First of source tests

---
 source_test.go | 89 ++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 89 insertions(+)
 create mode 100644 source_test.go

diff --git a/source_test.go b/source_test.go
new file mode 100644
index 0000000..bd05b3e
--- /dev/null
+++ b/source_test.go
@@ -0,0 +1,89 @@
+package gps
+
+import (
+	"io/ioutil"
+	"net/url"
+	"testing"
+)
+
+func TestGitVersionFetching(t *testing.T) {
+	// This test is quite slow, skip it on -short
+	if testing.Short() {
+		t.Skip("Skipping git source version fetching test in short mode")
+	}
+
+	cpath, err := ioutil.TempDir("", "smcache")
+	if err != nil {
+		t.Errorf("Failed to create temp dir: %s", err)
+	}
+	rf := func() {
+		err := removeAll(cpath)
+		if err != nil {
+			t.Errorf("removeAll failed: %s", err)
+		}
+	}
+
+	n := "github.com/Masterminds/VCSTestRepo"
+	u, err := url.Parse("https://" + n)
+	if err != nil {
+		t.Errorf("URL was bad, lolwut? errtext: %s", err)
+		rf()
+		t.FailNow()
+	}
+	mb := maybeGitSource{
+		n:   n,
+		url: u,
+	}
+
+	isrc, err := mb.try(cpath, naiveAnalyzer{})
+	if err != nil {
+		t.Errorf("Unexpected error while setting up gitSource for test repo: %s", err)
+		rf()
+		t.FailNow()
+	}
+	src, ok := isrc.(*gitSource)
+	if !ok {
+		t.Errorf("Expected a gitSource, got a %T", isrc)
+		rf()
+		t.FailNow()
+	}
+
+	vlist, err := src.listVersions()
+	if err != nil {
+		t.Errorf("Unexpected error getting version pairs from git repo: %s", err)
+		rf()
+		t.FailNow()
+	}
+
+	if src.ex.s&existsUpstream != existsUpstream {
+		t.Errorf("gitSource.listVersions() should have set the upstream existence bit for search")
+	}
+	if src.ex.f&existsUpstream != existsUpstream {
+		t.Errorf("gitSource.listVersions() should have set the upstream existence bit for found")
+	}
+	if src.ex.s&existsInCache != 0 {
+		t.Errorf("gitSource.listVersions() should not have set the cache existence bit for search")
+	}
+	if src.ex.f&existsInCache != 0 {
+		t.Errorf("gitSource.listVersions() should not have set the cache existence bit for found")
+	}
+
+	if len(vlist) != 3 {
+		t.Errorf("git test repo should've produced three versions, got %v", len(vlist))
+	} else {
+		v := NewBranch("master").Is(Revision("30605f6ac35fcb075ad0bfa9296f90a7d891523e"))
+		if vlist[0] != v {
+			t.Errorf("git pair fetch reported incorrect first version, got %s", vlist[0])
+		}
+
+		v = NewBranch("test").Is(Revision("30605f6ac35fcb075ad0bfa9296f90a7d891523e"))
+		if vlist[1] != v {
+			t.Errorf("git pair fetch reported incorrect second version, got %s", vlist[1])
+		}
+
+		v = NewVersion("1.0.0").Is(Revision("30605f6ac35fcb075ad0bfa9296f90a7d891523e"))
+		if vlist[2] != v {
+			t.Errorf("git pair fetch reported incorrect third version, got %s", vlist[2])
+		}
+	}
+}

From abd11df40f8ac5b745730a103fcd87b5caf3b2c9 Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Tue, 2 Aug 2016 14:01:28 -0400
Subject: [PATCH 19/71] Fix up git version parsing

---
 source.go      | 108 ++++++++++++++++++++++++++++++++++++++++++++++++-
 source_test.go |   8 ++--
 2 files changed, 111 insertions(+), 5 deletions(-)

diff --git a/source.go b/source.go
index a8b4011..2f6d166 100644
--- a/source.go
+++ b/source.go
@@ -1,9 +1,11 @@
 package gps
 
 import (
+	"bytes"
 	"fmt"
 	"net/url"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"strings"
 
@@ -58,7 +60,7 @@ type gitSource struct {
 
 func (m maybeGitSource) try(cachedir string, an ProjectAnalyzer) (source, error) {
 	path := filepath.Join(cachedir, "sources", sanitizer.Replace(m.url.String()))
-	r, err := vcs.NewGitRepo(path, m.url.String())
+	r, err := vcs.NewGitRepo(m.url.String(), path)
 	if err != nil {
 		return nil, err
 	}
@@ -358,3 +360,107 @@ func (s *gitSource) exportVersionTo(v Version, to string) error {
 	_, err = r.RunFromDir("git", "checkout-index", "-a", "--prefix="+to)
 	return err
 }
+
+func (s *gitSource) listVersions() (vlist []Version, err error) {
+	if s.cvsync {
+		vlist = make([]Version, len(s.dc.VMap))
+		k := 0
+		// TODO(sdboyer) key type of VMap should be string; recombine here
+		//for v, r := range s.dc.VMap {
+		for v := range s.dc.VMap {
+			vlist[k] = v
+			k++
+		}
+
+		return
+	}
+
+	r := s.crepo.r
+	var out []byte
+	c := exec.Command("git", "ls-remote", r.Remote())
+	// Ensure no terminal prompting for PWs
+	c.Env = mergeEnvLists([]string{"GIT_TERMINAL_PROMPT=0"}, os.Environ())
+	out, err = c.CombinedOutput()
+
+	all := bytes.Split(bytes.TrimSpace(out), []byte("\n"))
+	if err != nil || len(all) == 0 {
+		// TODO(sdboyer) remove this path? it really just complicates things, for
+		// probably not much benefit
+
+		// ls-remote failed, probably due to bad communication or a faulty
+		// upstream implementation. So fetch updates, then build the list
+		// locally
+		s.crepo.mut.Lock()
+		err = r.Update()
+		s.crepo.mut.Unlock()
+		if err != nil {
+			// Definitely have a problem, now - bail out
+			return
+		}
+
+		// Upstream and cache must exist for this to have worked, so add that to
+		// searched and found
+		s.ex.s |= existsUpstream | existsInCache
+		s.ex.f |= existsUpstream | existsInCache
+		// Also, local is definitely now synced
+		s.crepo.synced = true
+
+		out, err = r.RunFromDir("git", "show-ref", "--dereference")
+		if err != nil {
+			// TODO(sdboyer) More-er proper-er error
+			return
+		}
+
+		all = bytes.Split(bytes.TrimSpace(out), []byte("\n"))
+	}
+
+	// Local cache may not actually exist here, but upstream definitely does
+	s.ex.s |= existsUpstream
+	s.ex.f |= existsUpstream
+
+	tmap := make(map[string]PairedVersion)
+	for _, pair := range all {
+		var v PairedVersion
+		if string(pair[46:51]) == "heads" {
+			bname := string(pair[52:])
+			v = NewBranch(bname).Is(Revision(pair[:40])).(PairedVersion)
+			tmap["heads"+bname] = v
+		} else if string(pair[46:50]) == "tags" {
+			vstr := string(pair[51:])
+			if strings.HasSuffix(vstr, "^{}") {
+				// If the suffix is there, then we *know* this is the rev of
+				// the underlying commit object that we actually want
+				vstr = strings.TrimSuffix(vstr, "^{}")
+			} else if _, exists := tmap[vstr]; exists {
+				// Already saw the deref'd version of this tag, if one
+				// exists, so skip this.
+				continue
+				// Can only hit this branch if we somehow got the deref'd
+				// version first. Which should be impossible, but this
+				// covers us in case of weirdness, anyway.
+			}
+			v = NewVersion(vstr).Is(Revision(pair[:40])).(PairedVersion)
+			tmap["tags"+vstr] = v
+		}
+	}
+
+	// Process the version data into the cache
+	//
+	// reset the rmap and vmap, as they'll be fully repopulated by this
+	// TODO(sdboyer) detect out-of-sync pairings as we do this?
+	s.dc.VMap = make(map[Version]Revision)
+	s.dc.RMap = make(map[Revision][]Version)
+
+	vlist = make([]Version, len(tmap))
+	k := 0
+	for _, v := range tmap {
+		s.dc.VMap[v] = v.Underlying()
+		s.dc.RMap[v.Underlying()] = append(s.dc.RMap[v.Underlying()], v)
+		vlist[k] = v
+		k++
+	}
+	// Mark the cache as being in sync with upstream's version list
+	s.cvsync = true
+
+	return
+}
diff --git a/source_test.go b/source_test.go
index bd05b3e..d7c7dc7 100644
--- a/source_test.go
+++ b/source_test.go
@@ -69,21 +69,21 @@ func TestGitVersionFetching(t *testing.T) {
 	}
 
 	if len(vlist) != 3 {
-		t.Errorf("git test repo should've produced three versions, got %v", len(vlist))
+		t.Errorf("git test repo should've produced three versions, got %v: %s", len(vlist), vlist)
 	} else {
 		v := NewBranch("master").Is(Revision("30605f6ac35fcb075ad0bfa9296f90a7d891523e"))
 		if vlist[0] != v {
-			t.Errorf("git pair fetch reported incorrect first version, got %s", vlist[0])
+			t.Errorf("gitSource.listVersions() reported incorrect first version, got %s", vlist[0])
 		}
 
 		v = NewBranch("test").Is(Revision("30605f6ac35fcb075ad0bfa9296f90a7d891523e"))
 		if vlist[1] != v {
-			t.Errorf("git pair fetch reported incorrect second version, got %s", vlist[1])
+			t.Errorf("gitSource.listVersions() reported incorrect second version, got %s", vlist[1])
 		}
 
 		v = NewVersion("1.0.0").Is(Revision("30605f6ac35fcb075ad0bfa9296f90a7d891523e"))
 		if vlist[2] != v {
-			t.Errorf("git pair fetch reported incorrect third version, got %s", vlist[2])
+			t.Errorf("gitSource.listVersions() reported incorrect third version, got %s", vlist[2])
 		}
 	}
 }

From e46e3d48a6bda192df72785bc4076134f9546834 Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Tue, 2 Aug 2016 15:36:34 -0400
Subject: [PATCH 20/71] Only use map to deduplicate

This makes actually collecting the result slice more straightforward
later.
---
 source.go | 34 ++++++++++++++++++++--------------
 1 file changed, 20 insertions(+), 14 deletions(-)

diff --git a/source.go b/source.go
index 2f6d166..9f79a8c 100644
--- a/source.go
+++ b/source.go
@@ -412,26 +412,31 @@ func (s *gitSource) listVersions() (vlist []Version, err error) {
 		}
 
 		all = bytes.Split(bytes.TrimSpace(out), []byte("\n"))
+		if len(all) == 0 {
+			return nil, fmt.Errorf("No versions available for %s (this is weird)", r.Remote())
+		}
 	}
 
 	// Local cache may not actually exist here, but upstream definitely does
 	s.ex.s |= existsUpstream
 	s.ex.f |= existsUpstream
 
-	tmap := make(map[string]PairedVersion)
+	smap := make(map[string]bool)
+	uniq := 0
+	vlist = make([]Version, len(all)-1) // less 1, because always ignore HEAD
 	for _, pair := range all {
 		var v PairedVersion
 		if string(pair[46:51]) == "heads" {
-			bname := string(pair[52:])
-			v = NewBranch(bname).Is(Revision(pair[:40])).(PairedVersion)
-			tmap["heads"+bname] = v
+			v = NewBranch(string(pair[52:])).Is(Revision(pair[:40])).(PairedVersion)
+			vlist[uniq] = v
+			uniq++
 		} else if string(pair[46:50]) == "tags" {
 			vstr := string(pair[51:])
 			if strings.HasSuffix(vstr, "^{}") {
 				// If the suffix is there, then we *know* this is the rev of
 				// the underlying commit object that we actually want
 				vstr = strings.TrimSuffix(vstr, "^{}")
-			} else if _, exists := tmap[vstr]; exists {
+			} else if smap[vstr] {
 				// Already saw the deref'd version of this tag, if one
 				// exists, so skip this.
 				continue
@@ -440,10 +445,15 @@ func (s *gitSource) listVersions() (vlist []Version, err error) {
 				// covers us in case of weirdness, anyway.
 			}
 			v = NewVersion(vstr).Is(Revision(pair[:40])).(PairedVersion)
-			tmap["tags"+vstr] = v
+			smap[vstr] = true
+			vlist[uniq] = v
+			uniq++
 		}
 	}
 
+	// Trim off excess from the slice
+	vlist = vlist[:uniq]
+
 	// Process the version data into the cache
 	//
 	// reset the rmap and vmap, as they'll be fully repopulated by this
@@ -451,16 +461,12 @@ func (s *gitSource) listVersions() (vlist []Version, err error) {
 	s.dc.VMap = make(map[Version]Revision)
 	s.dc.RMap = make(map[Revision][]Version)
 
-	vlist = make([]Version, len(tmap))
-	k := 0
-	for _, v := range tmap {
-		s.dc.VMap[v] = v.Underlying()
-		s.dc.RMap[v.Underlying()] = append(s.dc.RMap[v.Underlying()], v)
-		vlist[k] = v
-		k++
+	for _, v := range vlist {
+		pv := v.(PairedVersion)
+		s.dc.VMap[v] = pv.Underlying()
+		s.dc.RMap[pv.Underlying()] = append(s.dc.RMap[pv.Underlying()], v)
 	}
 	// Mark the cache as being in sync with upstream's version list
 	s.cvsync = true
-
 	return
 }

From 98e9a02afa5978b5d02a143f47fe5f941037742f Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Tue, 2 Aug 2016 15:37:07 -0400
Subject: [PATCH 21/71] Fix ordering issues in gitSource tests

---
 source_test.go | 23 ++++++++++-------------
 1 file changed, 10 insertions(+), 13 deletions(-)

diff --git a/source_test.go b/source_test.go
index d7c7dc7..de2a8a0 100644
--- a/source_test.go
+++ b/source_test.go
@@ -3,6 +3,8 @@ package gps
 import (
 	"io/ioutil"
 	"net/url"
+	"reflect"
+	"sort"
 	"testing"
 )
 
@@ -69,21 +71,16 @@ func TestGitVersionFetching(t *testing.T) {
 	}
 
 	if len(vlist) != 3 {
-		t.Errorf("git test repo should've produced three versions, got %v: %s", len(vlist), vlist)
+		t.Errorf("git test repo should've produced three versions, got %v: vlist was %s", len(vlist), vlist)
 	} else {
-		v := NewBranch("master").Is(Revision("30605f6ac35fcb075ad0bfa9296f90a7d891523e"))
-		if vlist[0] != v {
-			t.Errorf("gitSource.listVersions() reported incorrect first version, got %s", vlist[0])
+		sort.Sort(upgradeVersionSorter(vlist))
+		evl := []Version{
+			NewVersion("1.0.0").Is(Revision("30605f6ac35fcb075ad0bfa9296f90a7d891523e")),
+			NewBranch("master").Is(Revision("30605f6ac35fcb075ad0bfa9296f90a7d891523e")),
+			NewBranch("test").Is(Revision("30605f6ac35fcb075ad0bfa9296f90a7d891523e")),
 		}
-
-		v = NewBranch("test").Is(Revision("30605f6ac35fcb075ad0bfa9296f90a7d891523e"))
-		if vlist[1] != v {
-			t.Errorf("gitSource.listVersions() reported incorrect second version, got %s", vlist[1])
-		}
-
-		v = NewVersion("1.0.0").Is(Revision("30605f6ac35fcb075ad0bfa9296f90a7d891523e"))
-		if vlist[2] != v {
-			t.Errorf("gitSource.listVersions() reported incorrect third version, got %s", vlist[2])
+		if !reflect.DeepEqual(vlist, evl) {
+			t.Errorf("Version list was not what we expected:\n\t(GOT): %s\n\t(WNT): %s", vlist, evl)
 		}
 	}
 }

From 41344d12ff2a82cb30bba72155aa69868b17e89c Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Tue, 2 Aug 2016 15:54:37 -0400
Subject: [PATCH 22/71] Add revisionPresentIn() impl to baseSource

---
 source.go | 29 +++++++++++++++++++++++++----
 1 file changed, 25 insertions(+), 4 deletions(-)

diff --git a/source.go b/source.go
index 9f79a8c..3e557be 100644
--- a/source.go
+++ b/source.go
@@ -18,7 +18,7 @@ type source interface {
 	getManifestAndLock(ProjectRoot, Version) (Manifest, Lock, error)
 	listPackages(ProjectRoot, Version) (PackageTree, error)
 	listVersions() ([]Version, error)
-	revisionPresentIn(ProjectRoot, Revision) (bool, error)
+	revisionPresentIn(Revision) (bool, error)
 }
 
 // TODO(sdboyer) de-export these fields
@@ -29,8 +29,8 @@ type projectDataCache struct {
 	VMap     map[Version]Revision     `json:"vmap"`
 	RMap     map[Revision][]Version   `json:"rmap"`
 	// granular mutexes for each map. this has major complexity costs, so we
-	// handle elsewhere - but keep these mutexes here as a TODO(sdboyer) to
-	// remind that we may want to do this eventually
+	// should handle elsewhere - but keep these mutexes here as a TODO(sdboyer)
+	// to remind that we may want to do this eventually
 	//imut, pmut, vmut, rmut sync.RWMutex
 }
 
@@ -149,7 +149,7 @@ func (bs *baseSource) getManifestAndLock(r ProjectRoot, v Version) (Manifest, Lo
 			l = prepLock(l)
 		}
 
-		// If m is nil, prebsanifest will provide an empty one.
+		// If m is nil, prepManifest will provide an empty one.
 		pi := projectInfo{
 			Manifest: prepManifest(m),
 			Lock:     l,
@@ -208,6 +208,27 @@ func (bs *baseSource) listVersions() (vlist []Version, err error) {
 	return
 }
 
+func (bs *baseSource) revisionPresentIn(r Revision) (bool, error) {
+	// First and fastest path is to check the data cache to see if the rev is
+	// present. This could give us false positives, but the cases where that can
+	// occur would require a type of cache staleness that seems *exceedingly*
+	// unlikely to occur.
+	if _, has := bs.dc.Infos[r]; has {
+		return true, nil
+	} else if _, has := bs.dc.RMap[r]; has {
+		return true, nil
+	}
+
+	err := bs.ensureCacheExistence()
+	if err != nil {
+		return false, err
+	}
+
+	bs.crepo.mut.RLock()
+	defer bs.crepo.mut.RUnlock()
+	return bs.crepo.r.IsReference(string(r)), nil
+}
+
 func (bs *baseSource) ensureCacheExistence() error {
 	// Technically, methods could could attempt to return straight from the
 	// metadata cache even if the repo cache doesn't exist on disk. But that

From c5d7e9ae22008826d725d1d735b19e649cb2640e Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Tue, 2 Aug 2016 16:04:06 -0400
Subject: [PATCH 23/71] s/projectDataCache/sourceMetaCache/

Also unexport its fields. We can revisit this later when we're
actually ready to start dealing with persisting the caches to disk.
---
 project_manager.go | 30 +++++++++----------
 source.go          | 72 ++++++++++++++++++++++------------------------
 source_manager.go  | 12 ++++----
 3 files changed, 55 insertions(+), 59 deletions(-)

diff --git a/project_manager.go b/project_manager.go
index 8631a51..ba306c1 100644
--- a/project_manager.go
+++ b/project_manager.go
@@ -38,7 +38,7 @@ type projectManager struct {
 	// The project metadata cache. This is persisted to disk, for reuse across
 	// solver runs.
 	// TODO(sdboyer) protect with mutex
-	dc *projectDataCache
+	dc *sourceMetaCache
 }
 
 type existence struct {
@@ -74,8 +74,8 @@ func (pm *projectManager) GetManifestAndLock(r ProjectRoot, v Version) (Manifest
 		return nil, nil, err
 	}
 
-	if r, exists := pm.dc.VMap[v]; exists {
-		if pi, exists := pm.dc.Infos[r]; exists {
+	if r, exists := pm.dc.vMap[v]; exists {
+		if pi, exists := pm.dc.infos[r]; exists {
 			return pi.Manifest, pi.Lock, nil
 		}
 	}
@@ -120,8 +120,8 @@ func (pm *projectManager) GetManifestAndLock(r ProjectRoot, v Version) (Manifest
 
 		// TODO(sdboyer) this just clobbers all over and ignores the paired/unpaired
 		// distinction; serious fix is needed
-		if r, exists := pm.dc.VMap[v]; exists {
-			pm.dc.Infos[r] = pi
+		if r, exists := pm.dc.vMap[v]; exists {
+			pm.dc.infos[r] = pi
 		}
 
 		return pi.Manifest, pi.Lock, nil
@@ -144,13 +144,13 @@ func (pm *projectManager) ListPackages(pr ProjectRoot, v Version) (ptree Package
 			r = v.(PairedVersion).Underlying()
 		}
 
-		if ptree, cached := pm.dc.Packages[r]; cached {
+		if ptree, cached := pm.dc.ptrees[r]; cached {
 			return ptree, nil
 		}
 	default:
 		var has bool
-		if r, has = pm.dc.VMap[v]; has {
-			if ptree, cached := pm.dc.Packages[r]; cached {
+		if r, has = pm.dc.vMap[v]; has {
+			if ptree, cached := pm.dc.ptrees[r]; cached {
 				return ptree, nil
 			}
 		}
@@ -182,7 +182,7 @@ func (pm *projectManager) ListPackages(pr ProjectRoot, v Version) (ptree Package
 
 	// TODO(sdboyer) cache errs?
 	if err != nil {
-		pm.dc.Packages[r] = ptree
+		pm.dc.ptrees[r] = ptree
 	}
 
 	return
@@ -236,16 +236,16 @@ func (pm *projectManager) ListVersions() (vlist []Version, err error) {
 		// Process the version data into the cache
 		// TODO(sdboyer) detect out-of-sync data as we do this?
 		for k, v := range vpairs {
-			pm.dc.VMap[v] = v.Underlying()
-			pm.dc.RMap[v.Underlying()] = append(pm.dc.RMap[v.Underlying()], v)
+			pm.dc.vMap[v] = v.Underlying()
+			pm.dc.rMap[v.Underlying()] = append(pm.dc.rMap[v.Underlying()], v)
 			vlist[k] = v
 		}
 	} else {
-		vlist = make([]Version, len(pm.dc.VMap))
+		vlist = make([]Version, len(pm.dc.vMap))
 		k := 0
 		// TODO(sdboyer) key type of VMap should be string; recombine here
 		//for v, r := range pm.dc.VMap {
-		for v := range pm.dc.VMap {
+		for v := range pm.dc.vMap {
 			vlist[k] = v
 			k++
 		}
@@ -259,9 +259,9 @@ func (pm *projectManager) RevisionPresentIn(pr ProjectRoot, r Revision) (bool, e
 	// present. This could give us false positives, but the cases where that can
 	// occur would require a type of cache staleness that seems *exceedingly*
 	// unlikely to occur.
-	if _, has := pm.dc.Infos[r]; has {
+	if _, has := pm.dc.infos[r]; has {
 		return true, nil
-	} else if _, has := pm.dc.RMap[r]; has {
+	} else if _, has := pm.dc.rMap[r]; has {
 		return true, nil
 	}
 
diff --git a/source.go b/source.go
index 3e557be..2452be2 100644
--- a/source.go
+++ b/source.go
@@ -21,25 +21,21 @@ type source interface {
 	revisionPresentIn(Revision) (bool, error)
 }
 
-// TODO(sdboyer) de-export these fields
-type projectDataCache struct {
-	Version  string                   `json:"version"` // TODO(sdboyer) use this
-	Infos    map[Revision]projectInfo `json:"infos"`
-	Packages map[Revision]PackageTree `json:"packages"`
-	VMap     map[Version]Revision     `json:"vmap"`
-	RMap     map[Revision][]Version   `json:"rmap"`
-	// granular mutexes for each map. this has major complexity costs, so we
-	// should handle elsewhere - but keep these mutexes here as a TODO(sdboyer)
-	// to remind that we may want to do this eventually
-	//imut, pmut, vmut, rmut sync.RWMutex
+type sourceMetaCache struct {
+	//Version  string                   // TODO(sdboyer) use this
+	infos  map[Revision]projectInfo
+	ptrees map[Revision]PackageTree
+	vMap   map[Version]Revision
+	rMap   map[Revision][]Version
+	// TODO(sdboyer) mutexes. actually probably just one, b/c complexity
 }
 
-func newDataCache() *projectDataCache {
-	return &projectDataCache{
-		Infos:    make(map[Revision]projectInfo),
-		Packages: make(map[Revision]PackageTree),
-		VMap:     make(map[Version]Revision),
-		RMap:     make(map[Revision][]Version),
+func newDataCache() *sourceMetaCache {
+	return &sourceMetaCache{
+		infos:  make(map[Revision]projectInfo),
+		ptrees: make(map[Revision]PackageTree),
+		vMap:   make(map[Version]Revision),
+		rMap:   make(map[Revision][]Version),
 	}
 }
 
@@ -103,7 +99,7 @@ type baseSource struct { // TODO(sdboyer) rename to baseVCSSource
 	// The project metadata cache. This is persisted to disk, for reuse across
 	// solver runs.
 	// TODO(sdboyer) protect with mutex
-	dc *projectDataCache
+	dc *sourceMetaCache
 }
 
 func (bs *baseSource) getManifestAndLock(r ProjectRoot, v Version) (Manifest, Lock, error) {
@@ -111,8 +107,8 @@ func (bs *baseSource) getManifestAndLock(r ProjectRoot, v Version) (Manifest, Lo
 		return nil, nil, err
 	}
 
-	if r, exists := bs.dc.VMap[v]; exists {
-		if pi, exists := bs.dc.Infos[r]; exists {
+	if r, exists := bs.dc.vMap[v]; exists {
+		if pi, exists := bs.dc.infos[r]; exists {
 			return pi.Manifest, pi.Lock, nil
 		}
 	}
@@ -157,8 +153,8 @@ func (bs *baseSource) getManifestAndLock(r ProjectRoot, v Version) (Manifest, Lo
 
 		// TODO(sdboyer) this just clobbers all over and ignores the paired/unpaired
 		// distinction; serious fix is needed
-		if r, exists := bs.dc.VMap[v]; exists {
-			bs.dc.Infos[r] = pi
+		if r, exists := bs.dc.vMap[v]; exists {
+			bs.dc.infos[r] = pi
 		}
 
 		return pi.Manifest, pi.Lock, nil
@@ -190,16 +186,16 @@ func (bs *baseSource) listVersions() (vlist []Version, err error) {
 		// Process the version data into the cache
 		// TODO(sdboyer) detect out-of-sync data as we do this?
 		for k, v := range vpairs {
-			bs.dc.VMap[v] = v.Underlying()
-			bs.dc.RMap[v.Underlying()] = append(bs.dc.RMap[v.Underlying()], v)
+			bs.dc.vMap[v] = v.Underlying()
+			bs.dc.rMap[v.Underlying()] = append(bs.dc.rMap[v.Underlying()], v)
 			vlist[k] = v
 		}
 	} else {
-		vlist = make([]Version, len(bs.dc.VMap))
+		vlist = make([]Version, len(bs.dc.vMap))
 		k := 0
 		// TODO(sdboyer) key type of VMap should be string; recombine here
 		//for v, r := range bs.dc.VMap {
-		for v := range bs.dc.VMap {
+		for v := range bs.dc.vMap {
 			vlist[k] = v
 			k++
 		}
@@ -213,9 +209,9 @@ func (bs *baseSource) revisionPresentIn(r Revision) (bool, error) {
 	// present. This could give us false positives, but the cases where that can
 	// occur would require a type of cache staleness that seems *exceedingly*
 	// unlikely to occur.
-	if _, has := bs.dc.Infos[r]; has {
+	if _, has := bs.dc.infos[r]; has {
 		return true, nil
-	} else if _, has := bs.dc.RMap[r]; has {
+	} else if _, has := bs.dc.rMap[r]; has {
 		return true, nil
 	}
 
@@ -301,13 +297,13 @@ func (bs *baseSource) listPackages(pr ProjectRoot, v Version) (ptree PackageTree
 			r = v.(PairedVersion).Underlying()
 		}
 
-		if ptree, cached := bs.dc.Packages[r]; cached {
+		if ptree, cached := bs.dc.ptrees[r]; cached {
 			return ptree, nil
 		}
 	default:
 		var has bool
-		if r, has = bs.dc.VMap[v]; has {
-			if ptree, cached := bs.dc.Packages[r]; cached {
+		if r, has = bs.dc.vMap[v]; has {
+			if ptree, cached := bs.dc.ptrees[r]; cached {
 				return ptree, nil
 			}
 		}
@@ -339,7 +335,7 @@ func (bs *baseSource) listPackages(pr ProjectRoot, v Version) (ptree PackageTree
 
 	// TODO(sdboyer) cache errs?
 	if err != nil {
-		bs.dc.Packages[r] = ptree
+		bs.dc.ptrees[r] = ptree
 	}
 
 	return
@@ -384,11 +380,11 @@ func (s *gitSource) exportVersionTo(v Version, to string) error {
 
 func (s *gitSource) listVersions() (vlist []Version, err error) {
 	if s.cvsync {
-		vlist = make([]Version, len(s.dc.VMap))
+		vlist = make([]Version, len(s.dc.vMap))
 		k := 0
 		// TODO(sdboyer) key type of VMap should be string; recombine here
 		//for v, r := range s.dc.VMap {
-		for v := range s.dc.VMap {
+		for v := range s.dc.vMap {
 			vlist[k] = v
 			k++
 		}
@@ -479,13 +475,13 @@ func (s *gitSource) listVersions() (vlist []Version, err error) {
 	//
 	// reset the rmap and vmap, as they'll be fully repopulated by this
 	// TODO(sdboyer) detect out-of-sync pairings as we do this?
-	s.dc.VMap = make(map[Version]Revision)
-	s.dc.RMap = make(map[Revision][]Version)
+	s.dc.vMap = make(map[Version]Revision)
+	s.dc.rMap = make(map[Revision][]Version)
 
 	for _, v := range vlist {
 		pv := v.(PairedVersion)
-		s.dc.VMap[v] = pv.Underlying()
-		s.dc.RMap[pv.Underlying()] = append(s.dc.RMap[pv.Underlying()], v)
+		s.dc.vMap[v] = pv.Underlying()
+		s.dc.rMap[pv.Underlying()] = append(s.dc.rMap[pv.Underlying()], v)
 	}
 	// Mark the cache as being in sync with upstream's version list
 	s.cvsync = true
diff --git a/source_manager.go b/source_manager.go
index 477e705..b66aa5e 100644
--- a/source_manager.go
+++ b/source_manager.go
@@ -350,7 +350,7 @@ decided:
 	pms := &pmState{}
 	cpath := filepath.Join(metadir, "cache.json")
 	fi, err := os.Stat(cpath)
-	var dc *projectDataCache
+	var dc *sourceMetaCache
 	if fi != nil {
 		pms.cf, err = os.OpenFile(cpath, os.O_RDWR, 0777)
 		if err != nil {
@@ -371,11 +371,11 @@ decided:
 		//return nil, fmt.Errorf("Err on creating metadata cache file: %s", err)
 		//}
 
-		dc = &projectDataCache{
-			Infos:    make(map[Revision]projectInfo),
-			Packages: make(map[Revision]PackageTree),
-			VMap:     make(map[Version]Revision),
-			RMap:     make(map[Revision][]Version),
+		dc = &sourceMetaCache{
+			infos:  make(map[Revision]projectInfo),
+			ptrees: make(map[Revision]PackageTree),
+			vMap:   make(map[Version]Revision),
+			rMap:   make(map[Revision][]Version),
 		}
 	}
 

From 43d62a888dd816061ffc4ced1fb93f8cec37e7b9 Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Tue, 2 Aug 2016 20:46:40 -0400
Subject: [PATCH 24/71] Move maybes into their own file

---
 maybe_source.go | 47 +++++++++++++++++++++++++++++++++++++++++++++++
 source.go       | 46 ++++------------------------------------------
 2 files changed, 51 insertions(+), 42 deletions(-)
 create mode 100644 maybe_source.go

diff --git a/maybe_source.go b/maybe_source.go
new file mode 100644
index 0000000..3c06b44
--- /dev/null
+++ b/maybe_source.go
@@ -0,0 +1,47 @@
+package gps
+
+import (
+	"net/url"
+	"path/filepath"
+
+	"github.com/Masterminds/vcs"
+)
+
+type maybeSource interface {
+	try(cachedir string, an ProjectAnalyzer) (source, error)
+}
+
+type maybeSources []maybeSource
+
+type maybeGitSource struct {
+	n   string
+	url *url.URL
+}
+
+func (m maybeGitSource) try(cachedir string, an ProjectAnalyzer) (source, error) {
+	path := filepath.Join(cachedir, "sources", sanitizer.Replace(m.url.String()))
+	r, err := vcs.NewGitRepo(m.url.String(), path)
+	if err != nil {
+		return nil, err
+	}
+
+	pm := &gitSource{
+		baseSource: baseSource{
+			an: an,
+			dc: newDataCache(),
+			crepo: &repo{
+				r:     r,
+				rpath: path,
+			},
+		},
+	}
+
+	_, err = pm.listVersions()
+	if err != nil {
+		return nil, err
+		//} else if pm.ex.f&existsUpstream == existsUpstream {
+		//return pm, nil
+	}
+
+	return pm, nil
+}
diff --git a/source.go b/source.go
index 2452be2..fae0285 100644
--- a/source.go
+++ b/source.go
@@ -3,13 +3,10 @@ package gps
 import (
 	"bytes"
 	"fmt"
-	"net/url"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"strings"
-
-	"github.com/Masterminds/vcs"
 )
 
 type source interface {
@@ -39,49 +36,10 @@ func newDataCache() *sourceMetaCache {
 	}
 }
 
-type maybeSource interface {
-	try(cachedir string, an ProjectAnalyzer) (source, error)
-}
-
-type maybeSources []maybeSource
-
-type maybeGitSource struct {
-	n   string
-	url *url.URL
-}
-
 type gitSource struct {
 	baseSource
 }
 
-func (m maybeGitSource) try(cachedir string, an ProjectAnalyzer) (source, error) {
-	path := filepath.Join(cachedir, "sources", sanitizer.Replace(m.url.String()))
-	r, err := vcs.NewGitRepo(m.url.String(), path)
-	if err != nil {
-		return nil, err
-	}
-
-	pm := &gitSource{
-		baseSource: baseSource{
-			an: an,
-			dc: newDataCache(),
-			crepo: &repo{
-				r:     r,
-				rpath: path,
-			},
-		},
-	}
-
-	_, err = pm.listVersions()
-	if err != nil {
-		return nil, err
-		//} else if pm.ex.f&existsUpstream == existsUpstream {
-		//return pm, nil
-	}
-
-	return pm, nil
-}
-
 type baseSource struct { // TODO(sdboyer) rename to baseVCSSource
 	// Object for the cache repository
 	crepo *repo
@@ -341,6 +299,10 @@ func (bs *baseSource) listPackages(pr ProjectRoot, v Version) (ptree PackageTree
 	return
 }
 
+func (bs *baseSource) exportVersionTo(v Version, to string) error {
+	return bs.crepo.exportVersionTo(v, to)
+}
+
 func (s *gitSource) exportVersionTo(v Version, to string) error {
 	s.crepo.mut.Lock()
 	defer s.crepo.mut.Unlock()

From 4c8445c937504806c93b8bb37fe946e553035bf7 Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Tue, 2 Aug 2016 22:51:12 -0400
Subject: [PATCH 25/71] Touchups around gitSource

---
 maybe_source.go |  6 +++---
 source.go       | 13 +++++++++----
 2 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/maybe_source.go b/maybe_source.go
index 3c06b44..6dd0c0f 100644
--- a/maybe_source.go
+++ b/maybe_source.go
@@ -25,7 +25,7 @@ func (m maybeGitSource) try(cachedir string, an ProjectAnalyzer) (source, error)
 		return nil, err
 	}
 
-	pm := &gitSource{
+	src := &gitSource{
 		baseSource: baseSource{
 			an: an,
 			dc: newDataCache(),
@@ -36,12 +36,12 @@ func (m maybeGitSource) try(cachedir string, an ProjectAnalyzer) (source, error)
 		},
 	}
 
-	_, err = pm.listVersions()
+	_, err = src.listVersions()
 	if err != nil {
 		return nil, err
 		//} else if pm.ex.f&existsUpstream == existsUpstream {
 		//return pm, nil
 	}
 
-	return pm, nil
+	return src, nil
 }
diff --git a/source.go b/source.go
index fae0285..7173a0d 100644
--- a/source.go
+++ b/source.go
@@ -36,10 +36,6 @@ func newDataCache() *sourceMetaCache {
 	}
 }
 
-type gitSource struct {
-	baseSource
-}
-
 type baseSource struct { // TODO(sdboyer) rename to baseVCSSource
 	// Object for the cache repository
 	crepo *repo
@@ -198,6 +194,7 @@ func (bs *baseSource) ensureCacheExistence() error {
 			if err != nil {
 				return fmt.Errorf("failed to create repository cache for %s", bs.crepo.r.Remote())
 			}
+			bs.crepo.synced = true
 			bs.ex.s |= existsInCache
 			bs.ex.f |= existsInCache
 		} else {
@@ -303,6 +300,12 @@ func (bs *baseSource) exportVersionTo(v Version, to string) error {
 	return bs.crepo.exportVersionTo(v, to)
 }
 
+// gitSource is a generic git repository implementation that should work with
+// all standard git remotes.
+type gitSource struct {
+	baseSource
+}
+
 func (s *gitSource) exportVersionTo(v Version, to string) error {
 	s.crepo.mut.Lock()
 	defer s.crepo.mut.Unlock()
@@ -384,7 +387,9 @@ func (s *gitSource) listVersions() (vlist []Version, err error) {
 		// Also, local is definitely now synced
 		s.crepo.synced = true
 
+		s.crepo.mut.RLock()
 		out, err = r.RunFromDir("git", "show-ref", "--dereference")
+		s.crepo.mut.RUnlock()
 		if err != nil {
 			// TODO(sdboyer) More-er proper-er error
 			return

From a7b9a6066c02e11890b9866e35aa9e067d384802 Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Tue, 2 Aug 2016 22:52:07 -0400
Subject: [PATCH 26/71] Add bzrSource and maybeBzrSource

---
 maybe_source.go | 28 +++++++++++++++++++
 source.go       | 74 +++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 102 insertions(+)

diff --git a/maybe_source.go b/maybe_source.go
index 6dd0c0f..c670e8d 100644
--- a/maybe_source.go
+++ b/maybe_source.go
@@ -1,6 +1,7 @@
 package gps
 
 import (
+	"fmt"
 	"net/url"
 	"path/filepath"
 
@@ -45,3 +46,30 @@ func (m maybeGitSource) try(cachedir string, an ProjectAnalyzer) (source, error)
 
 	return src, nil
 }
+
+type maybeBzrSource struct {
+	n   string
+	url *url.URL
+}
+
+func (m maybeBzrSource) try(cachedir string, an ProjectAnalyzer) (source, error) {
+	path := filepath.Join(cachedir, "sources", sanitizer.Replace(m.url.String()))
+	r, err := vcs.NewBzrRepo(m.url.String(), path)
+	if err != nil {
+		return nil, err
+	}
+	if !r.Ping() {
+		return nil, fmt.Errorf("Remote repository at %s does not exist, or is inaccessible", m.url.String())
+	}
+
+	return &bzrSource{
+		baseSource: baseSource{
+			an: an,
+			dc: newDataCache(),
+			crepo: &repo{
+				r:     r,
+				rpath: path,
+			},
+		},
+	}, nil
+}
diff --git a/source.go b/source.go
index 7173a0d..c30d142 100644
--- a/source.go
+++ b/source.go
@@ -454,3 +454,77 @@ func (s *gitSource) listVersions() (vlist []Version, err error) {
 	s.cvsync = true
 	return
 }
+
+// bzrSource is a generic bzr repository implementation that should work with
+// all standard git remotes.
+type bzrSource struct {
+	baseSource
+}
+
+func (s *bzrSource) listVersions() (vlist []Version, err error) {
+	if s.cvsync {
+		vlist = make([]Version, len(s.dc.vMap))
+		k := 0
+		// TODO(sdboyer) key type of VMap should be string; recombine here
+		//for v, r := range s.dc.VMap {
+		for v := range s.dc.vMap {
+			vlist[k] = v
+			k++
+		}
+
+		return
+	}
+
+	// Must first ensure cache checkout's existence
+	err = s.ensureCacheExistence()
+	if err != nil {
+		return
+	}
+
+	// Local repo won't have all the latest refs if ensureCacheExistence()
+	// didn't create it
+	if !s.crepo.synced {
+		r := s.crepo.r
+
+		s.crepo.mut.Lock()
+		err = r.Update()
+		s.crepo.mut.Unlock()
+		if err != nil {
+			return
+		}
+
+		s.crepo.synced = true
+	}
+
+	var out []byte
+
+	// Now, list all the tags
+	out, err = r.RunFromDir("bzr", "tags", "--show-ids", "-v")
+	if err != nil {
+		return
+	}
+
+	all := bytes.Split(bytes.TrimSpace(out), []byte("\n"))
+
+	// reset the rmap and vmap, as they'll be fully repopulated by this
+	// TODO(sdboyer) detect out-of-sync pairings as we do this?
+	s.dc.vMap = make(map[Version]Revision)
+	s.dc.rMap = make(map[Revision][]Version)
+
+	vlist = make([]Version, len(all))
+	k := 0
+	for _, line := range all {
+		idx := bytes.IndexByte(line, 32) // space
+		v := NewVersion(string(line[:idx]))
+		r := Revision(bytes.TrimSpace(line[idx:]))
+
+		s.dc.vMap[v] = r
+		s.dc.rMap[r] = append(s.dc.rMap[r], v)
+		vlist[k] = v.Is(r)
+		k++
+	}
+
+	// Cache is now in sync with upstream's version list
+	s.cvsync = true
+	return
+}

From 858aeaeaf9ea247aa3944e10ce3805af607e6cb5 Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Tue, 2 Aug 2016 23:08:53 -0400
Subject: [PATCH 27/71] Small bugfixes in bzrSource.listVersions()

- hoist scope on pulling repo into local var
- recombine unpaired versions into paired versions on cache hit
---
 source.go      | 8 +++-----
 source_test.go | 2 +-
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/source.go b/source.go
index c30d142..79b656d 100644
--- a/source.go
+++ b/source.go
@@ -465,10 +465,8 @@ func (s *bzrSource) listVersions() (vlist []Version, err error) {
 	if s.cvsync {
 		vlist = make([]Version, len(s.dc.vMap))
 		k := 0
-		// TODO(sdboyer) key type of VMap should be string; recombine here
-		//for v, r := range s.dc.VMap {
-		for v := range s.dc.vMap {
-			vlist[k] = v
+		for v, r := range s.dc.vMap {
+			vlist[k] = v.(UnpairedVersion).Is(r)
 			k++
 		}
 
@@ -480,11 +478,11 @@ func (s *bzrSource) listVersions() (vlist []Version, err error) {
 	if err != nil {
 		return
 	}
+	r := s.crepo.r
 
 	// Local repo won't have all the latest refs if ensureCacheExistence()
 	// didn't create it
 	if !s.crepo.synced {
-		r := s.crepo.r
 
 		s.crepo.mut.Lock()
 		err = r.Update()
diff --git a/source_test.go b/source_test.go
index de2a8a0..cb07e31 100644
--- a/source_test.go
+++ b/source_test.go
@@ -9,7 +9,7 @@ import (
 )
 
 func TestGitVersionFetching(t *testing.T) {
-	// This test is quite slow, skip it on -short
+	// This test is slowish, skip it on -short
 	if testing.Short() {
 		t.Skip("Skipping git source version fetching test in short mode")
 	}

From c53563b42aee6d8995421b1ba72a8e3ef6d72a47 Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Tue, 2 Aug 2016 23:37:19 -0400
Subject: [PATCH 28/71] Add basic tests for bzrSource

---
 source_test.go | 86 ++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 86 insertions(+)

diff --git a/source_test.go b/source_test.go
index cb07e31..6a36e3f 100644
--- a/source_test.go
+++ b/source_test.go
@@ -84,3 +84,89 @@ func TestGitVersionFetching(t *testing.T) {
 		}
 	}
 }
+
+func TestBzrVersionFetching(t *testing.T) {
+	// This test is quite slow (ugh bzr), so skip it on -short
+	if testing.Short() {
+		t.Skip("Skipping bzr source version fetching test in short mode")
+	}
+
+	cpath, err := ioutil.TempDir("", "smcache")
+	if err != nil {
+		t.Errorf("Failed to create temp dir: %s", err)
+	}
+	rf := func() {
+		err := removeAll(cpath)
+		if err != nil {
+			t.Errorf("removeAll failed: %s", err)
+		}
+	}
+
+	n := "launchpad.net/govcstestbzrrepo"
+	u, err := url.Parse("https://" + n)
+	if err != nil {
+		t.Errorf("URL was bad, lolwut? errtext: %s", err)
+		rf()
+		t.FailNow()
+	}
+	mb := maybeBzrSource{
+		n:   n,
+		url: u,
+	}
+
+	isrc, err := mb.try(cpath, naiveAnalyzer{})
+	if err != nil {
+		t.Errorf("Unexpected error while setting up bzrSource for test repo: %s", err)
+		rf()
+		t.FailNow()
+	}
+	src, ok := isrc.(*bzrSource)
+	if !ok {
+		t.Errorf("Expected a bzrSource, got a %T", isrc)
+		rf()
+		t.FailNow()
+	}
+
+	vlist, err := src.listVersions()
+	if err != nil {
+		t.Errorf("Unexpected error getting version pairs from bzr repo: %s", err)
+	}
+
+	if src.ex.s&existsUpstream|existsInCache != existsUpstream|existsInCache {
+		t.Errorf("bzrSource.listVersions() should have set the upstream and cache existence bits for search")
+	}
+	if src.ex.f&existsUpstream|existsInCache != existsUpstream|existsInCache {
+		t.Errorf("bzrSource.listVersions() should have set the upstream and cache existence bits for found")
+	}
+
+	if len(vlist) != 1 {
+		t.Errorf("bzr test repo should've produced one version, got %v", len(vlist))
+	} else {
+		v := NewVersion("1.0.0").Is(Revision("matt@mattfarina.com-20150731135137-pbphasfppmygpl68"))
+		if vlist[0] != v {
+			t.Errorf("bzr pair fetch reported incorrect first version, got %s", vlist[0])
+		}
+	}
+
+	// Run again, this time to ensure cache outputs correctly
+	vlist, err = src.listVersions()
+	if err != nil {
+		t.Errorf("Unexpected error getting version pairs from bzr repo: %s", err)
+	}
+
+	if src.ex.s&existsUpstream|existsInCache != existsUpstream|existsInCache {
+		t.Errorf("bzrSource.listVersions() should have set the upstream and cache existence bits for search")
+	}
+	if src.ex.f&existsUpstream|existsInCache != existsUpstream|existsInCache {
+		t.Errorf("bzrSource.listVersions() should have set the upstream and cache existence bits for found")
+	}
+
+	if len(vlist) != 1 {
+		t.Errorf("bzr test repo should've produced one version, got %v", len(vlist))
+	} else {
+		v := NewVersion("1.0.0").Is(Revision("matt@mattfarina.com-20150731135137-pbphasfppmygpl68"))
+		if vlist[0] != v {
+			t.Errorf("bzr pair fetch reported incorrect first version, got %s", vlist[0])
+		}
+	}
+}

From 72f1e0f7b650a00c4fdabcac9c3294d68115f07e Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Tue, 2 Aug 2016 23:55:55 -0400
Subject: [PATCH 29/71] Add tests for hgSource

---
 source_test.go | 90 ++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 90 insertions(+)

diff --git a/source_test.go b/source_test.go
index 6a36e3f..d5dd9c5 100644
--- a/source_test.go
+++ b/source_test.go
@@ -170,3 +170,93 @@ func TestBzrVersionFetching(t *testing.T) {
 		}
 	}
 }
+
+func TestHgVersionFetching(t *testing.T) {
+	// This test is slow, so skip it on -short
+	if testing.Short() {
+		t.Skip("Skipping hg source version fetching test in short mode")
+	}
+
+	cpath, err := ioutil.TempDir("", "smcache")
+	if err != nil {
+		t.Errorf("Failed to create temp dir: %s", err)
+	}
+	rf := func() {
+		err := removeAll(cpath)
+		if err != nil {
+			t.Errorf("removeAll failed: %s", err)
+		}
+	}
+
+	n := "bitbucket.org/mattfarina/testhgrepo"
+	u, err := url.Parse("https://" + n)
+	if err != nil {
+		t.Errorf("URL was bad, lolwut? errtext: %s", err)
+		rf()
+		t.FailNow()
+	}
+	mb := maybeHgSource{
+		n:   n,
+		url: u,
+	}
+
+	isrc, err := mb.try(cpath, naiveAnalyzer{})
+	if err != nil {
+		t.Errorf("Unexpected error while setting up hgSource for test repo: %s", err)
+		rf()
+		t.FailNow()
+	}
+	src, ok := isrc.(*hgSource)
+	if !ok {
+		t.Errorf("Expected a hgSource, got a %T", isrc)
+		rf()
+		t.FailNow()
+	}
+
+	vlist, err := src.listVersions()
+	if err != nil {
+		t.Errorf("Unexpected error getting version pairs from hg repo: %s", err)
+	}
+	evl := []Version{
+		NewVersion("1.0.0").Is(Revision("d680e82228d206935ab2eaa88612587abe68db07")),
+		NewBranch("test").Is(Revision("6c44ee3fe5d87763616c19bf7dbcadb24ff5a5ce")),
+	}
+
+	if src.ex.s&existsUpstream|existsInCache != existsUpstream|existsInCache {
+		t.Errorf("hgSource.listVersions() should have set the upstream and cache existence bits for search")
+	}
+	if src.ex.f&existsUpstream|existsInCache != existsUpstream|existsInCache {
+		t.Errorf("hgSource.listVersions() should have set the upstream and cache existence bits for found")
+	}
+
+	if len(vlist) != 2 {
+		t.Errorf("hg test repo should've produced one version, got %v", len(vlist))
+	} else {
+		sort.Sort(upgradeVersionSorter(vlist))
+		if !reflect.DeepEqual(vlist, evl) {
+			t.Errorf("Version list was not what we expected:\n\t(GOT): %s\n\t(WNT): %s", vlist, evl)
+		}
+	}
+
+	// Run again, this time to ensure cache outputs correctly
+	vlist, err = src.listVersions()
+	if err != nil {
+		t.Errorf("Unexpected error getting version pairs from hg repo: %s", err)
+	}
+
+	if src.ex.s&existsUpstream|existsInCache != existsUpstream|existsInCache {
+		t.Errorf("hgSource.listVersions() should have set the upstream and cache existence bits for search")
+	}
+	if src.ex.f&existsUpstream|existsInCache != existsUpstream|existsInCache {
+		t.Errorf("hgSource.listVersions() should have set the upstream and cache existence bits for found")
+	}
+
+	if len(vlist) != 2 {
+		t.Errorf("hg test repo should've produced one version, got %v", len(vlist))
+	} else {
+		sort.Sort(upgradeVersionSorter(vlist))
+		if !reflect.DeepEqual(vlist, evl) {
+			t.Errorf("Version list was not what we expected:\n\t(GOT): %s\n\t(WNT): %s", vlist, evl)
+		}
+	}
+}

From 56a6369740c49bf34acf26f1656eaed351152ba1 Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Wed, 3 Aug 2016 00:16:46 -0400
Subject: [PATCH 30/71] Add hgSource implementation, plus the maybe

---
 maybe_source.go |  27 ++++++++++++
 source.go       | 113 +++++++++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 138 insertions(+), 2 deletions(-)

diff --git a/maybe_source.go b/maybe_source.go
index c670e8d..270a493 100644
--- a/maybe_source.go
+++ b/maybe_source.go
@@ -73,3 +73,30 @@ func (m maybeBzrSource) try(cachedir string, an ProjectAnalyzer) (source, error)
 		},
 	}, nil
 }
+
+type maybeHgSource struct {
+	n   string
+	url *url.URL
+}
+
+func (m maybeHgSource) try(cachedir string, an ProjectAnalyzer) (source, error) {
+	path := filepath.Join(cachedir, "sources", sanitizer.Replace(m.url.String()))
+	r, err := vcs.NewHgRepo(m.url.String(), path)
+	if err != nil {
+		return nil, err
+	}
+	if !r.Ping() {
+		return nil, fmt.Errorf("Remote repository at %s does not exist, or is inaccessible", m.url.String())
+	}
+
+	return &hgSource{
+		baseSource: baseSource{
+			an: an,
+			dc: newDataCache(),
+			crepo: &repo{
+				r:     r,
+				rpath: path,
+			},
+		},
+	}, nil
+}
diff --git a/source.go b/source.go
index 79b656d..0f54c96 100644
--- a/source.go
+++ b/source.go
@@ -456,7 +456,7 @@ func (s *gitSource) listVersions() (vlist []Version, err error) {
 }
 
 // bzrSource is a generic bzr repository implementation that should work with
-// all standard git remotes.
+// all standard bazaar remotes.
 type bzrSource struct {
 	baseSource
 }
@@ -483,7 +483,6 @@ func (s *bzrSource) listVersions() (vlist []Version, err error) {
 	// Local repo won't have all the latest refs if ensureCacheExistence()
 	// didn't create it
 	if !s.crepo.synced {
-
 		s.crepo.mut.Lock()
 		err = r.Update()
 		s.crepo.mut.Unlock()
@@ -526,3 +525,113 @@ func (s *bzrSource) listVersions() (vlist []Version, err error) {
 	s.cvsync = true
 	return
 }
+
+// hgSource is a generic hg repository implementation that should work with
+// all standard mercurial servers.
+type hgSource struct {
+	baseSource
+}
+
+func (s *hgSource) listVersions() (vlist []Version, err error) {
+	if s.cvsync {
+		vlist = make([]Version, len(s.dc.vMap))
+		k := 0
+		for v := range s.dc.vMap {
+			vlist[k] = v
+			k++
+		}
+
+		return
+	}
+
+	// Must first ensure cache checkout's existence
+	err = s.ensureCacheExistence()
+	if err != nil {
+		return
+	}
+	r := s.crepo.r
+
+	// Local repo won't have all the latest refs if ensureCacheExistence()
+	// didn't create it
+	if !s.crepo.synced {
+		s.crepo.mut.Lock()
+		err = r.Update()
+		s.crepo.mut.Unlock()
+		if err != nil {
+			return
+		}
+
+		s.crepo.synced = true
+	}
+
+	var out []byte
+
+	// Now, list all the tags
+	out, err = r.RunFromDir("hg", "tags", "--debug", "--verbose")
+	if err != nil {
+		return
+	}
+
+	all := bytes.Split(bytes.TrimSpace(out), []byte("\n"))
+	lbyt := []byte("local")
+	nulrev := []byte("0000000000000000000000000000000000000000")
+	for _, line := range all {
+		if bytes.Equal(lbyt, line[len(line)-len(lbyt):]) {
+			// Skip local tags
+			continue
+		}
+
+		// tip is magic, don't include it
+		if bytes.HasPrefix(line, []byte("tip")) {
+			continue
+		}
+
+		// Split on colon; this gets us the rev and the tag plus local revno
+		pair := bytes.Split(line, []byte(":"))
+		if bytes.Equal(nulrev, pair[1]) {
+			// null rev indicates this tag is marked for deletion
+			continue
+		}
+
+		idx := bytes.IndexByte(pair[0], 32) // space
+		v := NewVersion(string(pair[0][:idx])).Is(Revision(pair[1])).(PairedVersion)
+		vlist = append(vlist, v)
+	}
+
+	out, err = r.RunFromDir("hg", "branches", "--debug", "--verbose")
+	if err != nil {
+		// better nothing than partial and misleading
+		vlist = nil
+		return
+	}
+
+	all = bytes.Split(bytes.TrimSpace(out), []byte("\n"))
+	lbyt = []byte("(inactive)")
+	for _, line := range all {
+		if bytes.Equal(lbyt, line[len(line)-len(lbyt):]) {
+			// Skip inactive branches
+			continue
+		}
+
+		// Split on colon; this gets us the rev and the branch plus local revno
+		pair := bytes.Split(line, []byte(":"))
+		idx := bytes.IndexByte(pair[0], 32) // space
+		v := NewBranch(string(pair[0][:idx])).Is(Revision(pair[1])).(PairedVersion)
+		vlist = append(vlist, v)
+	}
+
+	// reset the rmap and vmap, as they'll be fully repopulated by this
+	// TODO(sdboyer) detect out-of-sync pairings as we do this?
+	s.dc.vMap = make(map[Version]Revision)
+	s.dc.rMap = make(map[Revision][]Version)
+
+	for _, v := range vlist {
+		pv := v.(PairedVersion)
+		s.dc.vMap[v] = pv.Underlying()
+		s.dc.rMap[pv.Underlying()] = append(s.dc.rMap[pv.Underlying()], v)
+	}
+
+	// Cache is now in sync with upstream's version list
+	s.cvsync = true
+	return
+}

From eda0de9c5b851dc0d611b88fd8867a65e233e084 Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Wed, 3 Aug 2016 09:12:03 -0400
Subject: [PATCH 31/71] Add note to README about source inference choice

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index 89d7a78..6556779 100644
--- a/README.md
+++ b/README.md
@@ -69,6 +69,7 @@ There are also some current non-choices that we would like to push into the real
 
 * Different versions of packages from the same repository cannot be used
 * Importable projects that are not bound to the repository root
+* Source inference around different import path patterns (e.g., how `github.com/*` or `my_company/*` are handled)
 
 ### Choices
 

From fb085c7fbc47ad18404a2a8e8abe5d3ac74d869b Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Wed, 3 Aug 2016 09:34:41 -0400
Subject: [PATCH 32/71] Use UnpairedVersion in sourceMetaCache

This is incomplete/incremental, but important. When the meta caches only
kept a Version, we ended up caching results based on what the solver
happened to pass in, not something stable and predictable. This is much
better.
---
 maybe_source.go   |  6 ++--
 source.go         | 85 +++++++++++++++++++++++++++++++++++------------
 source_manager.go |  7 +---
 version.go        | 12 +++++++
 4 files changed, 79 insertions(+), 31 deletions(-)

diff --git a/maybe_source.go b/maybe_source.go
index 270a493..8b3596f 100644
--- a/maybe_source.go
+++ b/maybe_source.go
@@ -29,7 +29,7 @@ func (m maybeGitSource) try(cachedir string, an ProjectAnalyzer) (source, error)
 	src := &gitSource{
 		baseSource: baseSource{
 			an: an,
-			dc: newDataCache(),
+			dc: newMetaCache(),
 			crepo: &repo{
 				r:     r,
 				rpath: path,
@@ -65,7 +65,7 @@ func (m maybeBzrSource) try(cachedir string, an ProjectAnalyzer) (source, error)
 	return &bzrSource{
 		baseSource: baseSource{
 			an: an,
-			dc: newDataCache(),
+			dc: newMetaCache(),
 			crepo: &repo{
 				r:     r,
 				rpath: path,
@@ -92,7 +92,7 @@ func (m maybeHgSource) try(cachedir string, an ProjectAnalyzer) (source, error)
 	return &hgSource{
 		baseSource: baseSource{
 			an: an,
-			dc: newDataCache(),
+			dc: newMetaCache(),
 			crepo: &repo{
 				r:     r,
 				rpath: path,
diff --git a/source.go b/source.go
index 0f54c96..7297e5a 100644
--- a/source.go
+++ b/source.go
@@ -22,17 +22,17 @@ type sourceMetaCache struct {
 	//Version  string                   // TODO(sdboyer) use this
 	infos  map[Revision]projectInfo
 	ptrees map[Revision]PackageTree
-	vMap   map[Version]Revision
-	rMap   map[Revision][]Version
+	vMap   map[UnpairedVersion]Revision
+	rMap   map[Revision][]UnpairedVersion
 	// TODO(sdboyer) mutexes. actually probably just one, b/c complexity
 }
 
-func newDataCache() *sourceMetaCache {
+func newMetaCache() *sourceMetaCache {
 	return &sourceMetaCache{
 		infos:  make(map[Revision]projectInfo),
 		ptrees: make(map[Revision]PackageTree),
-		vMap:   make(map[Version]Revision),
-		rMap:   make(map[Revision][]Version),
+		vMap:   make(map[UnpairedVersion]Revision),
+		rMap:   make(map[Revision][]UnpairedVersion),
 	}
 }
 
@@ -117,6 +117,46 @@ func (bs *baseSource) getManifestAndLock(r ProjectRoot, v Version) (Manifest, Lo
 	return nil, nil, err
 }
 
+// toRevision turns a Version into a Revision, if doing so is possible based on
+// the information contained in the version itself, or in the cache maps.
+func (dc *sourceMetaCache) toRevision(v Version) Revision {
+	switch t := v.(type) {
+	case Revision:
+		return t
+	case PairedVersion:
+		return t.Underlying()
+	case UnpairedVersion:
+		// This will return the empty rev (empty string) if we don't have a
+		// record of it. It's up to the caller to decide, for example, if
+		// it's appropriate to update the cache.
+		return dc.vMap[t]
+	default:
+		panic(fmt.Sprintf("Unknown version type %T", v))
+	}
+}
+
+// toUnpaired turns a Version into an UnpairedVersion, if doing so is possible
+// based on the information contained in the version itself, or in the cache
+// maps.
+//
+// If the input is a revision and multiple UnpairedVersions are associated with
+// it, whatever happens to be the first is returned.
+func (dc *sourceMetaCache) toUnpaired(v Version) UnpairedVersion {
+	switch t := v.(type) {
+	case UnpairedVersion:
+		return t
+	case PairedVersion:
+		return t.Underlying()
+	case Revision:
+		if upv, has := dc.rMap[t]; has && len(upv) > 0 {
+			return upv[0]
+		}
+		return nil
+	default:
+		panic(fmt.Sprintf("Unknown version type %T", v))
+	}
+}
+
 func (bs *baseSource) listVersions() (vlist []Version, err error) {
 	if !bs.cvsync {
 		// This check only guarantees that the upstream exists, not the cache
@@ -140,15 +180,14 @@ func (bs *baseSource) listVersions() (vlist []Version, err error) {
 		// Process the version data into the cache
 		// TODO(sdboyer) detect out-of-sync data as we do this?
 		for k, v := range vpairs {
-			bs.dc.vMap[v] = v.Underlying()
-			bs.dc.rMap[v.Underlying()] = append(bs.dc.rMap[v.Underlying()], v)
+			u, r := v.Unpair(), v.Underlying()
+			bs.dc.vMap[u] = r
+			bs.dc.rMap[r] = append(bs.dc.rMap[r], u)
 			vlist[k] = v
 		}
 	} else {
 		vlist = make([]Version, len(bs.dc.vMap))
 		k := 0
-		// TODO(sdboyer) key type of VMap should be string; recombine here
-		//for v, r := range bs.dc.VMap {
 		for v := range bs.dc.vMap {
 			vlist[k] = v
 			k++
@@ -442,13 +481,14 @@ func (s *gitSource) listVersions() (vlist []Version, err error) {
 	//
 	// reset the rmap and vmap, as they'll be fully repopulated by this
 	// TODO(sdboyer) detect out-of-sync pairings as we do this?
-	s.dc.vMap = make(map[Version]Revision)
-	s.dc.rMap = make(map[Revision][]Version)
+	s.dc.vMap = make(map[UnpairedVersion]Revision)
+	s.dc.rMap = make(map[Revision][]UnpairedVersion)
 
 	for _, v := range vlist {
 		pv := v.(PairedVersion)
-		s.dc.vMap[v] = pv.Underlying()
-		s.dc.rMap[pv.Underlying()] = append(s.dc.rMap[pv.Underlying()], v)
+		u, r := pv.Unpair(), pv.Underlying()
+		s.dc.vMap[u] = r
+		s.dc.rMap[r] = append(s.dc.rMap[r], u)
 	}
 	// Mark the cache as being in sync with upstream's version list
 	s.cvsync = true
@@ -466,7 +506,7 @@ func (s *bzrSource) listVersions() (vlist []Version, err error) {
 		vlist = make([]Version, len(s.dc.vMap))
 		k := 0
 		for v, r := range s.dc.vMap {
-			vlist[k] = v.(UnpairedVersion).Is(r)
+			vlist[k] = v.Is(r)
 			k++
 		}
 
@@ -505,8 +545,8 @@ func (s *bzrSource) listVersions() (vlist []Version, err error) {
 
 	// reset the rmap and vmap, as they'll be fully repopulated by this
 	// TODO(sdboyer) detect out-of-sync pairings as we do this?
-	s.dc.vMap = make(map[Version]Revision)
-	s.dc.rMap = make(map[Revision][]Version)
+	s.dc.vMap = make(map[UnpairedVersion]Revision)
+	s.dc.rMap = make(map[Revision][]UnpairedVersion)
 
 	vlist = make([]Version, len(all))
 	k := 0
@@ -536,8 +576,8 @@ func (s *hgSource) listVersions() (vlist []Version, err error) {
 	if s.cvsync {
 		vlist = make([]Version, len(s.dc.vMap))
 		k := 0
-		for v := range s.dc.vMap {
-			vlist[k] = v
+		for v, r := range s.dc.vMap {
+			vlist[k] = v.Is(r)
 			k++
 		}
 
@@ -622,13 +662,14 @@ func (s *hgSource) listVersions() (vlist []Version, err error) {
 
 	// reset the rmap and vmap, as they'll be fully repopulated by this
 	// TODO(sdboyer) detect out-of-sync pairings as we do this?
-	s.dc.vMap = make(map[Version]Revision)
-	s.dc.rMap = make(map[Revision][]Version)
+	s.dc.vMap = make(map[UnpairedVersion]Revision)
+	s.dc.rMap = make(map[Revision][]UnpairedVersion)
 
 	for _, v := range vlist {
 		pv := v.(PairedVersion)
-		s.dc.vMap[v] = pv.Underlying()
-		s.dc.rMap[pv.Underlying()] = append(s.dc.rMap[pv.Underlying()], v)
+		u, r := pv.Unpair(), pv.Underlying()
+		s.dc.vMap[u] = r
+		s.dc.rMap[r] = append(s.dc.rMap[r], u)
 	}
 
 	// Cache is now in sync with upstream's version list
diff --git a/source_manager.go b/source_manager.go
index b66aa5e..87df464 100644
--- a/source_manager.go
+++ b/source_manager.go
@@ -371,12 +371,7 @@ decided:
 		//return nil, fmt.Errorf("Err on creating metadata cache file: %s", err)
 		//}
 
-		dc = &sourceMetaCache{
-			infos:  make(map[Revision]projectInfo),
-			ptrees: make(map[Revision]PackageTree),
-			vMap:   make(map[Version]Revision),
-			rMap:   make(map[Revision][]Version),
-		}
+		dc = newMetaCache()
 	}
 
 	pm := &projectManager{
diff --git a/version.go b/version.go
index 57d37ec..230e0ca 100644
--- a/version.go
+++ b/version.go
@@ -16,6 +16,7 @@ import "github.com/Masterminds/semver"
 // hiding behind the interface.
 type Version interface {
 	Constraint
+
 	// Indicates the type of version - Revision, Branch, Version, or Semver
 	Type() string
 }
@@ -24,8 +25,15 @@ type Version interface {
 // underlying Revision.
 type PairedVersion interface {
 	Version
+
 	// Underlying returns the immutable Revision that identifies this Version.
 	Underlying() Revision
+
+	// Unpair returns the surface-level UnpairedVersion that half of the pair.
+	//
+	// It does NOT modify the original PairedVersion
+	Unpair() UnpairedVersion
+
 	// Ensures it is impossible to be both a PairedVersion and an
 	// UnpairedVersion
 	_pair(int)
@@ -380,6 +388,10 @@ func (v versionPair) Underlying() Revision {
 	return v.r
 }
 
+func (v versionPair) Unpair() UnpairedVersion {
+	return v.v
+}
+
 func (v versionPair) Matches(v2 Version) bool {
 	switch tv2 := v2.(type) {
 	case versionTypeUnion:

From c72c3d6a543ef4ea375323c2844cb5b690b538c1 Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Wed, 3 Aug 2016 11:28:04 -0400
Subject: [PATCH 33/71] Refactor baseSource to use UnpairedV in metacache

---
 source.go | 82 +++++++++++++++++++++++++++++++++----------------------
 1 file changed, 49 insertions(+), 33 deletions(-)

diff --git a/source.go b/source.go
index 7297e5a..388c85d 100644
--- a/source.go
+++ b/source.go
@@ -50,10 +50,13 @@ type baseSource struct { // TODO(sdboyer) rename to baseVCSSource
 	// Whether the cache has the latest info on versions
 	cvsync bool
 
-	// The project metadata cache. This is persisted to disk, for reuse across
-	// solver runs.
-	// TODO(sdboyer) protect with mutex
+	// The project metadata cache. This is (or is intended to be) persisted to
+	// disk, for reuse across solver runs.
 	dc *sourceMetaCache
+
+	// lvfunc allows the other vcs source types that embed this type to inject
+	// their listVersions func into the baseSource, for use as needed.
+	lvfunc func() (vlist []Version, err error)
 }
 
 func (bs *baseSource) getManifestAndLock(r ProjectRoot, v Version) (Manifest, Lock, error) {
@@ -61,14 +64,17 @@ func (bs *baseSource) getManifestAndLock(r ProjectRoot, v Version) (Manifest, Lo
 		return nil, nil, err
 	}
 
-	if r, exists := bs.dc.vMap[v]; exists {
-		if pi, exists := bs.dc.infos[r]; exists {
-			return pi.Manifest, pi.Lock, nil
-		}
+	rev, err := bs.toRevOrErr(v)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// Return the info from the cache, if we already have it
+	if pi, exists := bs.dc.infos[rev]; exists {
+		return pi.Manifest, pi.Lock, nil
 	}
 
 	bs.crepo.mut.Lock()
-	var err error
 	if !bs.crepo.synced {
 		err = bs.crepo.r.Update()
 		if err != nil {
@@ -84,6 +90,7 @@ func (bs *baseSource) getManifestAndLock(r ProjectRoot, v Version) (Manifest, Lo
 		err = bs.crepo.r.UpdateVersion(v.String())
 	}
 	bs.crepo.mut.Unlock()
+
 	if err != nil {
 		// TODO(sdboyer) More-er proper-er error
 		panic(fmt.Sprintf("canary - why is checkout/whatever failing: %s %s %s", bs.crepo.r.LocalPath(), v.String(), err))
@@ -105,11 +112,7 @@ func (bs *baseSource) getManifestAndLock(r ProjectRoot, v Version) (Manifest, Lo
 			Lock:     l,
 		}
 
-		// TODO(sdboyer) this just clobbers all over and ignores the paired/unpaired
-		// distinction; serious fix is needed
-		if r, exists := bs.dc.vMap[v]; exists {
-			bs.dc.infos[r] = pi
-		}
+		bs.dc.infos[rev] = pi
 
 		return pi.Manifest, pi.Lock, nil
 	}
@@ -146,7 +149,7 @@ func (dc *sourceMetaCache) toUnpaired(v Version) UnpairedVersion {
 	case UnpairedVersion:
 		return t
 	case PairedVersion:
-		return t.Underlying()
+		return t.Unpair()
 	case Revision:
 		if upv, has := dc.rMap[t]; has && len(upv) > 0 {
 			return upv[0]
@@ -282,28 +285,16 @@ func (bs *baseSource) listPackages(pr ProjectRoot, v Version) (ptree PackageTree
 		return
 	}
 
-	// See if we can find it in the cache
 	var r Revision
-	switch v.(type) {
-	case Revision, PairedVersion:
-		var ok bool
-		if r, ok = v.(Revision); !ok {
-			r = v.(PairedVersion).Underlying()
-		}
-
-		if ptree, cached := bs.dc.ptrees[r]; cached {
-			return ptree, nil
-		}
-	default:
-		var has bool
-		if r, has = bs.dc.vMap[v]; has {
-			if ptree, cached := bs.dc.ptrees[r]; cached {
-				return ptree, nil
-			}
-		}
+	if r, err = bs.toRevOrErr(v); err != nil {
+		return
 	}
 
-	// TODO(sdboyer) handle the case where we have a version w/out rev, and not in cache
+	// Return the ptree from the cache, if we already have it
+	var exists bool
+	if ptree, exists = bs.dc.ptrees[r]; exists {
+		return
+	}
 
 	// Not in the cache; check out the version and do the analysis
 	bs.crepo.mut.Lock()
@@ -335,6 +326,31 @@ func (bs *baseSource) listPackages(pr ProjectRoot, v Version) (ptree PackageTree
 	return
 }
 
+// toRevOrErr makes all efforts to convert a Version into a rev, including
+// updating the cache repo (if needed). It does not guarantee that the returned
+// Revision actually exists in the repository (as one of the cheaper methods may
+// have had bad data).
+func (bs *baseSource) toRevOrErr(v Version) (r Revision, err error) {
+	r = bs.dc.toRevision(v)
+	if r == "" {
+		// Rev can be empty if:
+		//  - The cache is unsynced
+		//  - A version was passed that used to exist, but no longer does
+		//  - A garbage version was passed. (Functionally indistinguishable from
+		//  the previous)
+		if !bs.cvsync {
+			// call the lvfunc to sync the meta cache
+			_, err = bs.lvfunc()
+		}
+		// If we still don't have a rev, then the version's no good
+		if r == "" {
+			err = fmt.Errorf("Version %s does not exist in source %s", v, bs.crepo.r.Remote())
+		}
+	}
+
+	return
+}
+
 func (bs *baseSource) exportVersionTo(v Version, to string) error {
 	return bs.crepo.exportVersionTo(v, to)
 }

From d62bc08c69ae0013b55e6736fc6fb4e73f1c72c1 Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Wed, 3 Aug 2016 11:42:02 -0400
Subject: [PATCH 34/71] At least make projectManager compile again

We're going to remove it, but still better to have it compiling and
testable in the interim.
---
 project_manager.go | 82 ++++++++++++++++++++++++++--------------------
 source.go          | 16 ++++-----
 2 files changed, 54 insertions(+), 44 deletions(-)

diff --git a/project_manager.go b/project_manager.go
index ba306c1..417b8fc 100644
--- a/project_manager.go
+++ b/project_manager.go
@@ -74,18 +74,21 @@ func (pm *projectManager) GetManifestAndLock(r ProjectRoot, v Version) (Manifest
 		return nil, nil, err
 	}
 
-	if r, exists := pm.dc.vMap[v]; exists {
-		if pi, exists := pm.dc.infos[r]; exists {
-			return pi.Manifest, pi.Lock, nil
-		}
+	rev, err := pm.toRevOrErr(v)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// Return the info from the cache, if we already have it
+	if pi, exists := pm.dc.infos[rev]; exists {
+		return pi.Manifest, pi.Lock, nil
 	}
 
 	pm.crepo.mut.Lock()
-	var err error
 	if !pm.crepo.synced {
 		err = pm.crepo.r.Update()
 		if err != nil {
-			return nil, nil, fmt.Errorf("Could not fetch latest updates into repository")
+			return nil, nil, fmt.Errorf("could not fetch latest updates into repository")
 		}
 		pm.crepo.synced = true
 	}
@@ -120,9 +123,7 @@ func (pm *projectManager) GetManifestAndLock(r ProjectRoot, v Version) (Manifest
 
 		// TODO(sdboyer) this just clobbers all over and ignores the paired/unpaired
 		// distinction; serious fix is needed
-		if r, exists := pm.dc.vMap[v]; exists {
-			pm.dc.infos[r] = pi
-		}
+		pm.dc.infos[rev] = pi
 
 		return pi.Manifest, pi.Lock, nil
 	}
@@ -135,28 +136,16 @@ func (pm *projectManager) ListPackages(pr ProjectRoot, v Version) (ptree Package
 		return
 	}
 
-	// See if we can find it in the cache
 	var r Revision
-	switch v.(type) {
-	case Revision, PairedVersion:
-		var ok bool
-		if r, ok = v.(Revision); !ok {
-			r = v.(PairedVersion).Underlying()
-		}
-
-		if ptree, cached := pm.dc.ptrees[r]; cached {
-			return ptree, nil
-		}
-	default:
-		var has bool
-		if r, has = pm.dc.vMap[v]; has {
-			if ptree, cached := pm.dc.ptrees[r]; cached {
-				return ptree, nil
-			}
-		}
+	if r, err = pm.toRevOrErr(v); err != nil {
+		return
 	}
 
-	// TODO(sdboyer) handle the case where we have a version w/out rev, and not in cache
+	// Return the ptree from the cache, if we already have it
+	var exists bool
+	if ptree, exists = pm.dc.ptrees[r]; exists {
+		return
+	}
 
 	// Not in the cache; check out the version and do the analysis
 	pm.crepo.mut.Lock()
@@ -170,7 +159,7 @@ func (pm *projectManager) ListPackages(pr ProjectRoot, v Version) (ptree Package
 		if !pm.crepo.synced {
 			err = pm.crepo.r.Update()
 			if err != nil {
-				return PackageTree{}, fmt.Errorf("Could not fetch latest updates into repository: %s", err)
+				return PackageTree{}, fmt.Errorf("could not fetch latest updates into repository: %s", err)
 			}
 			pm.crepo.synced = true
 		}
@@ -236,17 +225,16 @@ func (pm *projectManager) ListVersions() (vlist []Version, err error) {
 		// Process the version data into the cache
 		// TODO(sdboyer) detect out-of-sync data as we do this?
 		for k, v := range vpairs {
-			pm.dc.vMap[v] = v.Underlying()
-			pm.dc.rMap[v.Underlying()] = append(pm.dc.rMap[v.Underlying()], v)
+			u, r := v.Unpair(), v.Underlying()
+			pm.dc.vMap[u] = r
+			pm.dc.rMap[r] = append(pm.dc.rMap[r], u)
 			vlist[k] = v
 		}
 	} else {
 		vlist = make([]Version, len(pm.dc.vMap))
 		k := 0
-		// TODO(sdboyer) key type of VMap should be string; recombine here
-		//for v, r := range pm.dc.VMap {
-		for v := range pm.dc.vMap {
-			vlist[k] = v
+		for v, r := range pm.dc.vMap {
+			vlist[k] = v.Is(r)
 			k++
 		}
 	}
@@ -254,6 +242,30 @@ func (pm *projectManager) ListVersions() (vlist []Version, err error) {
 	return
 }
 
+// toRevOrErr makes all efforts to convert a Version into a rev, including
+// updating the cache repo (if needed). It does not guarantee that the returned
+// Revision actually exists in the repository (as one of the cheaper methods may
+// have had bad data).
+func (pm *projectManager) toRevOrErr(v Version) (r Revision, err error) {
+	r = pm.dc.toRevision(v)
+	if r == "" {
+		// Rev can be empty if:
+		//  - The cache is unsynced
+		//  - A version was passed that used to exist, but no longer does
+		//  - A garbage version was passed. (Functionally indistinguishable from
+		//  the previous)
+		if !pm.cvsync {
+			_, err = pm.ListVersions()
+		}
+		// If we still don't have a rev, then the version's no good
+		if r == "" {
+			err = fmt.Errorf("version %s does not exist in source %s", v, pm.crepo.r.Remote())
+		}
+	}
+
+	return
+}
+
 func (pm *projectManager) RevisionPresentIn(pr ProjectRoot, r Revision) (bool, error) {
 	// First and fastest path is to check the data cache to see if the rev is
 	// present. This could give us false positives, but the cases where that can
diff --git a/source.go b/source.go
index 388c85d..22848b3 100644
--- a/source.go
+++ b/source.go
@@ -78,7 +78,7 @@ func (bs *baseSource) getManifestAndLock(r ProjectRoot, v Version) (Manifest, Lo
 	if !bs.crepo.synced {
 		err = bs.crepo.r.Update()
 		if err != nil {
-			return nil, nil, fmt.Errorf("Could not fetch latest updates into repository")
+			return nil, nil, fmt.Errorf("could not fetch latest updates into repository")
 		}
 		bs.crepo.synced = true
 	}
@@ -156,7 +156,7 @@ func (dc *sourceMetaCache) toUnpaired(v Version) UnpairedVersion {
 		}
 		return nil
 	default:
-		panic(fmt.Sprintf("Unknown version type %T", v))
+		panic(fmt.Sprintf("unknown version type %T", v))
 	}
 }
 
@@ -308,7 +308,7 @@ func (bs *baseSource) listPackages(pr ProjectRoot, v Version) (ptree PackageTree
 		if !bs.crepo.synced {
 			err = bs.crepo.r.Update()
 			if err != nil {
-				return PackageTree{}, fmt.Errorf("Could not fetch latest updates into repository: %s", err)
+				return PackageTree{}, fmt.Errorf("could not fetch latest updates into repository: %s", err)
 			}
 			bs.crepo.synced = true
 		}
@@ -344,7 +344,7 @@ func (bs *baseSource) toRevOrErr(v Version) (r Revision, err error) {
 		}
 		// If we still don't have a rev, then the version's no good
 		if r == "" {
-			err = fmt.Errorf("Version %s does not exist in source %s", v, bs.crepo.r.Remote())
+			err = fmt.Errorf("version %s does not exist in source %s", v, bs.crepo.r.Remote())
 		}
 	}
 
@@ -402,10 +402,8 @@ func (s *gitSource) listVersions() (vlist []Version, err error) {
 	if s.cvsync {
 		vlist = make([]Version, len(s.dc.vMap))
 		k := 0
-		// TODO(sdboyer) key type of VMap should be string; recombine here
-		//for v, r := range s.dc.VMap {
-		for v := range s.dc.vMap {
-			vlist[k] = v
+		for v, r := range s.dc.vMap {
+			vlist[k] = v.Is(r)
 			k++
 		}
 
@@ -452,7 +450,7 @@ func (s *gitSource) listVersions() (vlist []Version, err error) {
 
 		all = bytes.Split(bytes.TrimSpace(out), []byte("\n"))
 		if len(all) == 0 {
-			return nil, fmt.Errorf("No versions available for %s (this is weird)", r.Remote())
+			return nil, fmt.Errorf("no versions available for %s (this is weird)", r.Remote())
 		}
 	}
 

From 7fdd4c47455195c16fe61d255986133383cd5d17 Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Wed, 3 Aug 2016 11:48:08 -0400
Subject: [PATCH 35/71] Forgot to try to rederive Revision after syncing

---
 project_manager.go | 5 +++++
 source.go          | 5 +++++
 2 files changed, 10 insertions(+)

diff --git a/project_manager.go b/project_manager.go
index 417b8fc..b514dd3 100644
--- a/project_manager.go
+++ b/project_manager.go
@@ -256,7 +256,12 @@ func (pm *projectManager) toRevOrErr(v Version) (r Revision, err error) {
 		//  the previous)
 		if !pm.cvsync {
 			_, err = pm.ListVersions()
+			if err != nil {
+				return
+			}
 		}
+
+		r = pm.dc.toRevision(v)
 		// If we still don't have a rev, then the version's no good
 		if r == "" {
 			err = fmt.Errorf("version %s does not exist in source %s", v, pm.crepo.r.Remote())
diff --git a/source.go b/source.go
index 22848b3..5ec86f7 100644
--- a/source.go
+++ b/source.go
@@ -341,7 +341,12 @@ func (bs *baseSource) toRevOrErr(v Version) (r Revision, err error) {
 		if !bs.cvsync {
 			// call the lvfunc to sync the meta cache
 			_, err = bs.lvfunc()
+			if err != nil {
+				return
+			}
 		}
+
+		r = bs.dc.toRevision(v)
 		// If we still don't have a rev, then the version's no good
 		if r == "" {
 			err = fmt.Errorf("version %s does not exist in source %s", v, bs.crepo.r.Remote())

From 631f70855c82aae5a1f82859f86fa5803e73d720 Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Wed, 3 Aug 2016 11:53:51 -0400
Subject: [PATCH 36/71] s/baseSource/baseVCSSource/

---
 maybe_source.go |  6 +++---
 source.go       | 24 ++++++++++++------------
 2 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/maybe_source.go b/maybe_source.go
index 8b3596f..1c9180b 100644
--- a/maybe_source.go
+++ b/maybe_source.go
@@ -27,7 +27,7 @@ func (m maybeGitSource) try(cachedir string, an ProjectAnalyzer) (source, error)
 	}
 
 	src := &gitSource{
-		baseSource: baseSource{
+		baseVCSSource: baseVCSSource{
 			an: an,
 			dc: newMetaCache(),
 			crepo: &repo{
@@ -63,7 +63,7 @@ func (m maybeBzrSource) try(cachedir string, an ProjectAnalyzer) (source, error)
 	}
 
 	return &bzrSource{
-		baseSource: baseSource{
+		baseVCSSource: baseVCSSource{
 			an: an,
 			dc: newMetaCache(),
 			crepo: &repo{
@@ -90,7 +90,7 @@ func (m maybeHgSource) try(cachedir string, an ProjectAnalyzer) (source, error)
 	}
 
 	return &hgSource{
-		baseSource: baseSource{
+		baseVCSSource: baseVCSSource{
 			an: an,
 			dc: newMetaCache(),
 			crepo: &repo{
diff --git a/source.go b/source.go
index 5ec86f7..01ef29f 100644
--- a/source.go
+++ b/source.go
@@ -36,7 +36,7 @@ func newMetaCache() *sourceMetaCache {
 	}
 }
 
-type baseSource struct { // TODO(sdboyer) rename to baseVCSSource
+type baseVCSSource struct {
 	// Object for the cache repository
 	crepo *repo
 
@@ -59,7 +59,7 @@ type baseSource struct { // TODO(sdboyer) rename to baseVCSSource
 	lvfunc func() (vlist []Version, err error)
 }
 
-func (bs *baseSource) getManifestAndLock(r ProjectRoot, v Version) (Manifest, Lock, error) {
+func (bs *baseVCSSource) getManifestAndLock(r ProjectRoot, v Version) (Manifest, Lock, error) {
 	if err := bs.ensureCacheExistence(); err != nil {
 		return nil, nil, err
 	}
@@ -160,7 +160,7 @@ func (dc *sourceMetaCache) toUnpaired(v Version) UnpairedVersion {
 	}
 }
 
-func (bs *baseSource) listVersions() (vlist []Version, err error) {
+func (bs *baseVCSSource) listVersions() (vlist []Version, err error) {
 	if !bs.cvsync {
 		// This check only guarantees that the upstream exists, not the cache
 		bs.ex.s |= existsUpstream
@@ -200,7 +200,7 @@ func (bs *baseSource) listVersions() (vlist []Version, err error) {
 	return
 }
 
-func (bs *baseSource) revisionPresentIn(r Revision) (bool, error) {
+func (bs *baseVCSSource) revisionPresentIn(r Revision) (bool, error) {
 	// First and fastest path is to check the data cache to see if the rev is
 	// present. This could give us false positives, but the cases where that can
 	// occur would require a type of cache staleness that seems *exceedingly*
@@ -221,7 +221,7 @@ func (bs *baseSource) revisionPresentIn(r Revision) (bool, error) {
 	return bs.crepo.r.IsReference(string(r)), nil
 }
 
-func (bs *baseSource) ensureCacheExistence() error {
+func (bs *baseVCSSource) ensureCacheExistence() error {
 	// Technically, methods could could attempt to return straight from the
 	// metadata cache even if the repo cache doesn't exist on disk. But that
 	// would allow weird state inconsistencies (cache exists, but no repo...how
@@ -254,7 +254,7 @@ func (bs *baseSource) ensureCacheExistence() error {
 // Note that this may perform read-ish operations on the cache repo, and it
 // takes a lock accordingly. This makes it unsafe to call from a segment where
 // the cache repo mutex is already write-locked, as deadlock will occur.
-func (bs *baseSource) checkExistence(ex projectExistence) bool {
+func (bs *baseVCSSource) checkExistence(ex projectExistence) bool {
 	if bs.ex.s&ex != ex {
 		if ex&existsInVendorRoot != 0 && bs.ex.s&existsInVendorRoot == 0 {
 			panic("should now be implemented in bridge")
@@ -280,7 +280,7 @@ func (bs *baseSource) checkExistence(ex projectExistence) bool {
 	return ex&bs.ex.f == ex
 }
 
-func (bs *baseSource) listPackages(pr ProjectRoot, v Version) (ptree PackageTree, err error) {
+func (bs *baseVCSSource) listPackages(pr ProjectRoot, v Version) (ptree PackageTree, err error) {
 	if err = bs.ensureCacheExistence(); err != nil {
 		return
 	}
@@ -330,7 +330,7 @@ func (bs *baseSource) listPackages(pr ProjectRoot, v Version) (ptree PackageTree
 // updating the cache repo (if needed). It does not guarantee that the returned
 // Revision actually exists in the repository (as one of the cheaper methods may
 // have had bad data).
-func (bs *baseSource) toRevOrErr(v Version) (r Revision, err error) {
+func (bs *baseVCSSource) toRevOrErr(v Version) (r Revision, err error) {
 	r = bs.dc.toRevision(v)
 	if r == "" {
 		// Rev can be empty if:
@@ -356,14 +356,14 @@ func (bs *baseSource) toRevOrErr(v Version) (r Revision, err error) {
 	return
 }
 
-func (bs *baseSource) exportVersionTo(v Version, to string) error {
+func (bs *baseVCSSource) exportVersionTo(v Version, to string) error {
 	return bs.crepo.exportVersionTo(v, to)
 }
 
 // gitSource is a generic git repository implementation that should work with
 // all standard git remotes.
 type gitSource struct {
-	baseSource
+	baseVCSSource
 }
 
 func (s *gitSource) exportVersionTo(v Version, to string) error {
@@ -517,7 +517,7 @@ func (s *gitSource) listVersions() (vlist []Version, err error) {
 // bzrSource is a generic bzr repository implementation that should work with
 // all standard bazaar remotes.
 type bzrSource struct {
-	baseSource
+	baseVCSSource
 }
 
 func (s *bzrSource) listVersions() (vlist []Version, err error) {
@@ -588,7 +588,7 @@ func (s *bzrSource) listVersions() (vlist []Version, err error) {
 // hgSource is a generic hg repository implementation that should work with
 // all standard mercurial servers.
 type hgSource struct {
-	baseSource
+	baseVCSSource
 }
 
 func (s *hgSource) listVersions() (vlist []Version, err error) {

From e57f31187c3d116162d6b7cf38a6264586b53d10 Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Wed, 3 Aug 2016 14:20:15 -0400
Subject: [PATCH 37/71] Move vcs source subtypes into their own file

---
 flags.go           |   6 +-
 project_manager.go | 290 +--------------------
 source.go          | 349 +------------------------
 vcs_source.go      | 635 +++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 644 insertions(+), 636 deletions(-)
 create mode 100644 vcs_source.go

diff --git a/flags.go b/flags.go
index a7172c1..d9a3a1d 100644
--- a/flags.go
+++ b/flags.go
@@ -1,7 +1,7 @@
 package gps
 
-// projectExistence values represent the extent to which a project "exists."
-type projectExistence uint8
+// sourceExistence values represent the extent to which a project "exists."
+type sourceExistence uint8
 
 const (
 	// ExistsInVendorRoot indicates that a project exists in a vendor directory
@@ -19,7 +19,7 @@ const (
 	//
 	// In short, the information encoded in this flag should not be construed as
 	// exhaustive.
-	existsInVendorRoot projectExistence = 1 << iota
+	existsInVendorRoot sourceExistence = 1 << iota
 
 	// ExistsInCache indicates that a project exists on-disk in the local cache.
 	// It does not guarantee that an upstream exists, thus it cannot imply
diff --git a/project_manager.go b/project_manager.go
index b514dd3..992f6f3 100644
--- a/project_manager.go
+++ b/project_manager.go
@@ -1,17 +1,8 @@
 package gps
 
 import (
-	"bytes"
 	"fmt"
 	"go/build"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"strings"
-	"sync"
-
-	"github.com/Masterminds/vcs"
-	"github.com/termie/go-shutil"
 )
 
 type projectManager struct {
@@ -43,10 +34,10 @@ type projectManager struct {
 
 type existence struct {
 	// The existence levels for which a search/check has been performed
-	s projectExistence
+	s sourceExistence
 
 	// The existence levels verified to be present through searching
-	f projectExistence
+	f sourceExistence
 }
 
 // projectInfo holds manifest and lock
@@ -55,20 +46,6 @@ type projectInfo struct {
 	Lock
 }
 
-type repo struct {
-	// Path to the root of the default working copy (NOT the repo itself)
-	rpath string
-
-	// Mutex controlling general access to the repo
-	mut sync.RWMutex
-
-	// Object for direct repo interaction
-	r vcs.Repo
-
-	// Whether or not the cache repo is in sync (think dvcs) with upstream
-	synced bool
-}
-
 func (pm *projectManager) GetManifestAndLock(r ProjectRoot, v Version) (Manifest, Lock, error) {
 	if err := pm.ensureCacheExistence(); err != nil {
 		return nil, nil, err
@@ -297,7 +274,7 @@ func (pm *projectManager) RevisionPresentIn(pr ProjectRoot, r Revision) (bool, e
 // Note that this may perform read-ish operations on the cache repo, and it
 // takes a lock accordingly. Deadlock may result from calling it during a
 // segment where the cache repo mutex is already write-locked.
-func (pm *projectManager) CheckExistence(ex projectExistence) bool {
+func (pm *projectManager) CheckExistence(ex sourceExistence) bool {
 	if pm.ex.s&ex != ex {
 		if ex&existsInVendorRoot != 0 && pm.ex.s&existsInVendorRoot == 0 {
 			panic("should now be implemented in bridge")
@@ -326,264 +303,3 @@ func (pm *projectManager) CheckExistence(ex projectExistence) bool {
 func (pm *projectManager) ExportVersionTo(v Version, to string) error {
 	return pm.crepo.exportVersionTo(v, to)
 }
-
-func (r *repo) getCurrentVersionPairs() (vlist []PairedVersion, exbits projectExistence, err error) {
-	r.mut.Lock()
-	defer r.mut.Unlock()
-
-	switch r.r.(type) {
-	case *vcs.GitRepo:
-		var out []byte
-		c := exec.Command("git", "ls-remote", r.r.Remote())
-		// Ensure no terminal prompting for PWs
-		c.Env = mergeEnvLists([]string{"GIT_TERMINAL_PROMPT=0"}, os.Environ())
-		out, err = c.CombinedOutput()
-
-		all := bytes.Split(bytes.TrimSpace(out), []byte("\n"))
-		if err != nil || len(all) == 0 {
-			// TODO(sdboyer) remove this path? it really just complicates things, for
-			// probably not much benefit
-
-			// ls-remote failed, probably due to bad communication or a faulty
-			// upstream implementation. So fetch updates, then build the list
-			// locally
-			err = r.r.Update()
-			if err != nil {
-				// Definitely have a problem, now - bail out
-				return
-			}
-
-			// Upstream and cache must exist, so add that to exbits
-			exbits |= existsUpstream | existsInCache
-			// Also, local is definitely now synced
-			r.synced = true
-
-			out, err = r.r.RunFromDir("git", "show-ref", "--dereference")
-			if err != nil {
-				return
-			}
-
-			all = bytes.Split(bytes.TrimSpace(out), []byte("\n"))
-		}
-		// Local cache may not actually exist here, but upstream definitely does
-		exbits |= existsUpstream
-
-		tmap := make(map[string]PairedVersion)
-		for _, pair := range all {
-			var v PairedVersion
-			if string(pair[46:51]) == "heads" {
-				v = NewBranch(string(pair[52:])).Is(Revision(pair[:40])).(PairedVersion)
-				vlist = append(vlist, v)
-			} else if string(pair[46:50]) == "tags" {
-				vstr := string(pair[51:])
-				if strings.HasSuffix(vstr, "^{}") {
-					// If the suffix is there, then we *know* this is the rev of
-					// the underlying commit object that we actually want
-					vstr = strings.TrimSuffix(vstr, "^{}")
-				} else if _, exists := tmap[vstr]; exists {
-					// Already saw the deref'd version of this tag, if one
-					// exists, so skip this.
-					continue
-					// Can only hit this branch if we somehow got the deref'd
-					// version first. Which should be impossible, but this
-					// covers us in case of weirdness, anyway.
-				}
-				v = NewVersion(vstr).Is(Revision(pair[:40])).(PairedVersion)
-				tmap[vstr] = v
-			}
-		}
-
-		// Append all the deref'd (if applicable) tags into the list
-		for _, v := range tmap {
-			vlist = append(vlist, v)
-		}
-	case *vcs.BzrRepo:
-		var out []byte
-		// Update the local first
-		err = r.r.Update()
-		if err != nil {
-			return
-		}
-		// Upstream and cache must exist, so add that to exbits
-		exbits |= existsUpstream | existsInCache
-		// Also, local is definitely now synced
-		r.synced = true
-
-		// Now, list all the tags
-		out, err = r.r.RunFromDir("bzr", "tags", "--show-ids", "-v")
-		if err != nil {
-			return
-		}
-
-		all := bytes.Split(bytes.TrimSpace(out), []byte("\n"))
-		for _, line := range all {
-			idx := bytes.IndexByte(line, 32) // space
-			v := NewVersion(string(line[:idx])).Is(Revision(bytes.TrimSpace(line[idx:]))).(PairedVersion)
-			vlist = append(vlist, v)
-		}
-
-	case *vcs.HgRepo:
-		var out []byte
-		err = r.r.Update()
-		if err != nil {
-			return
-		}
-
-		// Upstream and cache must exist, so add that to exbits
-		exbits |= existsUpstream | existsInCache
-		// Also, local is definitely now synced
-		r.synced = true
-
-		out, err = r.r.RunFromDir("hg", "tags", "--debug", "--verbose")
-		if err != nil {
-			return
-		}
-
-		all := bytes.Split(bytes.TrimSpace(out), []byte("\n"))
-		lbyt := []byte("local")
-		nulrev := []byte("0000000000000000000000000000000000000000")
-		for _, line := range all {
-			if bytes.Equal(lbyt, line[len(line)-len(lbyt):]) {
-				// Skip local tags
-				continue
-			}
-
-			// tip is magic, don't include it
-			if bytes.HasPrefix(line, []byte("tip")) {
-				continue
-			}
-
-			// Split on colon; this gets us the rev and the tag plus local revno
-			pair := bytes.Split(line, []byte(":"))
-			if bytes.Equal(nulrev, pair[1]) {
-				// null rev indicates this tag is marked for deletion
-				continue
-			}
-
-			idx := bytes.IndexByte(pair[0], 32) // space
-			v := NewVersion(string(pair[0][:idx])).Is(Revision(pair[1])).(PairedVersion)
-			vlist = append(vlist, v)
-		}
-
-		out, err = r.r.RunFromDir("hg", "branches", "--debug", "--verbose")
-		if err != nil {
-			// better nothing than incomplete
-			vlist = nil
-			return
-		}
-
-		all = bytes.Split(bytes.TrimSpace(out), []byte("\n"))
-		lbyt = []byte("(inactive)")
-		for _, line := range all {
-			if bytes.Equal(lbyt, line[len(line)-len(lbyt):]) {
-				// Skip inactive branches
-				continue
-			}
-
-			// Split on colon; this gets us the rev and the branch plus local revno
-			pair := bytes.Split(line, []byte(":"))
-			idx := bytes.IndexByte(pair[0], 32) // space
-			v := NewBranch(string(pair[0][:idx])).Is(Revision(pair[1])).(PairedVersion)
-			vlist = append(vlist, v)
-		}
-	case *vcs.SvnRepo:
-		// TODO(sdboyer) is it ok to return empty vlist and no error?
-		// TODO(sdboyer) ...gotta do something for svn, right?
-	default:
-		panic("unknown repo type")
-	}
-
-	return
-}
-
-func (r *repo) exportVersionTo(v Version, to string) error {
-	r.mut.Lock()
-	defer r.mut.Unlock()
-
-	switch r.r.(type) {
-	case *vcs.GitRepo:
-		// Back up original index
-		idx, bak := filepath.Join(r.rpath, ".git", "index"), filepath.Join(r.rpath, ".git", "origindex")
-		err := os.Rename(idx, bak)
-		if err != nil {
-			return err
-		}
-
-		// TODO(sdboyer) could have an err here
-		defer os.Rename(bak, idx)
-
-		vstr := v.String()
-		if rv, ok := v.(PairedVersion); ok {
-			vstr = rv.Underlying().String()
-		}
-		_, err = r.r.RunFromDir("git", "read-tree", vstr)
-		if err != nil {
-			return err
-		}
-
-		// Ensure we have exactly one trailing slash
-		to = strings.TrimSuffix(to, string(os.PathSeparator)) + string(os.PathSeparator)
-		// Checkout from our temporary index to the desired target location on disk;
-		// now it's git's job to make it fast. Sadly, this approach *does* also
-		// write out vendor dirs. There doesn't appear to be a way to make
-		// checkout-index respect sparse checkout rules (-a supercedes it);
-		// the alternative is using plain checkout, though we have a bunch of
-		// housekeeping to do to set up, then tear down, the sparse checkout
-		// controls, as well as restore the original index and HEAD.
-		_, err = r.r.RunFromDir("git", "checkout-index", "-a", "--prefix="+to)
-		return err
-	default:
-		// TODO(sdboyer) This is a dumb, slow approach, but we're punting on making these
-		// fast for now because git is the OVERWHELMING case
-		r.r.UpdateVersion(v.String())
-
-		cfg := &shutil.CopyTreeOptions{
-			Symlinks:     true,
-			CopyFunction: shutil.Copy,
-			Ignore: func(src string, contents []os.FileInfo) (ignore []string) {
-				for _, fi := range contents {
-					if !fi.IsDir() {
-						continue
-					}
-					n := fi.Name()
-					switch n {
-					case "vendor", ".bzr", ".svn", ".hg":
-						ignore = append(ignore, n)
-					}
-				}
-
-				return
-			},
-		}
-
-		return shutil.CopyTree(r.rpath, to, cfg)
-	}
-}
-
-// This func copied from Masterminds/vcs so we can exec our own commands
-func mergeEnvLists(in, out []string) []string {
-NextVar:
-	for _, inkv := range in {
-		k := strings.SplitAfterN(inkv, "=", 2)[0]
-		for i, outkv := range out {
-			if strings.HasPrefix(outkv, k) {
-				out[i] = inkv
-				continue NextVar
-			}
-		}
-		out = append(out, inkv)
-	}
-	return out
-}
-
-func stripVendor(path string, info os.FileInfo, err error) error {
-	if info.Name() == "vendor" {
-		if _, err := os.Lstat(path); err == nil {
-			if info.IsDir() {
-				return removeAll(path)
-			}
-		}
-	}
-
-	return nil
-}
diff --git a/source.go b/source.go
index 01ef29f..1d431bc 100644
--- a/source.go
+++ b/source.go
@@ -1,16 +1,9 @@
 package gps
 
-import (
-	"bytes"
-	"fmt"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"strings"
-)
+import "fmt"
 
 type source interface {
-	checkExistence(projectExistence) bool
+	checkExistence(sourceExistence) bool
 	exportVersionTo(Version, string) error
 	getManifestAndLock(ProjectRoot, Version) (Manifest, Lock, error)
 	listPackages(ProjectRoot, Version) (PackageTree, error)
@@ -254,7 +247,7 @@ func (bs *baseVCSSource) ensureCacheExistence() error {
 // Note that this may perform read-ish operations on the cache repo, and it
 // takes a lock accordingly. This makes it unsafe to call from a segment where
 // the cache repo mutex is already write-locked, as deadlock will occur.
-func (bs *baseVCSSource) checkExistence(ex projectExistence) bool {
+func (bs *baseVCSSource) checkExistence(ex sourceExistence) bool {
 	if bs.ex.s&ex != ex {
 		if ex&existsInVendorRoot != 0 && bs.ex.s&existsInVendorRoot == 0 {
 			panic("should now be implemented in bridge")
@@ -359,339 +352,3 @@ func (bs *baseVCSSource) toRevOrErr(v Version) (r Revision, err error) {
 func (bs *baseVCSSource) exportVersionTo(v Version, to string) error {
 	return bs.crepo.exportVersionTo(v, to)
 }
-
-// gitSource is a generic git repository implementation that should work with
-// all standard git remotes.
-type gitSource struct {
-	baseVCSSource
-}
-
-func (s *gitSource) exportVersionTo(v Version, to string) error {
-	s.crepo.mut.Lock()
-	defer s.crepo.mut.Unlock()
-
-	r := s.crepo.r
-	// Back up original index
-	idx, bak := filepath.Join(r.LocalPath(), ".git", "index"), filepath.Join(r.LocalPath(), ".git", "origindex")
-	err := os.Rename(idx, bak)
-	if err != nil {
-		return err
-	}
-
-	// TODO(sdboyer) could have an err here
-	defer os.Rename(bak, idx)
-
-	vstr := v.String()
-	if rv, ok := v.(PairedVersion); ok {
-		vstr = rv.Underlying().String()
-	}
-	_, err = r.RunFromDir("git", "read-tree", vstr)
-	if err != nil {
-		return err
-	}
-
-	// Ensure we have exactly one trailing slash
-	to = strings.TrimSuffix(to, string(os.PathSeparator)) + string(os.PathSeparator)
-	// Checkout from our temporary index to the desired target location on disk;
-	// now it's git's job to make it fast. Sadly, this approach *does* also
-	// write out vendor dirs. There doesn't appear to be a way to make
-	// checkout-index respect sparse checkout rules (-a supercedes it);
-	// the alternative is using plain checkout, though we have a bunch of
-	// housekeeping to do to set up, then tear down, the sparse checkout
-	// controls, as well as restore the original index and HEAD.
-	_, err = r.RunFromDir("git", "checkout-index", "-a", "--prefix="+to)
-	return err
-}
-
-func (s *gitSource) listVersions() (vlist []Version, err error) {
-	if s.cvsync {
-		vlist = make([]Version, len(s.dc.vMap))
-		k := 0
-		for v, r := range s.dc.vMap {
-			vlist[k] = v.Is(r)
-			k++
-		}
-
-		return
-	}
-
-	r := s.crepo.r
-	var out []byte
-	c := exec.Command("git", "ls-remote", r.Remote())
-	// Ensure no terminal prompting for PWs
-	c.Env = mergeEnvLists([]string{"GIT_TERMINAL_PROMPT=0"}, os.Environ())
-	out, err = c.CombinedOutput()
-
-	all := bytes.Split(bytes.TrimSpace(out), []byte("\n"))
-	if err != nil || len(all) == 0 {
-		// TODO(sdboyer) remove this path? it really just complicates things, for
-		// probably not much benefit
-
-		// ls-remote failed, probably due to bad communication or a faulty
-		// upstream implementation. So fetch updates, then build the list
-		// locally
-		s.crepo.mut.Lock()
-		err = r.Update()
-		s.crepo.mut.Unlock()
-		if err != nil {
-			// Definitely have a problem, now - bail out
-			return
-		}
-
-		// Upstream and cache must exist for this to have worked, so add that to
-		// searched and found
-		s.ex.s |= existsUpstream | existsInCache
-		s.ex.f |= existsUpstream | existsInCache
-		// Also, local is definitely now synced
-		s.crepo.synced = true
-
-		s.crepo.mut.RLock()
-		out, err = r.RunFromDir("git", "show-ref", "--dereference")
-		s.crepo.mut.RUnlock()
-		if err != nil {
-			// TODO(sdboyer) More-er proper-er error
-			return
-		}
-
-		all = bytes.Split(bytes.TrimSpace(out), []byte("\n"))
-		if len(all) == 0 {
-			return nil, fmt.Errorf("no versions available for %s (this is weird)", r.Remote())
-		}
-	}
-
-	// Local cache may not actually exist here, but upstream definitely does
-	s.ex.s |= existsUpstream
-	s.ex.f |= existsUpstream
-
-	smap := make(map[string]bool)
-	uniq := 0
-	vlist = make([]Version, len(all)-1) // less 1, because always ignore HEAD
-	for _, pair := range all {
-		var v PairedVersion
-		if string(pair[46:51]) == "heads" {
-			v = NewBranch(string(pair[52:])).Is(Revision(pair[:40])).(PairedVersion)
-			vlist[uniq] = v
-			uniq++
-		} else if string(pair[46:50]) == "tags" {
-			vstr := string(pair[51:])
-			if strings.HasSuffix(vstr, "^{}") {
-				// If the suffix is there, then we *know* this is the rev of
-				// the underlying commit object that we actually want
-				vstr = strings.TrimSuffix(vstr, "^{}")
-			} else if smap[vstr] {
-				// Already saw the deref'd version of this tag, if one
-				// exists, so skip this.
-				continue
-				// Can only hit this branch if we somehow got the deref'd
-				// version first. Which should be impossible, but this
-				// covers us in case of weirdness, anyway.
-			}
-			v = NewVersion(vstr).Is(Revision(pair[:40])).(PairedVersion)
-			smap[vstr] = true
-			vlist[uniq] = v
-			uniq++
-		}
-	}
-
-	// Trim off excess from the slice
-	vlist = vlist[:uniq]
-
-	// Process the version data into the cache
-	//
-	// reset the rmap and vmap, as they'll be fully repopulated by this
-	// TODO(sdboyer) detect out-of-sync pairings as we do this?
-	s.dc.vMap = make(map[UnpairedVersion]Revision)
-	s.dc.rMap = make(map[Revision][]UnpairedVersion)
-
-	for _, v := range vlist {
-		pv := v.(PairedVersion)
-		u, r := pv.Unpair(), pv.Underlying()
-		s.dc.vMap[u] = r
-		s.dc.rMap[r] = append(s.dc.rMap[r], u)
-	}
-	// Mark the cache as being in sync with upstream's version list
-	s.cvsync = true
-	return
-}
-
-// bzrSource is a generic bzr repository implementation that should work with
-// all standard bazaar remotes.
-type bzrSource struct {
-	baseVCSSource
-}
-
-func (s *bzrSource) listVersions() (vlist []Version, err error) {
-	if s.cvsync {
-		vlist = make([]Version, len(s.dc.vMap))
-		k := 0
-		for v, r := range s.dc.vMap {
-			vlist[k] = v.Is(r)
-			k++
-		}
-
-		return
-	}
-
-	// Must first ensure cache checkout's existence
-	err = s.ensureCacheExistence()
-	if err != nil {
-		return
-	}
-	r := s.crepo.r
-
-	// Local repo won't have all the latest refs if ensureCacheExistence()
-	// didn't create it
-	if !s.crepo.synced {
-		s.crepo.mut.Lock()
-		err = r.Update()
-		s.crepo.mut.Unlock()
-		if err != nil {
-			return
-		}
-
-		s.crepo.synced = true
-	}
-
-	var out []byte
-
-	// Now, list all the tags
-	out, err = r.RunFromDir("bzr", "tags", "--show-ids", "-v")
-	if err != nil {
-		return
-	}
-
-	all := bytes.Split(bytes.TrimSpace(out), []byte("\n"))
-
-	// reset the rmap and vmap, as they'll be fully repopulated by this
-	// TODO(sdboyer) detect out-of-sync pairings as we do this?
-	s.dc.vMap = make(map[UnpairedVersion]Revision)
-	s.dc.rMap = make(map[Revision][]UnpairedVersion)
-
-	vlist = make([]Version, len(all))
-	k := 0
-	for _, line := range all {
-		idx := bytes.IndexByte(line, 32) // space
-		v := NewVersion(string(line[:idx]))
-		r := Revision(bytes.TrimSpace(line[idx:]))
-
-		s.dc.vMap[v] = r
-		s.dc.rMap[r] = append(s.dc.rMap[r], v)
-		vlist[k] = v.Is(r)
-		k++
-	}
-
-	// Cache is now in sync with upstream's version list
-	s.cvsync = true
-	return
-}
-
-// hgSource is a generic hg repository implementation that should work with
-// all standard mercurial servers.
-type hgSource struct {
-	baseVCSSource
-}
-
-func (s *hgSource) listVersions() (vlist []Version, err error) {
-	if s.cvsync {
-		vlist = make([]Version, len(s.dc.vMap))
-		k := 0
-		for v, r := range s.dc.vMap {
-			vlist[k] = v.Is(r)
-			k++
-		}
-
-		return
-	}
-
-	// Must first ensure cache checkout's existence
-	err = s.ensureCacheExistence()
-	if err != nil {
-		return
-	}
-	r := s.crepo.r
-
-	// Local repo won't have all the latest refs if ensureCacheExistence()
-	// didn't create it
-	if !s.crepo.synced {
-		s.crepo.mut.Lock()
-		err = r.Update()
-		s.crepo.mut.Unlock()
-		if err != nil {
-			return
-		}
-
-		s.crepo.synced = true
-	}
-
-	var out []byte
-
-	// Now, list all the tags
-	out, err = r.RunFromDir("hg", "tags", "--debug", "--verbose")
-	if err != nil {
-		return
-	}
-
-	all := bytes.Split(bytes.TrimSpace(out), []byte("\n"))
-	lbyt := []byte("local")
-	nulrev := []byte("0000000000000000000000000000000000000000")
-	for _, line := range all {
-		if bytes.Equal(lbyt, line[len(line)-len(lbyt):]) {
-			// Skip local tags
-			continue
-		}
-
-		// tip is magic, don't include it
-		if bytes.HasPrefix(line, []byte("tip")) {
-			continue
-		}
-
-		// Split on colon; this gets us the rev and the tag plus local revno
-		pair := bytes.Split(line, []byte(":"))
-		if bytes.Equal(nulrev, pair[1]) {
-			// null rev indicates this tag is marked for deletion
-			continue
-		}
-
-		idx := bytes.IndexByte(pair[0], 32) // space
-		v := NewVersion(string(pair[0][:idx])).Is(Revision(pair[1])).(PairedVersion)
-		vlist = append(vlist, v)
-	}
-
-	out, err = r.RunFromDir("hg", "branches", "--debug", "--verbose")
-	if err != nil {
-		// better nothing than partial and misleading
-		vlist = nil
-		return
-	}
-
-	all = bytes.Split(bytes.TrimSpace(out), []byte("\n"))
-	lbyt = []byte("(inactive)")
-	for _, line := range all {
-		if bytes.Equal(lbyt, line[len(line)-len(lbyt):]) {
-			// Skip inactive branches
-			continue
-		}
-
-		// Split on colon; this gets us the rev and the branch plus local revno
-		pair := bytes.Split(line, []byte(":"))
-		idx := bytes.IndexByte(pair[0], 32) // space
-		v := NewBranch(string(pair[0][:idx])).Is(Revision(pair[1])).(PairedVersion)
-		vlist = append(vlist, v)
-	}
-
-	// reset the rmap and vmap, as they'll be fully repopulated by this
-	// TODO(sdboyer) detect out-of-sync pairings as we do this?
-	s.dc.vMap = make(map[UnpairedVersion]Revision)
-	s.dc.rMap = make(map[Revision][]UnpairedVersion)
-
-	for _, v := range vlist {
-		pv := v.(PairedVersion)
-		u, r := pv.Unpair(), pv.Underlying()
-		s.dc.vMap[u] = r
-		s.dc.rMap[r] = append(s.dc.rMap[r], u)
-	}
-
-	// Cache is now in sync with upstream's version list
-	s.cvsync = true
-	return
-}
diff --git a/vcs_source.go b/vcs_source.go
new file mode 100644
index 0000000..3591c0d
--- /dev/null
+++ b/vcs_source.go
@@ -0,0 +1,635 @@
+package gps
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"sync"
+
+	"github.com/Masterminds/vcs"
+	"github.com/termie/go-shutil"
+)
+
+type vcsSource interface {
+	syncLocal() error
+	listLocalVersionPairs() ([]PairedVersion, sourceExistence, error)
+	listUpstreamVersionPairs() ([]PairedVersion, sourceExistence, error)
+	revisionPresentIn(Revision) (bool, error)
+	checkout(Version) error
+	ping() bool
+	ensureCacheExistence() error
+}
+
+// gitSource is a generic git repository implementation that should work with
+// all standard git remotes.
+type gitSource struct {
+	baseVCSSource
+}
+
+func (s *gitSource) exportVersionTo(v Version, to string) error {
+	s.crepo.mut.Lock()
+	defer s.crepo.mut.Unlock()
+
+	r := s.crepo.r
+	// Back up original index
+	idx, bak := filepath.Join(r.LocalPath(), ".git", "index"), filepath.Join(r.LocalPath(), ".git", "origindex")
+	err := os.Rename(idx, bak)
+	if err != nil {
+		return err
+	}
+
+	// TODO(sdboyer) could have an err here
+	defer os.Rename(bak, idx)
+
+	vstr := v.String()
+	if rv, ok := v.(PairedVersion); ok {
+		vstr = rv.Underlying().String()
+	}
+	_, err = r.RunFromDir("git", "read-tree", vstr)
+	if err != nil {
+		return err
+	}
+
+	// Ensure we have exactly one trailing slash
+	to = strings.TrimSuffix(to, string(os.PathSeparator)) + string(os.PathSeparator)
+	// Checkout from our temporary index to the desired target location on disk;
+	// now it's git's job to make it fast. Sadly, this approach *does* also
+	// write out vendor dirs. There doesn't appear to be a way to make
+	// checkout-index respect sparse checkout rules (-a supercedes it);
+	// the alternative is using plain checkout, though we have a bunch of
+	// housekeeping to do to set up, then tear down, the sparse checkout
+	// controls, as well as restore the original index and HEAD.
+	_, err = r.RunFromDir("git", "checkout-index", "-a", "--prefix="+to)
+	return err
+}
+
+func (s *gitSource) listVersions() (vlist []Version, err error) {
+	if s.cvsync {
+		vlist = make([]Version, len(s.dc.vMap))
+		k := 0
+		for v, r := range s.dc.vMap {
+			vlist[k] = v.Is(r)
+			k++
+		}
+
+		return
+	}
+
+	r := s.crepo.r
+	var out []byte
+	c := exec.Command("git", "ls-remote", r.Remote())
+	// Ensure no terminal prompting for PWs
+	c.Env = mergeEnvLists([]string{"GIT_TERMINAL_PROMPT=0"}, os.Environ())
+	out, err = c.CombinedOutput()
+
+	all := bytes.Split(bytes.TrimSpace(out), []byte("\n"))
+	if err != nil || len(all) == 0 {
+		// TODO(sdboyer) remove this path? it really just complicates things, for
+		// probably not much benefit
+
+		// ls-remote failed, probably due to bad communication or a faulty
+		// upstream implementation. So fetch updates, then build the list
+		// locally
+		s.crepo.mut.Lock()
+		err = r.Update()
+		s.crepo.mut.Unlock()
+		if err != nil {
+			// Definitely have a problem, now - bail out
+			return
+		}
+
+		// Upstream and cache must exist for this to have worked, so add that to
+		// searched and found
+		s.ex.s |= existsUpstream | existsInCache
+		s.ex.f |= existsUpstream | existsInCache
+		// Also, local is definitely now synced
+		s.crepo.synced = true
+
+		s.crepo.mut.RLock()
+		out, err = r.RunFromDir("git", "show-ref", "--dereference")
+		s.crepo.mut.RUnlock()
+		if err != nil {
+			// TODO(sdboyer) More-er proper-er error
+			return
+		}
+
+		all = bytes.Split(bytes.TrimSpace(out), []byte("\n"))
+		if len(all) == 0 {
+			return nil, fmt.Errorf("no versions available for %s (this is weird)", r.Remote())
+		}
+	}
+
+	// Local cache may not actually exist here, but upstream definitely does
+	s.ex.s |= existsUpstream
+	s.ex.f |= existsUpstream
+
+	smap := make(map[string]bool)
+	uniq := 0
+	vlist = make([]Version, len(all)-1) // less 1, because always ignore HEAD
+	for _, pair := range all {
+		var v PairedVersion
+		if string(pair[46:51]) == "heads" {
+			v = NewBranch(string(pair[52:])).Is(Revision(pair[:40])).(PairedVersion)
+			vlist[uniq] = v
+			uniq++
+		} else if string(pair[46:50]) == "tags" {
+			vstr := string(pair[51:])
+			if strings.HasSuffix(vstr, "^{}") {
+				// If the suffix is there, then we *know* this is the rev of
+				// the underlying commit object that we actually want
+				vstr = strings.TrimSuffix(vstr, "^{}")
+			} else if smap[vstr] {
+				// Already saw the deref'd version of this tag, if one
+				// exists, so skip this.
+				continue
+				// Can only hit this branch if we somehow got the deref'd
+				// version first. Which should be impossible, but this
+				// covers us in case of weirdness, anyway.
+			}
+			v = NewVersion(vstr).Is(Revision(pair[:40])).(PairedVersion)
+			smap[vstr] = true
+			vlist[uniq] = v
+			uniq++
+		}
+	}
+
+	// Trim off excess from the slice
+	vlist = vlist[:uniq]
+
+	// Process the version data into the cache
+	//
+	// reset the rmap and vmap, as they'll be fully repopulated by this
+	// TODO(sdboyer) detect out-of-sync pairings as we do this?
+	s.dc.vMap = make(map[UnpairedVersion]Revision)
+	s.dc.rMap = make(map[Revision][]UnpairedVersion)
+
+	for _, v := range vlist {
+		pv := v.(PairedVersion)
+		u, r := pv.Unpair(), pv.Underlying()
+		s.dc.vMap[u] = r
+		s.dc.rMap[r] = append(s.dc.rMap[r], u)
+	}
+	// Mark the cache as being in sync with upstream's version list
+	s.cvsync = true
+	return
+}
+
+// bzrSource is a generic bzr repository implementation that should work with
+// all standard bazaar remotes.
+type bzrSource struct {
+	baseVCSSource
+}
+
+func (s *bzrSource) listVersions() (vlist []Version, err error) {
+	if s.cvsync {
+		vlist = make([]Version, len(s.dc.vMap))
+		k := 0
+		for v, r := range s.dc.vMap {
+			vlist[k] = v.Is(r)
+			k++
+		}
+
+		return
+	}
+
+	// Must first ensure cache checkout's existence
+	err = s.ensureCacheExistence()
+	if err != nil {
+		return
+	}
+	r := s.crepo.r
+
+	// Local repo won't have all the latest refs if ensureCacheExistence()
+	// didn't create it
+	if !s.crepo.synced {
+		s.crepo.mut.Lock()
+		err = r.Update()
+		s.crepo.mut.Unlock()
+		if err != nil {
+			return
+		}
+
+		s.crepo.synced = true
+	}
+
+	var out []byte
+
+	// Now, list all the tags
+	out, err = r.RunFromDir("bzr", "tags", "--show-ids", "-v")
+	if err != nil {
+		return
+	}
+
+	all := bytes.Split(bytes.TrimSpace(out), []byte("\n"))
+
+	// reset the rmap and vmap, as they'll be fully repopulated by this
+	// TODO(sdboyer) detect out-of-sync pairings as we do this?
+	s.dc.vMap = make(map[UnpairedVersion]Revision)
+	s.dc.rMap = make(map[Revision][]UnpairedVersion)
+
+	vlist = make([]Version, len(all))
+	k := 0
+	for _, line := range all {
+		idx := bytes.IndexByte(line, 32) // space
+		v := NewVersion(string(line[:idx]))
+		r := Revision(bytes.TrimSpace(line[idx:]))
+
+		s.dc.vMap[v] = r
+		s.dc.rMap[r] = append(s.dc.rMap[r], v)
+		vlist[k] = v.Is(r)
+		k++
+	}
+
+	// Cache is now in sync with upstream's version list
+	s.cvsync = true
+	return
+}
+
+// hgSource is a generic hg repository implementation that should work with
+// all standard mercurial servers.
+type hgSource struct {
+	baseVCSSource
+}
+
+func (s *hgSource) listVersions() (vlist []Version, err error) {
+	if s.cvsync {
+		vlist = make([]Version, len(s.dc.vMap))
+		k := 0
+		for v, r := range s.dc.vMap {
+			vlist[k] = v.Is(r)
+			k++
+		}
+
+		return
+	}
+
+	// Must first ensure cache checkout's existence
+	err = s.ensureCacheExistence()
+	if err != nil {
+		return
+	}
+	r := s.crepo.r
+
+	// Local repo won't have all the latest refs if ensureCacheExistence()
+	// didn't create it
+	if !s.crepo.synced {
+		s.crepo.mut.Lock()
+		err = r.Update()
+		s.crepo.mut.Unlock()
+		if err != nil {
+			return
+		}
+
+		s.crepo.synced = true
+	}
+
+	var out []byte
+
+	// Now, list all the tags
+	out, err = r.RunFromDir("hg", "tags", "--debug", "--verbose")
+	if err != nil {
+		return
+	}
+
+	all := bytes.Split(bytes.TrimSpace(out), []byte("\n"))
+	lbyt := []byte("local")
+	nulrev := []byte("0000000000000000000000000000000000000000")
+	for _, line := range all {
+		if bytes.Equal(lbyt, line[len(line)-len(lbyt):]) {
+			// Skip local tags
+			continue
+		}
+
+		// tip is magic, don't include it
+		if bytes.HasPrefix(line, []byte("tip")) {
+			continue
+		}
+
+		// Split on colon; this gets us the rev and the tag plus local revno
+		pair := bytes.Split(line, []byte(":"))
+		if bytes.Equal(nulrev, pair[1]) {
+			// null rev indicates this tag is marked for deletion
+			continue
+		}
+
+		idx := bytes.IndexByte(pair[0], 32) // space
+		v := NewVersion(string(pair[0][:idx])).Is(Revision(pair[1])).(PairedVersion)
+		vlist = append(vlist, v)
+	}
+
+	out, err = r.RunFromDir("hg", "branches", "--debug", "--verbose")
+	if err != nil {
+		// better nothing than partial and misleading
+		vlist = nil
+		return
+	}
+
+	all = bytes.Split(bytes.TrimSpace(out), []byte("\n"))
+	lbyt = []byte("(inactive)")
+	for _, line := range all {
+		if bytes.Equal(lbyt, line[len(line)-len(lbyt):]) {
+			// Skip inactive branches
+			continue
+		}
+
+		// Split on colon; this gets us the rev and the branch plus local revno
+		pair := bytes.Split(line, []byte(":"))
+		idx := bytes.IndexByte(pair[0], 32) // space
+		v := NewBranch(string(pair[0][:idx])).Is(Revision(pair[1])).(PairedVersion)
+		vlist = append(vlist, v)
+	}
+
+	// reset the rmap and vmap, as they'll be fully repopulated by this
+	// TODO(sdboyer) detect out-of-sync pairings as we do this?
+	s.dc.vMap = make(map[UnpairedVersion]Revision)
+	s.dc.rMap = make(map[Revision][]UnpairedVersion)
+
+	for _, v := range vlist {
+		pv := v.(PairedVersion)
+		u, r := pv.Unpair(), pv.Underlying()
+		s.dc.vMap[u] = r
+		s.dc.rMap[r] = append(s.dc.rMap[r], u)
+	}
+
+	// Cache is now in sync with upstream's version list
+	s.cvsync = true
+	return
+}
+
+type repo struct {
+	// Path to the root of the default working copy (NOT the repo itself)
+	rpath string
+
+	// Mutex controlling general access to the repo
+	mut sync.RWMutex
+
+	// Object for direct repo interaction
+	r vcs.Repo
+
+	// Whether or not the cache repo is in sync (think dvcs) with upstream
+	synced bool
+}
+
+func (r *repo) getCurrentVersionPairs() (vlist []PairedVersion, exbits sourceExistence, err error) {
+	r.mut.Lock()
+	defer r.mut.Unlock()
+
+	switch r.r.(type) {
+	case *vcs.GitRepo:
+		var out []byte
+		c := exec.Command("git", "ls-remote", r.r.Remote())
+		// Ensure no terminal prompting for PWs
+		c.Env = mergeEnvLists([]string{"GIT_TERMINAL_PROMPT=0"}, os.Environ())
+		out, err = c.CombinedOutput()
+
+		all := bytes.Split(bytes.TrimSpace(out), []byte("\n"))
+		if err != nil || len(all) == 0 {
+			// TODO(sdboyer) remove this path? it really just complicates things, for
+			// probably not much benefit
+
+			// ls-remote failed, probably due to bad communication or a faulty
+			// upstream implementation. So fetch updates, then build the list
+			// locally
+			err = r.r.Update()
+			if err != nil {
+				// Definitely have a problem, now - bail out
+				return
+			}
+
+			// Upstream and cache must exist, so add that to exbits
+			exbits |= existsUpstream | existsInCache
+			// Also, local is definitely now synced
+			r.synced = true
+
+			out, err = r.r.RunFromDir("git", "show-ref", "--dereference")
+			if err != nil {
+				return
+			}
+
+			all = bytes.Split(bytes.TrimSpace(out), []byte("\n"))
+		}
+		// Local cache may not actually exist here, but upstream definitely does
+		exbits |= existsUpstream
+
+		tmap := make(map[string]PairedVersion)
+		for _, pair := range all {
+			var v PairedVersion
+			if string(pair[46:51]) == "heads" {
+				v = NewBranch(string(pair[52:])).Is(Revision(pair[:40])).(PairedVersion)
+				vlist = append(vlist, v)
+			} else if string(pair[46:50]) == "tags" {
+				vstr := string(pair[51:])
+				if strings.HasSuffix(vstr, "^{}") {
+					// If the suffix is there, then we *know* this is the rev of
+					// the underlying commit object that we actually want
+					vstr = strings.TrimSuffix(vstr, "^{}")
+				} else if _, exists := tmap[vstr]; exists {
+					// Already saw the deref'd version of this tag, if one
+					// exists, so skip this.
+					continue
+					// Can only hit this branch if we somehow got the deref'd
+					// version first. Which should be impossible, but this
+					// covers us in case of weirdness, anyway.
+				}
+				v = NewVersion(vstr).Is(Revision(pair[:40])).(PairedVersion)
+				tmap[vstr] = v
+			}
+		}
+
+		// Append all the deref'd (if applicable) tags into the list
+		for _, v := range tmap {
+			vlist = append(vlist, v)
+		}
+	case *vcs.BzrRepo:
+		var out []byte
+		// Update the local first
+		err = r.r.Update()
+		if err != nil {
+			return
+		}
+		// Upstream and cache must exist, so add that to exbits
+		exbits |= existsUpstream | existsInCache
+		// Also, local is definitely now synced
+		r.synced = true
+
+		// Now, list all the tags
+		out, err = r.r.RunFromDir("bzr", "tags", "--show-ids", "-v")
+		if err != nil {
+			return
+		}
+
+		all := bytes.Split(bytes.TrimSpace(out), []byte("\n"))
+		for _, line := range all {
+			idx := bytes.IndexByte(line, 32) // space
+			v := NewVersion(string(line[:idx])).Is(Revision(bytes.TrimSpace(line[idx:]))).(PairedVersion)
+			vlist = append(vlist, v)
+		}
+
+	case *vcs.HgRepo:
+		var out []byte
+		err = r.r.Update()
+		if err != nil {
+			return
+		}
+
+		// Upstream and cache must exist, so add that to exbits
+		exbits |= existsUpstream | existsInCache
+		// Also, local is definitely now synced
+		r.synced = true
+
+		out, err = r.r.RunFromDir("hg", "tags", "--debug", "--verbose")
+		if err != nil {
+			return
+		}
+
+		all := bytes.Split(bytes.TrimSpace(out), []byte("\n"))
+		lbyt := []byte("local")
+		nulrev := []byte("0000000000000000000000000000000000000000")
+		for _, line := range all {
+			if bytes.Equal(lbyt, line[len(line)-len(lbyt):]) {
+				// Skip local tags
+				continue
+			}
+
+			// tip is magic, don't include it
+			if bytes.HasPrefix(line, []byte("tip")) {
+				continue
+			}
+
+			// Split on colon; this gets us the rev and the tag plus local revno
+			pair := bytes.Split(line, []byte(":"))
+			if bytes.Equal(nulrev, pair[1]) {
+				// null rev indicates this tag is marked for deletion
+				continue
+			}
+
+			idx := bytes.IndexByte(pair[0], 32) // space
+			v := NewVersion(string(pair[0][:idx])).Is(Revision(pair[1])).(PairedVersion)
+			vlist = append(vlist, v)
+		}
+
+		out, err = r.r.RunFromDir("hg", "branches", "--debug", "--verbose")
+		if err != nil {
+			// better nothing than incomplete
+			vlist = nil
+			return
+		}
+
+		all = bytes.Split(bytes.TrimSpace(out), []byte("\n"))
+		lbyt = []byte("(inactive)")
+		for _, line := range all {
+			if bytes.Equal(lbyt, line[len(line)-len(lbyt):]) {
+				// Skip inactive branches
+				continue
+			}
+
+			// Split on colon; this gets us the rev and the branch plus local revno
+			pair := bytes.Split(line, []byte(":"))
+			idx := bytes.IndexByte(pair[0], 32) // space
+			v := NewBranch(string(pair[0][:idx])).Is(Revision(pair[1])).(PairedVersion)
+			vlist = append(vlist, v)
+		}
+	case *vcs.SvnRepo:
+		// TODO(sdboyer) is it ok to return empty vlist and no error?
+		// TODO(sdboyer) ...gotta do something for svn, right?
+	default:
+		panic("unknown repo type")
+	}
+
+	return
+}
+
+func (r *repo) exportVersionTo(v Version, to string) error {
+	r.mut.Lock()
+	defer r.mut.Unlock()
+
+	switch r.r.(type) {
+	case *vcs.GitRepo:
+		// Back up original index
+		idx, bak := filepath.Join(r.rpath, ".git", "index"), filepath.Join(r.rpath, ".git", "origindex")
+		err := os.Rename(idx, bak)
+		if err != nil {
+			return err
+		}
+
+		// TODO(sdboyer) could have an err here
+		defer os.Rename(bak, idx)
+
+		vstr := v.String()
+		if rv, ok := v.(PairedVersion); ok {
+			vstr = rv.Underlying().String()
+		}
+		_, err = r.r.RunFromDir("git", "read-tree", vstr)
+		if err != nil {
+			return err
+		}
+
+		// Ensure we have exactly one trailing slash
+		to = strings.TrimSuffix(to, string(os.PathSeparator)) + string(os.PathSeparator)
+		// Checkout from our temporary index to the desired target location on disk;
+		// now it's git's job to make it fast. Sadly, this approach *does* also
+		// write out vendor dirs. There doesn't appear to be a way to make
+		// checkout-index respect sparse checkout rules (-a supercedes it);
+		// the alternative is using plain checkout, though we have a bunch of
+		// housekeeping to do to set up, then tear down, the sparse checkout
+		// controls, as well as restore the original index and HEAD.
+		_, err = r.r.RunFromDir("git", "checkout-index", "-a", "--prefix="+to)
+		return err
+	default:
+		// TODO(sdboyer) This is a dumb, slow approach, but we're punting on making these
+		// fast for now because git is the OVERWHELMING case
+		r.r.UpdateVersion(v.String())
+
+		cfg := &shutil.CopyTreeOptions{
+			Symlinks:     true,
+			CopyFunction: shutil.Copy,
+			Ignore: func(src string, contents []os.FileInfo) (ignore []string) {
+				for _, fi := range contents {
+					if !fi.IsDir() {
+						continue
+					}
+					n := fi.Name()
+					switch n {
+					case "vendor", ".bzr", ".svn", ".hg":
+						ignore = append(ignore, n)
+					}
+				}
+
+				return
+			},
+		}
+
+		return shutil.CopyTree(r.rpath, to, cfg)
+	}
+}
+
+// This func copied from Masterminds/vcs so we can exec our own commands
+func mergeEnvLists(in, out []string) []string {
+NextVar:
+	for _, inkv := range in {
+		k := strings.SplitAfterN(inkv, "=", 2)[0]
+		for i, outkv := range out {
+			if strings.HasPrefix(outkv, k) {
+				out[i] = inkv
+				continue NextVar
+			}
+		}
+		out = append(out, inkv)
+	}
+	return out
+}
+
+func stripVendor(path string, info os.FileInfo, err error) error {
+	if info.Name() == "vendor" {
+		if _, err := os.Lstat(path); err == nil {
+			if info.IsDir() {
+				return removeAll(path)
+			}
+		}
+	}
+
+	return nil
+}

From 4072c90c565531aca5cc7991986295065fa6ae0b Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Wed, 3 Aug 2016 22:07:34 -0400
Subject: [PATCH 38/71] Remove build.Context entirely from SourceMgr

Liberation!
---
 source_manager.go | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/source_manager.go b/source_manager.go
index 87df464..cc0799c 100644
--- a/source_manager.go
+++ b/source_manager.go
@@ -3,7 +3,6 @@ package gps
 import (
 	"encoding/json"
 	"fmt"
-	"go/build"
 	"os"
 	"path/filepath"
 	"strings"
@@ -85,7 +84,6 @@ type SourceMgr struct {
 	}
 	rmut sync.RWMutex
 	an   ProjectAnalyzer
-	ctx  build.Context
 }
 
 var _ SourceManager = &SourceMgr{}
@@ -136,10 +134,6 @@ func NewSourceManager(an ProjectAnalyzer, cachedir string, force bool) (*SourceM
 		return nil, fmt.Errorf("failed to create global cache lock file at %s with err %s", glpath, err)
 	}
 
-	ctx := build.Default
-	// Replace GOPATH with our cache dir
-	ctx.GOPATH = cachedir
-
 	return &SourceMgr{
 		cachedir: cachedir,
 		pms:      make(map[string]*pmState),
@@ -147,8 +141,7 @@ func NewSourceManager(an ProjectAnalyzer, cachedir string, force bool) (*SourceM
 			rr  *remoteRepo
 			err error
 		}),
-		ctx: ctx,
-		an:  an,
+		an: an,
 	}, nil
 }
 

From 6f5bcdd1d28533abde1145ccb510aa17b47d9ac7 Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Fri, 5 Aug 2016 00:57:22 -0400
Subject: [PATCH 39/71] Add sourceFailures to hold multiple try() fails

---
 maybe_source.go | 28 +++++++++++++++++++++++++---
 1 file changed, 25 insertions(+), 3 deletions(-)

diff --git a/maybe_source.go b/maybe_source.go
index 1c9180b..19fb961 100644
--- a/maybe_source.go
+++ b/maybe_source.go
@@ -1,6 +1,7 @@
 package gps
 
 import (
+	"bytes"
 	"fmt"
 	"net/url"
 	"path/filepath"
@@ -14,8 +15,31 @@ type maybeSource interface {
 
 type maybeSources []maybeSource
 
+func (mbs maybeSources) try(cachedir string, an ProjectAnalyzer) (source, error) {
+	var e sourceFailures
+	for _, mb := range mbs {
+		src, err := mb.try(cachedir, an)
+		if err == nil {
+			return src, nil
+		}
+		e = append(e, err)
+	}
+	return nil, e
+}
+
+type sourceFailures []error
+
+func (sf sourceFailures) Error() string {
+	var buf bytes.Buffer
+	fmt.Fprintf(&buf, "No valid source could be created:\n")
+	for _, e := range sf {
+		fmt.Fprintf(&buf, "\t%s", e.Error())
+	}
+
+	return buf.String()
+}
+
 type maybeGitSource struct {
-	n   string
 	url *url.URL
 }
 
@@ -48,7 +72,6 @@ func (m maybeGitSource) try(cachedir string, an ProjectAnalyzer) (source, error)
 }
 
 type maybeBzrSource struct {
-	n   string
 	url *url.URL
 }
 
@@ -75,7 +98,6 @@ func (m maybeBzrSource) try(cachedir string, an ProjectAnalyzer) (source, error)
 }
 
 type maybeHgSource struct {
-	n   string
 	url *url.URL
 }
 

From 058b587cd1e8f3fecc0dc0bba1eab988c098ec08 Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Fri, 5 Aug 2016 00:58:45 -0400
Subject: [PATCH 40/71] Introduce futures for import path interpretation

It's important for prefetchingthat we can defer the discovery process
of sources for import paths (which inevitably involves some network interaction)
into a background goroutine. At the same time, it's also crucial that we
can choose whether or not to parallelize such things from outside (the
caller). This futures implementation satisfies both requirements.
---
 remote.go         | 745 +++++++++++++++++++++++++++++++++++-----------
 source.go         |  42 +++
 source_manager.go |   9 +-
 source_test.go    |   3 -
 4 files changed, 621 insertions(+), 178 deletions(-)

diff --git a/remote.go b/remote.go
index d28c5e9..e041cd1 100644
--- a/remote.go
+++ b/remote.go
@@ -29,6 +29,29 @@ var (
 	svnSchemes = []string{"https", "http", "svn", "svn+ssh"}
 )
 
+func validateVCSScheme(scheme, typ string) bool {
+	var schemes []string
+	switch typ {
+	case "git":
+		schemes = gitSchemes
+	case "bzr":
+		schemes = bzrSchemes
+	case "hg":
+		schemes = hgSchemes
+	case "svn":
+		schemes = svnSchemes
+	default:
+		panic(fmt.Sprint("unsupported vcs type", scheme))
+	}
+
+	for _, valid := range schemes {
+		if scheme == valid {
+			return true
+		}
+	}
+	return false
+}
+
 // Regexes for the different known import path flavors
 var (
 	// This regex allowed some usernames that github currently disallows. They
@@ -43,9 +66,9 @@ var (
 	//glpRegex = regexp.MustCompile(`^(?P<root>git\.launchpad\.net/([A-Za-z0-9_.\-]+)|~[A-Za-z0-9_.\-]+/(\+git|[A-Za-z0-9_.\-]+)/[A-Za-z0-9_.\-]+)$`)
 	glpRegex = regexp.MustCompile(`^(?P<root>git\.launchpad\.net/([A-Za-z0-9_.\-]+))((?:/[A-Za-z0-9_.\-]+)*)$`)
 	//gcRegex      = regexp.MustCompile(`^(?P<root>code\.google\.com/[pr]/(?P<project>[a-z0-9\-]+)(\.(?P<subrepo>[a-z0-9\-]+))?)(/[A-Za-z0-9_.\-]+)*$`)
-	jazzRegex    = regexp.MustCompile(`^(?P<root>hub\.jazz\.net/(git/[a-z0-9]+/[A-Za-z0-9_.\-]+))((?:/[A-Za-z0-9_.\-]+)*)$`)
-	apacheRegex  = regexp.MustCompile(`^(?P<root>git\.apache\.org/([a-z0-9_.\-]+\.git))((?:/[A-Za-z0-9_.\-]+)*)$`)
-	genericRegex = regexp.MustCompile(`^(?P<root>(?P<repo>([a-z0-9.\-]+\.)+[a-z0-9.\-]+(:[0-9]+)?/[A-Za-z0-9_.\-/~]*?)\.(?P<vcs>bzr|git|hg|svn))((?:/[A-Za-z0-9_.\-]+)*)$`)
+	jazzRegex         = regexp.MustCompile(`^(?P<root>hub\.jazz\.net/(git/[a-z0-9]+/[A-Za-z0-9_.\-]+))((?:/[A-Za-z0-9_.\-]+)*)$`)
+	apacheRegex       = regexp.MustCompile(`^(?P<root>git\.apache\.org/([a-z0-9_.\-]+\.git))((?:/[A-Za-z0-9_.\-]+)*)$`)
+	vcsExtensionRegex = regexp.MustCompile(`^(?P<root>(?P<repo>([a-z0-9.\-]+\.)+[a-z0-9.\-]+(:[0-9]+)?/[A-Za-z0-9_.\-/~]*?)\.(?P<vcs>bzr|git|hg|svn))((?:/[A-Za-z0-9_.\-]+)*)$`)
 )
 
 // Other helper regexes
@@ -54,221 +77,599 @@ var (
 	pathvld     = regexp.MustCompile(`^([A-Za-z0-9-]+)(\.[A-Za-z0-9-]+)+(/[A-Za-z0-9-_.~]+)*$`)
 )
 
-// deduceRemoteRepo takes a potential import path and returns a RemoteRepo
-// representing the remote location of the source of an import path. Remote
-// repositories can be bare import paths, or urls including a checkout scheme.
-func deduceRemoteRepo(path string) (rr *remoteRepo, err error) {
-	rr = &remoteRepo{}
-	if m := scpSyntaxRe.FindStringSubmatch(path); m != nil {
-		// Match SCP-like syntax and convert it to a URL.
-		// Eg, "git@github.com:user/repo" becomes
-		// "ssh://git@github.com/user/repo".
-		rr.CloneURL = &url.URL{
-			Scheme: "ssh",
-			User:   url.User(m[1]),
-			Host:   m[2],
-			Path:   "/" + m[3],
-			// TODO(sdboyer) This is what stdlib sets; grok why better
-			//RawPath: m[3],
+func simpleStringFuture(s string) futureString {
+	return func() (string, error) {
+		return s, nil
+	}
+}
+
+func sourceFutureFactory(mb maybeSource) func(string, ProjectAnalyzer) futureSource {
+	return func(cachedir string, an ProjectAnalyzer) futureSource {
+		var src source
+		var err error
+
+		c := make(chan struct{}, 1)
+		go func() {
+			defer close(c)
+			src, err = mb.try(cachedir, an)
+		}()
+
+		return func() (source, error) {
+			<-c
+			return src, err
 		}
-	} else {
-		rr.CloneURL, err = url.Parse(path)
-		if err != nil {
-			return nil, fmt.Errorf("%q is not a valid import path", path)
+	}
+}
+
+type matcher interface {
+	deduceRoot(string) (futureString, error)
+	deduceSource(string, *url.URL) (func(string, ProjectAnalyzer) futureSource, error)
+}
+
+type githubMatcher struct {
+	regexp *regexp.Regexp
+}
+
+func (m githubMatcher) deduceRoot(path string) (futureString, error) {
+	v := m.regexp.FindStringSubmatch(path)
+	if v == nil {
+		return nil, fmt.Errorf("%s is not a valid path for a source on github.com", path)
+	}
+
+	return simpleStringFuture("github.com/" + v[2]), nil
+}
+
+func (m githubMatcher) deduceSource(path string, u *url.URL) (func(string, ProjectAnalyzer) futureSource, error) {
+	v := m.regexp.FindStringSubmatch(path)
+	if v == nil {
+		return nil, fmt.Errorf("%s is not a valid path for a source on github.com", path)
+	}
+
+	u.Path = v[2]
+	if u.Scheme != "" {
+		if !validateVCSScheme(u.Scheme, "git") {
+			return nil, fmt.Errorf("%s is not a valid scheme for accessing a git repository", u.Scheme)
 		}
+		return sourceFutureFactory(maybeGitSource{url: u}), nil
 	}
 
-	if rr.CloneURL.Host != "" {
-		path = rr.CloneURL.Host + "/" + strings.TrimPrefix(rr.CloneURL.Path, "/")
+	mb := make(maybeSources, len(gitSchemes))
+	for k, scheme := range gitSchemes {
+		u2 := *u
+		u2.Scheme = scheme
+		mb[k] = maybeGitSource{url: &u2}
+	}
+
+	return sourceFutureFactory(mb), nil
+}
+
+type bitbucketMatcher struct {
+	regexp *regexp.Regexp
+}
+
+func (m bitbucketMatcher) deduceRoot(path string) (futureString, error) {
+	v := m.regexp.FindStringSubmatch(path)
+	if v == nil {
+		return nil, fmt.Errorf("%s is not a valid path for a source on bitbucket.org", path)
+	}
+
+	return simpleStringFuture("bitbucket.org/" + v[2]), nil
+}
+
+func (m bitbucketMatcher) deduceSource(path string, u *url.URL) (func(string, ProjectAnalyzer) futureSource, error) {
+	v := m.regexp.FindStringSubmatch(path)
+	if v == nil {
+		return nil, fmt.Errorf("%s is not a valid path for a source on bitbucket.org", path)
+	}
+	u.Path = v[2]
+
+	// This isn't definitive, but it'll probably catch most
+	isgit := strings.HasSuffix(u.Path, ".git") || u.User.Username() == "git"
+	ishg := strings.HasSuffix(u.Path, ".hg") || u.User.Username() == "hg"
+
+	if u.Scheme != "" {
+		validgit, validhg := validateVCSScheme(u.Scheme, "git"), validateVCSScheme(u.Scheme, "hg")
+		if isgit {
+			if !validgit {
+				return nil, fmt.Errorf("%s is not a valid scheme for accessing a git repository", u.Scheme)
+			}
+			return sourceFutureFactory(maybeGitSource{url: u}), nil
+		} else if ishg {
+			if !validhg {
+				return nil, fmt.Errorf("%s is not a valid scheme for accessing an hg repository", u.Scheme)
+			}
+			return sourceFutureFactory(maybeHgSource{url: u}), nil
+		} else if !validgit && !validhg {
+			return nil, fmt.Errorf("%s is not a valid scheme for accessing either a git or hg repository", u.Scheme)
+		}
+
+		// No other choice, make an option for both git and hg
+		return sourceFutureFactory(maybeSources{
+			// Git first, because it's a) faster and b) git
+			maybeGitSource{url: u},
+			maybeHgSource{url: u},
+		}), nil
+	}
+
+	mb := make(maybeSources, 0)
+	if !ishg {
+		for _, scheme := range gitSchemes {
+			u2 := *u
+			u2.Scheme = scheme
+			mb = append(mb, maybeGitSource{url: &u2})
+		}
+	}
+
+	if !isgit {
+		for _, scheme := range hgSchemes {
+			u2 := *u
+			u2.Scheme = scheme
+			mb = append(mb, maybeHgSource{url: &u2})
+		}
+	}
+
+	return sourceFutureFactory(mb), nil
+}
+
+type gopkginMatcher struct {
+	regexp *regexp.Regexp
+}
+
+func (m gopkginMatcher) deduceRoot(path string) (futureString, error) {
+	v := m.regexp.FindStringSubmatch(path)
+	if v == nil {
+		return nil, fmt.Errorf("%s is not a valid path for a source on gopkg.in", path)
+	}
+
+	return simpleStringFuture("gopkg.in/" + v[2]), nil
+}
+
+func (m gopkginMatcher) deduceSource(path string, u *url.URL) (func(string, ProjectAnalyzer) futureSource, error) {
+
+	v := m.regexp.FindStringSubmatch(path)
+	if v == nil {
+		return nil, fmt.Errorf("%s is not a valid path for a source on gopkg.in", path)
+	}
+
+	// Duplicate some logic from the gopkg.in server in order to validate
+	// the import path string without having to hit the server
+	if strings.Contains(v[4], ".") {
+		return nil, fmt.Errorf("%q is not a valid import path; gopkg.in only allows major versions (%q instead of %q)",
+			path, v[4][:strings.Index(v[4], ".")], v[4])
+	}
+
+	// Putting a scheme on gopkg.in would be really weird, disallow it
+	if u.Scheme != "" {
+		return nil, fmt.Errorf("Specifying alternate schemes on gopkg.in imports is not permitted")
+	}
+
+	// gopkg.in is always backed by github
+	u.Host = "github.com"
+	// If the third position is empty, it's the shortened form that expands
+	// to the go-pkg github user
+	if v[2] == "" {
+		u.Path = "go-pkg/" + v[3]
 	} else {
-		path = rr.CloneURL.Path
+		u.Path = v[2] + "/" + v[3]
 	}
 
-	if !pathvld.MatchString(path) {
-		return nil, fmt.Errorf("%q is not a valid import path", path)
+	mb := make(maybeSources, len(gitSchemes))
+	for k, scheme := range gitSchemes {
+		u2 := *u
+		u2.Scheme = scheme
+		mb[k] = maybeGitSource{url: &u2}
 	}
 
-	if rr.CloneURL.Scheme != "" {
-		rr.Schemes = []string{rr.CloneURL.Scheme}
+	return sourceFutureFactory(mb), nil
+}
+
+type launchpadMatcher struct {
+	regexp *regexp.Regexp
+}
+
+func (m launchpadMatcher) deduceRoot(path string) (futureString, error) {
+	// TODO(sdboyer) lp handling is nasty - there's ambiguities which can only really
+	// be resolved with a metadata request. See https://github.com/golang/go/issues/11436
+	v := m.regexp.FindStringSubmatch(path)
+	if v == nil {
+		return nil, fmt.Errorf("%s is not a valid path for a source on launchpad.net", path)
 	}
 
-	// TODO(sdboyer) instead of a switch, encode base domain in radix tree and pick
-	// detector from there; if failure, then fall back on metadata work
+	return simpleStringFuture("launchpad.net/" + v[2]), nil
+}
+
+func (m launchpadMatcher) deduceSource(path string, u *url.URL) (func(string, ProjectAnalyzer) futureSource, error) {
+	v := m.regexp.FindStringSubmatch(path)
+	if v == nil {
+		return nil, fmt.Errorf("%s is not a valid path for a source on launchpad.net", path)
+	}
 
-	switch {
-	case ghRegex.MatchString(path):
-		v := ghRegex.FindStringSubmatch(path)
-
-		rr.CloneURL.Host = "github.com"
-		rr.CloneURL.Path = v[2]
-		rr.Base = v[1]
-		rr.RelPkg = strings.TrimPrefix(v[3], "/")
-		rr.VCS = []string{"git"}
-		// If no scheme was already recorded, then add the possible schemes for github
-		if rr.Schemes == nil {
-			rr.Schemes = gitSchemes
+	u.Path = v[2]
+	if u.Scheme != "" {
+		if !validateVCSScheme(u.Scheme, "bzr") {
+			return nil, fmt.Errorf("%s is not a valid scheme for accessing a bzr repository", u.Scheme)
 		}
+		return sourceFutureFactory(maybeBzrSource{url: u}), nil
+	}
 
-		return
+	mb := make(maybeSources, len(bzrSchemes))
+	for k, scheme := range bzrSchemes {
+		u2 := *u
+		u2.Scheme = scheme
+		mb[k] = maybeBzrSource{url: &u2}
+	}
 
-	case gpinNewRegex.MatchString(path):
-		v := gpinNewRegex.FindStringSubmatch(path)
-		// Duplicate some logic from the gopkg.in server in order to validate
-		// the import path string without having to hit the server
-		if strings.Contains(v[4], ".") {
-			return nil, fmt.Errorf("%q is not a valid import path; gopkg.in only allows major versions (%q instead of %q)",
-				path, v[4][:strings.Index(v[4], ".")], v[4])
+	return sourceFutureFactory(mb), nil
+}
+
+type launchpadGitMatcher struct {
+	regexp *regexp.Regexp
+}
+
+func (m launchpadGitMatcher) deduceRoot(path string) (futureString, error) {
+	// TODO(sdboyer) same ambiguity issues as with normal bzr lp
+	v := m.regexp.FindStringSubmatch(path)
+	if v == nil {
+		return nil, fmt.Errorf("%s is not a valid path for a source on git.launchpad.net", path)
+	}
+
+	return simpleStringFuture("git.launchpad.net/" + v[2]), nil
+}
+
+func (m launchpadGitMatcher) deduceSource(path string, u *url.URL) (func(string, ProjectAnalyzer) futureSource, error) {
+	v := m.regexp.FindStringSubmatch(path)
+	if v == nil {
+		return nil, fmt.Errorf("%s is not a valid path for a source on git.launchpad.net", path)
+	}
+
+	u.Path = v[2]
+	if u.Scheme != "" {
+		if !validateVCSScheme(u.Scheme, "git") {
+			return nil, fmt.Errorf("%s is not a valid scheme for accessing a git repository", u.Scheme)
 		}
+		return sourceFutureFactory(maybeGitSource{url: u}), nil
+	}
+
+	mb := make(maybeSources, len(bzrSchemes))
+	for k, scheme := range bzrSchemes {
+		u2 := *u
+		u2.Scheme = scheme
+		mb[k] = maybeGitSource{url: &u2}
+	}
 
-		// gopkg.in is always backed by github
-		rr.CloneURL.Host = "github.com"
-		// If the third position is empty, it's the shortened form that expands
-		// to the go-pkg github user
-		if v[2] == "" {
-			rr.CloneURL.Path = "go-pkg/" + v[3]
-		} else {
-			rr.CloneURL.Path = v[2] + "/" + v[3]
+	return sourceFutureFactory(mb), nil
+}
+
+type jazzMatcher struct {
+	regexp *regexp.Regexp
+}
+
+func (m jazzMatcher) deduceRoot(path string) (futureString, error) {
+	v := m.regexp.FindStringSubmatch(path)
+	if v == nil {
+		return nil, fmt.Errorf("%s is not a valid path for a source on hub.jazz.net", path)
+	}
+
+	return simpleStringFuture("hub.jazz.net/" + v[2]), nil
+}
+
+func (m jazzMatcher) deduceSource(path string, u *url.URL) (func(string, ProjectAnalyzer) futureSource, error) {
+	v := m.regexp.FindStringSubmatch(path)
+	if v == nil {
+		return nil, fmt.Errorf("%s is not a valid path for a source on hub.jazz.net", path)
+	}
+
+	u.Path = v[2]
+	if u.Scheme != "" {
+		if !validateVCSScheme(u.Scheme, "git") {
+			return nil, fmt.Errorf("%s is not a valid scheme for accessing a git repository", u.Scheme)
 		}
-		rr.Base = v[1]
-		rr.RelPkg = strings.TrimPrefix(v[6], "/")
-		rr.VCS = []string{"git"}
-		// If no scheme was already recorded, then add the possible schemes for github
-		if rr.Schemes == nil {
-			rr.Schemes = gitSchemes
+		return sourceFutureFactory(maybeGitSource{url: u}), nil
+	}
+
+	mb := make(maybeSources, len(gitSchemes))
+	for k, scheme := range gitSchemes {
+		u2 := *u
+		u2.Scheme = scheme
+		mb[k] = maybeGitSource{url: &u2}
+	}
+
+	return sourceFutureFactory(mb), nil
+}
+
+type apacheMatcher struct {
+	regexp *regexp.Regexp
+}
+
+func (m apacheMatcher) deduceRoot(path string) (futureString, error) {
+	v := m.regexp.FindStringSubmatch(path)
+	if v == nil {
+		return nil, fmt.Errorf("%s is not a valid path for a source on git.apache.org", path)
+	}
+
+	return simpleStringFuture("git.apache.org/" + v[2]), nil
+}
+
+func (m apacheMatcher) deduceSource(path string, u *url.URL) (func(string, ProjectAnalyzer) futureSource, error) {
+	v := m.regexp.FindStringSubmatch(path)
+	if v == nil {
+		return nil, fmt.Errorf("%s is not a valid path for a source on git.apache.org", path)
+	}
+
+	u.Path = v[2]
+	if u.Scheme != "" {
+		if !validateVCSScheme(u.Scheme, "git") {
+			return nil, fmt.Errorf("%s is not a valid scheme for accessing a git repository", u.Scheme)
 		}
+		return sourceFutureFactory(maybeGitSource{url: u}), nil
+	}
 
-		return
-	//case gpinOldRegex.MatchString(path):
-
-	case bbRegex.MatchString(path):
-		v := bbRegex.FindStringSubmatch(path)
-
-		rr.CloneURL.Host = "bitbucket.org"
-		rr.CloneURL.Path = v[2]
-		rr.Base = v[1]
-		rr.RelPkg = strings.TrimPrefix(v[3], "/")
-		rr.VCS = []string{"git", "hg"}
-		// FIXME(sdboyer) this ambiguity of vcs kills us on schemes, as schemes
-		// are inherently vcs-specific. Fixing this requires a wider refactor.
-		// For now, we only allow the intersection, which is just the hg schemes
-		if rr.Schemes == nil {
-			rr.Schemes = hgSchemes
+	mb := make(maybeSources, len(gitSchemes))
+	for k, scheme := range gitSchemes {
+		u2 := *u
+		u2.Scheme = scheme
+		mb[k] = maybeGitSource{url: &u2}
+	}
+
+	return sourceFutureFactory(mb), nil
+}
+
+type vcsExtensionMatcher struct {
+	regexp *regexp.Regexp
+}
+
+func (m vcsExtensionMatcher) deduceRoot(path string) (futureString, error) {
+	v := m.regexp.FindStringSubmatch(path)
+	if v == nil {
+		return nil, fmt.Errorf("%s contains no vcs extension hints for matching", path)
+	}
+
+	return simpleStringFuture(v[1]), nil
+}
+
+func (m vcsExtensionMatcher) deduceSource(path string, u *url.URL) (func(string, ProjectAnalyzer) futureSource, error) {
+	v := m.regexp.FindStringSubmatch(path)
+	if v == nil {
+		return nil, fmt.Errorf("%s contains no vcs extension hints for matching", path)
+	}
+
+	switch v[5] {
+	case "git", "hg", "bzr":
+		x := strings.SplitN(v[1], "/", 2)
+		// TODO(sdboyer) is this actually correct for bzr?
+		u.Host = x[0]
+		u.Path = x[1]
+
+		if u.Scheme != "" {
+			if !validateVCSScheme(u.Scheme, v[5]) {
+				return nil, fmt.Errorf("%s is not a valid scheme for accessing %s repositories (path %s)", u.Scheme, v[5], path)
+			}
+
+			switch v[5] {
+			case "git":
+				return sourceFutureFactory(maybeGitSource{url: u}), nil
+			case "bzr":
+				return sourceFutureFactory(maybeBzrSource{url: u}), nil
+			case "hg":
+				return sourceFutureFactory(maybeHgSource{url: u}), nil
+			}
 		}
 
-		return
+		var schemes []string
+		var mb maybeSources
+		var f func(k int, u *url.URL)
+		switch v[5] {
+		case "git":
+			schemes = gitSchemes
+			f = func(k int, u *url.URL) {
+				mb[k] = maybeGitSource{url: u}
+			}
+		case "bzr":
+			schemes = bzrSchemes
+			f = func(k int, u *url.URL) {
+				mb[k] = maybeBzrSource{url: u}
+			}
+		case "hg":
+			schemes = hgSchemes
+			f = func(k int, u *url.URL) {
+				mb[k] = maybeHgSource{url: u}
+			}
+		}
+		mb = make(maybeSources, len(schemes))
 
-	//case gcRegex.MatchString(path):
-	//v := gcRegex.FindStringSubmatch(path)
-
-	//rr.CloneURL.Host = "code.google.com"
-	//rr.CloneURL.Path = "p/" + v[2]
-	//rr.Base = v[1]
-	//rr.RelPkg = strings.TrimPrefix(v[5], "/")
-	//rr.VCS = []string{"hg", "git"}
-
-	//return
-
-	case lpRegex.MatchString(path):
-		// TODO(sdboyer) lp handling is nasty - there's ambiguities which can only really
-		// be resolved with a metadata request. See https://github.com/golang/go/issues/11436
-		v := lpRegex.FindStringSubmatch(path)
-
-		rr.CloneURL.Host = "launchpad.net"
-		rr.CloneURL.Path = v[2]
-		rr.Base = v[1]
-		rr.RelPkg = strings.TrimPrefix(v[3], "/")
-		rr.VCS = []string{"bzr"}
-		if rr.Schemes == nil {
-			rr.Schemes = bzrSchemes
+		for k, scheme := range gitSchemes {
+			u2 := *u
+			u2.Scheme = scheme
+			f(k, &u2)
 		}
 
-		return
+		return sourceFutureFactory(mb), nil
+	default:
+		return nil, fmt.Errorf("unknown repository type: %q", v[5])
+	}
+}
+
+type doubleFut struct {
+	root futureString
+	src  func(string, ProjectAnalyzer) futureSource
+}
+
+func (fut doubleFut) importRoot() (string, error) {
+	return fut.root()
+}
+
+func (fut doubleFut) source(cachedir string, an ProjectAnalyzer) (source, error) {
+	return fut.src(cachedir, an)()
+}
 
-	case glpRegex.MatchString(path):
-		// TODO(sdboyer) same ambiguity issues as with normal bzr lp
-		v := glpRegex.FindStringSubmatch(path)
-
-		rr.CloneURL.Host = "git.launchpad.net"
-		rr.CloneURL.Path = v[2]
-		rr.Base = v[1]
-		rr.RelPkg = strings.TrimPrefix(v[3], "/")
-		rr.VCS = []string{"git"}
-		if rr.Schemes == nil {
-			rr.Schemes = gitSchemes
+// deduceFromPath takes an import path and converts it into a valid source root.
+//
+// The result is wrapped in a future, as some import path patterns may require
+// network activity to correctly determine them via the parsing of "go get" HTTP
+// meta tags.
+func (sm *SourceMgr) deduceFromPath(path string) (sourceFuture, error) {
+	u, err := normalizeURI(path)
+	if err != nil {
+		return nil, err
+	}
+
+	df := doubleFut{}
+	// First, try the root path-based matches
+	if _, mtchi, has := sm.rootxt.LongestPrefix(path); has {
+		mtch := mtchi.(matcher)
+		df.root, err = mtch.deduceRoot(path)
+		if err != nil {
+			return nil, err
+		}
+		df.src, err = mtch.deduceSource(path, u)
+		if err != nil {
+			return nil, err
 		}
 
-		return
+		return df, nil
+	}
 
-	case jazzRegex.MatchString(path):
-		v := jazzRegex.FindStringSubmatch(path)
+	// Next, try the vcs extension-based (infix) matcher
+	exm := vcsExtensionMatcher{regexp: vcsExtensionRegex}
+	if df.root, err = exm.deduceRoot(path); err == nil {
+		df.src, err = exm.deduceSource(path, u)
+		if err != nil {
+			return nil, err
+		}
+	}
 
-		rr.CloneURL.Host = "hub.jazz.net"
-		rr.CloneURL.Path = v[2]
-		rr.Base = v[1]
-		rr.RelPkg = strings.TrimPrefix(v[3], "/")
-		rr.VCS = []string{"git"}
-		if rr.Schemes == nil {
-			rr.Schemes = gitSchemes
+	// Still no luck. Fall back on "go get"-style metadata
+	var importroot, vcs, reporoot string
+	df.root = stringFuture(func() (string, error) {
+		var err error
+		importroot, vcs, reporoot, err = parseMetadata(path)
+		if err != nil {
+			return "", fmt.Errorf("unable to deduce repository and source type for: %q", path)
 		}
 
-		return
+		// If we got something back at all, then it supercedes the actual input for
+		// the real URL to hit
+		_, err = url.Parse(reporoot)
+		if err != nil {
+			return "", fmt.Errorf("server returned bad URL when searching for vanity import: %q", reporoot)
+		}
 
-	case apacheRegex.MatchString(path):
-		v := apacheRegex.FindStringSubmatch(path)
+		return importroot, nil
+	})
 
-		rr.CloneURL.Host = "git.apache.org"
-		rr.CloneURL.Path = v[2]
-		rr.Base = v[1]
-		rr.RelPkg = strings.TrimPrefix(v[3], "/")
-		rr.VCS = []string{"git"}
-		if rr.Schemes == nil {
-			rr.Schemes = gitSchemes
+	df.src = srcFuture(func(cachedir string, an ProjectAnalyzer) (source, error) {
+		// make sure the metadata future is finished, and without errors
+		_, err := df.root()
+		if err != nil {
+			return nil, err
 		}
 
-		return
+		// we know it can't error b/c it already parsed successfully in the
+		// other future
+		u, _ := url.Parse(reporoot)
 
-	// try the general syntax
-	case genericRegex.MatchString(path):
-		v := genericRegex.FindStringSubmatch(path)
-		switch v[5] {
-		case "git", "hg", "bzr":
-			x := strings.SplitN(v[1], "/", 2)
-			// TODO(sdboyer) is this actually correct for bzr?
-			rr.CloneURL.Host = x[0]
-			rr.CloneURL.Path = x[1]
-			rr.VCS = []string{v[5]}
-			rr.Base = v[1]
-			rr.RelPkg = strings.TrimPrefix(v[6], "/")
-
-			if rr.Schemes == nil {
-				if v[5] == "git" {
-					rr.Schemes = gitSchemes
-				} else if v[5] == "bzr" {
-					rr.Schemes = bzrSchemes
-				} else if v[5] == "hg" {
-					rr.Schemes = hgSchemes
-				}
+		switch vcs {
+		case "git":
+			m := maybeGitSource{
+				url: u,
 			}
-
-			return
+			return m.try(cachedir, an)
+		case "bzr":
+			m := maybeBzrSource{
+				url: u,
+			}
+			return m.try(cachedir, an)
+		case "hg":
+			m := maybeHgSource{
+				url: u,
+			}
+			return m.try(cachedir, an)
 		default:
-			return nil, fmt.Errorf("unknown repository type: %q", v[5])
+			return nil, fmt.Errorf("unsupported vcs type %s", vcs)
+		}
+	})
+
+	return df, nil
+}
+
+func normalizeURI(path string) (u *url.URL, err error) {
+	if m := scpSyntaxRe.FindStringSubmatch(path); m != nil {
+		// Match SCP-like syntax and convert it to a URL.
+		// Eg, "git@github.com:user/repo" becomes
+		// "ssh://git@github.com/user/repo".
+		u = &url.URL{
+			Scheme: "ssh",
+			User:   url.User(m[1]),
+			Host:   m[2],
+			Path:   "/" + m[3],
+			// TODO(sdboyer) This is what stdlib sets; grok why better
+			//RawPath: m[3],
+		}
+	} else {
+		u, err = url.Parse(path)
+		if err != nil {
+			return nil, fmt.Errorf("%q is not a valid URI", path)
 		}
 	}
 
-	// No luck so far. maybe it's one of them vanity imports?
-	importroot, vcs, reporoot, err := parseMetadata(path)
-	if err != nil {
-		return nil, fmt.Errorf("unable to deduce repository and source type for: %q", path)
+	if u.Host != "" {
+		path = u.Host + "/" + strings.TrimPrefix(u.Path, "/")
+	} else {
+		path = u.Path
 	}
 
-	// If we got something back at all, then it supercedes the actual input for
-	// the real URL to hit
-	rr.CloneURL, err = url.Parse(reporoot)
-	if err != nil {
-		return nil, fmt.Errorf("server returned bad URL when searching for vanity import: %q", reporoot)
+	if !pathvld.MatchString(path) {
+		return nil, fmt.Errorf("%q is not a valid import path", path)
+	}
+
+	return
+}
+
+// deduceRemoteRepo takes a potential import path and returns a RemoteRepo
+// representing the remote location of the source of an import path. Remote
+// repositories can be bare import paths, or urls including a checkout scheme.
+func deduceRemoteRepo(path string) (rr *remoteRepo, err error) {
+	rr = &remoteRepo{}
+	if m := scpSyntaxRe.FindStringSubmatch(path); m != nil {
+		// Match SCP-like syntax and convert it to a URL.
+		// Eg, "git@github.com:user/repo" becomes
+		// "ssh://git@github.com/user/repo".
+		rr.CloneURL = &url.URL{
+			Scheme: "ssh",
+			User:   url.User(m[1]),
+			Host:   m[2],
+			Path:   "/" + m[3],
+			// TODO(sdboyer) This is what stdlib sets; grok why better
+			//RawPath: m[3],
+		}
+	} else {
+		rr.CloneURL, err = url.Parse(path)
+		if err != nil {
+			return nil, fmt.Errorf("%q is not a valid import path", path)
+		}
 	}
 
+	if rr.CloneURL.Host != "" {
+		path = rr.CloneURL.Host + "/" + strings.TrimPrefix(rr.CloneURL.Path, "/")
+	} else {
+		path = rr.CloneURL.Path
+	}
+
+	if !pathvld.MatchString(path) {
+		return nil, fmt.Errorf("%q is not a valid import path", path)
+	}
+
+	if rr.CloneURL.Scheme != "" {
+		rr.Schemes = []string{rr.CloneURL.Scheme}
+	}
+
+	// TODO(sdboyer) instead of a switch, encode base domain in radix tree and pick
+	// detector from there; if failure, then fall back on metadata work
+
+	// No luck so far. maybe it's one of them vanity imports?
+	// We have to get a little fancier for the metadata lookup - wrap a future
+	// around a future
+	var importroot, vcs string
 	// We have a real URL. Set the other values and return.
 	rr.Base = importroot
 	rr.RelPkg = strings.TrimPrefix(path[len(importroot):], "/")
diff --git a/source.go b/source.go
index 1d431bc..8c4c37f 100644
--- a/source.go
+++ b/source.go
@@ -29,6 +29,48 @@ func newMetaCache() *sourceMetaCache {
 	}
 }
 
+type futureString func() (string, error)
+type futureSource func() (source, error)
+
+func stringFuture(f func() (string, error)) func() (string, error) {
+	var result string
+	var err error
+
+	c := make(chan struct{}, 1)
+	go func() {
+		defer close(c)
+		result, err = f()
+	}()
+
+	return func() (string, error) {
+		<-c
+		return result, err
+	}
+}
+
+func srcFuture(f func(string, ProjectAnalyzer) (source, error)) func(string, ProjectAnalyzer) futureSource {
+	return func(cachedir string, an ProjectAnalyzer) futureSource {
+		var src source
+		var err error
+
+		c := make(chan struct{}, 1)
+		go func() {
+			defer close(c)
+			src, err = f(cachedir, an)
+		}()
+
+		return func() (source, error) {
+			<-c
+			return src, err
+		}
+	}
+}
+
+type sourceFuture interface {
+	importRoot() (string, error)
+	source(string, ProjectAnalyzer) (source, error)
+}
+
 type baseVCSSource struct {
 	// Object for the cache repository
 	crepo *repo
diff --git a/source_manager.go b/source_manager.go
index cc0799c..b6abef1 100644
--- a/source_manager.go
+++ b/source_manager.go
@@ -10,6 +10,7 @@ import (
 
 	"github.com/Masterminds/semver"
 	"github.com/Masterminds/vcs"
+	"github.com/armon/go-radix"
 )
 
 // Used to compute a friendly filepath from a URL-shaped input
@@ -82,8 +83,9 @@ type SourceMgr struct {
 		rr  *remoteRepo
 		err error
 	}
-	rmut sync.RWMutex
-	an   ProjectAnalyzer
+	rmut   sync.RWMutex
+	an     ProjectAnalyzer
+	rootxt *radix.Tree
 }
 
 var _ SourceManager = &SourceMgr{}
@@ -141,7 +143,8 @@ func NewSourceManager(an ProjectAnalyzer, cachedir string, force bool) (*SourceM
 			rr  *remoteRepo
 			err error
 		}),
-		an: an,
+		an:     an,
+		rootxt: radix.New(),
 	}, nil
 }
 
diff --git a/source_test.go b/source_test.go
index d5dd9c5..57a9394 100644
--- a/source_test.go
+++ b/source_test.go
@@ -33,7 +33,6 @@ func TestGitVersionFetching(t *testing.T) {
 		t.FailNow()
 	}
 	mb := maybeGitSource{
-		n:   n,
 		url: u,
 	}
 
@@ -110,7 +109,6 @@ func TestBzrVersionFetching(t *testing.T) {
 		t.FailNow()
 	}
 	mb := maybeBzrSource{
-		n:   n,
 		url: u,
 	}
 
@@ -196,7 +194,6 @@ func TestHgVersionFetching(t *testing.T) {
 		t.FailNow()
 	}
 	mb := maybeHgSource{
-		n:   n,
 		url: u,
 	}
 

From fc7f8a717682f5c06d758ef16039102d39687dfe Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Fri, 5 Aug 2016 09:41:27 -0400
Subject: [PATCH 41/71] Fail immediately if test can't load a project

This avoids an unnecessary panic condition later.
---
 manager_test.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/manager_test.go b/manager_test.go
index b8e3039..4351445 100644
--- a/manager_test.go
+++ b/manager_test.go
@@ -182,6 +182,7 @@ func TestProjectManagerInit(t *testing.T) {
 	pms, err := sm.getProjectManager(id)
 	if err != nil {
 		t.Errorf("Error on grabbing project manager obj: %s", err)
+		t.FailNow()
 	}
 
 	// Check upstream existence flag

From 6f40c2d7b745209130e18395fa565d0b5a71953c Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Fri, 5 Aug 2016 09:42:08 -0400
Subject: [PATCH 42/71] Get rid of unnecessary interface/struct for return

Fewer types is more costly than a 3-ary (vs. 2-ary) return.
---
 remote.go | 43 +++++++++++++++----------------------------
 source.go |  1 +
 2 files changed, 16 insertions(+), 28 deletions(-)

diff --git a/remote.go b/remote.go
index e041cd1..86ace8c 100644
--- a/remote.go
+++ b/remote.go
@@ -489,58 +489,45 @@ func (m vcsExtensionMatcher) deduceSource(path string, u *url.URL) (func(string,
 	}
 }
 
-type doubleFut struct {
-	root futureString
-	src  func(string, ProjectAnalyzer) futureSource
-}
-
-func (fut doubleFut) importRoot() (string, error) {
-	return fut.root()
-}
-
-func (fut doubleFut) source(cachedir string, an ProjectAnalyzer) (source, error) {
-	return fut.src(cachedir, an)()
-}
-
 // deduceFromPath takes an import path and converts it into a valid source root.
 //
 // The result is wrapped in a future, as some import path patterns may require
 // network activity to correctly determine them via the parsing of "go get" HTTP
 // meta tags.
-func (sm *SourceMgr) deduceFromPath(path string) (sourceFuture, error) {
+func (sm *SourceMgr) deduceFromPath(path string) (root futureString, src deferredFutureSource, err error) {
 	u, err := normalizeURI(path)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
-	df := doubleFut{}
 	// First, try the root path-based matches
 	if _, mtchi, has := sm.rootxt.LongestPrefix(path); has {
 		mtch := mtchi.(matcher)
-		df.root, err = mtch.deduceRoot(path)
+		root, err = mtch.deduceRoot(path)
 		if err != nil {
-			return nil, err
+			return nil, nil, err
 		}
-		df.src, err = mtch.deduceSource(path, u)
+		src, err = mtch.deduceSource(path, u)
 		if err != nil {
-			return nil, err
+			return nil, nil, err
 		}
 
-		return df, nil
+		return
 	}
 
 	// Next, try the vcs extension-based (infix) matcher
 	exm := vcsExtensionMatcher{regexp: vcsExtensionRegex}
-	if df.root, err = exm.deduceRoot(path); err == nil {
-		df.src, err = exm.deduceSource(path, u)
+	if root, err = exm.deduceRoot(path); err == nil {
+		src, err = exm.deduceSource(path, u)
 		if err != nil {
-			return nil, err
+			root, src = nil, nil
 		}
+		return
 	}
 
 	// Still no luck. Fall back on "go get"-style metadata
 	var importroot, vcs, reporoot string
-	df.root = stringFuture(func() (string, error) {
+	root = stringFuture(func() (string, error) {
 		var err error
 		importroot, vcs, reporoot, err = parseMetadata(path)
 		if err != nil {
@@ -557,9 +544,9 @@ func (sm *SourceMgr) deduceFromPath(path string) (sourceFuture, error) {
 		return importroot, nil
 	})
 
-	df.src = srcFuture(func(cachedir string, an ProjectAnalyzer) (source, error) {
+	src = srcFuture(func(cachedir string, an ProjectAnalyzer) (source, error) {
 		// make sure the metadata future is finished, and without errors
-		_, err := df.root()
+		_, err := root()
 		if err != nil {
 			return nil, err
 		}
@@ -589,7 +576,7 @@ func (sm *SourceMgr) deduceFromPath(path string) (sourceFuture, error) {
 		}
 	})
 
-	return df, nil
+	return
 }
 
 func normalizeURI(path string) (u *url.URL, err error) {
diff --git a/source.go b/source.go
index 8c4c37f..5ea242d 100644
--- a/source.go
+++ b/source.go
@@ -31,6 +31,7 @@ func newMetaCache() *sourceMetaCache {
 
 type futureString func() (string, error)
 type futureSource func() (source, error)
+type deferredFutureSource func(string, ProjectAnalyzer) futureSource
 
 func stringFuture(f func() (string, error)) func() (string, error) {
 	var result string

From 621d3baa993a37cfcdfdd059f7d035cc2b02a98d Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Fri, 5 Aug 2016 10:36:59 -0400
Subject: [PATCH 43/71] Get rid of unnecessary futurizing funcs

There's only the one use case right now, so it's better and clearer to
just create the futures in local scope than rely on an abstraction to
create them for us.
---
 remote.go | 103 ++++++++++++++++++++++++++++++++++--------------------
 source.go |  43 -----------------------
 2 files changed, 65 insertions(+), 81 deletions(-)

diff --git a/remote.go b/remote.go
index 86ace8c..293d59b 100644
--- a/remote.go
+++ b/remote.go
@@ -22,6 +22,10 @@ type remoteRepo struct {
 	VCS      []string
 }
 
+type futureString func() (string, error)
+type futureSource func() (source, error)
+type deferredFutureSource func(string, ProjectAnalyzer) futureSource
+
 var (
 	gitSchemes = []string{"https", "ssh", "git", "http"}
 	bzrSchemes = []string{"https", "bzr+ssh", "bzr", "http"}
@@ -525,56 +529,79 @@ func (sm *SourceMgr) deduceFromPath(path string) (root futureString, src deferre
 		return
 	}
 
-	// Still no luck. Fall back on "go get"-style metadata
-	var importroot, vcs, reporoot string
-	root = stringFuture(func() (string, error) {
-		var err error
-		importroot, vcs, reporoot, err = parseMetadata(path)
-		if err != nil {
-			return "", fmt.Errorf("unable to deduce repository and source type for: %q", path)
+	// No luck so far. maybe it's one of them vanity imports?
+	// We have to get a little fancier for the metadata lookup by chaining the
+	// source future onto the metadata future
+
+	// Declare these out here so they're available for the source future
+	var vcs string
+	var ru *url.URL
+
+	// Kick off the vanity metadata fetch
+	var importroot string
+	var futerr error
+	c := make(chan struct{}, 1)
+	go func() {
+		defer close(c)
+		var reporoot string
+		importroot, vcs, reporoot, futerr = parseMetadata(path)
+		if futerr != nil {
+			futerr = fmt.Errorf("unable to deduce repository and source type for: %q", path)
+			return
 		}
 
 		// If we got something back at all, then it supercedes the actual input for
 		// the real URL to hit
-		_, err = url.Parse(reporoot)
-		if err != nil {
-			return "", fmt.Errorf("server returned bad URL when searching for vanity import: %q", reporoot)
+		ru, futerr = url.Parse(reporoot)
+		if futerr != nil {
+			futerr = fmt.Errorf("server returned bad URL when searching for vanity import: %q", reporoot)
+			importroot = ""
+			return
 		}
+	}()
 
-		return importroot, nil
-	})
-
-	src = srcFuture(func(cachedir string, an ProjectAnalyzer) (source, error) {
-		// make sure the metadata future is finished, and without errors
-		_, err := root()
-		if err != nil {
-			return nil, err
-		}
+	// Set up the root func to catch the result
+	root = func() (string, error) {
+		<-c
+		return importroot, futerr
+	}
 
-		// we know it can't error b/c it already parsed successfully in the
-		// other future
-		u, _ := url.Parse(reporoot)
+	src = func(cachedir string, an ProjectAnalyzer) futureSource {
+		var src source
+		var err error
 
-		switch vcs {
-		case "git":
-			m := maybeGitSource{
-				url: u,
+		c := make(chan struct{}, 1)
+		go func() {
+			defer close(c)
+			// make sure the metadata future is finished (without errors), thus
+			// guaranteeing that ru and vcs will be populated
+			_, err := root()
+			if err != nil {
+				return
 			}
-			return m.try(cachedir, an)
-		case "bzr":
-			m := maybeBzrSource{
-				url: u,
+
+			var m maybeSource
+			switch vcs {
+			case "git":
+				m = maybeGitSource{url: ru}
+			case "bzr":
+				m = maybeBzrSource{url: ru}
+			case "hg":
+				m = maybeHgSource{url: ru}
 			}
-			return m.try(cachedir, an)
-		case "hg":
-			m := maybeHgSource{
-				url: u,
+
+			if m != nil {
+				src, err = m.try(cachedir, an)
+			} else {
+				err = fmt.Errorf("unsupported vcs type %s", vcs)
 			}
-			return m.try(cachedir, an)
-		default:
-			return nil, fmt.Errorf("unsupported vcs type %s", vcs)
+		}()
+
+		return func() (source, error) {
+			<-c
+			return src, err
 		}
-	})
+	}
 
 	return
 }
diff --git a/source.go b/source.go
index 5ea242d..1d431bc 100644
--- a/source.go
+++ b/source.go
@@ -29,49 +29,6 @@ func newMetaCache() *sourceMetaCache {
 	}
 }
 
-type futureString func() (string, error)
-type futureSource func() (source, error)
-type deferredFutureSource func(string, ProjectAnalyzer) futureSource
-
-func stringFuture(f func() (string, error)) func() (string, error) {
-	var result string
-	var err error
-
-	c := make(chan struct{}, 1)
-	go func() {
-		defer close(c)
-		result, err = f()
-	}()
-
-	return func() (string, error) {
-		<-c
-		return result, err
-	}
-}
-
-func srcFuture(f func(string, ProjectAnalyzer) (source, error)) func(string, ProjectAnalyzer) futureSource {
-	return func(cachedir string, an ProjectAnalyzer) futureSource {
-		var src source
-		var err error
-
-		c := make(chan struct{}, 1)
-		go func() {
-			defer close(c)
-			src, err = f(cachedir, an)
-		}()
-
-		return func() (source, error) {
-			<-c
-			return src, err
-		}
-	}
-}
-
-type sourceFuture interface {
-	importRoot() (string, error)
-	source(string, ProjectAnalyzer) (source, error)
-}
-
 type baseVCSSource struct {
 	// Object for the cache repository
 	crepo *repo

From ada3344f6caa07a77befcef3b721bcfba3dced2c Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Fri, 5 Aug 2016 10:46:16 -0400
Subject: [PATCH 44/71] Several type naming improvements

---
 remote.go | 74 +++++++++++++++++++++++++++----------------------------
 1 file changed, 37 insertions(+), 37 deletions(-)

diff --git a/remote.go b/remote.go
index 293d59b..c0f19ec 100644
--- a/remote.go
+++ b/remote.go
@@ -22,9 +22,9 @@ type remoteRepo struct {
 	VCS      []string
 }
 
-type futureString func() (string, error)
-type futureSource func() (source, error)
-type deferredFutureSource func(string, ProjectAnalyzer) futureSource
+type stringFuture func() (string, error)
+type sourceFuture func() (source, error)
+type partialSourceFuture func(string, ProjectAnalyzer) sourceFuture
 
 var (
 	gitSchemes = []string{"https", "ssh", "git", "http"}
@@ -81,14 +81,14 @@ var (
 	pathvld     = regexp.MustCompile(`^([A-Za-z0-9-]+)(\.[A-Za-z0-9-]+)+(/[A-Za-z0-9-_.~]+)*$`)
 )
 
-func simpleStringFuture(s string) futureString {
+func simpleStringFuture(s string) stringFuture {
 	return func() (string, error) {
 		return s, nil
 	}
 }
 
-func sourceFutureFactory(mb maybeSource) func(string, ProjectAnalyzer) futureSource {
-	return func(cachedir string, an ProjectAnalyzer) futureSource {
+func sourceFutureFactory(mb maybeSource) func(string, ProjectAnalyzer) sourceFuture {
+	return func(cachedir string, an ProjectAnalyzer) sourceFuture {
 		var src source
 		var err error
 
@@ -105,16 +105,16 @@ func sourceFutureFactory(mb maybeSource) func(string, ProjectAnalyzer) futureSou
 	}
 }
 
-type matcher interface {
-	deduceRoot(string) (futureString, error)
-	deduceSource(string, *url.URL) (func(string, ProjectAnalyzer) futureSource, error)
+type pathDeducer interface {
+	deduceRoot(string) (stringFuture, error)
+	deduceSource(string, *url.URL) (func(string, ProjectAnalyzer) sourceFuture, error)
 }
 
-type githubMatcher struct {
+type githubDeducer struct {
 	regexp *regexp.Regexp
 }
 
-func (m githubMatcher) deduceRoot(path string) (futureString, error) {
+func (m githubDeducer) deduceRoot(path string) (stringFuture, error) {
 	v := m.regexp.FindStringSubmatch(path)
 	if v == nil {
 		return nil, fmt.Errorf("%s is not a valid path for a source on github.com", path)
@@ -123,7 +123,7 @@ func (m githubMatcher) deduceRoot(path string) (futureString, error) {
 	return simpleStringFuture("github.com/" + v[2]), nil
 }
 
-func (m githubMatcher) deduceSource(path string, u *url.URL) (func(string, ProjectAnalyzer) futureSource, error) {
+func (m githubDeducer) deduceSource(path string, u *url.URL) (func(string, ProjectAnalyzer) sourceFuture, error) {
 	v := m.regexp.FindStringSubmatch(path)
 	if v == nil {
 		return nil, fmt.Errorf("%s is not a valid path for a source on github.com", path)
@@ -147,11 +147,11 @@ func (m githubMatcher) deduceSource(path string, u *url.URL) (func(string, Proje
 	return sourceFutureFactory(mb), nil
 }
 
-type bitbucketMatcher struct {
+type bitbucketDeducer struct {
 	regexp *regexp.Regexp
 }
 
-func (m bitbucketMatcher) deduceRoot(path string) (futureString, error) {
+func (m bitbucketDeducer) deduceRoot(path string) (stringFuture, error) {
 	v := m.regexp.FindStringSubmatch(path)
 	if v == nil {
 		return nil, fmt.Errorf("%s is not a valid path for a source on bitbucket.org", path)
@@ -160,7 +160,7 @@ func (m bitbucketMatcher) deduceRoot(path string) (futureString, error) {
 	return simpleStringFuture("bitbucket.org/" + v[2]), nil
 }
 
-func (m bitbucketMatcher) deduceSource(path string, u *url.URL) (func(string, ProjectAnalyzer) futureSource, error) {
+func (m bitbucketDeducer) deduceSource(path string, u *url.URL) (func(string, ProjectAnalyzer) sourceFuture, error) {
 	v := m.regexp.FindStringSubmatch(path)
 	if v == nil {
 		return nil, fmt.Errorf("%s is not a valid path for a source on bitbucket.org", path)
@@ -215,11 +215,11 @@ func (m bitbucketMatcher) deduceSource(path string, u *url.URL) (func(string, Pr
 	return sourceFutureFactory(mb), nil
 }
 
-type gopkginMatcher struct {
+type gopkginDeducer struct {
 	regexp *regexp.Regexp
 }
 
-func (m gopkginMatcher) deduceRoot(path string) (futureString, error) {
+func (m gopkginDeducer) deduceRoot(path string) (stringFuture, error) {
 	v := m.regexp.FindStringSubmatch(path)
 	if v == nil {
 		return nil, fmt.Errorf("%s is not a valid path for a source on gopkg.in", path)
@@ -228,7 +228,7 @@ func (m gopkginMatcher) deduceRoot(path string) (futureString, error) {
 	return simpleStringFuture("gopkg.in/" + v[2]), nil
 }
 
-func (m gopkginMatcher) deduceSource(path string, u *url.URL) (func(string, ProjectAnalyzer) futureSource, error) {
+func (m gopkginDeducer) deduceSource(path string, u *url.URL) (func(string, ProjectAnalyzer) sourceFuture, error) {
 
 	v := m.regexp.FindStringSubmatch(path)
 	if v == nil {
@@ -267,11 +267,11 @@ func (m gopkginMatcher) deduceSource(path string, u *url.URL) (func(string, Proj
 	return sourceFutureFactory(mb), nil
 }
 
-type launchpadMatcher struct {
+type launchpadDeducer struct {
 	regexp *regexp.Regexp
 }
 
-func (m launchpadMatcher) deduceRoot(path string) (futureString, error) {
+func (m launchpadDeducer) deduceRoot(path string) (stringFuture, error) {
 	// TODO(sdboyer) lp handling is nasty - there's ambiguities which can only really
 	// be resolved with a metadata request. See https://github.com/golang/go/issues/11436
 	v := m.regexp.FindStringSubmatch(path)
@@ -282,7 +282,7 @@ func (m launchpadMatcher) deduceRoot(path string) (futureString, error) {
 	return simpleStringFuture("launchpad.net/" + v[2]), nil
 }
 
-func (m launchpadMatcher) deduceSource(path string, u *url.URL) (func(string, ProjectAnalyzer) futureSource, error) {
+func (m launchpadDeducer) deduceSource(path string, u *url.URL) (func(string, ProjectAnalyzer) sourceFuture, error) {
 	v := m.regexp.FindStringSubmatch(path)
 	if v == nil {
 		return nil, fmt.Errorf("%s is not a valid path for a source on launchpad.net", path)
@@ -306,11 +306,11 @@ func (m launchpadMatcher) deduceSource(path string, u *url.URL) (func(string, Pr
 	return sourceFutureFactory(mb), nil
 }
 
-type launchpadGitMatcher struct {
+type launchpadGitDeducer struct {
 	regexp *regexp.Regexp
 }
 
-func (m launchpadGitMatcher) deduceRoot(path string) (futureString, error) {
+func (m launchpadGitDeducer) deduceRoot(path string) (stringFuture, error) {
 	// TODO(sdboyer) same ambiguity issues as with normal bzr lp
 	v := m.regexp.FindStringSubmatch(path)
 	if v == nil {
@@ -320,7 +320,7 @@ func (m launchpadGitMatcher) deduceRoot(path string) (futureString, error) {
 	return simpleStringFuture("git.launchpad.net/" + v[2]), nil
 }
 
-func (m launchpadGitMatcher) deduceSource(path string, u *url.URL) (func(string, ProjectAnalyzer) futureSource, error) {
+func (m launchpadGitDeducer) deduceSource(path string, u *url.URL) (func(string, ProjectAnalyzer) sourceFuture, error) {
 	v := m.regexp.FindStringSubmatch(path)
 	if v == nil {
 		return nil, fmt.Errorf("%s is not a valid path for a source on git.launchpad.net", path)
@@ -344,11 +344,11 @@ func (m launchpadGitMatcher) deduceSource(path string, u *url.URL) (func(string,
 	return sourceFutureFactory(mb), nil
 }
 
-type jazzMatcher struct {
+type jazzDeducer struct {
 	regexp *regexp.Regexp
 }
 
-func (m jazzMatcher) deduceRoot(path string) (futureString, error) {
+func (m jazzDeducer) deduceRoot(path string) (stringFuture, error) {
 	v := m.regexp.FindStringSubmatch(path)
 	if v == nil {
 		return nil, fmt.Errorf("%s is not a valid path for a source on hub.jazz.net", path)
@@ -357,7 +357,7 @@ func (m jazzMatcher) deduceRoot(path string) (futureString, error) {
 	return simpleStringFuture("hub.jazz.net/" + v[2]), nil
 }
 
-func (m jazzMatcher) deduceSource(path string, u *url.URL) (func(string, ProjectAnalyzer) futureSource, error) {
+func (m jazzDeducer) deduceSource(path string, u *url.URL) (func(string, ProjectAnalyzer) sourceFuture, error) {
 	v := m.regexp.FindStringSubmatch(path)
 	if v == nil {
 		return nil, fmt.Errorf("%s is not a valid path for a source on hub.jazz.net", path)
@@ -381,11 +381,11 @@ func (m jazzMatcher) deduceSource(path string, u *url.URL) (func(string, Project
 	return sourceFutureFactory(mb), nil
 }
 
-type apacheMatcher struct {
+type apacheDeducer struct {
 	regexp *regexp.Regexp
 }
 
-func (m apacheMatcher) deduceRoot(path string) (futureString, error) {
+func (m apacheDeducer) deduceRoot(path string) (stringFuture, error) {
 	v := m.regexp.FindStringSubmatch(path)
 	if v == nil {
 		return nil, fmt.Errorf("%s is not a valid path for a source on git.apache.org", path)
@@ -394,7 +394,7 @@ func (m apacheMatcher) deduceRoot(path string) (futureString, error) {
 	return simpleStringFuture("git.apache.org/" + v[2]), nil
 }
 
-func (m apacheMatcher) deduceSource(path string, u *url.URL) (func(string, ProjectAnalyzer) futureSource, error) {
+func (m apacheDeducer) deduceSource(path string, u *url.URL) (func(string, ProjectAnalyzer) sourceFuture, error) {
 	v := m.regexp.FindStringSubmatch(path)
 	if v == nil {
 		return nil, fmt.Errorf("%s is not a valid path for a source on git.apache.org", path)
@@ -418,11 +418,11 @@ func (m apacheMatcher) deduceSource(path string, u *url.URL) (func(string, Proje
 	return sourceFutureFactory(mb), nil
 }
 
-type vcsExtensionMatcher struct {
+type vcsExtensionDeducer struct {
 	regexp *regexp.Regexp
 }
 
-func (m vcsExtensionMatcher) deduceRoot(path string) (futureString, error) {
+func (m vcsExtensionDeducer) deduceRoot(path string) (stringFuture, error) {
 	v := m.regexp.FindStringSubmatch(path)
 	if v == nil {
 		return nil, fmt.Errorf("%s contains no vcs extension hints for matching", path)
@@ -431,7 +431,7 @@ func (m vcsExtensionMatcher) deduceRoot(path string) (futureString, error) {
 	return simpleStringFuture(v[1]), nil
 }
 
-func (m vcsExtensionMatcher) deduceSource(path string, u *url.URL) (func(string, ProjectAnalyzer) futureSource, error) {
+func (m vcsExtensionDeducer) deduceSource(path string, u *url.URL) (func(string, ProjectAnalyzer) sourceFuture, error) {
 	v := m.regexp.FindStringSubmatch(path)
 	if v == nil {
 		return nil, fmt.Errorf("%s contains no vcs extension hints for matching", path)
@@ -498,7 +498,7 @@ func (m vcsExtensionMatcher) deduceSource(path string, u *url.URL) (func(string,
 // The result is wrapped in a future, as some import path patterns may require
 // network activity to correctly determine them via the parsing of "go get" HTTP
 // meta tags.
-func (sm *SourceMgr) deduceFromPath(path string) (root futureString, src deferredFutureSource, err error) {
+func (sm *SourceMgr) deduceFromPath(path string) (root stringFuture, src partialSourceFuture, err error) {
 	u, err := normalizeURI(path)
 	if err != nil {
 		return nil, nil, err
@@ -506,7 +506,7 @@ func (sm *SourceMgr) deduceFromPath(path string) (root futureString, src deferre
 
 	// First, try the root path-based matches
 	if _, mtchi, has := sm.rootxt.LongestPrefix(path); has {
-		mtch := mtchi.(matcher)
+		mtch := mtchi.(pathDeducer)
 		root, err = mtch.deduceRoot(path)
 		if err != nil {
 			return nil, nil, err
@@ -520,7 +520,7 @@ func (sm *SourceMgr) deduceFromPath(path string) (root futureString, src deferre
 	}
 
 	// Next, try the vcs extension-based (infix) matcher
-	exm := vcsExtensionMatcher{regexp: vcsExtensionRegex}
+	exm := vcsExtensionDeducer{regexp: vcsExtensionRegex}
 	if root, err = exm.deduceRoot(path); err == nil {
 		src, err = exm.deduceSource(path, u)
 		if err != nil {
@@ -566,7 +566,7 @@ func (sm *SourceMgr) deduceFromPath(path string) (root futureString, src deferre
 		return importroot, futerr
 	}
 
-	src = func(cachedir string, an ProjectAnalyzer) futureSource {
+	src = func(cachedir string, an ProjectAnalyzer) sourceFuture {
 		var src source
 		var err error
 

From 0b089553cbc18923b6254396ca5fdb1a17189fd0 Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Fri, 5 Aug 2016 11:31:55 -0400
Subject: [PATCH 45/71] Don't return futures from pathDeducers

This is good architecture in general - the pathDeducers don't need to
return futures themselves, as long as they return futurizable results -
but it's particularly important for testing, as there are no facilities
by which we can inspect and validate the results of pathDeducers' work
when that state is held in future closures.
---
 remote.go | 200 ++++++++++++++++++++++++++++--------------------------
 1 file changed, 102 insertions(+), 98 deletions(-)

diff --git a/remote.go b/remote.go
index c0f19ec..6cdaddc 100644
--- a/remote.go
+++ b/remote.go
@@ -22,10 +22,6 @@ type remoteRepo struct {
 	VCS      []string
 }
 
-type stringFuture func() (string, error)
-type sourceFuture func() (source, error)
-type partialSourceFuture func(string, ProjectAnalyzer) sourceFuture
-
 var (
 	gitSchemes = []string{"https", "ssh", "git", "http"}
 	bzrSchemes = []string{"https", "bzr+ssh", "bzr", "http"}
@@ -81,49 +77,25 @@ var (
 	pathvld     = regexp.MustCompile(`^([A-Za-z0-9-]+)(\.[A-Za-z0-9-]+)+(/[A-Za-z0-9-_.~]+)*$`)
 )
 
-func simpleStringFuture(s string) stringFuture {
-	return func() (string, error) {
-		return s, nil
-	}
-}
-
-func sourceFutureFactory(mb maybeSource) func(string, ProjectAnalyzer) sourceFuture {
-	return func(cachedir string, an ProjectAnalyzer) sourceFuture {
-		var src source
-		var err error
-
-		c := make(chan struct{}, 1)
-		go func() {
-			defer close(c)
-			src, err = mb.try(cachedir, an)
-		}()
-
-		return func() (source, error) {
-			<-c
-			return src, err
-		}
-	}
-}
-
 type pathDeducer interface {
-	deduceRoot(string) (stringFuture, error)
-	deduceSource(string, *url.URL) (func(string, ProjectAnalyzer) sourceFuture, error)
+	deduceRoot(string) (string, error)
+	deduceSource(string, *url.URL) (maybeSource, error)
 }
 
 type githubDeducer struct {
 	regexp *regexp.Regexp
 }
 
-func (m githubDeducer) deduceRoot(path string) (stringFuture, error) {
+func (m githubDeducer) deduceRoot(path string) (string, error) {
 	v := m.regexp.FindStringSubmatch(path)
 	if v == nil {
-		return nil, fmt.Errorf("%s is not a valid path for a source on github.com", path)
+		return "", fmt.Errorf("%s is not a valid path for a source on github.com", path)
 	}
 
-	return simpleStringFuture("github.com/" + v[2]), nil
+	return "github.com/" + v[2], nil
 }
 
-func (m githubDeducer) deduceSource(path string, u *url.URL) (func(string, ProjectAnalyzer) sourceFuture, error) {
+func (m githubDeducer) deduceSource(path string, u *url.URL) (maybeSource, error) {
 	v := m.regexp.FindStringSubmatch(path)
 	if v == nil {
 		return nil, fmt.Errorf("%s is not a valid path for a source on github.com", path)
@@ -134,7 +106,7 @@ func (m githubDeducer) deduceSource(path string, u *url.URL) (func(string, Proje
 		if !validateVCSScheme(u.Scheme, "git") {
 			return nil, fmt.Errorf("%s is not a valid scheme for accessing a git repository", u.Scheme)
 		}
-		return sourceFutureFactory(maybeGitSource{url: u}), nil
+		return maybeGitSource{url: u}, nil
 	}
 
 	mb := make(maybeSources, len(gitSchemes))
@@ -144,23 +116,23 @@ func (m githubDeducer) deduceSource(path string, u *url.URL) (func(string, Proje
 		mb[k] = maybeGitSource{url: &u2}
 	}
 
-	return sourceFutureFactory(mb), nil
+	return mb, nil
 }
 
 type bitbucketDeducer struct {
 	regexp *regexp.Regexp
 }
 
-func (m bitbucketDeducer) deduceRoot(path string) (stringFuture, error) {
+func (m bitbucketDeducer) deduceRoot(path string) (string, error) {
 	v := m.regexp.FindStringSubmatch(path)
 	if v == nil {
-		return nil, fmt.Errorf("%s is not a valid path for a source on bitbucket.org", path)
+		return "", fmt.Errorf("%s is not a valid path for a source on bitbucket.org", path)
 	}
 
-	return simpleStringFuture("bitbucket.org/" + v[2]), nil
+	return "bitbucket.org/" + v[2], nil
 }
 
-func (m bitbucketDeducer) deduceSource(path string, u *url.URL) (func(string, ProjectAnalyzer) sourceFuture, error) {
+func (m bitbucketDeducer) deduceSource(path string, u *url.URL) (maybeSource, error) {
 	v := m.regexp.FindStringSubmatch(path)
 	if v == nil {
 		return nil, fmt.Errorf("%s is not a valid path for a source on bitbucket.org", path)
@@ -177,22 +149,22 @@ func (m bitbucketDeducer) deduceSource(path string, u *url.URL) (func(string, Pr
 			if !validgit {
 				return nil, fmt.Errorf("%s is not a valid scheme for accessing a git repository", u.Scheme)
 			}
-			return sourceFutureFactory(maybeGitSource{url: u}), nil
+			return maybeGitSource{url: u}, nil
 		} else if ishg {
 			if !validhg {
 				return nil, fmt.Errorf("%s is not a valid scheme for accessing an hg repository", u.Scheme)
 			}
-			return sourceFutureFactory(maybeHgSource{url: u}), nil
+			return maybeHgSource{url: u}, nil
 		} else if !validgit && !validhg {
 			return nil, fmt.Errorf("%s is not a valid scheme for accessing either a git or hg repository", u.Scheme)
 		}
 
 		// No other choice, make an option for both git and hg
-		return sourceFutureFactory(maybeSources{
+		return maybeSources{
 			// Git first, because it's a) faster and b) git
 			maybeGitSource{url: u},
 			maybeHgSource{url: u},
-		}), nil
+		}, nil
 	}
 
 	mb := make(maybeSources, 0)
@@ -212,24 +184,23 @@ func (m bitbucketDeducer) deduceSource(path string, u *url.URL) (func(string, Pr
 		}
 	}
 
-	return sourceFutureFactory(mb), nil
+	return mb, nil
 }
 
 type gopkginDeducer struct {
 	regexp *regexp.Regexp
 }
 
-func (m gopkginDeducer) deduceRoot(path string) (stringFuture, error) {
+func (m gopkginDeducer) deduceRoot(path string) (string, error) {
 	v := m.regexp.FindStringSubmatch(path)
 	if v == nil {
-		return nil, fmt.Errorf("%s is not a valid path for a source on gopkg.in", path)
+		return "", fmt.Errorf("%s is not a valid path for a source on gopkg.in", path)
 	}
 
-	return simpleStringFuture("gopkg.in/" + v[2]), nil
+	return "gopkg.in/" + v[2], nil
 }
 
-func (m gopkginDeducer) deduceSource(path string, u *url.URL) (func(string, ProjectAnalyzer) sourceFuture, error) {
-
+func (m gopkginDeducer) deduceSource(path string, u *url.URL) (maybeSource, error) {
 	v := m.regexp.FindStringSubmatch(path)
 	if v == nil {
 		return nil, fmt.Errorf("%s is not a valid path for a source on gopkg.in", path)
@@ -264,25 +235,25 @@ func (m gopkginDeducer) deduceSource(path string, u *url.URL) (func(string, Proj
 		mb[k] = maybeGitSource{url: &u2}
 	}
 
-	return sourceFutureFactory(mb), nil
+	return mb, nil
 }
 
 type launchpadDeducer struct {
 	regexp *regexp.Regexp
 }
 
-func (m launchpadDeducer) deduceRoot(path string) (stringFuture, error) {
+func (m launchpadDeducer) deduceRoot(path string) (string, error) {
 	// TODO(sdboyer) lp handling is nasty - there's ambiguities which can only really
 	// be resolved with a metadata request. See https://github.com/golang/go/issues/11436
 	v := m.regexp.FindStringSubmatch(path)
 	if v == nil {
-		return nil, fmt.Errorf("%s is not a valid path for a source on launchpad.net", path)
+		return "", fmt.Errorf("%s is not a valid path for a source on launchpad.net", path)
 	}
 
-	return simpleStringFuture("launchpad.net/" + v[2]), nil
+	return "launchpad.net/" + v[2], nil
 }
 
-func (m launchpadDeducer) deduceSource(path string, u *url.URL) (func(string, ProjectAnalyzer) sourceFuture, error) {
+func (m launchpadDeducer) deduceSource(path string, u *url.URL) (maybeSource, error) {
 	v := m.regexp.FindStringSubmatch(path)
 	if v == nil {
 		return nil, fmt.Errorf("%s is not a valid path for a source on launchpad.net", path)
@@ -293,7 +264,7 @@ func (m launchpadDeducer) deduceSource(path string, u *url.URL) (func(string, Pr
 		if !validateVCSScheme(u.Scheme, "bzr") {
 			return nil, fmt.Errorf("%s is not a valid scheme for accessing a bzr repository", u.Scheme)
 		}
-		return sourceFutureFactory(maybeBzrSource{url: u}), nil
+		return maybeBzrSource{url: u}, nil
 	}
 
 	mb := make(maybeSources, len(bzrSchemes))
@@ -303,24 +274,24 @@ func (m launchpadDeducer) deduceSource(path string, u *url.URL) (func(string, Pr
 		mb[k] = maybeBzrSource{url: &u2}
 	}
 
-	return sourceFutureFactory(mb), nil
+	return mb, nil
 }
 
 type launchpadGitDeducer struct {
 	regexp *regexp.Regexp
 }
 
-func (m launchpadGitDeducer) deduceRoot(path string) (stringFuture, error) {
+func (m launchpadGitDeducer) deduceRoot(path string) (string, error) {
 	// TODO(sdboyer) same ambiguity issues as with normal bzr lp
 	v := m.regexp.FindStringSubmatch(path)
 	if v == nil {
-		return nil, fmt.Errorf("%s is not a valid path for a source on git.launchpad.net", path)
+		return "", fmt.Errorf("%s is not a valid path for a source on git.launchpad.net", path)
 	}
 
-	return simpleStringFuture("git.launchpad.net/" + v[2]), nil
+	return "git.launchpad.net/" + v[2], nil
 }
 
-func (m launchpadGitDeducer) deduceSource(path string, u *url.URL) (func(string, ProjectAnalyzer) sourceFuture, error) {
+func (m launchpadGitDeducer) deduceSource(path string, u *url.URL) (maybeSource, error) {
 	v := m.regexp.FindStringSubmatch(path)
 	if v == nil {
 		return nil, fmt.Errorf("%s is not a valid path for a source on git.launchpad.net", path)
@@ -331,7 +302,7 @@ func (m launchpadGitDeducer) deduceSource(path string, u *url.URL) (func(string,
 		if !validateVCSScheme(u.Scheme, "git") {
 			return nil, fmt.Errorf("%s is not a valid scheme for accessing a git repository", u.Scheme)
 		}
-		return sourceFutureFactory(maybeGitSource{url: u}), nil
+		return maybeGitSource{url: u}, nil
 	}
 
 	mb := make(maybeSources, len(bzrSchemes))
@@ -341,23 +312,23 @@ func (m launchpadGitDeducer) deduceSource(path string, u *url.URL) (func(string,
 		mb[k] = maybeGitSource{url: &u2}
 	}
 
-	return sourceFutureFactory(mb), nil
+	return mb, nil
 }
 
 type jazzDeducer struct {
 	regexp *regexp.Regexp
 }
 
-func (m jazzDeducer) deduceRoot(path string) (stringFuture, error) {
+func (m jazzDeducer) deduceRoot(path string) (string, error) {
 	v := m.regexp.FindStringSubmatch(path)
 	if v == nil {
-		return nil, fmt.Errorf("%s is not a valid path for a source on hub.jazz.net", path)
+		return "", fmt.Errorf("%s is not a valid path for a source on hub.jazz.net", path)
 	}
 
-	return simpleStringFuture("hub.jazz.net/" + v[2]), nil
+	return "hub.jazz.net/" + v[2], nil
 }
 
-func (m jazzDeducer) deduceSource(path string, u *url.URL) (func(string, ProjectAnalyzer) sourceFuture, error) {
+func (m jazzDeducer) deduceSource(path string, u *url.URL) (maybeSource, error) {
 	v := m.regexp.FindStringSubmatch(path)
 	if v == nil {
 		return nil, fmt.Errorf("%s is not a valid path for a source on hub.jazz.net", path)
@@ -368,7 +339,7 @@ func (m jazzDeducer) deduceSource(path string, u *url.URL) (func(string, Project
 		if !validateVCSScheme(u.Scheme, "git") {
 			return nil, fmt.Errorf("%s is not a valid scheme for accessing a git repository", u.Scheme)
 		}
-		return sourceFutureFactory(maybeGitSource{url: u}), nil
+		return maybeGitSource{url: u}, nil
 	}
 
 	mb := make(maybeSources, len(gitSchemes))
@@ -378,23 +349,23 @@ func (m jazzDeducer) deduceSource(path string, u *url.URL) (func(string, Project
 		mb[k] = maybeGitSource{url: &u2}
 	}
 
-	return sourceFutureFactory(mb), nil
+	return mb, nil
 }
 
 type apacheDeducer struct {
 	regexp *regexp.Regexp
 }
 
-func (m apacheDeducer) deduceRoot(path string) (stringFuture, error) {
+func (m apacheDeducer) deduceRoot(path string) (string, error) {
 	v := m.regexp.FindStringSubmatch(path)
 	if v == nil {
-		return nil, fmt.Errorf("%s is not a valid path for a source on git.apache.org", path)
+		return "", fmt.Errorf("%s is not a valid path for a source on git.apache.org", path)
 	}
 
-	return simpleStringFuture("git.apache.org/" + v[2]), nil
+	return "git.apache.org/" + v[2], nil
 }
 
-func (m apacheDeducer) deduceSource(path string, u *url.URL) (func(string, ProjectAnalyzer) sourceFuture, error) {
+func (m apacheDeducer) deduceSource(path string, u *url.URL) (maybeSource, error) {
 	v := m.regexp.FindStringSubmatch(path)
 	if v == nil {
 		return nil, fmt.Errorf("%s is not a valid path for a source on git.apache.org", path)
@@ -405,7 +376,7 @@ func (m apacheDeducer) deduceSource(path string, u *url.URL) (func(string, Proje
 		if !validateVCSScheme(u.Scheme, "git") {
 			return nil, fmt.Errorf("%s is not a valid scheme for accessing a git repository", u.Scheme)
 		}
-		return sourceFutureFactory(maybeGitSource{url: u}), nil
+		return maybeGitSource{url: u}, nil
 	}
 
 	mb := make(maybeSources, len(gitSchemes))
@@ -415,23 +386,23 @@ func (m apacheDeducer) deduceSource(path string, u *url.URL) (func(string, Proje
 		mb[k] = maybeGitSource{url: &u2}
 	}
 
-	return sourceFutureFactory(mb), nil
+	return mb, nil
 }
 
 type vcsExtensionDeducer struct {
 	regexp *regexp.Regexp
 }
 
-func (m vcsExtensionDeducer) deduceRoot(path string) (stringFuture, error) {
+func (m vcsExtensionDeducer) deduceRoot(path string) (string, error) {
 	v := m.regexp.FindStringSubmatch(path)
 	if v == nil {
-		return nil, fmt.Errorf("%s contains no vcs extension hints for matching", path)
+		return "", fmt.Errorf("%s contains no vcs extension hints for matching", path)
 	}
 
-	return simpleStringFuture(v[1]), nil
+	return v[1], nil
 }
 
-func (m vcsExtensionDeducer) deduceSource(path string, u *url.URL) (func(string, ProjectAnalyzer) sourceFuture, error) {
+func (m vcsExtensionDeducer) deduceSource(path string, u *url.URL) (maybeSource, error) {
 	v := m.regexp.FindStringSubmatch(path)
 	if v == nil {
 		return nil, fmt.Errorf("%s contains no vcs extension hints for matching", path)
@@ -451,11 +422,11 @@ func (m vcsExtensionDeducer) deduceSource(path string, u *url.URL) (func(string,
 
 			switch v[5] {
 			case "git":
-				return sourceFutureFactory(maybeGitSource{url: u}), nil
+				return maybeGitSource{url: u}, nil
 			case "bzr":
-				return sourceFutureFactory(maybeBzrSource{url: u}), nil
+				return maybeBzrSource{url: u}, nil
 			case "hg":
-				return sourceFutureFactory(maybeHgSource{url: u}), nil
+				return maybeHgSource{url: u}, nil
 			}
 		}
 
@@ -487,46 +458,79 @@ func (m vcsExtensionDeducer) deduceSource(path string, u *url.URL) (func(string,
 			f(k, &u2)
 		}
 
-		return sourceFutureFactory(mb), nil
+		return mb, nil
 	default:
 		return nil, fmt.Errorf("unknown repository type: %q", v[5])
 	}
 }
 
-// deduceFromPath takes an import path and converts it into a valid source root.
+type stringFuture func() (string, error)
+type sourceFuture func() (source, error)
+type partialSourceFuture func(string, ProjectAnalyzer) sourceFuture
+
+// deduceFromPath takes an import path and attempts to deduce various
+// metadata about it - what type of source should handle it, and where its
+// "root" is (for vcs repositories, the repository root).
 //
-// The result is wrapped in a future, as some import path patterns may require
-// network activity to correctly determine them via the parsing of "go get" HTTP
-// meta tags.
-func (sm *SourceMgr) deduceFromPath(path string) (root stringFuture, src partialSourceFuture, err error) {
+// The results are wrapped in futures, as most of these operations require at
+// least some network activity to complete. For the first return value, network
+// activity will be triggered when the future is called. For the second,
+// network activity is triggered only when calling the sourceFuture returned
+// from the partialSourceFuture.
+func (sm *SourceMgr) deduceFromPath(path string) (stringFuture, partialSourceFuture, error) {
 	u, err := normalizeURI(path)
 	if err != nil {
 		return nil, nil, err
 	}
 
+	// Helpers to futurize the results from deducers
+	strfut := func(s string) stringFuture {
+		return func() (string, error) {
+			return s, nil
+		}
+	}
+
+	srcfut := func(mb maybeSource) func(string, ProjectAnalyzer) sourceFuture {
+		return func(cachedir string, an ProjectAnalyzer) sourceFuture {
+			var src source
+			var err error
+
+			c := make(chan struct{}, 1)
+			go func() {
+				defer close(c)
+				src, err = mb.try(cachedir, an)
+			}()
+
+			return func() (source, error) {
+				<-c
+				return src, err
+			}
+		}
+	}
+
 	// First, try the root path-based matches
 	if _, mtchi, has := sm.rootxt.LongestPrefix(path); has {
 		mtch := mtchi.(pathDeducer)
-		root, err = mtch.deduceRoot(path)
+		root, err := mtch.deduceRoot(path)
 		if err != nil {
 			return nil, nil, err
 		}
-		src, err = mtch.deduceSource(path, u)
+		mb, err := mtch.deduceSource(path, u)
 		if err != nil {
 			return nil, nil, err
 		}
 
-		return
+		return strfut(root), srcfut(mb), nil
 	}
 
 	// Next, try the vcs extension-based (infix) matcher
 	exm := vcsExtensionDeducer{regexp: vcsExtensionRegex}
-	if root, err = exm.deduceRoot(path); err == nil {
-		src, err = exm.deduceSource(path, u)
+	if root, err := exm.deduceRoot(path); err == nil {
+		mb, err := exm.deduceSource(path, u)
 		if err != nil {
-			root, src = nil, nil
+			return nil, nil, err
 		}
-		return
+		return strfut(root), srcfut(mb), nil
 	}
 
 	// No luck so far. maybe it's one of them vanity imports?
@@ -561,12 +565,12 @@ func (sm *SourceMgr) deduceFromPath(path string) (root stringFuture, src partial
 	}()
 
 	// Set up the root func to catch the result
-	root = func() (string, error) {
+	root := func() (string, error) {
 		<-c
 		return importroot, futerr
 	}
 
-	src = func(cachedir string, an ProjectAnalyzer) sourceFuture {
+	src := func(cachedir string, an ProjectAnalyzer) sourceFuture {
 		var src source
 		var err error
 
@@ -603,7 +607,7 @@ func (sm *SourceMgr) deduceFromPath(path string) (root stringFuture, src partial
 		}
 	}
 
-	return
+	return root, src, nil
 }
 
 func normalizeURI(path string) (u *url.URL, err error) {

From b89117630001b4dc8e73d44e3cfe1f6d0bfab7fa Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Fri, 5 Aug 2016 12:57:06 -0400
Subject: [PATCH 46/71] Add gopkg.in/yaml special case

This is starting to make having a deducer for gopkg.in look like a
really bad idea. We'll chuck it and fall back on vanity import detection
via metadata if we find one more exception.
---
 remote.go      | 11 ++++++++++-
 remote_test.go |  4 ++--
 2 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/remote.go b/remote.go
index 6cdaddc..9692746 100644
--- a/remote.go
+++ b/remote.go
@@ -223,7 +223,16 @@ func (m gopkginDeducer) deduceSource(path string, u *url.URL) (maybeSource, erro
 	// If the third position is empty, it's the shortened form that expands
 	// to the go-pkg github user
 	if v[2] == "" {
-		u.Path = "go-pkg/" + v[3]
+		var inter string
+		// Apparently gopkg.in special-cases gopkg.in/yaml, violating its own rules?
+		// If we find one more exception, chuck this and just rely on vanity
+		// metadata resolving.
+		if strings.HasPrefix(path, "gopkg.in/yaml") {
+			inter = "go-yaml"
+		} else {
+			inter = "go-pkg"
+		}
+		u.Path = inter + v[3]
 	} else {
 		u.Path = v[2] + "/" + v[3]
 	}
diff --git a/remote_test.go b/remote_test.go
index 6f5cb62..808019c 100644
--- a/remote_test.go
+++ b/remote_test.go
@@ -148,7 +148,7 @@ func TestDeduceRemotes(t *testing.T) {
 				RelPkg: "",
 				CloneURL: &url.URL{
 					Host: "github.com",
-					Path: "go-pkg/yaml",
+					Path: "go-yaml/yaml",
 				},
 				Schemes: gitSchemes,
 				VCS:     []string{"git"},
@@ -161,7 +161,7 @@ func TestDeduceRemotes(t *testing.T) {
 				RelPkg: "foo/bar",
 				CloneURL: &url.URL{
 					Host: "github.com",
-					Path: "go-pkg/yaml",
+					Path: "go-yaml/yaml",
 				},
 				Schemes: gitSchemes,
 				VCS:     []string{"git"},

From aaa93900e5ce041d2c3be8827f55771af0d56859 Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Mon, 8 Aug 2016 13:26:09 -0400
Subject: [PATCH 47/71] Convert remote deduction fixtures (todo: checking)

This converts the old remote repo deduction fixtures into something
based on the new system. A bit more refactoring is still needed to fully
flesh out exactly where these fixtures are applied, but for historical
clarity, it's worth breaking the conversion into a single commit.
---
 remote_test.go | 645 +++++++++++++++++++++++--------------------------
 1 file changed, 308 insertions(+), 337 deletions(-)

diff --git a/remote_test.go b/remote_test.go
index 808019c..4e9f804 100644
--- a/remote_test.go
+++ b/remote_test.go
@@ -1,346 +1,337 @@
 package gps
 
 import (
+	"errors"
 	"fmt"
+	"io/ioutil"
 	"net/url"
-	"reflect"
 	"testing"
 )
 
-func TestDeduceRemotes(t *testing.T) {
+func TestDeduceFromPath(t *testing.T) {
 	if testing.Short() {
 		t.Skip("Skipping remote deduction test in short mode")
 	}
 
+	cpath, err := ioutil.TempDir("", "smcache")
+	if err != nil {
+		t.Errorf("Failed to create temp dir: %s", err)
+	}
+	sm, err := NewSourceManager(naiveAnalyzer{}, cpath, false)
+
+	if err != nil {
+		t.Errorf("Unexpected error on SourceManager creation: %s", err)
+		t.FailNow()
+	}
+	defer func() {
+		err := removeAll(cpath)
+		if err != nil {
+			t.Errorf("removeAll failed: %s", err)
+		}
+	}()
+	defer sm.Release()
+
+	// helper func to generate testing *url.URLs, panicking on err
+	mkurl := func(s string) (u *url.URL) {
+		var err error
+		u, err = url.Parse(s)
+		if err != nil {
+			panic(fmt.Sprint("string is not a valid URL:", s))
+		}
+		return
+	}
+
 	fixtures := []struct {
-		path string
-		want *remoteRepo
+		in     string
+		root   string
+		rerr   error
+		mb     maybeSource
+		srcerr error
 	}{
 		{
-			"github.com/sdboyer/gps",
-			&remoteRepo{
-				Base:   "github.com/sdboyer/gps",
-				RelPkg: "",
-				CloneURL: &url.URL{
-					Host: "github.com",
-					Path: "sdboyer/gps",
-				},
-				Schemes: gitSchemes,
-				VCS:     []string{"git"},
+			in:   "github.com/sdboyer/gps",
+			root: "github.com/sdboyer/gps",
+			mb: maybeSources{
+				&maybeGitSource{url: mkurl("https://github.com/sdboyer/gps")},
+				&maybeGitSource{url: mkurl("ssh://git@github.com/sdboyer/gps")},
+				&maybeGitSource{url: mkurl("git://github.com/sdboyer/gps")},
+				&maybeGitSource{url: mkurl("http://github.com/sdboyer/gps")},
 			},
 		},
 		{
-			"github.com/sdboyer/gps/foo",
-			&remoteRepo{
-				Base:   "github.com/sdboyer/gps",
-				RelPkg: "foo",
-				CloneURL: &url.URL{
-					Host: "github.com",
-					Path: "sdboyer/gps",
-				},
-				Schemes: gitSchemes,
-				VCS:     []string{"git"},
+			in:   "github.com/sdboyer/gps/foo",
+			root: "github.com/sdboyer/gps",
+			mb: maybeSources{
+				&maybeGitSource{url: mkurl("https://github.com/sdboyer/gps")},
+				&maybeGitSource{url: mkurl("ssh://git@github.com/sdboyer/gps")},
+				&maybeGitSource{url: mkurl("git://github.com/sdboyer/gps")},
+				&maybeGitSource{url: mkurl("http://github.com/sdboyer/gps")},
 			},
 		},
 		{
-			"git@github.com:sdboyer/gps",
-			&remoteRepo{
-				Base:   "github.com/sdboyer/gps",
-				RelPkg: "",
-				CloneURL: &url.URL{
-					Scheme: "ssh",
-					User:   url.User("git"),
-					Host:   "github.com",
-					Path:   "sdboyer/gps",
-				},
-				Schemes: []string{"ssh"},
-				VCS:     []string{"git"},
+			in:   "github.com/sdboyer/gps.git/foo",
+			root: "github.com/sdboyer/gps",
+			mb: maybeSources{
+				&maybeGitSource{url: mkurl("https://github.com/sdboyer/gps")},
+				&maybeGitSource{url: mkurl("ssh://git@github.com/sdboyer/gps")},
+				&maybeGitSource{url: mkurl("git://github.com/sdboyer/gps")},
+				&maybeGitSource{url: mkurl("http://github.com/sdboyer/gps")},
 			},
 		},
 		{
-			"https://github.com/sdboyer/gps/foo",
-			&remoteRepo{
-				Base:   "github.com/sdboyer/gps",
-				RelPkg: "foo",
-				CloneURL: &url.URL{
-					Scheme: "https",
-					Host:   "github.com",
-					Path:   "sdboyer/gps",
-				},
-				Schemes: []string{"https"},
-				VCS:     []string{"git"},
-			},
+			in:   "git@github.com:sdboyer/gps",
+			root: "github.com/sdboyer/gps",
+			mb:   &maybeGitSource{url: mkurl("ssh://git@github.com/sdboyer/gps")},
 		},
 		{
-			"https://github.com/sdboyer/gps/foo/bar",
-			&remoteRepo{
-				Base:   "github.com/sdboyer/gps",
-				RelPkg: "foo/bar",
-				CloneURL: &url.URL{
-					Scheme: "https",
-					Host:   "github.com",
-					Path:   "sdboyer/gps",
-				},
-				Schemes: []string{"https"},
-				VCS:     []string{"git"},
-			},
+			in:   "https://github.com/sdboyer/gps",
+			root: "github.com/sdboyer/gps",
+			mb:   &maybeGitSource{url: mkurl("https://github.com/sdboyer/gps")},
+		},
+		{
+			in:   "https://github.com/sdboyer/gps/foo/bar",
+			root: "github.com/sdboyer/gps",
+			mb:   &maybeGitSource{url: mkurl("https://github.com/sdboyer/gps")},
 		},
 		// some invalid github username patterns
 		{
-			"github.com/-sdboyer/gps/foo",
-			nil,
+			in:   "github.com/-sdboyer/gps/foo",
+			rerr: errors.New("github.com/-sdboyer/gps/foo is not a valid path for a source on github.com"),
 		},
 		{
-			"github.com/sdboyer-/gps/foo",
-			nil,
+			in:   "github.com/sdboyer-/gps/foo",
+			rerr: errors.New("github.com/sdboyer-/gps/foo is not a valid path for a source on github.com"),
 		},
 		{
-			"github.com/sdbo.yer/gps/foo",
-			nil,
+			in:   "github.com/sdbo.yer/gps/foo",
+			rerr: errors.New("github.com/sdbo.yer/gps/foo is not a valid path for a source on github.com"),
 		},
 		{
-			"github.com/sdbo_yer/gps/foo",
-			nil,
+			in:   "github.com/sdbo_yer/gps/foo",
+			rerr: errors.New("github.com/sdbo_yer/gps/foo is not a valid path for a source on github.com"),
 		},
 		{
-			"gopkg.in/sdboyer/gps.v0",
-			&remoteRepo{
-				Base:   "gopkg.in/sdboyer/gps.v0",
-				RelPkg: "",
-				CloneURL: &url.URL{
-					Host: "github.com",
-					Path: "sdboyer/gps",
-				},
-				Schemes: gitSchemes,
-				VCS:     []string{"git"},
+			in:   "gopkg.in/sdboyer/gps.v0",
+			root: "gopkg.in/sdboyer/gps.v0",
+			mb: maybeSources{
+				&maybeGitSource{url: mkurl("https://github.com/sdboyer/gps")},
+				&maybeGitSource{url: mkurl("ssh://git@github.com/sdboyer/gps")},
+				&maybeGitSource{url: mkurl("git://github.com/sdboyer/gps")},
+				&maybeGitSource{url: mkurl("http://github.com/sdboyer/gps")},
 			},
 		},
 		{
-			"gopkg.in/sdboyer/gps.v0/foo",
-			&remoteRepo{
-				Base:   "gopkg.in/sdboyer/gps.v0",
-				RelPkg: "foo",
-				CloneURL: &url.URL{
-					Host: "github.com",
-					Path: "sdboyer/gps",
-				},
-				Schemes: gitSchemes,
-				VCS:     []string{"git"},
+			in:   "gopkg.in/sdboyer/gps.v0/foo",
+			root: "gopkg.in/sdboyer/gps.v0",
+			mb: maybeSources{
+				&maybeGitSource{url: mkurl("https://github.com/sdboyer/gps")},
+				&maybeGitSource{url: mkurl("ssh://git@github.com/sdboyer/gps")},
+				&maybeGitSource{url: mkurl("git://github.com/sdboyer/gps")},
+				&maybeGitSource{url: mkurl("http://github.com/sdboyer/gps")},
 			},
 		},
 		{
-			"gopkg.in/sdboyer/gps.v0/foo/bar",
-			&remoteRepo{
-				Base:   "gopkg.in/sdboyer/gps.v0",
-				RelPkg: "foo/bar",
-				CloneURL: &url.URL{
-					Host: "github.com",
-					Path: "sdboyer/gps",
-				},
-				Schemes: gitSchemes,
-				VCS:     []string{"git"},
+			in:   "gopkg.in/sdboyer/gps.v1/foo/bar",
+			root: "gopkg.in/sdboyer/gps.v1",
+			mb: maybeSources{
+				&maybeGitSource{url: mkurl("https://github.com/sdboyer/gps")},
+				&maybeGitSource{url: mkurl("ssh://git@github.com/sdboyer/gps")},
+				&maybeGitSource{url: mkurl("git://github.com/sdboyer/gps")},
+				&maybeGitSource{url: mkurl("http://github.com/sdboyer/gps")},
 			},
 		},
 		{
-			"gopkg.in/yaml.v1",
-			&remoteRepo{
-				Base:   "gopkg.in/yaml.v1",
-				RelPkg: "",
-				CloneURL: &url.URL{
-					Host: "github.com",
-					Path: "go-yaml/yaml",
-				},
-				Schemes: gitSchemes,
-				VCS:     []string{"git"},
-			},
+			in:   "gopkg.in/yaml.v1",
+			root: "gopkg.in/yaml.v1",
+			mb:   &maybeGitSource{url: mkurl("https://github.com/go-yaml/yaml")},
 		},
 		{
-			"gopkg.in/yaml.v1/foo/bar",
-			&remoteRepo{
-				Base:   "gopkg.in/yaml.v1",
-				RelPkg: "foo/bar",
-				CloneURL: &url.URL{
-					Host: "github.com",
-					Path: "go-yaml/yaml",
-				},
-				Schemes: gitSchemes,
-				VCS:     []string{"git"},
-			},
+			in:   "gopkg.in/yaml.v1/foo/bar",
+			root: "gopkg.in/yaml.v1",
+			mb:   &maybeGitSource{url: mkurl("https://github.com/go-yaml/yaml")},
 		},
 		{
 			// gopkg.in only allows specifying major version in import path
-			"gopkg.in/yaml.v1.2",
-			nil,
+			root: "gopkg.in/yaml.v1.2",
+			rerr: errors.New("gopkg.in/yaml.v1.2 is not a valid path; gopkg.in only allows major versions (\"v1\" instead of \"v1.2\")"),
 		},
 		// IBM hub devops services - fixtures borrowed from go get
 		{
-			"hub.jazz.net/git/user1/pkgname",
-			&remoteRepo{
-				Base:   "hub.jazz.net/git/user1/pkgname",
-				RelPkg: "",
-				CloneURL: &url.URL{
-					Host: "hub.jazz.net",
-					Path: "git/user1/pkgname",
-				},
-				Schemes: gitSchemes,
-				VCS:     []string{"git"},
+			in:   "hub.jazz.net/git/user1/pkgname",
+			root: "hub.jazz.net/git/user1/pkgname",
+			mb: maybeSources{
+				&maybeGitSource{url: mkurl("https://hub.jazz.net/git/user1/pkgname")},
+				&maybeGitSource{url: mkurl("ssh://git@hub.jazz.net/git/user1/pkgname")},
+				&maybeGitSource{url: mkurl("git://hub.jazz.net/git/user1/pkgname")},
+				&maybeGitSource{url: mkurl("http://hub.jazz.net/git/user1/pkgname")},
 			},
 		},
 		{
-			"hub.jazz.net/git/user1/pkgname/submodule/submodule/submodule",
-			&remoteRepo{
-				Base:   "hub.jazz.net/git/user1/pkgname",
-				RelPkg: "submodule/submodule/submodule",
-				CloneURL: &url.URL{
-					Host: "hub.jazz.net",
-					Path: "git/user1/pkgname",
-				},
-				Schemes: gitSchemes,
-				VCS:     []string{"git"},
+			in:   "hub.jazz.net/git/user1/pkgname/submodule/submodule/submodule",
+			root: "hub.jazz.net/git/user1/pkgname",
+			mb: maybeSources{
+				&maybeGitSource{url: mkurl("https://hub.jazz.net/git/user1/pkgname")},
+				&maybeGitSource{url: mkurl("ssh://git@hub.jazz.net/git/user1/pkgname")},
+				&maybeGitSource{url: mkurl("git://hub.jazz.net/git/user1/pkgname")},
+				&maybeGitSource{url: mkurl("http://hub.jazz.net/git/user1/pkgname")},
 			},
 		},
 		{
-			"hub.jazz.net",
-			nil,
+			in:   "hub.jazz.net",
+			rerr: errors.New("unable to deduce repository and source type for: \"hub.jazz.net\""),
 		},
 		{
-			"hub2.jazz.net",
-			nil,
+			in:   "hub2.jazz.net",
+			rerr: errors.New("unable to deduce repository and source type for: \"hub2.jazz.net\""),
 		},
 		{
-			"hub.jazz.net/someotherprefix",
-			nil,
+			in:   "hub.jazz.net/someotherprefix",
+			rerr: errors.New("unable to deduce repository and source type for: \"hub.jazz.net/someotherprefix\""),
 		},
 		{
-			"hub.jazz.net/someotherprefix/user1/pkgname",
-			nil,
+			in:   "hub.jazz.net/someotherprefix/user1/packagename",
+			rerr: errors.New("unable to deduce repository and source type for: \"hub.jazz.net/someotherprefix/user1/packagename\""),
 		},
 		// Spaces are not valid in user names or package names
 		{
-			"hub.jazz.net/git/User 1/pkgname",
-			nil,
+			in:   "hub.jazz.net/git/User 1/pkgname",
+			rerr: errors.New("hub.jazz.net/git/User 1/pkgname is not a valid path for a source on hub.jazz.net"),
 		},
 		{
-			"hub.jazz.net/git/user1/pkg name",
-			nil,
+			in:   "hub.jazz.net/git/user1/pkg name",
+			rerr: errors.New("hub.jazz.net/git/user1/pkg name is not a valid path for a source on hub.jazz.net"),
 		},
 		// Dots are not valid in user names
 		{
-			"hub.jazz.net/git/user.1/pkgname",
-			nil,
+			in:   "hub.jazz.net/git/user.1/pkgname",
+			rerr: errors.New("hub.jazz.net/git/user.1/pkgname is not a valid path for a source on hub.jazz.net"),
 		},
 		{
-			"hub.jazz.net/git/user/pkg.name",
-			&remoteRepo{
-				Base:   "hub.jazz.net/git/user/pkg.name",
-				RelPkg: "",
-				CloneURL: &url.URL{
-					Host: "hub.jazz.net",
-					Path: "git/user/pkg.name",
-				},
-				Schemes: gitSchemes,
-				VCS:     []string{"git"},
+			in:   "hub.jazz.net/git/user/pkg.name",
+			root: "hub.jazz.net/git/user/pkg.name",
+			mb: maybeSources{
+				&maybeGitSource{url: mkurl("https://hub.jazz.net/git/user1/pkgname")},
+				&maybeGitSource{url: mkurl("ssh://git@hub.jazz.net/git/user1/pkgname")},
+				&maybeGitSource{url: mkurl("git://hub.jazz.net/git/user1/pkgname")},
+				&maybeGitSource{url: mkurl("http://hub.jazz.net/git/user1/pkgname")},
 			},
 		},
 		// User names cannot have uppercase letters
 		{
-			"hub.jazz.net/git/USER/pkgname",
-			nil,
+			in:   "hub.jazz.net/git/USER/pkgname",
+			rerr: errors.New("hub.jazz.net/git/USER/pkgname is not a valid path for a source on hub.jazz.net"),
 		},
 		{
-			"bitbucket.org/sdboyer/reporoot",
-			&remoteRepo{
-				Base:   "bitbucket.org/sdboyer/reporoot",
-				RelPkg: "",
-				CloneURL: &url.URL{
-					Host: "bitbucket.org",
-					Path: "sdboyer/reporoot",
-				},
-				Schemes: hgSchemes,
-				VCS:     []string{"git", "hg"},
+			in:   "bitbucket.org/sdboyer/reporoot",
+			root: "bitbucket.org/sdboyer/reporoot",
+			mb: maybeSources{
+				&maybeGitSource{url: mkurl("https://bitbucket.org/sdboyer/reporoot")},
+				&maybeGitSource{url: mkurl("ssh://git@bitbucket.org/sdboyer/reporoot")},
+				&maybeGitSource{url: mkurl("git://bitbucket.org/sdboyer/reporoot")},
+				&maybeGitSource{url: mkurl("http://bitbucket.org/sdboyer/reporoot")},
+				&maybeHgSource{url: mkurl("https://bitbucket.org/sdboyer/reporoot")},
+				&maybeHgSource{url: mkurl("ssh://hg@bitbucket.org/sdboyer/reporoot")},
+				&maybeHgSource{url: mkurl("http://bitbucket.org/sdboyer/reporoot")},
 			},
 		},
 		{
-			"bitbucket.org/sdboyer/reporoot/foo/bar",
-			&remoteRepo{
-				Base:   "bitbucket.org/sdboyer/reporoot",
-				RelPkg: "foo/bar",
-				CloneURL: &url.URL{
-					Host: "bitbucket.org",
-					Path: "sdboyer/reporoot",
-				},
-				Schemes: hgSchemes,
-				VCS:     []string{"git", "hg"},
+			in:   "bitbucket.org/sdboyer/reporoot/foo/bar",
+			root: "bitbucket.org/sdboyer/reporoot",
+			mb: maybeSources{
+				&maybeGitSource{url: mkurl("https://bitbucket.org/sdboyer/reporoot")},
+				&maybeGitSource{url: mkurl("ssh://git@bitbucket.org/sdboyer/reporoot")},
+				&maybeGitSource{url: mkurl("git://bitbucket.org/sdboyer/reporoot")},
+				&maybeGitSource{url: mkurl("http://bitbucket.org/sdboyer/reporoot")},
+				&maybeHgSource{url: mkurl("https://bitbucket.org/sdboyer/reporoot")},
+				&maybeHgSource{url: mkurl("ssh://hg@bitbucket.org/sdboyer/reporoot")},
+				&maybeHgSource{url: mkurl("http://bitbucket.org/sdboyer/reporoot")},
 			},
 		},
 		{
-			"https://bitbucket.org/sdboyer/reporoot/foo/bar",
-			&remoteRepo{
-				Base:   "bitbucket.org/sdboyer/reporoot",
-				RelPkg: "foo/bar",
-				CloneURL: &url.URL{
-					Scheme: "https",
-					Host:   "bitbucket.org",
-					Path:   "sdboyer/reporoot",
-				},
-				Schemes: []string{"https"},
-				VCS:     []string{"git", "hg"},
+			in:   "https://bitbucket.org/sdboyer/reporoot/foo/bar",
+			root: "bitbucket.org/sdboyer/reporoot",
+			mb: maybeSources{
+				&maybeGitSource{url: mkurl("https://bitbucket.org/sdboyer/reporoot")},
+				&maybeHgSource{url: mkurl("https://bitbucket.org/sdboyer/reporoot")},
 			},
 		},
+		// Less standard behaviors possible due to the hg/git ambiguity
 		{
-			"launchpad.net/govcstestbzrrepo",
-			&remoteRepo{
-				Base:   "launchpad.net/govcstestbzrrepo",
-				RelPkg: "",
-				CloneURL: &url.URL{
-					Host: "launchpad.net",
-					Path: "govcstestbzrrepo",
-				},
-				Schemes: bzrSchemes,
-				VCS:     []string{"bzr"},
+			in:   "bitbucket.org/sdboyer/reporoot.git",
+			root: "bitbucket.org/sdboyer/reporoot.git",
+			mb: maybeSources{
+				&maybeGitSource{url: mkurl("https://bitbucket.org/sdboyer/reporoot")},
+				&maybeGitSource{url: mkurl("ssh://git@bitbucket.org/sdboyer/reporoot")},
+				&maybeGitSource{url: mkurl("git://bitbucket.org/sdboyer/reporoot")},
+				&maybeGitSource{url: mkurl("http://bitbucket.org/sdboyer/reporoot")},
 			},
 		},
 		{
-			"launchpad.net/govcstestbzrrepo/foo/bar",
-			&remoteRepo{
-				Base:   "launchpad.net/govcstestbzrrepo",
-				RelPkg: "foo/bar",
-				CloneURL: &url.URL{
-					Host: "launchpad.net",
-					Path: "govcstestbzrrepo",
-				},
-				Schemes: bzrSchemes,
-				VCS:     []string{"bzr"},
+			in:   "git@bitbucket.org:sdboyer/reporoot.git",
+			root: "bitbucket.org/sdboyer/reporoot.git",
+			mb:   &maybeGitSource{url: mkurl("ssh://git@bitbucket.org/sdboyer/reporoot")},
+		},
+		{
+			in:   "bitbucket.org/sdboyer/reporoot.hg",
+			root: "bitbucket.org/sdboyer/reporoot.hg",
+			mb: maybeSources{
+				&maybeHgSource{url: mkurl("https://bitbucket.org/sdboyer/reporoot")},
+				&maybeHgSource{url: mkurl("ssh://hg@bitbucket.org/sdboyer/reporoot")},
+				&maybeHgSource{url: mkurl("http://bitbucket.org/sdboyer/reporoot")},
+			},
+		},
+		{
+			in:   "hg@bitbucket.org:sdboyer/reporoot",
+			root: "bitbucket.org/sdboyer/reporoot",
+			mb:   &maybeHgSource{url: mkurl("ssh://hg@bitbucket.org/sdboyer/reporoot")},
+		},
+		{
+			in:     "git://bitbucket.org/sdboyer/reporoot.hg",
+			root:   "bitbucket.org/sdboyer/reporoot.hg",
+			srcerr: errors.New("git is not a valid scheme for accessing an hg repository"),
+		},
+		// tests for launchpad, mostly bazaar
+		// TODO(sdboyer) need more tests to deal w/launchpad's oddities
+		{
+			in:   "launchpad.net/govcstestbzrrepo",
+			root: "launchpad.net/govcstestbzrrepo",
+			mb: maybeSources{
+				&maybeBzrSource{url: mkurl("https://launchpad.net/govcstestbzrrepo")},
+				&maybeBzrSource{url: mkurl("bzr://launchpad.net/govcstestbzrrepo")},
+				&maybeBzrSource{url: mkurl("http://launchpad.net/govcstestbzrrepo")},
+			},
+		},
+		{
+			in:   "launchpad.net/govcstestbzrrepo/foo/bar",
+			root: "launchpad.net/govcstestbzrrepo",
+			mb: maybeSources{
+				&maybeBzrSource{url: mkurl("https://launchpad.net/govcstestbzrrepo")},
+				&maybeBzrSource{url: mkurl("bzr://launchpad.net/govcstestbzrrepo")},
+				&maybeBzrSource{url: mkurl("http://launchpad.net/govcstestbzrrepo")},
 			},
 		},
 		{
 			"launchpad.net/repo root",
-			nil,
+			rerr: errors.New("launchpad.net/repo root is not a valid path for a source on launchpad.net"),
 		},
 		{
-			"git.launchpad.net/reporoot",
-			&remoteRepo{
-				Base:   "git.launchpad.net/reporoot",
-				RelPkg: "",
-				CloneURL: &url.URL{
-					Host: "git.launchpad.net",
-					Path: "reporoot",
-				},
-				Schemes: gitSchemes,
-				VCS:     []string{"git"},
+			in:   "git.launchpad.net/reporoot",
+			root: "git.launchpad.net/reporoot",
+			mb: maybeSources{
+				&maybeGitSource{url: mkurl("https://git.launchpad.net/reporoot")},
+				&maybeGitSource{url: mkurl("ssh://git@git.launchpad.net/reporoot")},
+				&maybeGitSource{url: mkurl("git://git.launchpad.net/reporoot")},
+				&maybeGitSource{url: mkurl("http://git.launchpad.net/reporoot")},
 			},
 		},
 		{
-			"git.launchpad.net/reporoot/foo/bar",
-			&remoteRepo{
-				Base:   "git.launchpad.net/reporoot",
-				RelPkg: "foo/bar",
-				CloneURL: &url.URL{
-					Host: "git.launchpad.net",
-					Path: "reporoot",
-				},
-				Schemes: gitSchemes,
-				VCS:     []string{"git"},
+			in:   "git.launchpad.net/reporoot/foo/bar",
+			root: "git.launchpad.net/reporoot",
+			mb: maybeSources{
+				&maybeGitSource{url: mkurl("https://git.launchpad.net/reporoot")},
+				&maybeGitSource{url: mkurl("ssh://git@git.launchpad.net/reporoot")},
+				&maybeGitSource{url: mkurl("git://git.launchpad.net/reporoot")},
+				&maybeGitSource{url: mkurl("http://git.launchpad.net/reporoot")},
 			},
 		},
 		{
@@ -358,126 +349,106 @@ func TestDeduceRemotes(t *testing.T) {
 		},
 		{
 			"git.launchpad.net/repo root",
-			nil,
+			rerr: errors.New("git.launchpad.net/repo root is not a valid path for a source on launchpad.net"),
 		},
 		{
-			"git.apache.org/package-name.git",
-			&remoteRepo{
-				Base:   "git.apache.org/package-name.git",
-				RelPkg: "",
-				CloneURL: &url.URL{
-					Host: "git.apache.org",
-					Path: "package-name.git",
-				},
-				Schemes: gitSchemes,
-				VCS:     []string{"git"},
+			in:   "git.apache.org/package-name.git",
+			root: "git.apache.org/package-name.git",
+			mb: maybeSources{
+				&maybeGitSource{url: mkurl("https://git.apache.org/package-name.git")},
+				&maybeGitSource{url: mkurl("ssh://git@git.apache.org/package-name.git")},
+				&maybeGitSource{url: mkurl("git://git.apache.org/package-name.git")},
+				&maybeGitSource{url: mkurl("http://git.apache.org/package-name.git")},
 			},
 		},
 		{
-			"git.apache.org/package-name.git/foo/bar",
-			&remoteRepo{
-				Base:   "git.apache.org/package-name.git",
-				RelPkg: "foo/bar",
-				CloneURL: &url.URL{
-					Host: "git.apache.org",
-					Path: "package-name.git",
-				},
-				Schemes: gitSchemes,
-				VCS:     []string{"git"},
+			in:   "git.apache.org/package-name.git/foo/bar",
+			root: "git.apache.org/package-name.git",
+			mb: maybeSources{
+				&maybeGitSource{url: mkurl("https://git.apache.org/package-name.git")},
+				&maybeGitSource{url: mkurl("ssh://git@git.apache.org/package-name.git")},
+				&maybeGitSource{url: mkurl("git://git.apache.org/package-name.git")},
+				&maybeGitSource{url: mkurl("http://git.apache.org/package-name.git")},
 			},
 		},
 		// Vanity imports
 		{
-			"golang.org/x/exp",
-			&remoteRepo{
-				Base:   "golang.org/x/exp",
-				RelPkg: "",
-				CloneURL: &url.URL{
-					Scheme: "https",
-					Host:   "go.googlesource.com",
-					Path:   "/exp",
-				},
-				Schemes: []string{"https"},
-				VCS:     []string{"git"},
-			},
+			in:   "golang.org/x/exp",
+			root: "golang.org/x/exp",
+			mb:   &maybeGitSource{url: mkurl("https://go.googlesource.com/exp")},
 		},
 		{
-			"golang.org/x/exp/inotify",
-			&remoteRepo{
-				Base:   "golang.org/x/exp",
-				RelPkg: "inotify",
-				CloneURL: &url.URL{
-					Scheme: "https",
-					Host:   "go.googlesource.com",
-					Path:   "/exp",
-				},
-				Schemes: []string{"https"},
-				VCS:     []string{"git"},
-			},
+			in:   "golang.org/x/exp/inotify",
+			root: "golang.org/x/exp",
+			mb:   &maybeGitSource{url: mkurl("https://go.googlesource.com/exp")},
 		},
 		{
-			"rsc.io/pdf",
-			&remoteRepo{
-				Base:   "rsc.io/pdf",
-				RelPkg: "",
-				CloneURL: &url.URL{
-					Scheme: "https",
-					Host:   "github.com",
-					Path:   "/rsc/pdf",
-				},
-				Schemes: []string{"https"},
-				VCS:     []string{"git"},
-			},
+			in:   "rsc.io/pdf",
+			root: "rsc.io/pdf",
+			mb:   &maybeGitSource{url: mkurl("https://github.com/rsc/pdf")},
 		},
 		// Regression - gh does allow two-letter usernames
 		{
-			"github.com/kr/pretty",
-			&remoteRepo{
-				Base:   "github.com/kr/pretty",
-				RelPkg: "",
-				CloneURL: &url.URL{
-					Host: "github.com",
-					Path: "kr/pretty",
-				},
-				Schemes: gitSchemes,
-				VCS:     []string{"git"},
+			in:   "github.com/kr/pretty",
+			root: "github.com/kr/pretty",
+			mb: maybeSources{
+				&maybeGitSource{url: mkurl("https://github.com/kr/pretty")},
+				&maybeGitSource{url: mkurl("ssh://git@github.com/kr/pretty")},
+				&maybeGitSource{url: mkurl("git://github.com/kr/pretty")},
+				&maybeGitSource{url: mkurl("http://github.com/kr/pretty")},
+			},
+		},
+		// VCS extension-based syntax
+		{
+			in:   "foobar/baz.git",
+			root: "foobar/baz.git",
+			mb: maybeSources{
+				&maybeGitSource{url: mkurl("https://foobar/baz.git")},
+				&maybeGitSource{url: mkurl("git://foobar/baz.git")},
+				&maybeGitSource{url: mkurl("http://foobar/baz.git")},
 			},
 		},
+		{
+			in:   "foobar/baz.git/quark/quizzle.git",
+			rerr: errors.New("not allowed: foobar/baz.git/quark/quizzle.git contains multiple vcs extension hints"),
+		},
 	}
 
-	for _, fix := range fixtures {
-		got, err := deduceRemoteRepo(fix.path)
-		want := fix.want
+	// TODO(sdboyer) this is all the old checking logic; convert it
+	//for _, fix := range fixtures {
+	//got, err := deduceRemoteRepo(fix.path)
+	//want := fix.want
 
-		if want == nil {
-			if err == nil {
-				t.Errorf("deduceRemoteRepo(%q): Error expected but not received", fix.path)
-			}
-			continue
-		}
+	//if want == nil {
+	//if err == nil {
+	//t.Errorf("deduceRemoteRepo(%q): Error expected but not received", fix.path)
+	//}
+	//continue
+	//}
 
-		if err != nil {
-			t.Errorf("deduceRemoteRepo(%q): %v", fix.path, err)
-			continue
-		}
+	//if err != nil {
+	//t.Errorf("deduceRemoteRepo(%q): %v", fix.path, err)
+	//continue
+	//}
 
-		if got.Base != want.Base {
-			t.Errorf("deduceRemoteRepo(%q): Base was %s, wanted %s", fix.path, got.Base, want.Base)
-		}
-		if got.RelPkg != want.RelPkg {
-			t.Errorf("deduceRemoteRepo(%q): RelPkg was %s, wanted %s", fix.path, got.RelPkg, want.RelPkg)
-		}
-		if !reflect.DeepEqual(got.CloneURL, want.CloneURL) {
-			// misspelling things is cool when it makes columns line up
-			t.Errorf("deduceRemoteRepo(%q): CloneURL disagreement:\n(GOT) %s\n(WNT) %s", fix.path, ufmt(got.CloneURL), ufmt(want.CloneURL))
-		}
-		if !reflect.DeepEqual(got.VCS, want.VCS) {
-			t.Errorf("deduceRemoteRepo(%q): VCS was %s, wanted %s", fix.path, got.VCS, want.VCS)
-		}
-		if !reflect.DeepEqual(got.Schemes, want.Schemes) {
-			t.Errorf("deduceRemoteRepo(%q): Schemes was %s, wanted %s", fix.path, got.Schemes, want.Schemes)
-		}
-	}
+	//if got.Base != want.Base {
+	//t.Errorf("deduceRemoteRepo(%q): Base was %s, wanted %s", fix.path, got.Base, want.Base)
+	//}
+	//if got.RelPkg != want.RelPkg {
+	//t.Errorf("deduceRemoteRepo(%q): RelPkg was %s, wanted %s", fix.path, got.RelPkg, want.RelPkg)
+	//}
+	//if !reflect.DeepEqual(got.CloneURL, want.CloneURL) {
+	//// misspelling things is cool when it makes columns line up
+	//t.Errorf("deduceRemoteRepo(%q): CloneURL disagreement:\n(GOT) %s\n(WNT) %s", fix.path, ufmt(got.CloneURL), ufmt(want.CloneURL))
+	//}
+	//if !reflect.DeepEqual(got.VCS, want.VCS) {
+	//t.Errorf("deduceRemoteRepo(%q): VCS was %s, wanted %s", fix.path, got.VCS, want.VCS)
+	//}
+	//if !reflect.DeepEqual(got.Schemes, want.Schemes) {
+	//t.Errorf("deduceRemoteRepo(%q): Schemes was %s, wanted %s", fix.path, got.Schemes, want.Schemes)
+	//}
+	//}
+	t.Error("TODO implement checking of new path deduction fixtures")
 }
 
 // borrow from stdlib

From 32658fb94fedb4ef680edf3a0fad6845d0ed5304 Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Mon, 8 Aug 2016 13:26:44 -0400
Subject: [PATCH 48/71] Compat comments; use an explicit type on return

---
 remote.go      |  5 ++++-
 remote_test.go | 17 ++---------------
 2 files changed, 6 insertions(+), 16 deletions(-)

diff --git a/remote.go b/remote.go
index 9692746..0fa7e9b 100644
--- a/remote.go
+++ b/remote.go
@@ -147,6 +147,8 @@ func (m bitbucketDeducer) deduceSource(path string, u *url.URL) (maybeSource, er
 		validgit, validhg := validateVCSScheme(u.Scheme, "git"), validateVCSScheme(u.Scheme, "hg")
 		if isgit {
 			if !validgit {
+				// This is unreachable for now, as the git schemes are a
+				// superset of the hg schemes
 				return nil, fmt.Errorf("%s is not a valid scheme for accessing a git repository", u.Scheme)
 			}
 			return maybeGitSource{url: u}, nil
@@ -277,6 +279,7 @@ func (m launchpadDeducer) deduceSource(path string, u *url.URL) (maybeSource, er
 	}
 
 	mb := make(maybeSources, len(bzrSchemes))
+	// TODO(sdboyer) is there a generic ssh user for lp? if not, drop bzr+ssh
 	for k, scheme := range bzrSchemes {
 		u2 := *u
 		u2.Scheme = scheme
@@ -499,7 +502,7 @@ func (sm *SourceMgr) deduceFromPath(path string) (stringFuture, partialSourceFut
 		}
 	}
 
-	srcfut := func(mb maybeSource) func(string, ProjectAnalyzer) sourceFuture {
+	srcfut := func(mb maybeSource) partialSourceFuture {
 		return func(cachedir string, an ProjectAnalyzer) sourceFuture {
 			var src source
 			var err error
diff --git a/remote_test.go b/remote_test.go
index 4e9f804..417d80e 100644
--- a/remote_test.go
+++ b/remote_test.go
@@ -311,7 +311,7 @@ func TestDeduceFromPath(t *testing.T) {
 			},
 		},
 		{
-			"launchpad.net/repo root",
+			in:   "launchpad.net/repo root",
 			rerr: errors.New("launchpad.net/repo root is not a valid path for a source on launchpad.net"),
 		},
 		{
@@ -335,20 +335,7 @@ func TestDeduceFromPath(t *testing.T) {
 			},
 		},
 		{
-			"git.launchpad.net/reporoot",
-			&remoteRepo{
-				Base:   "git.launchpad.net/reporoot",
-				RelPkg: "",
-				CloneURL: &url.URL{
-					Host: "git.launchpad.net",
-					Path: "reporoot",
-				},
-				Schemes: gitSchemes,
-				VCS:     []string{"git"},
-			},
-		},
-		{
-			"git.launchpad.net/repo root",
+			in:   "git.launchpad.net/repo root",
 			rerr: errors.New("git.launchpad.net/repo root is not a valid path for a source on launchpad.net"),
 		},
 		{

From ca638c418b4948e9c22f073ac1de76a7ac5296f7 Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Mon, 8 Aug 2016 23:41:28 -0400
Subject: [PATCH 49/71] Implement checking for new

Still many failures. This also reveals some necessary refactors.
---
 remote.go      |  21 +--
 remote_test.go | 474 +++++++++++++++++++++++++++++--------------------
 2 files changed, 293 insertions(+), 202 deletions(-)

diff --git a/remote.go b/remote.go
index 0fa7e9b..dd6c0f7 100644
--- a/remote.go
+++ b/remote.go
@@ -140,8 +140,8 @@ func (m bitbucketDeducer) deduceSource(path string, u *url.URL) (maybeSource, er
 	u.Path = v[2]
 
 	// This isn't definitive, but it'll probably catch most
-	isgit := strings.HasSuffix(u.Path, ".git") || u.User.Username() == "git"
-	ishg := strings.HasSuffix(u.Path, ".hg") || u.User.Username() == "hg"
+	isgit := strings.HasSuffix(u.Path, ".git") || (u.User != nil && u.User.Username() == "git")
+	ishg := strings.HasSuffix(u.Path, ".hg") || (u.User != nil && u.User.Username() == "hg")
 
 	if u.Scheme != "" {
 		validgit, validhg := validateVCSScheme(u.Scheme, "git"), validateVCSScheme(u.Scheme, "hg")
@@ -490,7 +490,8 @@ type partialSourceFuture func(string, ProjectAnalyzer) sourceFuture
 // network activity is triggered only when calling the sourceFuture returned
 // from the partialSourceFuture.
 func (sm *SourceMgr) deduceFromPath(path string) (stringFuture, partialSourceFuture, error) {
-	u, err := normalizeURI(path)
+	opath := path
+	u, path, err := normalizeURI(path)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -562,7 +563,7 @@ func (sm *SourceMgr) deduceFromPath(path string) (stringFuture, partialSourceFut
 		var reporoot string
 		importroot, vcs, reporoot, futerr = parseMetadata(path)
 		if futerr != nil {
-			futerr = fmt.Errorf("unable to deduce repository and source type for: %q", path)
+			futerr = fmt.Errorf("unable to deduce repository and source type for: %q", opath)
 			return
 		}
 
@@ -622,7 +623,7 @@ func (sm *SourceMgr) deduceFromPath(path string) (stringFuture, partialSourceFut
 	return root, src, nil
 }
 
-func normalizeURI(path string) (u *url.URL, err error) {
+func normalizeURI(path string) (u *url.URL, newpath string, err error) {
 	if m := scpSyntaxRe.FindStringSubmatch(path); m != nil {
 		// Match SCP-like syntax and convert it to a URL.
 		// Eg, "git@github.com:user/repo" becomes
@@ -638,18 +639,18 @@ func normalizeURI(path string) (u *url.URL, err error) {
 	} else {
 		u, err = url.Parse(path)
 		if err != nil {
-			return nil, fmt.Errorf("%q is not a valid URI", path)
+			return nil, "", fmt.Errorf("%q is not a valid URI", path)
 		}
 	}
 
 	if u.Host != "" {
-		path = u.Host + "/" + strings.TrimPrefix(u.Path, "/")
+		newpath = u.Host + "/" + strings.TrimPrefix(u.Path, "/")
 	} else {
-		path = u.Path
+		newpath = u.Path
 	}
 
-	if !pathvld.MatchString(path) {
-		return nil, fmt.Errorf("%q is not a valid import path", path)
+	if !pathvld.MatchString(newpath) {
+		return nil, "", fmt.Errorf("%q is not a valid import path", newpath)
 	}
 
 	return
diff --git a/remote_test.go b/remote_test.go
index 417d80e..a188d73 100644
--- a/remote_test.go
+++ b/remote_test.go
@@ -1,97 +1,81 @@
 package gps
 
 import (
+	"bytes"
 	"errors"
 	"fmt"
 	"io/ioutil"
 	"net/url"
+	"reflect"
 	"testing"
-)
 
-func TestDeduceFromPath(t *testing.T) {
-	if testing.Short() {
-		t.Skip("Skipping remote deduction test in short mode")
-	}
+	"github.com/davecgh/go-spew/spew"
+)
 
-	cpath, err := ioutil.TempDir("", "smcache")
-	if err != nil {
-		t.Errorf("Failed to create temp dir: %s", err)
-	}
-	sm, err := NewSourceManager(naiveAnalyzer{}, cpath, false)
+type pathDeductionFixture struct {
+	in     string
+	root   string
+	rerr   error
+	mb     maybeSource
+	srcerr error
+}
 
+// helper func to generate testing *url.URLs, panicking on err
+func mkurl(s string) (u *url.URL) {
+	var err error
+	u, err = url.Parse(s)
 	if err != nil {
-		t.Errorf("Unexpected error on SourceManager creation: %s", err)
-		t.FailNow()
-	}
-	defer func() {
-		err := removeAll(cpath)
-		if err != nil {
-			t.Errorf("removeAll failed: %s", err)
-		}
-	}()
-	defer sm.Release()
-
-	// helper func to generate testing *url.URLs, panicking on err
-	mkurl := func(s string) (u *url.URL) {
-		var err error
-		u, err = url.Parse(s)
-		if err != nil {
-			panic(fmt.Sprint("string is not a valid URL:", s))
-		}
-		return
+		panic(fmt.Sprint("string is not a valid URL:", s))
 	}
+	return
+}
 
-	fixtures := []struct {
-		in     string
-		root   string
-		rerr   error
-		mb     maybeSource
-		srcerr error
-	}{
+var pathDeductionFixtures = map[string][]pathDeductionFixture{
+	"github": []pathDeductionFixture{
 		{
 			in:   "github.com/sdboyer/gps",
 			root: "github.com/sdboyer/gps",
 			mb: maybeSources{
-				&maybeGitSource{url: mkurl("https://github.com/sdboyer/gps")},
-				&maybeGitSource{url: mkurl("ssh://git@github.com/sdboyer/gps")},
-				&maybeGitSource{url: mkurl("git://github.com/sdboyer/gps")},
-				&maybeGitSource{url: mkurl("http://github.com/sdboyer/gps")},
+				maybeGitSource{url: mkurl("https://github.com/sdboyer/gps")},
+				maybeGitSource{url: mkurl("ssh://git@github.com/sdboyer/gps")},
+				maybeGitSource{url: mkurl("git://github.com/sdboyer/gps")},
+				maybeGitSource{url: mkurl("http://github.com/sdboyer/gps")},
 			},
 		},
 		{
 			in:   "github.com/sdboyer/gps/foo",
 			root: "github.com/sdboyer/gps",
 			mb: maybeSources{
-				&maybeGitSource{url: mkurl("https://github.com/sdboyer/gps")},
-				&maybeGitSource{url: mkurl("ssh://git@github.com/sdboyer/gps")},
-				&maybeGitSource{url: mkurl("git://github.com/sdboyer/gps")},
-				&maybeGitSource{url: mkurl("http://github.com/sdboyer/gps")},
+				maybeGitSource{url: mkurl("https://github.com/sdboyer/gps")},
+				maybeGitSource{url: mkurl("ssh://git@github.com/sdboyer/gps")},
+				maybeGitSource{url: mkurl("git://github.com/sdboyer/gps")},
+				maybeGitSource{url: mkurl("http://github.com/sdboyer/gps")},
 			},
 		},
 		{
 			in:   "github.com/sdboyer/gps.git/foo",
 			root: "github.com/sdboyer/gps",
 			mb: maybeSources{
-				&maybeGitSource{url: mkurl("https://github.com/sdboyer/gps")},
-				&maybeGitSource{url: mkurl("ssh://git@github.com/sdboyer/gps")},
-				&maybeGitSource{url: mkurl("git://github.com/sdboyer/gps")},
-				&maybeGitSource{url: mkurl("http://github.com/sdboyer/gps")},
+				maybeGitSource{url: mkurl("https://github.com/sdboyer/gps")},
+				maybeGitSource{url: mkurl("ssh://git@github.com/sdboyer/gps")},
+				maybeGitSource{url: mkurl("git://github.com/sdboyer/gps")},
+				maybeGitSource{url: mkurl("http://github.com/sdboyer/gps")},
 			},
 		},
 		{
 			in:   "git@github.com:sdboyer/gps",
 			root: "github.com/sdboyer/gps",
-			mb:   &maybeGitSource{url: mkurl("ssh://git@github.com/sdboyer/gps")},
+			mb:   maybeGitSource{url: mkurl("ssh://git@github.com/sdboyer/gps")},
 		},
 		{
 			in:   "https://github.com/sdboyer/gps",
 			root: "github.com/sdboyer/gps",
-			mb:   &maybeGitSource{url: mkurl("https://github.com/sdboyer/gps")},
+			mb:   maybeGitSource{url: mkurl("https://github.com/sdboyer/gps")},
 		},
 		{
 			in:   "https://github.com/sdboyer/gps/foo/bar",
 			root: "github.com/sdboyer/gps",
-			mb:   &maybeGitSource{url: mkurl("https://github.com/sdboyer/gps")},
+			mb:   maybeGitSource{url: mkurl("https://github.com/sdboyer/gps")},
 		},
 		// some invalid github username patterns
 		{
@@ -110,80 +94,87 @@ func TestDeduceFromPath(t *testing.T) {
 			in:   "github.com/sdbo_yer/gps/foo",
 			rerr: errors.New("github.com/sdbo_yer/gps/foo is not a valid path for a source on github.com"),
 		},
+		// Regression - gh does allow two-letter usernames
+		{
+			in:   "github.com/kr/pretty",
+			root: "github.com/kr/pretty",
+			mb: maybeSources{
+				maybeGitSource{url: mkurl("https://github.com/kr/pretty")},
+				maybeGitSource{url: mkurl("ssh://git@github.com/kr/pretty")},
+				maybeGitSource{url: mkurl("git://github.com/kr/pretty")},
+				maybeGitSource{url: mkurl("http://github.com/kr/pretty")},
+			},
+		},
+	},
+	"gopkg.in": []pathDeductionFixture{
 		{
 			in:   "gopkg.in/sdboyer/gps.v0",
 			root: "gopkg.in/sdboyer/gps.v0",
 			mb: maybeSources{
-				&maybeGitSource{url: mkurl("https://github.com/sdboyer/gps")},
-				&maybeGitSource{url: mkurl("ssh://git@github.com/sdboyer/gps")},
-				&maybeGitSource{url: mkurl("git://github.com/sdboyer/gps")},
-				&maybeGitSource{url: mkurl("http://github.com/sdboyer/gps")},
+				maybeGitSource{url: mkurl("https://github.com/sdboyer/gps")},
+				maybeGitSource{url: mkurl("ssh://git@github.com/sdboyer/gps")},
+				maybeGitSource{url: mkurl("git://github.com/sdboyer/gps")},
+				maybeGitSource{url: mkurl("http://github.com/sdboyer/gps")},
 			},
 		},
 		{
 			in:   "gopkg.in/sdboyer/gps.v0/foo",
 			root: "gopkg.in/sdboyer/gps.v0",
 			mb: maybeSources{
-				&maybeGitSource{url: mkurl("https://github.com/sdboyer/gps")},
-				&maybeGitSource{url: mkurl("ssh://git@github.com/sdboyer/gps")},
-				&maybeGitSource{url: mkurl("git://github.com/sdboyer/gps")},
-				&maybeGitSource{url: mkurl("http://github.com/sdboyer/gps")},
+				maybeGitSource{url: mkurl("https://github.com/sdboyer/gps")},
+				maybeGitSource{url: mkurl("ssh://git@github.com/sdboyer/gps")},
+				maybeGitSource{url: mkurl("git://github.com/sdboyer/gps")},
+				maybeGitSource{url: mkurl("http://github.com/sdboyer/gps")},
 			},
 		},
 		{
 			in:   "gopkg.in/sdboyer/gps.v1/foo/bar",
 			root: "gopkg.in/sdboyer/gps.v1",
 			mb: maybeSources{
-				&maybeGitSource{url: mkurl("https://github.com/sdboyer/gps")},
-				&maybeGitSource{url: mkurl("ssh://git@github.com/sdboyer/gps")},
-				&maybeGitSource{url: mkurl("git://github.com/sdboyer/gps")},
-				&maybeGitSource{url: mkurl("http://github.com/sdboyer/gps")},
+				maybeGitSource{url: mkurl("https://github.com/sdboyer/gps")},
+				maybeGitSource{url: mkurl("ssh://git@github.com/sdboyer/gps")},
+				maybeGitSource{url: mkurl("git://github.com/sdboyer/gps")},
+				maybeGitSource{url: mkurl("http://github.com/sdboyer/gps")},
 			},
 		},
 		{
 			in:   "gopkg.in/yaml.v1",
 			root: "gopkg.in/yaml.v1",
-			mb:   &maybeGitSource{url: mkurl("https://github.com/go-yaml/yaml")},
+			mb:   maybeGitSource{url: mkurl("https://github.com/go-yaml/yaml")},
 		},
 		{
 			in:   "gopkg.in/yaml.v1/foo/bar",
 			root: "gopkg.in/yaml.v1",
-			mb:   &maybeGitSource{url: mkurl("https://github.com/go-yaml/yaml")},
+			mb:   maybeGitSource{url: mkurl("https://github.com/go-yaml/yaml")},
 		},
 		{
 			// gopkg.in only allows specifying major version in import path
-			root: "gopkg.in/yaml.v1.2",
+			in:   "gopkg.in/yaml.v1.2",
 			rerr: errors.New("gopkg.in/yaml.v1.2 is not a valid path; gopkg.in only allows major versions (\"v1\" instead of \"v1.2\")"),
 		},
+	},
+	"jazz": []pathDeductionFixture{
 		// IBM hub devops services - fixtures borrowed from go get
 		{
 			in:   "hub.jazz.net/git/user1/pkgname",
 			root: "hub.jazz.net/git/user1/pkgname",
 			mb: maybeSources{
-				&maybeGitSource{url: mkurl("https://hub.jazz.net/git/user1/pkgname")},
-				&maybeGitSource{url: mkurl("ssh://git@hub.jazz.net/git/user1/pkgname")},
-				&maybeGitSource{url: mkurl("git://hub.jazz.net/git/user1/pkgname")},
-				&maybeGitSource{url: mkurl("http://hub.jazz.net/git/user1/pkgname")},
+				maybeGitSource{url: mkurl("https://hub.jazz.net/git/user1/pkgname")},
+				maybeGitSource{url: mkurl("ssh://git@hub.jazz.net/git/user1/pkgname")},
+				maybeGitSource{url: mkurl("git://hub.jazz.net/git/user1/pkgname")},
+				maybeGitSource{url: mkurl("http://hub.jazz.net/git/user1/pkgname")},
 			},
 		},
 		{
 			in:   "hub.jazz.net/git/user1/pkgname/submodule/submodule/submodule",
 			root: "hub.jazz.net/git/user1/pkgname",
 			mb: maybeSources{
-				&maybeGitSource{url: mkurl("https://hub.jazz.net/git/user1/pkgname")},
-				&maybeGitSource{url: mkurl("ssh://git@hub.jazz.net/git/user1/pkgname")},
-				&maybeGitSource{url: mkurl("git://hub.jazz.net/git/user1/pkgname")},
-				&maybeGitSource{url: mkurl("http://hub.jazz.net/git/user1/pkgname")},
+				maybeGitSource{url: mkurl("https://hub.jazz.net/git/user1/pkgname")},
+				maybeGitSource{url: mkurl("ssh://git@hub.jazz.net/git/user1/pkgname")},
+				maybeGitSource{url: mkurl("git://hub.jazz.net/git/user1/pkgname")},
+				maybeGitSource{url: mkurl("http://hub.jazz.net/git/user1/pkgname")},
 			},
 		},
-		{
-			in:   "hub.jazz.net",
-			rerr: errors.New("unable to deduce repository and source type for: \"hub.jazz.net\""),
-		},
-		{
-			in:   "hub2.jazz.net",
-			rerr: errors.New("unable to deduce repository and source type for: \"hub2.jazz.net\""),
-		},
 		{
 			in:   "hub.jazz.net/someotherprefix",
 			rerr: errors.New("unable to deduce repository and source type for: \"hub.jazz.net/someotherprefix\""),
@@ -210,10 +201,10 @@ func TestDeduceFromPath(t *testing.T) {
 			in:   "hub.jazz.net/git/user/pkg.name",
 			root: "hub.jazz.net/git/user/pkg.name",
 			mb: maybeSources{
-				&maybeGitSource{url: mkurl("https://hub.jazz.net/git/user1/pkgname")},
-				&maybeGitSource{url: mkurl("ssh://git@hub.jazz.net/git/user1/pkgname")},
-				&maybeGitSource{url: mkurl("git://hub.jazz.net/git/user1/pkgname")},
-				&maybeGitSource{url: mkurl("http://hub.jazz.net/git/user1/pkgname")},
+				maybeGitSource{url: mkurl("https://hub.jazz.net/git/user1/pkgname")},
+				maybeGitSource{url: mkurl("ssh://git@hub.jazz.net/git/user1/pkgname")},
+				maybeGitSource{url: mkurl("git://hub.jazz.net/git/user1/pkgname")},
+				maybeGitSource{url: mkurl("http://hub.jazz.net/git/user1/pkgname")},
 			},
 		},
 		// User names cannot have uppercase letters
@@ -225,34 +216,36 @@ func TestDeduceFromPath(t *testing.T) {
 			in:   "bitbucket.org/sdboyer/reporoot",
 			root: "bitbucket.org/sdboyer/reporoot",
 			mb: maybeSources{
-				&maybeGitSource{url: mkurl("https://bitbucket.org/sdboyer/reporoot")},
-				&maybeGitSource{url: mkurl("ssh://git@bitbucket.org/sdboyer/reporoot")},
-				&maybeGitSource{url: mkurl("git://bitbucket.org/sdboyer/reporoot")},
-				&maybeGitSource{url: mkurl("http://bitbucket.org/sdboyer/reporoot")},
-				&maybeHgSource{url: mkurl("https://bitbucket.org/sdboyer/reporoot")},
-				&maybeHgSource{url: mkurl("ssh://hg@bitbucket.org/sdboyer/reporoot")},
-				&maybeHgSource{url: mkurl("http://bitbucket.org/sdboyer/reporoot")},
+				maybeGitSource{url: mkurl("https://bitbucket.org/sdboyer/reporoot")},
+				maybeGitSource{url: mkurl("ssh://git@bitbucket.org/sdboyer/reporoot")},
+				maybeGitSource{url: mkurl("git://bitbucket.org/sdboyer/reporoot")},
+				maybeGitSource{url: mkurl("http://bitbucket.org/sdboyer/reporoot")},
+				maybeHgSource{url: mkurl("https://bitbucket.org/sdboyer/reporoot")},
+				maybeHgSource{url: mkurl("ssh://hg@bitbucket.org/sdboyer/reporoot")},
+				maybeHgSource{url: mkurl("http://bitbucket.org/sdboyer/reporoot")},
 			},
 		},
+	},
+	"bitbucket": []pathDeductionFixture{
 		{
 			in:   "bitbucket.org/sdboyer/reporoot/foo/bar",
 			root: "bitbucket.org/sdboyer/reporoot",
 			mb: maybeSources{
-				&maybeGitSource{url: mkurl("https://bitbucket.org/sdboyer/reporoot")},
-				&maybeGitSource{url: mkurl("ssh://git@bitbucket.org/sdboyer/reporoot")},
-				&maybeGitSource{url: mkurl("git://bitbucket.org/sdboyer/reporoot")},
-				&maybeGitSource{url: mkurl("http://bitbucket.org/sdboyer/reporoot")},
-				&maybeHgSource{url: mkurl("https://bitbucket.org/sdboyer/reporoot")},
-				&maybeHgSource{url: mkurl("ssh://hg@bitbucket.org/sdboyer/reporoot")},
-				&maybeHgSource{url: mkurl("http://bitbucket.org/sdboyer/reporoot")},
+				maybeGitSource{url: mkurl("https://bitbucket.org/sdboyer/reporoot")},
+				maybeGitSource{url: mkurl("ssh://git@bitbucket.org/sdboyer/reporoot")},
+				maybeGitSource{url: mkurl("git://bitbucket.org/sdboyer/reporoot")},
+				maybeGitSource{url: mkurl("http://bitbucket.org/sdboyer/reporoot")},
+				maybeHgSource{url: mkurl("https://bitbucket.org/sdboyer/reporoot")},
+				maybeHgSource{url: mkurl("ssh://hg@bitbucket.org/sdboyer/reporoot")},
+				maybeHgSource{url: mkurl("http://bitbucket.org/sdboyer/reporoot")},
 			},
 		},
 		{
 			in:   "https://bitbucket.org/sdboyer/reporoot/foo/bar",
 			root: "bitbucket.org/sdboyer/reporoot",
 			mb: maybeSources{
-				&maybeGitSource{url: mkurl("https://bitbucket.org/sdboyer/reporoot")},
-				&maybeHgSource{url: mkurl("https://bitbucket.org/sdboyer/reporoot")},
+				maybeGitSource{url: mkurl("https://bitbucket.org/sdboyer/reporoot")},
+				maybeHgSource{url: mkurl("https://bitbucket.org/sdboyer/reporoot")},
 			},
 		},
 		// Less standard behaviors possible due to the hg/git ambiguity
@@ -260,182 +253,279 @@ func TestDeduceFromPath(t *testing.T) {
 			in:   "bitbucket.org/sdboyer/reporoot.git",
 			root: "bitbucket.org/sdboyer/reporoot.git",
 			mb: maybeSources{
-				&maybeGitSource{url: mkurl("https://bitbucket.org/sdboyer/reporoot")},
-				&maybeGitSource{url: mkurl("ssh://git@bitbucket.org/sdboyer/reporoot")},
-				&maybeGitSource{url: mkurl("git://bitbucket.org/sdboyer/reporoot")},
-				&maybeGitSource{url: mkurl("http://bitbucket.org/sdboyer/reporoot")},
+				maybeGitSource{url: mkurl("https://bitbucket.org/sdboyer/reporoot")},
+				maybeGitSource{url: mkurl("ssh://git@bitbucket.org/sdboyer/reporoot")},
+				maybeGitSource{url: mkurl("git://bitbucket.org/sdboyer/reporoot")},
+				maybeGitSource{url: mkurl("http://bitbucket.org/sdboyer/reporoot")},
 			},
 		},
 		{
 			in:   "git@bitbucket.org:sdboyer/reporoot.git",
 			root: "bitbucket.org/sdboyer/reporoot.git",
-			mb:   &maybeGitSource{url: mkurl("ssh://git@bitbucket.org/sdboyer/reporoot")},
+			mb:   maybeGitSource{url: mkurl("ssh://git@bitbucket.org/sdboyer/reporoot")},
 		},
 		{
 			in:   "bitbucket.org/sdboyer/reporoot.hg",
 			root: "bitbucket.org/sdboyer/reporoot.hg",
 			mb: maybeSources{
-				&maybeHgSource{url: mkurl("https://bitbucket.org/sdboyer/reporoot")},
-				&maybeHgSource{url: mkurl("ssh://hg@bitbucket.org/sdboyer/reporoot")},
-				&maybeHgSource{url: mkurl("http://bitbucket.org/sdboyer/reporoot")},
+				maybeHgSource{url: mkurl("https://bitbucket.org/sdboyer/reporoot")},
+				maybeHgSource{url: mkurl("ssh://hg@bitbucket.org/sdboyer/reporoot")},
+				maybeHgSource{url: mkurl("http://bitbucket.org/sdboyer/reporoot")},
 			},
 		},
 		{
 			in:   "hg@bitbucket.org:sdboyer/reporoot",
 			root: "bitbucket.org/sdboyer/reporoot",
-			mb:   &maybeHgSource{url: mkurl("ssh://hg@bitbucket.org/sdboyer/reporoot")},
+			mb:   maybeHgSource{url: mkurl("ssh://hg@bitbucket.org/sdboyer/reporoot")},
 		},
 		{
 			in:     "git://bitbucket.org/sdboyer/reporoot.hg",
 			root:   "bitbucket.org/sdboyer/reporoot.hg",
 			srcerr: errors.New("git is not a valid scheme for accessing an hg repository"),
 		},
+	},
+	"launchpad": []pathDeductionFixture{
 		// tests for launchpad, mostly bazaar
 		// TODO(sdboyer) need more tests to deal w/launchpad's oddities
 		{
 			in:   "launchpad.net/govcstestbzrrepo",
 			root: "launchpad.net/govcstestbzrrepo",
 			mb: maybeSources{
-				&maybeBzrSource{url: mkurl("https://launchpad.net/govcstestbzrrepo")},
-				&maybeBzrSource{url: mkurl("bzr://launchpad.net/govcstestbzrrepo")},
-				&maybeBzrSource{url: mkurl("http://launchpad.net/govcstestbzrrepo")},
+				maybeBzrSource{url: mkurl("https://launchpad.net/govcstestbzrrepo")},
+				maybeBzrSource{url: mkurl("bzr://launchpad.net/govcstestbzrrepo")},
+				maybeBzrSource{url: mkurl("http://launchpad.net/govcstestbzrrepo")},
 			},
 		},
 		{
 			in:   "launchpad.net/govcstestbzrrepo/foo/bar",
 			root: "launchpad.net/govcstestbzrrepo",
 			mb: maybeSources{
-				&maybeBzrSource{url: mkurl("https://launchpad.net/govcstestbzrrepo")},
-				&maybeBzrSource{url: mkurl("bzr://launchpad.net/govcstestbzrrepo")},
-				&maybeBzrSource{url: mkurl("http://launchpad.net/govcstestbzrrepo")},
+				maybeBzrSource{url: mkurl("https://launchpad.net/govcstestbzrrepo")},
+				maybeBzrSource{url: mkurl("bzr://launchpad.net/govcstestbzrrepo")},
+				maybeBzrSource{url: mkurl("http://launchpad.net/govcstestbzrrepo")},
 			},
 		},
 		{
 			in:   "launchpad.net/repo root",
 			rerr: errors.New("launchpad.net/repo root is not a valid path for a source on launchpad.net"),
 		},
+	},
+	"git.launchpad": []pathDeductionFixture{
 		{
 			in:   "git.launchpad.net/reporoot",
 			root: "git.launchpad.net/reporoot",
 			mb: maybeSources{
-				&maybeGitSource{url: mkurl("https://git.launchpad.net/reporoot")},
-				&maybeGitSource{url: mkurl("ssh://git@git.launchpad.net/reporoot")},
-				&maybeGitSource{url: mkurl("git://git.launchpad.net/reporoot")},
-				&maybeGitSource{url: mkurl("http://git.launchpad.net/reporoot")},
+				maybeGitSource{url: mkurl("https://git.launchpad.net/reporoot")},
+				maybeGitSource{url: mkurl("ssh://git@git.launchpad.net/reporoot")},
+				maybeGitSource{url: mkurl("git://git.launchpad.net/reporoot")},
+				maybeGitSource{url: mkurl("http://git.launchpad.net/reporoot")},
 			},
 		},
 		{
 			in:   "git.launchpad.net/reporoot/foo/bar",
 			root: "git.launchpad.net/reporoot",
 			mb: maybeSources{
-				&maybeGitSource{url: mkurl("https://git.launchpad.net/reporoot")},
-				&maybeGitSource{url: mkurl("ssh://git@git.launchpad.net/reporoot")},
-				&maybeGitSource{url: mkurl("git://git.launchpad.net/reporoot")},
-				&maybeGitSource{url: mkurl("http://git.launchpad.net/reporoot")},
+				maybeGitSource{url: mkurl("https://git.launchpad.net/reporoot")},
+				maybeGitSource{url: mkurl("ssh://git@git.launchpad.net/reporoot")},
+				maybeGitSource{url: mkurl("git://git.launchpad.net/reporoot")},
+				maybeGitSource{url: mkurl("http://git.launchpad.net/reporoot")},
 			},
 		},
 		{
 			in:   "git.launchpad.net/repo root",
 			rerr: errors.New("git.launchpad.net/repo root is not a valid path for a source on launchpad.net"),
 		},
+	},
+	"apache": []pathDeductionFixture{
 		{
 			in:   "git.apache.org/package-name.git",
 			root: "git.apache.org/package-name.git",
 			mb: maybeSources{
-				&maybeGitSource{url: mkurl("https://git.apache.org/package-name.git")},
-				&maybeGitSource{url: mkurl("ssh://git@git.apache.org/package-name.git")},
-				&maybeGitSource{url: mkurl("git://git.apache.org/package-name.git")},
-				&maybeGitSource{url: mkurl("http://git.apache.org/package-name.git")},
+				maybeGitSource{url: mkurl("https://git.apache.org/package-name.git")},
+				maybeGitSource{url: mkurl("ssh://git@git.apache.org/package-name.git")},
+				maybeGitSource{url: mkurl("git://git.apache.org/package-name.git")},
+				maybeGitSource{url: mkurl("http://git.apache.org/package-name.git")},
 			},
 		},
 		{
 			in:   "git.apache.org/package-name.git/foo/bar",
 			root: "git.apache.org/package-name.git",
 			mb: maybeSources{
-				&maybeGitSource{url: mkurl("https://git.apache.org/package-name.git")},
-				&maybeGitSource{url: mkurl("ssh://git@git.apache.org/package-name.git")},
-				&maybeGitSource{url: mkurl("git://git.apache.org/package-name.git")},
-				&maybeGitSource{url: mkurl("http://git.apache.org/package-name.git")},
+				maybeGitSource{url: mkurl("https://git.apache.org/package-name.git")},
+				maybeGitSource{url: mkurl("ssh://git@git.apache.org/package-name.git")},
+				maybeGitSource{url: mkurl("git://git.apache.org/package-name.git")},
+				maybeGitSource{url: mkurl("http://git.apache.org/package-name.git")},
 			},
 		},
-		// Vanity imports
-		{
-			in:   "golang.org/x/exp",
-			root: "golang.org/x/exp",
-			mb:   &maybeGitSource{url: mkurl("https://go.googlesource.com/exp")},
-		},
-		{
-			in:   "golang.org/x/exp/inotify",
-			root: "golang.org/x/exp",
-			mb:   &maybeGitSource{url: mkurl("https://go.googlesource.com/exp")},
-		},
+	},
+	"vcsext": []pathDeductionFixture{
+		// VCS extension-based syntax
 		{
-			in:   "rsc.io/pdf",
-			root: "rsc.io/pdf",
-			mb:   &maybeGitSource{url: mkurl("https://github.com/rsc/pdf")},
+			in:   "foobar/baz.git",
+			root: "foobar/baz.git",
+			mb: maybeSources{
+				maybeGitSource{url: mkurl("https://foobar/baz.git")},
+				maybeGitSource{url: mkurl("git://foobar/baz.git")},
+				maybeGitSource{url: mkurl("http://foobar/baz.git")},
+			},
 		},
-		// Regression - gh does allow two-letter usernames
 		{
-			in:   "github.com/kr/pretty",
-			root: "github.com/kr/pretty",
+			in:   "foobar/baz.bzr",
+			root: "foobar/baz.bzr",
 			mb: maybeSources{
-				&maybeGitSource{url: mkurl("https://github.com/kr/pretty")},
-				&maybeGitSource{url: mkurl("ssh://git@github.com/kr/pretty")},
-				&maybeGitSource{url: mkurl("git://github.com/kr/pretty")},
-				&maybeGitSource{url: mkurl("http://github.com/kr/pretty")},
+				maybeBzrSource{url: mkurl("https://foobar/baz.bzr")},
+				maybeBzrSource{url: mkurl("bzr://foobar/baz.bzr")},
+				maybeBzrSource{url: mkurl("http://foobar/baz.bzr")},
 			},
 		},
-		// VCS extension-based syntax
 		{
-			in:   "foobar/baz.git",
-			root: "foobar/baz.git",
+			in:   "foobar/baz.hg",
+			root: "foobar/baz.hg",
 			mb: maybeSources{
-				&maybeGitSource{url: mkurl("https://foobar/baz.git")},
-				&maybeGitSource{url: mkurl("git://foobar/baz.git")},
-				&maybeGitSource{url: mkurl("http://foobar/baz.git")},
+				maybeHgSource{url: mkurl("https://foobar/baz.hg")},
+				maybeHgSource{url: mkurl("http://foobar/baz.hg")},
 			},
 		},
 		{
 			in:   "foobar/baz.git/quark/quizzle.git",
 			rerr: errors.New("not allowed: foobar/baz.git/quark/quizzle.git contains multiple vcs extension hints"),
 		},
+	},
+	"vanity": []pathDeductionFixture{
+		// Vanity imports
+		{
+			in:   "golang.org/x/exp",
+			root: "golang.org/x/exp",
+			mb:   maybeGitSource{url: mkurl("https://go.googlesource.com/exp")},
+		},
+		{
+			in:   "golang.org/x/exp/inotify",
+			root: "golang.org/x/exp",
+			mb:   maybeGitSource{url: mkurl("https://go.googlesource.com/exp")},
+		},
+		{
+			in:   "rsc.io/pdf",
+			root: "rsc.io/pdf",
+			mb:   maybeGitSource{url: mkurl("https://github.com/rsc/pdf")},
+		},
+	},
+}
+
+func TestDeduceFromPath(t *testing.T) {
+	cpath, err := ioutil.TempDir("", "smcache")
+	if err != nil {
+		t.Errorf("Failed to create temp dir: %s", err)
 	}
+	sm, err := NewSourceManager(naiveAnalyzer{}, cpath, false)
 
-	// TODO(sdboyer) this is all the old checking logic; convert it
-	//for _, fix := range fixtures {
-	//got, err := deduceRemoteRepo(fix.path)
-	//want := fix.want
+	if err != nil {
+		t.Errorf("Unexpected error on SourceManager creation: %s", err)
+		t.FailNow()
+	}
+	defer func() {
+		err := removeAll(cpath)
+		if err != nil {
+			t.Errorf("removeAll failed: %s", err)
+		}
+	}()
+	defer sm.Release()
 
-	//if want == nil {
-	//if err == nil {
-	//t.Errorf("deduceRemoteRepo(%q): Error expected but not received", fix.path)
-	//}
-	//continue
-	//}
+	for typ, fixtures := range pathDeductionFixtures {
+		var deducer pathDeducer
+		switch typ {
+		case "github":
+			deducer = githubDeducer{regexp: ghRegex}
+		case "gopkg.in":
+			deducer = gopkginDeducer{regexp: gpinNewRegex}
+		case "jazz":
+			deducer = jazzDeducer{regexp: jazzRegex}
+		case "bitbucket":
+			deducer = bitbucketDeducer{regexp: bbRegex}
+		case "launchpad":
+			deducer = launchpadDeducer{regexp: lpRegex}
+		case "git.launchpad":
+			deducer = launchpadGitDeducer{regexp: glpRegex}
+		case "apache":
+			deducer = apacheDeducer{regexp: apacheRegex}
+		case "vcsext":
+			deducer = vcsExtensionDeducer{regexp: vcsExtensionRegex}
+		default:
+			// Should just be the vanity imports, which we do elsewhere
+			continue
+		}
 
-	//if err != nil {
-	//t.Errorf("deduceRemoteRepo(%q): %v", fix.path, err)
-	//continue
-	//}
+		var printmb func(mb maybeSource) string
+		printmb = func(mb maybeSource) string {
+			switch tmb := mb.(type) {
+			case maybeSources:
+				var buf bytes.Buffer
+				fmt.Fprintf(&buf, "%v maybeSources:", len(tmb))
+				for _, elem := range tmb {
+					fmt.Fprintf(&buf, "\n\t\t%s", printmb(elem))
+				}
+				return buf.String()
+			case maybeGitSource:
+				return fmt.Sprintf("%T: %s", tmb, ufmt(tmb.url))
+			case maybeBzrSource:
+				return fmt.Sprintf("%T: %s", tmb, ufmt(tmb.url))
+			case maybeHgSource:
+				return fmt.Sprintf("%T: %s", tmb, ufmt(tmb.url))
+			default:
+				t.Errorf("Unknown maybeSource type: %T", mb)
+				t.FailNow()
+			}
+			return ""
+		}
+
+		for _, fix := range fixtures {
+			u, in, uerr := normalizeURI(fix.in)
+			if uerr != nil {
+				if fix.rerr == nil {
+					t.Errorf("(in: %s) bad input URI %s", fix.in, uerr)
+				}
+				continue
+			}
+			if u == nil {
+				spew.Dump(fix, uerr)
+			}
+
+			root, rerr := deducer.deduceRoot(in)
+			if fix.rerr != nil {
+				if fix.rerr != rerr {
+					if rerr == nil {
+						t.Errorf("(in: %s, %T) Expected error on deducing root, got none:\n\t(WNT) %s", in, deducer, fix.rerr)
+					} else {
+						t.Errorf("(in: %s, %T) Got unexpected error on deducing root:\n\t(GOT) %s\n\t(WNT) %s", in, deducer, rerr, fix.rerr)
+					}
+				}
+			} else if rerr != nil {
+				t.Errorf("(in: %s, %T) Got unexpected error on deducing root:\n\t(GOT) %s", in, deducer, rerr)
+			} else if root != fix.root {
+				t.Errorf("(in: %s, %T) Deducer did not return expected root:\n\t(GOT) %s\n\t(WNT) %s", in, deducer, root, fix.root)
+			}
 
-	//if got.Base != want.Base {
-	//t.Errorf("deduceRemoteRepo(%q): Base was %s, wanted %s", fix.path, got.Base, want.Base)
-	//}
-	//if got.RelPkg != want.RelPkg {
-	//t.Errorf("deduceRemoteRepo(%q): RelPkg was %s, wanted %s", fix.path, got.RelPkg, want.RelPkg)
-	//}
-	//if !reflect.DeepEqual(got.CloneURL, want.CloneURL) {
-	//// misspelling things is cool when it makes columns line up
-	//t.Errorf("deduceRemoteRepo(%q): CloneURL disagreement:\n(GOT) %s\n(WNT) %s", fix.path, ufmt(got.CloneURL), ufmt(want.CloneURL))
-	//}
-	//if !reflect.DeepEqual(got.VCS, want.VCS) {
-	//t.Errorf("deduceRemoteRepo(%q): VCS was %s, wanted %s", fix.path, got.VCS, want.VCS)
-	//}
-	//if !reflect.DeepEqual(got.Schemes, want.Schemes) {
-	//t.Errorf("deduceRemoteRepo(%q): Schemes was %s, wanted %s", fix.path, got.Schemes, want.Schemes)
-	//}
-	//}
-	t.Error("TODO implement checking of new path deduction fixtures")
+			mb, mberr := deducer.deduceSource(fix.in, u)
+			if fix.srcerr != nil {
+				if fix.srcerr != mberr {
+					if mberr == nil {
+						t.Errorf("(in: %s, %T) Expected error on deducing source, got none:\n\t(WNT) %s", in, deducer, fix.srcerr)
+					} else {
+						t.Errorf("(in: %s, %T) Got unexpected error on deducing source:\n\t(GOT) %s\n\t(WNT) %s", in, deducer, mberr, fix.srcerr)
+					}
+				}
+			} else if mberr != nil && fix.rerr == nil { // don't complain the fix already expected an rerr
+				t.Errorf("(in: %s, %T) Got unexpected error on deducing source:\n\t(GOT) %s", in, deducer, mberr)
+			} else if !reflect.DeepEqual(mb, fix.mb) {
+				if mb == nil {
+					t.Errorf("(in: %s, %T) Deducer returned source maybes, but none expected:\n\t(GOT) (none)\n\t(WNT) %s", in, deducer, printmb(fix.mb))
+				} else if fix.mb == nil {
+					t.Errorf("(in: %s, %T) Deducer returned source maybes, but none expected:\n\t(GOT) %s\n\t(WNT) (none)", in, deducer, printmb(mb))
+				} else {
+					t.Errorf("(in: %s, %T) Deducer did not return expected source:\n\t(GOT) %s\n\t(WNT) %s", in, deducer, printmb(mb), printmb(fix.mb))
+				}
+			}
+		}
+	}
 }
 
 // borrow from stdlib

From 449ec8731cb023152106d1b8574e3bdf13f60482 Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Wed, 10 Aug 2016 00:31:41 -0400
Subject: [PATCH 50/71] Fix all the new import path deduction unit tests

Tons of refactoring, but this gets us a long way towards complete
handling for import paths.
---
 remote.go      | 169 +++++++++++++++++++++++++----------------
 remote_test.go | 199 +++++++++++++++++++++++++++++--------------------
 2 files changed, 223 insertions(+), 145 deletions(-)

diff --git a/remote.go b/remote.go
index dd6c0f7..c55218e 100644
--- a/remote.go
+++ b/remote.go
@@ -5,6 +5,7 @@ import (
 	"io"
 	"net/http"
 	"net/url"
+	"path"
 	"regexp"
 	"strings"
 )
@@ -30,6 +31,11 @@ var (
 )
 
 func validateVCSScheme(scheme, typ string) bool {
+	// everything allows plain ssh
+	if scheme == "ssh" {
+		return true
+	}
+
 	var schemes []string
 	switch typ {
 	case "git":
@@ -57,18 +63,18 @@ var (
 	// This regex allowed some usernames that github currently disallows. They
 	// may have allowed them in the past; keeping it in case we need to revert.
 	//ghRegex      = regexp.MustCompile(`^(?P<root>github\.com/([A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+))(/[A-Za-z0-9_.\-]+)*$`)
-	ghRegex      = regexp.MustCompile(`^(?P<root>github\.com/([A-Za-z0-9][-A-Za-z0-9]*[A-Za-z0-9]/[A-Za-z0-9_.\-]+))((?:/[A-Za-z0-9_.\-]+)*)$`)
-	gpinNewRegex = regexp.MustCompile(`^(?P<root>gopkg\.in/(?:([a-zA-Z0-9][-a-zA-Z0-9]+)/)?([a-zA-Z][-.a-zA-Z0-9]*)\.((?:v0|v[1-9][0-9]*)(?:\.0|\.[1-9][0-9]*){0,2}(-unstable)?)(?:\.git)?)((?:/[a-zA-Z0-9][-.a-zA-Z0-9]*)*)$`)
+	ghRegex      = regexp.MustCompile(`^(?P<root>github\.com(/[A-Za-z0-9][-A-Za-z0-9]*[A-Za-z0-9]/[A-Za-z0-9_.\-]+))((?:/[A-Za-z0-9_.\-]+)*)$`)
+	gpinNewRegex = regexp.MustCompile(`^(?P<root>gopkg\.in(?:(/[a-zA-Z0-9][-a-zA-Z0-9]+)?)(/[a-zA-Z][-.a-zA-Z0-9]*)\.((?:v0|v[1-9][0-9]*)(?:\.0|\.[1-9][0-9]*){0,2}(?:-unstable)?)(?:\.git)?)((?:/[a-zA-Z0-9][-.a-zA-Z0-9]*)*)$`)
 	//gpinOldRegex = regexp.MustCompile(`^(?P<root>gopkg\.in/(?:([a-z0-9][-a-z0-9]+)/)?((?:v0|v[1-9][0-9]*)(?:\.0|\.[1-9][0-9]*){0,2}(-unstable)?)/([a-zA-Z][-a-zA-Z0-9]*)(?:\.git)?)((?:/[a-zA-Z][-a-zA-Z0-9]*)*)$`)
-	bbRegex = regexp.MustCompile(`^(?P<root>bitbucket\.org/(?P<bitname>[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+))((?:/[A-Za-z0-9_.\-]+)*)$`)
+	bbRegex = regexp.MustCompile(`^(?P<root>bitbucket\.org(?P<bitname>/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+))((?:/[A-Za-z0-9_.\-]+)*)$`)
 	//lpRegex = regexp.MustCompile(`^(?P<root>launchpad\.net/([A-Za-z0-9-._]+)(/[A-Za-z0-9-._]+)?)(/.+)?`)
-	lpRegex = regexp.MustCompile(`^(?P<root>launchpad\.net/([A-Za-z0-9-._]+))((?:/[A-Za-z0-9_.\-]+)*)?`)
+	lpRegex = regexp.MustCompile(`^(?P<root>launchpad\.net(/[A-Za-z0-9-._]+))((?:/[A-Za-z0-9_.\-]+)*)?`)
 	//glpRegex = regexp.MustCompile(`^(?P<root>git\.launchpad\.net/([A-Za-z0-9_.\-]+)|~[A-Za-z0-9_.\-]+/(\+git|[A-Za-z0-9_.\-]+)/[A-Za-z0-9_.\-]+)$`)
-	glpRegex = regexp.MustCompile(`^(?P<root>git\.launchpad\.net/([A-Za-z0-9_.\-]+))((?:/[A-Za-z0-9_.\-]+)*)$`)
+	glpRegex = regexp.MustCompile(`^(?P<root>git\.launchpad\.net(/[A-Za-z0-9_.\-]+))((?:/[A-Za-z0-9_.\-]+)*)$`)
 	//gcRegex      = regexp.MustCompile(`^(?P<root>code\.google\.com/[pr]/(?P<project>[a-z0-9\-]+)(\.(?P<subrepo>[a-z0-9\-]+))?)(/[A-Za-z0-9_.\-]+)*$`)
-	jazzRegex         = regexp.MustCompile(`^(?P<root>hub\.jazz\.net/(git/[a-z0-9]+/[A-Za-z0-9_.\-]+))((?:/[A-Za-z0-9_.\-]+)*)$`)
-	apacheRegex       = regexp.MustCompile(`^(?P<root>git\.apache\.org/([a-z0-9_.\-]+\.git))((?:/[A-Za-z0-9_.\-]+)*)$`)
-	vcsExtensionRegex = regexp.MustCompile(`^(?P<root>(?P<repo>([a-z0-9.\-]+\.)+[a-z0-9.\-]+(:[0-9]+)?/[A-Za-z0-9_.\-/~]*?)\.(?P<vcs>bzr|git|hg|svn))((?:/[A-Za-z0-9_.\-]+)*)$`)
+	jazzRegex         = regexp.MustCompile(`^(?P<root>hub\.jazz\.net(/git/[a-z0-9]+/[A-Za-z0-9_.\-]+))((?:/[A-Za-z0-9_.\-]+)*)$`)
+	apacheRegex       = regexp.MustCompile(`^(?P<root>git\.apache\.org(/[a-z0-9_.\-]+\.git))((?:/[A-Za-z0-9_.\-]+)*)$`)
+	vcsExtensionRegex = regexp.MustCompile(`^(?P<root>([a-z0-9.\-]+\.)+[a-z0-9.\-]+(:[0-9]+)?/[A-Za-z0-9_.\-/~]*?\.(?P<vcs>bzr|git|hg|svn))((?:/[A-Za-z0-9_.\-]+)*)$`)
 )
 
 // Other helper regexes
@@ -92,7 +98,7 @@ func (m githubDeducer) deduceRoot(path string) (string, error) {
 		return "", fmt.Errorf("%s is not a valid path for a source on github.com", path)
 	}
 
-	return "github.com/" + v[2], nil
+	return "github.com" + v[2], nil
 }
 
 func (m githubDeducer) deduceSource(path string, u *url.URL) (maybeSource, error) {
@@ -101,17 +107,27 @@ func (m githubDeducer) deduceSource(path string, u *url.URL) (maybeSource, error
 		return nil, fmt.Errorf("%s is not a valid path for a source on github.com", path)
 	}
 
+	u.Host = "github.com"
 	u.Path = v[2]
-	if u.Scheme != "" {
+
+	if u.Scheme == "ssh" && u.User != nil && u.User.Username() != "git" {
+		return nil, fmt.Errorf("github ssh must be accessed via the 'git' user; %s was provided", u.User.Username())
+	} else if u.Scheme != "" {
 		if !validateVCSScheme(u.Scheme, "git") {
 			return nil, fmt.Errorf("%s is not a valid scheme for accessing a git repository", u.Scheme)
 		}
+		if u.Scheme == "ssh" {
+			u.User = url.User("git")
+		}
 		return maybeGitSource{url: u}, nil
 	}
 
 	mb := make(maybeSources, len(gitSchemes))
 	for k, scheme := range gitSchemes {
 		u2 := *u
+		if scheme == "ssh" {
+			u2.User = url.User("git")
+		}
 		u2.Scheme = scheme
 		mb[k] = maybeGitSource{url: &u2}
 	}
@@ -129,7 +145,7 @@ func (m bitbucketDeducer) deduceRoot(path string) (string, error) {
 		return "", fmt.Errorf("%s is not a valid path for a source on bitbucket.org", path)
 	}
 
-	return "bitbucket.org/" + v[2], nil
+	return "bitbucket.org" + v[2], nil
 }
 
 func (m bitbucketDeducer) deduceSource(path string, u *url.URL) (maybeSource, error) {
@@ -137,6 +153,8 @@ func (m bitbucketDeducer) deduceSource(path string, u *url.URL) (maybeSource, er
 	if v == nil {
 		return nil, fmt.Errorf("%s is not a valid path for a source on bitbucket.org", path)
 	}
+
+	u.Host = "bitbucket.org"
 	u.Path = v[2]
 
 	// This isn't definitive, but it'll probably catch most
@@ -173,6 +191,9 @@ func (m bitbucketDeducer) deduceSource(path string, u *url.URL) (maybeSource, er
 	if !ishg {
 		for _, scheme := range gitSchemes {
 			u2 := *u
+			if scheme == "ssh" {
+				u2.User = url.User("git")
+			}
 			u2.Scheme = scheme
 			mb = append(mb, maybeGitSource{url: &u2})
 		}
@@ -181,6 +202,9 @@ func (m bitbucketDeducer) deduceSource(path string, u *url.URL) (maybeSource, er
 	if !isgit {
 		for _, scheme := range hgSchemes {
 			u2 := *u
+			if scheme == "ssh" {
+				u2.User = url.User("hg")
+			}
 			u2.Scheme = scheme
 			mb = append(mb, maybeHgSource{url: &u2})
 		}
@@ -193,26 +217,36 @@ type gopkginDeducer struct {
 	regexp *regexp.Regexp
 }
 
-func (m gopkginDeducer) deduceRoot(path string) (string, error) {
-	v := m.regexp.FindStringSubmatch(path)
-	if v == nil {
-		return "", fmt.Errorf("%s is not a valid path for a source on gopkg.in", path)
+func (m gopkginDeducer) deduceRoot(p string) (string, error) {
+	v, err := m.parseAndValidatePath(p)
+	if err != nil {
+		return "", err
 	}
 
-	return "gopkg.in/" + v[2], nil
+	return v[1], nil
 }
 
-func (m gopkginDeducer) deduceSource(path string, u *url.URL) (maybeSource, error) {
-	v := m.regexp.FindStringSubmatch(path)
+func (m gopkginDeducer) parseAndValidatePath(p string) ([]string, error) {
+	v := m.regexp.FindStringSubmatch(p)
 	if v == nil {
-		return nil, fmt.Errorf("%s is not a valid path for a source on gopkg.in", path)
+		return nil, fmt.Errorf("%s is not a valid path for a source on gopkg.in", p)
 	}
 
-	// Duplicate some logic from the gopkg.in server in order to validate
-	// the import path string without having to hit the server
+	// We duplicate some logic from the gopkg.in server in order to validate the
+	// import path string without having to make a network request
 	if strings.Contains(v[4], ".") {
-		return nil, fmt.Errorf("%q is not a valid import path; gopkg.in only allows major versions (%q instead of %q)",
-			path, v[4][:strings.Index(v[4], ".")], v[4])
+		return nil, fmt.Errorf("%s is not a valid import path; gopkg.in only allows major versions (%q instead of %q)",
+			p, v[4][:strings.Index(v[4], ".")], v[4])
+	}
+
+	return v, nil
+}
+
+func (m gopkginDeducer) deduceSource(p string, u *url.URL) (maybeSource, error) {
+	// Reuse root detection logic for initial validation
+	v, err := m.parseAndValidatePath(p)
+	if err != nil {
+		return nil, err
 	}
 
 	// Putting a scheme on gopkg.in would be really weird, disallow it
@@ -225,23 +259,24 @@ func (m gopkginDeducer) deduceSource(path string, u *url.URL) (maybeSource, erro
 	// If the third position is empty, it's the shortened form that expands
 	// to the go-pkg github user
 	if v[2] == "" {
-		var inter string
 		// Apparently gopkg.in special-cases gopkg.in/yaml, violating its own rules?
 		// If we find one more exception, chuck this and just rely on vanity
 		// metadata resolving.
-		if strings.HasPrefix(path, "gopkg.in/yaml") {
-			inter = "go-yaml"
+		if v[3] == "/yaml" {
+			u.Path = "/go-yaml/yaml"
 		} else {
-			inter = "go-pkg"
+			u.Path = path.Join("/go-pkg", v[3])
 		}
-		u.Path = inter + v[3]
 	} else {
-		u.Path = v[2] + "/" + v[3]
+		u.Path = path.Join(v[2], v[3])
 	}
 
 	mb := make(maybeSources, len(gitSchemes))
 	for k, scheme := range gitSchemes {
 		u2 := *u
+		if scheme == "ssh" {
+			u2.User = url.User("git")
+		}
 		u2.Scheme = scheme
 		mb[k] = maybeGitSource{url: &u2}
 	}
@@ -261,7 +296,7 @@ func (m launchpadDeducer) deduceRoot(path string) (string, error) {
 		return "", fmt.Errorf("%s is not a valid path for a source on launchpad.net", path)
 	}
 
-	return "launchpad.net/" + v[2], nil
+	return "launchpad.net" + v[2], nil
 }
 
 func (m launchpadDeducer) deduceSource(path string, u *url.URL) (maybeSource, error) {
@@ -270,7 +305,9 @@ func (m launchpadDeducer) deduceSource(path string, u *url.URL) (maybeSource, er
 		return nil, fmt.Errorf("%s is not a valid path for a source on launchpad.net", path)
 	}
 
+	u.Host = "launchpad.net"
 	u.Path = v[2]
+
 	if u.Scheme != "" {
 		if !validateVCSScheme(u.Scheme, "bzr") {
 			return nil, fmt.Errorf("%s is not a valid scheme for accessing a bzr repository", u.Scheme)
@@ -279,7 +316,6 @@ func (m launchpadDeducer) deduceSource(path string, u *url.URL) (maybeSource, er
 	}
 
 	mb := make(maybeSources, len(bzrSchemes))
-	// TODO(sdboyer) is there a generic ssh user for lp? if not, drop bzr+ssh
 	for k, scheme := range bzrSchemes {
 		u2 := *u
 		u2.Scheme = scheme
@@ -300,7 +336,7 @@ func (m launchpadGitDeducer) deduceRoot(path string) (string, error) {
 		return "", fmt.Errorf("%s is not a valid path for a source on git.launchpad.net", path)
 	}
 
-	return "git.launchpad.net/" + v[2], nil
+	return "git.launchpad.net" + v[2], nil
 }
 
 func (m launchpadGitDeducer) deduceSource(path string, u *url.URL) (maybeSource, error) {
@@ -309,7 +345,9 @@ func (m launchpadGitDeducer) deduceSource(path string, u *url.URL) (maybeSource,
 		return nil, fmt.Errorf("%s is not a valid path for a source on git.launchpad.net", path)
 	}
 
+	u.Host = "git.launchpad.net"
 	u.Path = v[2]
+
 	if u.Scheme != "" {
 		if !validateVCSScheme(u.Scheme, "git") {
 			return nil, fmt.Errorf("%s is not a valid scheme for accessing a git repository", u.Scheme)
@@ -317,8 +355,8 @@ func (m launchpadGitDeducer) deduceSource(path string, u *url.URL) (maybeSource,
 		return maybeGitSource{url: u}, nil
 	}
 
-	mb := make(maybeSources, len(bzrSchemes))
-	for k, scheme := range bzrSchemes {
+	mb := make(maybeSources, len(gitSchemes))
+	for k, scheme := range gitSchemes {
 		u2 := *u
 		u2.Scheme = scheme
 		mb[k] = maybeGitSource{url: &u2}
@@ -337,7 +375,7 @@ func (m jazzDeducer) deduceRoot(path string) (string, error) {
 		return "", fmt.Errorf("%s is not a valid path for a source on hub.jazz.net", path)
 	}
 
-	return "hub.jazz.net/" + v[2], nil
+	return "hub.jazz.net" + v[2], nil
 }
 
 func (m jazzDeducer) deduceSource(path string, u *url.URL) (maybeSource, error) {
@@ -346,22 +384,18 @@ func (m jazzDeducer) deduceSource(path string, u *url.URL) (maybeSource, error)
 		return nil, fmt.Errorf("%s is not a valid path for a source on hub.jazz.net", path)
 	}
 
+	u.Host = "hub.jazz.net"
 	u.Path = v[2]
-	if u.Scheme != "" {
-		if !validateVCSScheme(u.Scheme, "git") {
-			return nil, fmt.Errorf("%s is not a valid scheme for accessing a git repository", u.Scheme)
-		}
-		return maybeGitSource{url: u}, nil
-	}
 
-	mb := make(maybeSources, len(gitSchemes))
-	for k, scheme := range gitSchemes {
-		u2 := *u
-		u2.Scheme = scheme
-		mb[k] = maybeGitSource{url: &u2}
+	switch u.Scheme {
+	case "":
+		u.Scheme = "https"
+		fallthrough
+	case "https":
+		return maybeGitSource{url: u}, nil
+	default:
+		return nil, fmt.Errorf("IBM's jazz hub only supports https, %s is not allowed", u.String())
 	}
-
-	return mb, nil
 }
 
 type apacheDeducer struct {
@@ -374,7 +408,7 @@ func (m apacheDeducer) deduceRoot(path string) (string, error) {
 		return "", fmt.Errorf("%s is not a valid path for a source on git.apache.org", path)
 	}
 
-	return "git.apache.org/" + v[2], nil
+	return "git.apache.org" + v[2], nil
 }
 
 func (m apacheDeducer) deduceSource(path string, u *url.URL) (maybeSource, error) {
@@ -383,7 +417,9 @@ func (m apacheDeducer) deduceSource(path string, u *url.URL) (maybeSource, error
 		return nil, fmt.Errorf("%s is not a valid path for a source on git.apache.org", path)
 	}
 
+	u.Host = "git.apache.org"
 	u.Path = v[2]
+
 	if u.Scheme != "" {
 		if !validateVCSScheme(u.Scheme, "git") {
 			return nil, fmt.Errorf("%s is not a valid scheme for accessing a git repository", u.Scheme)
@@ -420,19 +456,19 @@ func (m vcsExtensionDeducer) deduceSource(path string, u *url.URL) (maybeSource,
 		return nil, fmt.Errorf("%s contains no vcs extension hints for matching", path)
 	}
 
-	switch v[5] {
+	switch v[4] {
 	case "git", "hg", "bzr":
 		x := strings.SplitN(v[1], "/", 2)
 		// TODO(sdboyer) is this actually correct for bzr?
 		u.Host = x[0]
-		u.Path = x[1]
+		u.Path = "/" + x[1]
 
 		if u.Scheme != "" {
-			if !validateVCSScheme(u.Scheme, v[5]) {
-				return nil, fmt.Errorf("%s is not a valid scheme for accessing %s repositories (path %s)", u.Scheme, v[5], path)
+			if !validateVCSScheme(u.Scheme, v[4]) {
+				return nil, fmt.Errorf("%s is not a valid scheme for accessing %s repositories (path %s)", u.Scheme, v[4], path)
 			}
 
-			switch v[5] {
+			switch v[4] {
 			case "git":
 				return maybeGitSource{url: u}, nil
 			case "bzr":
@@ -445,7 +481,8 @@ func (m vcsExtensionDeducer) deduceSource(path string, u *url.URL) (maybeSource,
 		var schemes []string
 		var mb maybeSources
 		var f func(k int, u *url.URL)
-		switch v[5] {
+
+		switch v[4] {
 		case "git":
 			schemes = gitSchemes
 			f = func(k int, u *url.URL) {
@@ -462,9 +499,9 @@ func (m vcsExtensionDeducer) deduceSource(path string, u *url.URL) (maybeSource,
 				mb[k] = maybeHgSource{url: u}
 			}
 		}
-		mb = make(maybeSources, len(schemes))
 
-		for k, scheme := range gitSchemes {
+		mb = make(maybeSources, len(schemes))
+		for k, scheme := range schemes {
 			u2 := *u
 			u2.Scheme = scheme
 			f(k, &u2)
@@ -472,7 +509,7 @@ func (m vcsExtensionDeducer) deduceSource(path string, u *url.URL) (maybeSource,
 
 		return mb, nil
 	default:
-		return nil, fmt.Errorf("unknown repository type: %q", v[5])
+		return nil, fmt.Errorf("unknown repository type: %q", v[4])
 	}
 }
 
@@ -623,8 +660,8 @@ func (sm *SourceMgr) deduceFromPath(path string) (stringFuture, partialSourceFut
 	return root, src, nil
 }
 
-func normalizeURI(path string) (u *url.URL, newpath string, err error) {
-	if m := scpSyntaxRe.FindStringSubmatch(path); m != nil {
+func normalizeURI(p string) (u *url.URL, newpath string, err error) {
+	if m := scpSyntaxRe.FindStringSubmatch(p); m != nil {
 		// Match SCP-like syntax and convert it to a URL.
 		// Eg, "git@github.com:user/repo" becomes
 		// "ssh://git@github.com/user/repo".
@@ -637,16 +674,18 @@ func normalizeURI(path string) (u *url.URL, newpath string, err error) {
 			//RawPath: m[3],
 		}
 	} else {
-		u, err = url.Parse(path)
+		u, err = url.Parse(p)
 		if err != nil {
-			return nil, "", fmt.Errorf("%q is not a valid URI", path)
+			return nil, "", fmt.Errorf("%q is not a valid URI", p)
 		}
 	}
 
-	if u.Host != "" {
-		newpath = u.Host + "/" + strings.TrimPrefix(u.Path, "/")
+	// If no scheme was passed, then the entire path will have been put into
+	// u.Path. Either way, construct the normalized path correctly.
+	if u.Host == "" {
+		newpath = p
 	} else {
-		newpath = u.Path
+		newpath = path.Join(u.Host, u.Path)
 	}
 
 	if !pathvld.MatchString(newpath) {
diff --git a/remote_test.go b/remote_test.go
index a188d73..6d88ff1 100644
--- a/remote_test.go
+++ b/remote_test.go
@@ -8,8 +8,6 @@ import (
 	"net/url"
 	"reflect"
 	"testing"
-
-	"github.com/davecgh/go-spew/spew"
 )
 
 type pathDeductionFixture struct {
@@ -53,13 +51,15 @@ var pathDeductionFixtures = map[string][]pathDeductionFixture{
 			},
 		},
 		{
+			// TODO(sdboyer) is this a problem for enforcing uniqueness? do we
+			// need to collapse these extensions?
 			in:   "github.com/sdboyer/gps.git/foo",
-			root: "github.com/sdboyer/gps",
+			root: "github.com/sdboyer/gps.git",
 			mb: maybeSources{
-				maybeGitSource{url: mkurl("https://github.com/sdboyer/gps")},
-				maybeGitSource{url: mkurl("ssh://git@github.com/sdboyer/gps")},
-				maybeGitSource{url: mkurl("git://github.com/sdboyer/gps")},
-				maybeGitSource{url: mkurl("http://github.com/sdboyer/gps")},
+				maybeGitSource{url: mkurl("https://github.com/sdboyer/gps.git")},
+				maybeGitSource{url: mkurl("ssh://git@github.com/sdboyer/gps.git")},
+				maybeGitSource{url: mkurl("git://github.com/sdboyer/gps.git")},
+				maybeGitSource{url: mkurl("http://github.com/sdboyer/gps.git")},
 			},
 		},
 		{
@@ -140,17 +140,27 @@ var pathDeductionFixtures = map[string][]pathDeductionFixture{
 		{
 			in:   "gopkg.in/yaml.v1",
 			root: "gopkg.in/yaml.v1",
-			mb:   maybeGitSource{url: mkurl("https://github.com/go-yaml/yaml")},
+			mb: maybeSources{
+				maybeGitSource{url: mkurl("https://github.com/go-yaml/yaml")},
+				maybeGitSource{url: mkurl("ssh://git@github.com/go-yaml/yaml")},
+				maybeGitSource{url: mkurl("git://github.com/go-yaml/yaml")},
+				maybeGitSource{url: mkurl("http://github.com/go-yaml/yaml")},
+			},
 		},
 		{
 			in:   "gopkg.in/yaml.v1/foo/bar",
 			root: "gopkg.in/yaml.v1",
-			mb:   maybeGitSource{url: mkurl("https://github.com/go-yaml/yaml")},
+			mb: maybeSources{
+				maybeGitSource{url: mkurl("https://github.com/go-yaml/yaml")},
+				maybeGitSource{url: mkurl("ssh://git@github.com/go-yaml/yaml")},
+				maybeGitSource{url: mkurl("git://github.com/go-yaml/yaml")},
+				maybeGitSource{url: mkurl("http://github.com/go-yaml/yaml")},
+			},
 		},
 		{
 			// gopkg.in only allows specifying major version in import path
 			in:   "gopkg.in/yaml.v1.2",
-			rerr: errors.New("gopkg.in/yaml.v1.2 is not a valid path; gopkg.in only allows major versions (\"v1\" instead of \"v1.2\")"),
+			rerr: errors.New("gopkg.in/yaml.v1.2 is not a valid import path; gopkg.in only allows major versions (\"v1\" instead of \"v1.2\")"),
 		},
 	},
 	"jazz": []pathDeductionFixture{
@@ -158,30 +168,20 @@ var pathDeductionFixtures = map[string][]pathDeductionFixture{
 		{
 			in:   "hub.jazz.net/git/user1/pkgname",
 			root: "hub.jazz.net/git/user1/pkgname",
-			mb: maybeSources{
-				maybeGitSource{url: mkurl("https://hub.jazz.net/git/user1/pkgname")},
-				maybeGitSource{url: mkurl("ssh://git@hub.jazz.net/git/user1/pkgname")},
-				maybeGitSource{url: mkurl("git://hub.jazz.net/git/user1/pkgname")},
-				maybeGitSource{url: mkurl("http://hub.jazz.net/git/user1/pkgname")},
-			},
+			mb:   maybeGitSource{url: mkurl("https://hub.jazz.net/git/user1/pkgname")},
 		},
 		{
 			in:   "hub.jazz.net/git/user1/pkgname/submodule/submodule/submodule",
 			root: "hub.jazz.net/git/user1/pkgname",
-			mb: maybeSources{
-				maybeGitSource{url: mkurl("https://hub.jazz.net/git/user1/pkgname")},
-				maybeGitSource{url: mkurl("ssh://git@hub.jazz.net/git/user1/pkgname")},
-				maybeGitSource{url: mkurl("git://hub.jazz.net/git/user1/pkgname")},
-				maybeGitSource{url: mkurl("http://hub.jazz.net/git/user1/pkgname")},
-			},
+			mb:   maybeGitSource{url: mkurl("https://hub.jazz.net/git/user1/pkgname")},
 		},
 		{
 			in:   "hub.jazz.net/someotherprefix",
-			rerr: errors.New("unable to deduce repository and source type for: \"hub.jazz.net/someotherprefix\""),
+			rerr: errors.New("hub.jazz.net/someotherprefix is not a valid path for a source on hub.jazz.net"),
 		},
 		{
 			in:   "hub.jazz.net/someotherprefix/user1/packagename",
-			rerr: errors.New("unable to deduce repository and source type for: \"hub.jazz.net/someotherprefix/user1/packagename\""),
+			rerr: errors.New("hub.jazz.net/someotherprefix/user1/packagename is not a valid path for a source on hub.jazz.net"),
 		},
 		// Spaces are not valid in user names or package names
 		{
@@ -198,20 +198,17 @@ var pathDeductionFixtures = map[string][]pathDeductionFixture{
 			rerr: errors.New("hub.jazz.net/git/user.1/pkgname is not a valid path for a source on hub.jazz.net"),
 		},
 		{
-			in:   "hub.jazz.net/git/user/pkg.name",
-			root: "hub.jazz.net/git/user/pkg.name",
-			mb: maybeSources{
-				maybeGitSource{url: mkurl("https://hub.jazz.net/git/user1/pkgname")},
-				maybeGitSource{url: mkurl("ssh://git@hub.jazz.net/git/user1/pkgname")},
-				maybeGitSource{url: mkurl("git://hub.jazz.net/git/user1/pkgname")},
-				maybeGitSource{url: mkurl("http://hub.jazz.net/git/user1/pkgname")},
-			},
+			in:   "hub.jazz.net/git/user1/pkg.name",
+			root: "hub.jazz.net/git/user1/pkg.name",
+			mb:   maybeGitSource{url: mkurl("https://hub.jazz.net/git/user1/pkg.name")},
 		},
 		// User names cannot have uppercase letters
 		{
 			in:   "hub.jazz.net/git/USER/pkgname",
 			rerr: errors.New("hub.jazz.net/git/USER/pkgname is not a valid path for a source on hub.jazz.net"),
 		},
+	},
+	"bitbucket": []pathDeductionFixture{
 		{
 			in:   "bitbucket.org/sdboyer/reporoot",
 			root: "bitbucket.org/sdboyer/reporoot",
@@ -225,8 +222,6 @@ var pathDeductionFixtures = map[string][]pathDeductionFixture{
 				maybeHgSource{url: mkurl("http://bitbucket.org/sdboyer/reporoot")},
 			},
 		},
-	},
-	"bitbucket": []pathDeductionFixture{
 		{
 			in:   "bitbucket.org/sdboyer/reporoot/foo/bar",
 			root: "bitbucket.org/sdboyer/reporoot",
@@ -253,24 +248,24 @@ var pathDeductionFixtures = map[string][]pathDeductionFixture{
 			in:   "bitbucket.org/sdboyer/reporoot.git",
 			root: "bitbucket.org/sdboyer/reporoot.git",
 			mb: maybeSources{
-				maybeGitSource{url: mkurl("https://bitbucket.org/sdboyer/reporoot")},
-				maybeGitSource{url: mkurl("ssh://git@bitbucket.org/sdboyer/reporoot")},
-				maybeGitSource{url: mkurl("git://bitbucket.org/sdboyer/reporoot")},
-				maybeGitSource{url: mkurl("http://bitbucket.org/sdboyer/reporoot")},
+				maybeGitSource{url: mkurl("https://bitbucket.org/sdboyer/reporoot.git")},
+				maybeGitSource{url: mkurl("ssh://git@bitbucket.org/sdboyer/reporoot.git")},
+				maybeGitSource{url: mkurl("git://bitbucket.org/sdboyer/reporoot.git")},
+				maybeGitSource{url: mkurl("http://bitbucket.org/sdboyer/reporoot.git")},
 			},
 		},
 		{
 			in:   "git@bitbucket.org:sdboyer/reporoot.git",
 			root: "bitbucket.org/sdboyer/reporoot.git",
-			mb:   maybeGitSource{url: mkurl("ssh://git@bitbucket.org/sdboyer/reporoot")},
+			mb:   maybeGitSource{url: mkurl("ssh://git@bitbucket.org/sdboyer/reporoot.git")},
 		},
 		{
 			in:   "bitbucket.org/sdboyer/reporoot.hg",
 			root: "bitbucket.org/sdboyer/reporoot.hg",
 			mb: maybeSources{
-				maybeHgSource{url: mkurl("https://bitbucket.org/sdboyer/reporoot")},
-				maybeHgSource{url: mkurl("ssh://hg@bitbucket.org/sdboyer/reporoot")},
-				maybeHgSource{url: mkurl("http://bitbucket.org/sdboyer/reporoot")},
+				maybeHgSource{url: mkurl("https://bitbucket.org/sdboyer/reporoot.hg")},
+				maybeHgSource{url: mkurl("ssh://hg@bitbucket.org/sdboyer/reporoot.hg")},
+				maybeHgSource{url: mkurl("http://bitbucket.org/sdboyer/reporoot.hg")},
 			},
 		},
 		{
@@ -292,6 +287,7 @@ var pathDeductionFixtures = map[string][]pathDeductionFixture{
 			root: "launchpad.net/govcstestbzrrepo",
 			mb: maybeSources{
 				maybeBzrSource{url: mkurl("https://launchpad.net/govcstestbzrrepo")},
+				maybeBzrSource{url: mkurl("bzr+ssh://launchpad.net/govcstestbzrrepo")},
 				maybeBzrSource{url: mkurl("bzr://launchpad.net/govcstestbzrrepo")},
 				maybeBzrSource{url: mkurl("http://launchpad.net/govcstestbzrrepo")},
 			},
@@ -301,6 +297,7 @@ var pathDeductionFixtures = map[string][]pathDeductionFixture{
 			root: "launchpad.net/govcstestbzrrepo",
 			mb: maybeSources{
 				maybeBzrSource{url: mkurl("https://launchpad.net/govcstestbzrrepo")},
+				maybeBzrSource{url: mkurl("bzr+ssh://launchpad.net/govcstestbzrrepo")},
 				maybeBzrSource{url: mkurl("bzr://launchpad.net/govcstestbzrrepo")},
 				maybeBzrSource{url: mkurl("http://launchpad.net/govcstestbzrrepo")},
 			},
@@ -316,7 +313,7 @@ var pathDeductionFixtures = map[string][]pathDeductionFixture{
 			root: "git.launchpad.net/reporoot",
 			mb: maybeSources{
 				maybeGitSource{url: mkurl("https://git.launchpad.net/reporoot")},
-				maybeGitSource{url: mkurl("ssh://git@git.launchpad.net/reporoot")},
+				maybeGitSource{url: mkurl("ssh://git.launchpad.net/reporoot")},
 				maybeGitSource{url: mkurl("git://git.launchpad.net/reporoot")},
 				maybeGitSource{url: mkurl("http://git.launchpad.net/reporoot")},
 			},
@@ -326,7 +323,7 @@ var pathDeductionFixtures = map[string][]pathDeductionFixture{
 			root: "git.launchpad.net/reporoot",
 			mb: maybeSources{
 				maybeGitSource{url: mkurl("https://git.launchpad.net/reporoot")},
-				maybeGitSource{url: mkurl("ssh://git@git.launchpad.net/reporoot")},
+				maybeGitSource{url: mkurl("ssh://git.launchpad.net/reporoot")},
 				maybeGitSource{url: mkurl("git://git.launchpad.net/reporoot")},
 				maybeGitSource{url: mkurl("http://git.launchpad.net/reporoot")},
 			},
@@ -342,7 +339,7 @@ var pathDeductionFixtures = map[string][]pathDeductionFixture{
 			root: "git.apache.org/package-name.git",
 			mb: maybeSources{
 				maybeGitSource{url: mkurl("https://git.apache.org/package-name.git")},
-				maybeGitSource{url: mkurl("ssh://git@git.apache.org/package-name.git")},
+				maybeGitSource{url: mkurl("ssh://git.apache.org/package-name.git")},
 				maybeGitSource{url: mkurl("git://git.apache.org/package-name.git")},
 				maybeGitSource{url: mkurl("http://git.apache.org/package-name.git")},
 			},
@@ -352,7 +349,7 @@ var pathDeductionFixtures = map[string][]pathDeductionFixture{
 			root: "git.apache.org/package-name.git",
 			mb: maybeSources{
 				maybeGitSource{url: mkurl("https://git.apache.org/package-name.git")},
-				maybeGitSource{url: mkurl("ssh://git@git.apache.org/package-name.git")},
+				maybeGitSource{url: mkurl("ssh://git.apache.org/package-name.git")},
 				maybeGitSource{url: mkurl("git://git.apache.org/package-name.git")},
 				maybeGitSource{url: mkurl("http://git.apache.org/package-name.git")},
 			},
@@ -361,34 +358,80 @@ var pathDeductionFixtures = map[string][]pathDeductionFixture{
 	"vcsext": []pathDeductionFixture{
 		// VCS extension-based syntax
 		{
-			in:   "foobar/baz.git",
-			root: "foobar/baz.git",
+			in:   "foobar.com/baz.git",
+			root: "foobar.com/baz.git",
+			mb: maybeSources{
+				maybeGitSource{url: mkurl("https://foobar.com/baz.git")},
+				maybeGitSource{url: mkurl("ssh://foobar.com/baz.git")},
+				maybeGitSource{url: mkurl("git://foobar.com/baz.git")},
+				maybeGitSource{url: mkurl("http://foobar.com/baz.git")},
+			},
+		},
+		{
+			in:   "foobar.com/baz.git/extra/path",
+			root: "foobar.com/baz.git",
 			mb: maybeSources{
-				maybeGitSource{url: mkurl("https://foobar/baz.git")},
-				maybeGitSource{url: mkurl("git://foobar/baz.git")},
-				maybeGitSource{url: mkurl("http://foobar/baz.git")},
+				maybeGitSource{url: mkurl("https://foobar.com/baz.git")},
+				maybeGitSource{url: mkurl("ssh://foobar.com/baz.git")},
+				maybeGitSource{url: mkurl("git://foobar.com/baz.git")},
+				maybeGitSource{url: mkurl("http://foobar.com/baz.git")},
 			},
 		},
 		{
-			in:   "foobar/baz.bzr",
-			root: "foobar/baz.bzr",
+			in:   "foobar.com/baz.bzr",
+			root: "foobar.com/baz.bzr",
 			mb: maybeSources{
-				maybeBzrSource{url: mkurl("https://foobar/baz.bzr")},
-				maybeBzrSource{url: mkurl("bzr://foobar/baz.bzr")},
-				maybeBzrSource{url: mkurl("http://foobar/baz.bzr")},
+				maybeBzrSource{url: mkurl("https://foobar.com/baz.bzr")},
+				maybeBzrSource{url: mkurl("bzr+ssh://foobar.com/baz.bzr")},
+				maybeBzrSource{url: mkurl("bzr://foobar.com/baz.bzr")},
+				maybeBzrSource{url: mkurl("http://foobar.com/baz.bzr")},
 			},
 		},
 		{
-			in:   "foobar/baz.hg",
-			root: "foobar/baz.hg",
+			in:   "foo-bar.com/baz.hg",
+			root: "foo-bar.com/baz.hg",
 			mb: maybeSources{
-				maybeHgSource{url: mkurl("https://foobar/baz.hg")},
-				maybeHgSource{url: mkurl("http://foobar/baz.hg")},
+				maybeHgSource{url: mkurl("https://foo-bar.com/baz.hg")},
+				maybeHgSource{url: mkurl("ssh://foo-bar.com/baz.hg")},
+				maybeHgSource{url: mkurl("http://foo-bar.com/baz.hg")},
 			},
 		},
 		{
-			in:   "foobar/baz.git/quark/quizzle.git",
-			rerr: errors.New("not allowed: foobar/baz.git/quark/quizzle.git contains multiple vcs extension hints"),
+			in:   "git@foobar.com:baz.git",
+			root: "foobar.com/baz.git",
+			mb:   maybeGitSource{url: mkurl("ssh://git@foobar.com/baz.git")},
+		},
+		{
+			in:   "bzr+ssh://foobar.com/baz.bzr",
+			root: "foobar.com/baz.bzr",
+			mb:   maybeBzrSource{url: mkurl("bzr+ssh://foobar.com/baz.bzr")},
+		},
+		{
+			in:   "ssh://foobar.com/baz.bzr",
+			root: "foobar.com/baz.bzr",
+			mb:   maybeBzrSource{url: mkurl("ssh://foobar.com/baz.bzr")},
+		},
+		{
+			in:   "https://foobar.com/baz.hg",
+			root: "foobar.com/baz.hg",
+			mb:   maybeHgSource{url: mkurl("https://foobar.com/baz.hg")},
+		},
+		{
+			in:     "git://foobar.com/baz.hg",
+			root:   "foobar.com/baz.hg",
+			srcerr: errors.New("git is not a valid scheme for accessing hg repositories (path foobar.com/baz.hg)"),
+		},
+		// who knows why anyone would do this, but having a second vcs ext
+		// shouldn't throw us off - only the first one counts
+		{
+			in:   "foobar.com/baz.git/quark/quizzle.bzr/quorum",
+			root: "foobar.com/baz.git",
+			mb: maybeSources{
+				maybeGitSource{url: mkurl("https://foobar.com/baz.git")},
+				maybeGitSource{url: mkurl("ssh://foobar.com/baz.git")},
+				maybeGitSource{url: mkurl("git://foobar.com/baz.git")},
+				maybeGitSource{url: mkurl("http://foobar.com/baz.git")},
+			},
 		},
 	},
 	"vanity": []pathDeductionFixture{
@@ -485,18 +528,13 @@ func TestDeduceFromPath(t *testing.T) {
 				}
 				continue
 			}
-			if u == nil {
-				spew.Dump(fix, uerr)
-			}
 
 			root, rerr := deducer.deduceRoot(in)
 			if fix.rerr != nil {
-				if fix.rerr != rerr {
-					if rerr == nil {
-						t.Errorf("(in: %s, %T) Expected error on deducing root, got none:\n\t(WNT) %s", in, deducer, fix.rerr)
-					} else {
-						t.Errorf("(in: %s, %T) Got unexpected error on deducing root:\n\t(GOT) %s\n\t(WNT) %s", in, deducer, rerr, fix.rerr)
-					}
+				if rerr == nil {
+					t.Errorf("(in: %s, %T) Expected error on deducing root, got none:\n\t(WNT) %s", in, deducer, fix.rerr)
+				} else if fix.rerr.Error() != rerr.Error() {
+					t.Errorf("(in: %s, %T) Got unexpected error on deducing root:\n\t(GOT) %s\n\t(WNT) %s", in, deducer, rerr, fix.rerr)
 				}
 			} else if rerr != nil {
 				t.Errorf("(in: %s, %T) Got unexpected error on deducing root:\n\t(GOT) %s", in, deducer, rerr)
@@ -504,17 +542,18 @@ func TestDeduceFromPath(t *testing.T) {
 				t.Errorf("(in: %s, %T) Deducer did not return expected root:\n\t(GOT) %s\n\t(WNT) %s", in, deducer, root, fix.root)
 			}
 
-			mb, mberr := deducer.deduceSource(fix.in, u)
+			mb, mberr := deducer.deduceSource(in, u)
 			if fix.srcerr != nil {
-				if fix.srcerr != mberr {
-					if mberr == nil {
-						t.Errorf("(in: %s, %T) Expected error on deducing source, got none:\n\t(WNT) %s", in, deducer, fix.srcerr)
-					} else {
-						t.Errorf("(in: %s, %T) Got unexpected error on deducing source:\n\t(GOT) %s\n\t(WNT) %s", in, deducer, mberr, fix.srcerr)
-					}
+				if mberr == nil {
+					t.Errorf("(in: %s, %T) Expected error on deducing source, got none:\n\t(WNT) %s", in, deducer, fix.srcerr)
+				} else if fix.srcerr.Error() != mberr.Error() {
+					t.Errorf("(in: %s, %T) Got unexpected error on deducing source:\n\t(GOT) %s\n\t(WNT) %s", in, deducer, mberr, fix.srcerr)
+				}
+			} else if mberr != nil {
+				// don't complain the fix already expected an rerr
+				if fix.rerr == nil {
+					t.Errorf("(in: %s, %T) Got unexpected error on deducing source:\n\t(GOT) %s", in, deducer, mberr)
 				}
-			} else if mberr != nil && fix.rerr == nil { // don't complain the fix already expected an rerr
-				t.Errorf("(in: %s, %T) Got unexpected error on deducing source:\n\t(GOT) %s", in, deducer, mberr)
 			} else if !reflect.DeepEqual(mb, fix.mb) {
 				if mb == nil {
 					t.Errorf("(in: %s, %T) Deducer returned source maybes, but none expected:\n\t(GOT) (none)\n\t(WNT) %s", in, deducer, printmb(fix.mb))

From 593b12fe40f23681f8e9aeee0e31de941717c237 Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Wed, 10 Aug 2016 01:45:17 -0400
Subject: [PATCH 51/71] s/RepoExists()/SourceExists()/

---
 bridge.go           | 4 ++--
 manager_test.go     | 4 ++--
 solve_basic_test.go | 2 +-
 solver.go           | 4 ++--
 source_manager.go   | 9 ++++-----
 5 files changed, 11 insertions(+), 12 deletions(-)

diff --git a/bridge.go b/bridge.go
index 00fb839..0591ad5 100644
--- a/bridge.go
+++ b/bridge.go
@@ -107,8 +107,8 @@ func (b *bridge) RevisionPresentIn(id ProjectIdentifier, r Revision) (bool, erro
 	return b.sm.RevisionPresentIn(id, r)
 }
 
-func (b *bridge) RepoExists(id ProjectIdentifier) (bool, error) {
-	return b.sm.RepoExists(id)
+func (b *bridge) SourceExists(id ProjectIdentifier) (bool, error) {
+	return b.sm.SourceExists(id)
 }
 
 func (b *bridge) vendorCodeExists(id ProjectIdentifier) (bool, error) {
diff --git a/manager_test.go b/manager_test.go
index 4351445..7c49593 100644
--- a/manager_test.go
+++ b/manager_test.go
@@ -170,9 +170,9 @@ func TestProjectManagerInit(t *testing.T) {
 
 	// Ensure project existence values are what we expect
 	var exists bool
-	exists, err = sm.RepoExists(id)
+	exists, err = sm.SourceExists(id)
 	if err != nil {
-		t.Errorf("Error on checking RepoExists: %s", err)
+		t.Errorf("Error on checking SourceExists: %s", err)
 	}
 	if !exists {
 		t.Error("Repo should exist after non-erroring call to ListVersions")
diff --git a/solve_basic_test.go b/solve_basic_test.go
index c493b19..b4b6fac 100644
--- a/solve_basic_test.go
+++ b/solve_basic_test.go
@@ -1283,7 +1283,7 @@ func (sm *depspecSourceManager) RevisionPresentIn(id ProjectIdentifier, r Revisi
 	return false, fmt.Errorf("Project %s has no revision %s", id.errString(), r)
 }
 
-func (sm *depspecSourceManager) RepoExists(id ProjectIdentifier) (bool, error) {
+func (sm *depspecSourceManager) SourceExists(id ProjectIdentifier) (bool, error) {
 	for _, ds := range sm.specs {
 		if id.ProjectRoot == ds.n {
 			return true, nil
diff --git a/solver.go b/solver.go
index eab3b42..e11f69c 100644
--- a/solver.go
+++ b/solver.go
@@ -639,7 +639,7 @@ func (s *solver) createVersionQueue(bmi bimodalIdentifier) (*versionQueue, error
 		return newVersionQueue(id, nil, nil, s.b)
 	}
 
-	exists, err := s.b.RepoExists(id)
+	exists, err := s.b.SourceExists(id)
 	if err != nil {
 		return nil, err
 	}
@@ -816,7 +816,7 @@ func (s *solver) getLockVersionIfValid(id ProjectIdentifier) (Version, error) {
 		// to be found and attempted in the repository. If it's only in vendor,
 		// though, then we have to try to use what's in the lock, because that's
 		// the only version we'll be able to get.
-		if exist, _ := s.b.RepoExists(id); exist {
+		if exist, _ := s.b.SourceExists(id); exist {
 			return nil, nil
 		}
 
diff --git a/source_manager.go b/source_manager.go
index b6abef1..3e75aa3 100644
--- a/source_manager.go
+++ b/source_manager.go
@@ -26,10 +26,9 @@ var sanitizer = strings.NewReplacer(":", "-", "/", "-", "+", "-")
 // sufficient for any purpose. It provides some additional semantics around the
 // methods defined here.
 type SourceManager interface {
-	// RepoExists checks if a repository exists, either upstream or in the
+	// SourceExists checks if a repository exists, either upstream or in the
 	// SourceManager's central repository cache.
-	// TODO(sdboyer) rename to SourceExists
-	RepoExists(ProjectIdentifier) (bool, error)
+	SourceExists(ProjectIdentifier) (bool, error)
 
 	// ListVersions retrieves a list of the available versions for a given
 	// repository name.
@@ -219,9 +218,9 @@ func (sm *SourceMgr) RevisionPresentIn(id ProjectIdentifier, r Revision) (bool,
 	return pmc.pm.RevisionPresentIn(id.ProjectRoot, r)
 }
 
-// RepoExists checks if a repository exists, either upstream or in the cache,
+// SourceExists checks if a repository exists, either upstream or in the cache,
 // for the provided ProjectIdentifier.
-func (sm *SourceMgr) RepoExists(id ProjectIdentifier) (bool, error) {
+func (sm *SourceMgr) SourceExists(id ProjectIdentifier) (bool, error) {
 	pms, err := sm.getProjectManager(id)
 	if err != nil {
 		return false, err

From 149ef63d5a05918fadc88666848a2d85433a8fe9 Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Wed, 10 Aug 2016 21:40:04 -0400
Subject: [PATCH 52/71] Add deducerTrie (typed wrapper of radix.Tree)

This will hold the pathDeducers, to be used by the SourceManager when
performing path deduction.
---
 remote.go         | 16 ++++++++++-
 source_manager.go | 11 ++++----
 typed_radix.go    | 70 +++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 90 insertions(+), 7 deletions(-)
 create mode 100644 typed_radix.go

diff --git a/remote.go b/remote.go
index c55218e..4a6328e 100644
--- a/remote.go
+++ b/remote.go
@@ -83,6 +83,20 @@ var (
 	pathvld     = regexp.MustCompile(`^([A-Za-z0-9-]+)(\.[A-Za-z0-9-]+)+(/[A-Za-z0-9-_.~]+)*$`)
 )
 
+func pathDeducerTrie() deducerTrie {
+	dxt := newDeducerTrie()
+
+	dxt.Insert("github.com/", githubDeducer{regexp: ghRegex})
+	dxt.Insert("gopkg.in/", gopkginDeducer{regexp: gpinNewRegex})
+	dxt.Insert("bitbucket.org/", bitbucketDeducer{regexp: bbRegex})
+	dxt.Insert("launchpad.net/", launchpadDeducer{regexp: lpRegex})
+	dxt.Insert("git.launchpad.net/", launchpadGitDeducer{regexp: glpRegex})
+	dxt.Insert("hub.jazz.net/", jazzDeducer{regexp: jazzRegex})
+	dxt.Insert("git.apache.org/", apacheDeducer{regexp: apacheRegex})
+
+	return dxt
+}
+
 type pathDeducer interface {
 	deduceRoot(string) (string, error)
 	deduceSource(string, *url.URL) (maybeSource, error)
@@ -559,7 +573,7 @@ func (sm *SourceMgr) deduceFromPath(path string) (stringFuture, partialSourceFut
 	}
 
 	// First, try the root path-based matches
-	if _, mtchi, has := sm.rootxt.LongestPrefix(path); has {
+	if _, mtchi, has := sm.dxt.LongestPrefix(path); has {
 		mtch := mtchi.(pathDeducer)
 		root, err := mtch.deduceRoot(path)
 		if err != nil {
diff --git a/source_manager.go b/source_manager.go
index 3e75aa3..120ec24 100644
--- a/source_manager.go
+++ b/source_manager.go
@@ -10,7 +10,6 @@ import (
 
 	"github.com/Masterminds/semver"
 	"github.com/Masterminds/vcs"
-	"github.com/armon/go-radix"
 )
 
 // Used to compute a friendly filepath from a URL-shaped input
@@ -82,9 +81,9 @@ type SourceMgr struct {
 		rr  *remoteRepo
 		err error
 	}
-	rmut   sync.RWMutex
-	an     ProjectAnalyzer
-	rootxt *radix.Tree
+	rmut sync.RWMutex
+	an   ProjectAnalyzer
+	dxt  deducerTrie
 }
 
 var _ SourceManager = &SourceMgr{}
@@ -142,8 +141,8 @@ func NewSourceManager(an ProjectAnalyzer, cachedir string, force bool) (*SourceM
 			rr  *remoteRepo
 			err error
 		}),
-		an:     an,
-		rootxt: radix.New(),
+		an:  an,
+		dxt: pathDeducerTrie(),
 	}, nil
 }
 
diff --git a/typed_radix.go b/typed_radix.go
new file mode 100644
index 0000000..707397e
--- /dev/null
+++ b/typed_radix.go
@@ -0,0 +1,70 @@
+package gps
+
+import "github.com/armon/go-radix"
+
+// Typed implementations of radix trees. These are just simple wrappers that let
+// us avoid having to type assert anywhere else, cleaning up other code a bit.
+//
+// Some of the more annoying things to implement (like walks) aren't
+// implemented. They can be added if/when we actually need them.
+//
+// Oh generics, where art thou...
+
+type deducerTrie struct {
+	t *radix.Tree
+}
+
+func newDeducerTrie() deducerTrie {
+	return deducerTrie{
+		t: radix.New(),
+	}
+}
+
+// Delete is used to delete a key, returning the previous value and if it was deleted
+func (t deducerTrie) Delete(s string) (pathDeducer, bool) {
+	if v, had := t.t.Delete(s); had {
+		return v.(pathDeducer), had
+	}
+	return nil, false
+}
+
+// Get is used to lookup a specific key, returning the value and if it was found
+func (t deducerTrie) Get(s string) (pathDeducer, bool) {
+	if v, has := t.t.Get(s); has {
+		return v.(pathDeducer), has
+	}
+	return nil, false
+}
+
+// Insert is used to add a newentry or update an existing entry. Returns if updated.
+func (t deducerTrie) Insert(s string, v pathDeducer) (pathDeducer, bool) {
+	if v2, had := t.t.Insert(s, v); had {
+		return v2.(pathDeducer), had
+	}
+	return nil, false
+}
+
+// Len is used to return the number of elements in the tree
+func (t deducerTrie) Len() int {
+	return t.t.Len()
+}
+
+// LongestPrefix is like Get, but instead of an exact match, it will return the
+// longest prefix match.
+func (t deducerTrie) LongestPrefix(s string) (string, pathDeducer, bool) {
+	if p, v, has := t.t.LongestPrefix(s); has {
+		return p, v.(pathDeducer), has
+	}
+	return "", nil, false
+}
+
+// ToMap is used to walk the tree and convert it to a map.
+func (t deducerTrie) ToMap() map[string]pathDeducer {
+	m := make(map[string]pathDeducer)
+	t.t.Walk(func(s string, v interface{}) bool {
+		m[s] = v.(pathDeducer)
+		return false
+	})
+
+	return m
+}

From 1bc246912dc6d68a64cd8ab1a2cfcf3df5794d55 Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Thu, 11 Aug 2016 08:50:27 -0400
Subject: [PATCH 53/71] Introduce deductionFuture for deduction results

This adds one additional field, a flag indicating whether the root
future is *actually* async (there's currently only one case in deduction
where it actually does require network activity). This allows the main
handling process to be a little smarter about how it stores the
information.
---
 remote.go | 39 ++++++++++++++++++++++++++++++---------
 1 file changed, 30 insertions(+), 9 deletions(-)

diff --git a/remote.go b/remote.go
index 4a6328e..5ab73b4 100644
--- a/remote.go
+++ b/remote.go
@@ -531,6 +531,14 @@ type stringFuture func() (string, error)
 type sourceFuture func() (source, error)
 type partialSourceFuture func(string, ProjectAnalyzer) sourceFuture
 
+type deductionFuture struct {
+	// rslow indicates that the root future may be a slow call (that it has to
+	// hit the network for some reason)
+	rslow bool
+	root  stringFuture
+	psf   partialSourceFuture
+}
+
 // deduceFromPath takes an import path and attempts to deduce various
 // metadata about it - what type of source should handle it, and where its
 // "root" is (for vcs repositories, the repository root).
@@ -540,11 +548,11 @@ type partialSourceFuture func(string, ProjectAnalyzer) sourceFuture
 // activity will be triggered when the future is called. For the second,
 // network activity is triggered only when calling the sourceFuture returned
 // from the partialSourceFuture.
-func (sm *SourceMgr) deduceFromPath(path string) (stringFuture, partialSourceFuture, error) {
+func (sm *SourceMgr) deduceFromPath(path string) (deductionFuture, error) {
 	opath := path
 	u, path, err := normalizeURI(path)
 	if err != nil {
-		return nil, nil, err
+		return deductionFuture{}, err
 	}
 
 	// Helpers to futurize the results from deducers
@@ -577,14 +585,18 @@ func (sm *SourceMgr) deduceFromPath(path string) (stringFuture, partialSourceFut
 		mtch := mtchi.(pathDeducer)
 		root, err := mtch.deduceRoot(path)
 		if err != nil {
-			return nil, nil, err
+			return deductionFuture{}, err
 		}
 		mb, err := mtch.deduceSource(path, u)
 		if err != nil {
-			return nil, nil, err
+			return deductionFuture{}, err
 		}
 
-		return strfut(root), srcfut(mb), nil
+		return deductionFuture{
+			rslow: false,
+			root:  strfut(root),
+			psf:   srcfut(mb),
+		}, nil
 	}
 
 	// Next, try the vcs extension-based (infix) matcher
@@ -592,9 +604,14 @@ func (sm *SourceMgr) deduceFromPath(path string) (stringFuture, partialSourceFut
 	if root, err := exm.deduceRoot(path); err == nil {
 		mb, err := exm.deduceSource(path, u)
 		if err != nil {
-			return nil, nil, err
+			return deductionFuture{}, err
 		}
-		return strfut(root), srcfut(mb), nil
+
+		return deductionFuture{
+			rslow: false,
+			root:  strfut(root),
+			psf:   srcfut(mb),
+		}, nil
 	}
 
 	// No luck so far. maybe it's one of them vanity imports?
@@ -671,7 +688,11 @@ func (sm *SourceMgr) deduceFromPath(path string) (stringFuture, partialSourceFut
 		}
 	}
 
-	return root, src, nil
+	return deductionFuture{
+		rslow: true,
+		root:  root,
+		psf:   src,
+	}, nil
 }
 
 func normalizeURI(p string) (u *url.URL, newpath string, err error) {
@@ -766,7 +787,7 @@ func deduceRemoteRepo(path string) (rr *remoteRepo, err error) {
 	return rr, nil
 }
 
-// fetchMetadata fetchs the remote metadata for path.
+// fetchMetadata fetches the remote metadata for path.
 func fetchMetadata(path string) (rc io.ReadCloser, err error) {
 	defer func() {
 		if err != nil {

From 6cfedc35cb656a37ee5369976f0d22978bc7bf5f Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Fri, 12 Aug 2016 10:36:47 -0400
Subject: [PATCH 54/71] Also return string ident from sourceFuture

This helps resolve a problem that will probably only exist for vcs-type
sources, where there could be a difference between the input ident
(e.g., a plain import path) and the actual on-disk ident of the
resulting source (e.g., a full URL). The manager needs to know which
unique string ident is in use, because that will become an access path
for lookups (in addition to the input path).
---
 maybe_source.go | 65 ++++++++++++++++++++++++++++++-------------------
 remote.go       | 17 +++++++------
 source_test.go  | 24 +++++++++++++-----
 3 files changed, 68 insertions(+), 38 deletions(-)

diff --git a/maybe_source.go b/maybe_source.go
index 19fb961..8d4cf72 100644
--- a/maybe_source.go
+++ b/maybe_source.go
@@ -10,24 +10,36 @@ import (
 )
 
 type maybeSource interface {
-	try(cachedir string, an ProjectAnalyzer) (source, error)
+	try(cachedir string, an ProjectAnalyzer) (source, string, error)
 }
 
 type maybeSources []maybeSource
 
-func (mbs maybeSources) try(cachedir string, an ProjectAnalyzer) (source, error) {
+func (mbs maybeSources) try(cachedir string, an ProjectAnalyzer) (source, string, error) {
 	var e sourceFailures
 	for _, mb := range mbs {
-		src, err := mb.try(cachedir, an)
+		src, ident, err := mb.try(cachedir, an)
 		if err == nil {
-			return src, nil
+			return src, ident, nil
 		}
-		e = append(e, err)
+		e = append(e, sourceSetupFailure{
+			ident: ident,
+			err:   err,
+		})
 	}
-	return nil, e
+	return nil, "", e
 }
 
-type sourceFailures []error
+type sourceSetupFailure struct {
+	ident string
+	err   error
+}
+
+func (e sourceSetupFailure) Error() string {
+	return fmt.Sprintf("failed to set up %q, error %s", e.ident, e.err.Error())
+}
+
+type sourceFailures []sourceSetupFailure
 
 func (sf sourceFailures) Error() string {
 	var buf bytes.Buffer
@@ -43,11 +55,12 @@ type maybeGitSource struct {
 	url *url.URL
 }
 
-func (m maybeGitSource) try(cachedir string, an ProjectAnalyzer) (source, error) {
-	path := filepath.Join(cachedir, "sources", sanitizer.Replace(m.url.String()))
-	r, err := vcs.NewGitRepo(m.url.String(), path)
+func (m maybeGitSource) try(cachedir string, an ProjectAnalyzer) (source, string, error) {
+	ustr := m.url.String()
+	path := filepath.Join(cachedir, "sources", sanitizer.Replace(ustr))
+	r, err := vcs.NewGitRepo(ustr, path)
 	if err != nil {
-		return nil, err
+		return nil, "", err
 	}
 
 	src := &gitSource{
@@ -63,26 +76,27 @@ func (m maybeGitSource) try(cachedir string, an ProjectAnalyzer) (source, error)
 
 	_, err = src.listVersions()
 	if err != nil {
-		return nil, err
+		return nil, "", err
 		//} else if pm.ex.f&existsUpstream == existsUpstream {
 		//return pm, nil
 	}
 
-	return src, nil
+	return src, ustr, nil
 }
 
 type maybeBzrSource struct {
 	url *url.URL
 }
 
-func (m maybeBzrSource) try(cachedir string, an ProjectAnalyzer) (source, error) {
-	path := filepath.Join(cachedir, "sources", sanitizer.Replace(m.url.String()))
-	r, err := vcs.NewBzrRepo(m.url.String(), path)
+func (m maybeBzrSource) try(cachedir string, an ProjectAnalyzer) (source, string, error) {
+	ustr := m.url.String()
+	path := filepath.Join(cachedir, "sources", sanitizer.Replace(ustr))
+	r, err := vcs.NewBzrRepo(ustr, path)
 	if err != nil {
-		return nil, err
+		return nil, "", err
 	}
 	if !r.Ping() {
-		return nil, fmt.Errorf("Remote repository at %s does not exist, or is inaccessible", m.url.String())
+		return nil, "", fmt.Errorf("Remote repository at %s does not exist, or is inaccessible", ustr)
 	}
 
 	return &bzrSource{
@@ -94,21 +108,22 @@ func (m maybeBzrSource) try(cachedir string, an ProjectAnalyzer) (source, error)
 				rpath: path,
 			},
 		},
-	}, nil
+	}, ustr, nil
 }
 
 type maybeHgSource struct {
 	url *url.URL
 }
 
-func (m maybeHgSource) try(cachedir string, an ProjectAnalyzer) (source, error) {
-	path := filepath.Join(cachedir, "sources", sanitizer.Replace(m.url.String()))
-	r, err := vcs.NewHgRepo(m.url.String(), path)
+func (m maybeHgSource) try(cachedir string, an ProjectAnalyzer) (source, string, error) {
+	ustr := m.url.String()
+	path := filepath.Join(cachedir, "sources", sanitizer.Replace(ustr))
+	r, err := vcs.NewBzrRepo(ustr, path)
 	if err != nil {
-		return nil, err
+		return nil, "", err
 	}
 	if !r.Ping() {
-		return nil, fmt.Errorf("Remote repository at %s does not exist, or is inaccessible", m.url.String())
+		return nil, "", fmt.Errorf("Remote repository at %s does not exist, or is inaccessible", ustr)
 	}
 
 	return &hgSource{
@@ -120,5 +135,5 @@ func (m maybeHgSource) try(cachedir string, an ProjectAnalyzer) (source, error)
 				rpath: path,
 			},
 		},
-	}, nil
+	}, ustr, nil
 }
diff --git a/remote.go b/remote.go
index 5ab73b4..17fcf3a 100644
--- a/remote.go
+++ b/remote.go
@@ -528,7 +528,7 @@ func (m vcsExtensionDeducer) deduceSource(path string, u *url.URL) (maybeSource,
 }
 
 type stringFuture func() (string, error)
-type sourceFuture func() (source, error)
+type sourceFuture func() (source, string, error)
 type partialSourceFuture func(string, ProjectAnalyzer) sourceFuture
 
 type deductionFuture struct {
@@ -565,17 +565,18 @@ func (sm *SourceMgr) deduceFromPath(path string) (deductionFuture, error) {
 	srcfut := func(mb maybeSource) partialSourceFuture {
 		return func(cachedir string, an ProjectAnalyzer) sourceFuture {
 			var src source
+			var ident string
 			var err error
 
 			c := make(chan struct{}, 1)
 			go func() {
 				defer close(c)
-				src, err = mb.try(cachedir, an)
+				src, ident, err = mb.try(cachedir, an)
 			}()
 
-			return func() (source, error) {
+			return func() (source, string, error) {
 				<-c
-				return src, err
+				return src, ident, err
 			}
 		}
 	}
@@ -653,6 +654,7 @@ func (sm *SourceMgr) deduceFromPath(path string) (deductionFuture, error) {
 
 	src := func(cachedir string, an ProjectAnalyzer) sourceFuture {
 		var src source
+		var ident string
 		var err error
 
 		c := make(chan struct{}, 1)
@@ -664,6 +666,7 @@ func (sm *SourceMgr) deduceFromPath(path string) (deductionFuture, error) {
 			if err != nil {
 				return
 			}
+			ident = ru.String()
 
 			var m maybeSource
 			switch vcs {
@@ -676,15 +679,15 @@ func (sm *SourceMgr) deduceFromPath(path string) (deductionFuture, error) {
 			}
 
 			if m != nil {
-				src, err = m.try(cachedir, an)
+				src, ident, err = m.try(cachedir, an)
 			} else {
 				err = fmt.Errorf("unsupported vcs type %s", vcs)
 			}
 		}()
 
-		return func() (source, error) {
+		return func() (source, string, error) {
 			<-c
-			return src, err
+			return src, ident, err
 		}
 	}
 
diff --git a/source_test.go b/source_test.go
index 57a9394..33d2acb 100644
--- a/source_test.go
+++ b/source_test.go
@@ -26,7 +26,8 @@ func TestGitVersionFetching(t *testing.T) {
 	}
 
 	n := "github.com/Masterminds/VCSTestRepo"
-	u, err := url.Parse("https://" + n)
+	un := "https://" + n
+	u, err := url.Parse(un)
 	if err != nil {
 		t.Errorf("URL was bad, lolwut? errtext: %s", err)
 		rf()
@@ -36,7 +37,7 @@ func TestGitVersionFetching(t *testing.T) {
 		url: u,
 	}
 
-	isrc, err := mb.try(cpath, naiveAnalyzer{})
+	isrc, ident, err := mb.try(cpath, naiveAnalyzer{})
 	if err != nil {
 		t.Errorf("Unexpected error while setting up gitSource for test repo: %s", err)
 		rf()
@@ -48,6 +49,9 @@ func TestGitVersionFetching(t *testing.T) {
 		rf()
 		t.FailNow()
 	}
+	if ident != un {
+		t.Errorf("Expected %s as source ident, got %s", un, ident)
+	}
 
 	vlist, err := src.listVersions()
 	if err != nil {
@@ -102,7 +106,8 @@ func TestBzrVersionFetching(t *testing.T) {
 	}
 
 	n := "launchpad.net/govcstestbzrrepo"
-	u, err := url.Parse("https://" + n)
+	un := "https://" + n
+	u, err := url.Parse(un)
 	if err != nil {
 		t.Errorf("URL was bad, lolwut? errtext: %s", err)
 		rf()
@@ -112,7 +117,7 @@ func TestBzrVersionFetching(t *testing.T) {
 		url: u,
 	}
 
-	isrc, err := mb.try(cpath, naiveAnalyzer{})
+	isrc, ident, err := mb.try(cpath, naiveAnalyzer{})
 	if err != nil {
 		t.Errorf("Unexpected error while setting up bzrSource for test repo: %s", err)
 		rf()
@@ -124,6 +129,9 @@ func TestBzrVersionFetching(t *testing.T) {
 		rf()
 		t.FailNow()
 	}
+	if ident != un {
+		t.Errorf("Expected %s as source ident, got %s", un, ident)
+	}
 
 	vlist, err := src.listVersions()
 	if err != nil {
@@ -187,7 +195,8 @@ func TestHgVersionFetching(t *testing.T) {
 	}
 
 	n := "bitbucket.org/mattfarina/testhgrepo"
-	u, err := url.Parse("https://" + n)
+	un := "https://" + n
+	u, err := url.Parse(un)
 	if err != nil {
 		t.Errorf("URL was bad, lolwut? errtext: %s", err)
 		rf()
@@ -197,7 +206,7 @@ func TestHgVersionFetching(t *testing.T) {
 		url: u,
 	}
 
-	isrc, err := mb.try(cpath, naiveAnalyzer{})
+	isrc, ident, err := mb.try(cpath, naiveAnalyzer{})
 	if err != nil {
 		t.Errorf("Unexpected error while setting up hgSource for test repo: %s", err)
 		rf()
@@ -209,6 +218,9 @@ func TestHgVersionFetching(t *testing.T) {
 		rf()
 		t.FailNow()
 	}
+	if ident != un {
+		t.Errorf("Expected %s as source ident, got %s", un, ident)
+	}
 
 	vlist, err := src.listVersions()
 	if err != nil {

From c2e8b10784fdd85dc71b33deca16f6020920b740 Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Fri, 12 Aug 2016 23:32:11 -0400
Subject: [PATCH 55/71] Add helper func for creating SourceMgr w/tmp dir

---
 manager_test.go | 71 ++++++++++++++++++++++---------------------------
 remote_test.go  | 19 -------------
 result_test.go  |  8 +++---
 3 files changed, 35 insertions(+), 63 deletions(-)

diff --git a/manager_test.go b/manager_test.go
index 7c49593..7e8972a 100644
--- a/manager_test.go
+++ b/manager_test.go
@@ -36,6 +36,28 @@ func sv(s string) *semver.Version {
 	return sv
 }
 
+func mkNaiveSM(t *testing.T) (*SourceMgr, func()) {
+	cpath, err := ioutil.TempDir("", "smcache")
+	if err != nil {
+		t.Errorf("Failed to create temp dir: %s", err)
+		t.FailNow()
+	}
+
+	sm, err := NewSourceManager(naiveAnalyzer{}, cpath, false)
+	if err != nil {
+		t.Errorf("Unexpected error on SourceManager creation: %s", err)
+		t.FailNow()
+	}
+
+	return sm, func() {
+		sm.Release()
+		err := removeAll(cpath)
+		if err != nil {
+			t.Errorf("removeAll failed: %s", err)
+		}
+	}
+}
+
 func init() {
 	_, filename, _, _ := runtime.Caller(1)
 	bd = path.Dir(filename)
@@ -83,20 +105,22 @@ func TestProjectManagerInit(t *testing.T) {
 	cpath, err := ioutil.TempDir("", "smcache")
 	if err != nil {
 		t.Errorf("Failed to create temp dir: %s", err)
+		t.FailNow()
 	}
-	sm, err := NewSourceManager(naiveAnalyzer{}, cpath, false)
 
+	sm, err := NewSourceManager(naiveAnalyzer{}, cpath, false)
 	if err != nil {
 		t.Errorf("Unexpected error on SourceManager creation: %s", err)
 		t.FailNow()
 	}
+
 	defer func() {
+		sm.Release()
 		err := removeAll(cpath)
 		if err != nil {
 			t.Errorf("removeAll failed: %s", err)
 		}
 	}()
-	defer sm.Release()
 
 	id := mkPI("github.com/Masterminds/VCSTestRepo")
 	v, err := sm.ListVersions(id)
@@ -197,16 +221,7 @@ func TestRepoVersionFetching(t *testing.T) {
 		t.Skip("Skipping repo version fetching test in short mode")
 	}
 
-	cpath, err := ioutil.TempDir("", "smcache")
-	if err != nil {
-		t.Errorf("Failed to create temp dir: %s", err)
-	}
-
-	sm, err := NewSourceManager(naiveAnalyzer{}, cpath, false)
-	if err != nil {
-		t.Errorf("Unexpected error on SourceManager creation: %s", err)
-		t.FailNow()
-	}
+	sm, clean := mkNaiveSM(t)
 
 	upstreams := []ProjectIdentifier{
 		mkPI("github.com/Masterminds/VCSTestRepo"),
@@ -218,21 +233,14 @@ func TestRepoVersionFetching(t *testing.T) {
 	for k, u := range upstreams {
 		pmi, err := sm.getProjectManager(u)
 		if err != nil {
-			sm.Release()
-			removeAll(cpath)
+			clean()
 			t.Errorf("Unexpected error on ProjectManager creation: %s", err)
 			t.FailNow()
 		}
 		pms[k] = pmi.pm
 	}
 
-	defer func() {
-		err := removeAll(cpath)
-		if err != nil {
-			t.Errorf("removeAll failed: %s", err)
-		}
-	}()
-	defer sm.Release()
+	defer clean()
 
 	// test git first
 	vlist, exbits, err := pms[0].crepo.getCurrentVersionPairs()
@@ -309,29 +317,14 @@ func TestGetInfoListVersionsOrdering(t *testing.T) {
 		t.Skip("Skipping slow test in short mode")
 	}
 
-	cpath, err := ioutil.TempDir("", "smcache")
-	if err != nil {
-		t.Errorf("Failed to create temp dir: %s", err)
-	}
-	sm, err := NewSourceManager(naiveAnalyzer{}, cpath, false)
-
-	if err != nil {
-		t.Errorf("Unexpected error on SourceManager creation: %s", err)
-		t.FailNow()
-	}
-	defer func() {
-		err := removeAll(cpath)
-		if err != nil {
-			t.Errorf("removeAll failed: %s", err)
-		}
-	}()
-	defer sm.Release()
+	sm, clean := mkNaiveSM(t)
+	defer clean()
 
 	// setup done, now do the test
 
 	id := mkPI("github.com/Masterminds/VCSTestRepo")
 
-	_, _, err = sm.GetManifestAndLock(id, NewVersion("1.0.0"))
+	_, _, err := sm.GetManifestAndLock(id, NewVersion("1.0.0"))
 	if err != nil {
 		t.Errorf("Unexpected error from GetInfoAt %s", err)
 	}
diff --git a/remote_test.go b/remote_test.go
index 6d88ff1..bb18a76 100644
--- a/remote_test.go
+++ b/remote_test.go
@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
-	"io/ioutil"
 	"net/url"
 	"reflect"
 	"testing"
@@ -455,24 +454,6 @@ var pathDeductionFixtures = map[string][]pathDeductionFixture{
 }
 
 func TestDeduceFromPath(t *testing.T) {
-	cpath, err := ioutil.TempDir("", "smcache")
-	if err != nil {
-		t.Errorf("Failed to create temp dir: %s", err)
-	}
-	sm, err := NewSourceManager(naiveAnalyzer{}, cpath, false)
-
-	if err != nil {
-		t.Errorf("Unexpected error on SourceManager creation: %s", err)
-		t.FailNow()
-	}
-	defer func() {
-		err := removeAll(cpath)
-		if err != nil {
-			t.Errorf("removeAll failed: %s", err)
-		}
-	}()
-	defer sm.Release()
-
 	for typ, fixtures := range pathDeductionFixtures {
 		var deducer pathDeducer
 		switch typ {
diff --git a/result_test.go b/result_test.go
index 1a2a8ad..61c20f3 100644
--- a/result_test.go
+++ b/result_test.go
@@ -48,12 +48,10 @@ func TestResultCreateVendorTree(t *testing.T) {
 	tmp := path.Join(os.TempDir(), "vsolvtest")
 	os.RemoveAll(tmp)
 
-	sm, err := NewSourceManager(naiveAnalyzer{}, path.Join(tmp, "cache"), false)
-	if err != nil {
-		t.Errorf("NewSourceManager errored unexpectedly: %q", err)
-	}
+	sm, clean := mkNaiveSM(t)
+	defer clean()
 
-	err = CreateVendorTree(path.Join(tmp, "export"), r, sm, true)
+	err := CreateVendorTree(path.Join(tmp, "export"), r, sm, true)
 	if err != nil {
 		t.Errorf("Unexpected error while creating vendor tree: %s", err)
 	}

From b663ae8f1b24982f3c6027b3578a69850d5ee12f Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Sat, 13 Aug 2016 23:20:33 -0400
Subject: [PATCH 56/71] Add typed radix trie for project roots

---
 typed_radix.go | 83 +++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 82 insertions(+), 1 deletion(-)

diff --git a/typed_radix.go b/typed_radix.go
index 707397e..9f56a9b 100644
--- a/typed_radix.go
+++ b/typed_radix.go
@@ -1,6 +1,10 @@
 package gps
 
-import "github.com/armon/go-radix"
+import (
+	"strings"
+
+	"github.com/armon/go-radix"
+)
 
 // Typed implementations of radix trees. These are just simple wrappers that let
 // us avoid having to type assert anywhere else, cleaning up other code a bit.
@@ -68,3 +72,80 @@ func (t deducerTrie) ToMap() map[string]pathDeducer {
 
 	return m
 }
+
+type prTrie struct {
+	t *radix.Tree
+}
+
+func newProjectRootTrie() prTrie {
+	return prTrie{
+		t: radix.New(),
+	}
+}
+
+// Delete is used to delete a key, returning the previous value and if it was deleted
+func (t prTrie) Delete(s string) (ProjectRoot, bool) {
+	if v, had := t.t.Delete(s); had {
+		return v.(ProjectRoot), had
+	}
+	return "", false
+}
+
+// Get is used to lookup a specific key, returning the value and if it was found
+func (t prTrie) Get(s string) (ProjectRoot, bool) {
+	if v, has := t.t.Get(s); has {
+		return v.(ProjectRoot), has
+	}
+	return "", false
+}
+
+// Insert is used to add a newentry or update an existing entry. Returns if updated.
+func (t prTrie) Insert(s string, v ProjectRoot) (ProjectRoot, bool) {
+	if v2, had := t.t.Insert(s, v); had {
+		return v2.(ProjectRoot), had
+	}
+	return "", false
+}
+
+// Len is used to return the number of elements in the tree
+func (t prTrie) Len() int {
+	return t.t.Len()
+}
+
+// LongestPrefix is like Get, but instead of an exact match, it will return the
+// longest prefix match.
+func (t prTrie) LongestPrefix(s string) (string, ProjectRoot, bool) {
+	if p, v, has := t.t.LongestPrefix(s); has && isPathPrefixOrEqual(p, s) {
+		return p, v.(ProjectRoot), has
+	}
+	return "", "", false
+}
+
+// ToMap is used to walk the tree and convert it to a map.
+func (t prTrie) ToMap() map[string]ProjectRoot {
+	m := make(map[string]ProjectRoot)
+	t.t.Walk(func(s string, v interface{}) bool {
+		m[s] = v.(ProjectRoot)
+		return false
+	})
+
+	return m
+}
+
+// isPathPrefixOrEqual is an additional helper check to ensure that the literal
+// string prefix returned from a radix tree prefix match is also a tree match.
+//
+// The radix tree gets it mostly right, but we have to guard against
+// possibilities like this:
+//
+// github.com/sdboyer/foo
+// github.com/sdboyer/foobar/baz
+//
+// The latter would incorrectly be conflated with the former. As we know we're
+// operating on strings that describe paths, guard against this case by
+// verifying that either the input is the same length as the match (in which
+// case we know they're equal), or that the next character is a "/".
+func isPathPrefixOrEqual(pre, path string) bool {
+	prflen := len(pre)
+	return prflen == len(path) || strings.Index(path[:prflen], "/") == 0
+}

From 8aae7f23b88efc0b29c5e42494f88b091202d63e Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Sat, 13 Aug 2016 23:21:03 -0400
Subject: [PATCH 57/71] Create impls and tests for future-based deduction

---
 manager_test.go   | 139 ++++++++++++++++++++++++++++++++++
 source_manager.go | 184 ++++++++++++++++++++++++++++++++++++++++++++--
 2 files changed, 318 insertions(+), 5 deletions(-)

diff --git a/manager_test.go b/manager_test.go
index 7e8972a..ba946e4 100644
--- a/manager_test.go
+++ b/manager_test.go
@@ -7,6 +7,7 @@ import (
 	"path"
 	"runtime"
 	"sort"
+	"sync"
 	"testing"
 
 	"github.com/Masterminds/semver"
@@ -338,3 +339,141 @@ func TestGetInfoListVersionsOrdering(t *testing.T) {
 		t.Errorf("Expected three results from ListVersions, got %v", len(v))
 	}
 }
+
+func TestDeduceProjectRoot(t *testing.T) {
+	sm, clean := mkNaiveSM(t)
+	defer clean()
+
+	in := "github.com/sdboyer/gps"
+	pr, err := sm.DeduceProjectRoot(in)
+	if err != nil {
+		t.Errorf("Problem while detecting root of %q %s", in, err)
+	}
+	if string(pr) != in {
+		t.Errorf("Wrong project root was deduced;\n\t(GOT) %s\n\t(WNT) %s", pr, in)
+	}
+	if sm.rootxt.Len() != 1 {
+		t.Errorf("Root path trie should have one element after one deduction, has %v", sm.rootxt.Len())
+	}
+
+	pr, err = sm.DeduceProjectRoot(in)
+	if err != nil {
+		t.Errorf("Problem while detecting root of %q %s", in, err)
+	} else if string(pr) != in {
+		t.Errorf("Wrong project root was deduced;\n\t(GOT) %s\n\t(WNT) %s", pr, in)
+	}
+	if sm.rootxt.Len() != 1 {
+		t.Errorf("Root path trie should have one element after performing the same deduction twice; has %v", sm.rootxt.Len())
+	}
+
+	// Now do a subpath
+	sub := path.Join(in, "foo")
+	pr, err = sm.DeduceProjectRoot(sub)
+	if err != nil {
+		t.Errorf("Problem while detecting root of %q %s", sub, err)
+	} else if string(pr) != in {
+		t.Errorf("Wrong project root was deduced;\n\t(GOT) %s\n\t(WNT) %s", pr, in)
+	}
+	if sm.rootxt.Len() != 2 {
+		t.Errorf("Root path trie should have two elements, one for root and one for subpath; has %v", sm.rootxt.Len())
+	}
+
+	// Now do a fully different root, but still on github
+	in2 := "github.com/bagel/lox"
+	sub2 := path.Join(in2, "cheese")
+	pr, err = sm.DeduceProjectRoot(sub2)
+	if err != nil {
+		t.Errorf("Problem while detecting root of %q %s", sub2, err)
+	} else if string(pr) != in2 {
+		t.Errorf("Wrong project root was deduced;\n\t(GOT) %s\n\t(WNT) %s", pr, in)
+	}
+	if sm.rootxt.Len() != 4 {
+		t.Errorf("Root path trie should have four elements, one for each unique root and subpath; has %v", sm.rootxt.Len())
+	}
+
+	// Ensure that our prefixes are bounded by path separators
+	in4 := "github.com/bagel/loxx"
+	pr, err = sm.DeduceProjectRoot(in4)
+	if err != nil {
+		t.Errorf("Problem while detecting root of %q %s", in4, err)
+	} else if string(pr) != in4 {
+		t.Errorf("Wrong project root was deduced;\n\t(GOT) %s\n\t(WNT) %s", pr, in)
+	}
+	if sm.rootxt.Len() != 5 {
+		t.Errorf("Root path trie should have five elements, one for each unique root and subpath; has %v", sm.rootxt.Len())
+	}
+}
+
+// Test that the future returned from SourceMgr.deducePathAndProcess() is safe
+// to call concurrently.
+//
+// Obviously, this is just a heuristic; passage does not guarantee correctness
+// (though failure does guarantee incorrectness)
+func TestMultiDeduceThreadsafe(t *testing.T) {
+	sm, clean := mkNaiveSM(t)
+	defer clean()
+
+	in := "github.com/sdboyer/gps"
+	rootf, srcf, err := sm.deducePathAndProcess(in)
+	if err != nil {
+		t.Errorf("Known-good path %q had unexpected basic deduction error: %s", in, err)
+		t.FailNow()
+	}
+
+	cnum := 50
+	wg := &sync.WaitGroup{}
+
+	// Set up channel for everything else to block on
+	c := make(chan struct{}, 1)
+	f := func(rnum int) {
+		wg.Add(1)
+		defer func() {
+			if e := recover(); e != nil {
+				t.Errorf("goroutine number %v panicked with err: %s", rnum, e)
+			}
+		}()
+		<-c
+		_, err := rootf()
+		if err != nil {
+			t.Errorf("err was non-nil on root detection in goroutine number %v: %s", rnum, err)
+		}
+		wg.Done()
+	}
+
+	for k := range make([]struct{}, cnum) {
+		go f(k)
+		runtime.Gosched()
+	}
+	close(c)
+	wg.Wait()
+	if sm.rootxt.Len() != 1 {
+		t.Errorf("Root path trie should have just one element; has %v", sm.rootxt.Len())
+	}
+
+	// repeat for srcf
+	c = make(chan struct{}, 1)
+	f = func(rnum int) {
+		wg.Add(1)
+		defer func() {
+			if e := recover(); e != nil {
+				t.Errorf("goroutine number %v panicked with err: %s", rnum, e)
+			}
+		}()
+		<-c
+		_, _, err := srcf()
+		if err != nil {
+			t.Errorf("err was non-nil on root detection in goroutine number %v: %s", rnum, err)
+		}
+		wg.Done()
+	}
+
+	for k := range make([]struct{}, cnum) {
+		go f(k)
+		runtime.Gosched()
+	}
+	close(c)
+	wg.Wait()
+	if len(sm.srcs) != 2 {
+		t.Errorf("Sources map should have just two elements, but has %v", len(sm.srcs))
+	}
+}
diff --git a/source_manager.go b/source_manager.go
index 120ec24..027ac66 100644
--- a/source_manager.go
+++ b/source_manager.go
@@ -7,6 +7,7 @@ import (
 	"path/filepath"
 	"strings"
 	"sync"
+	"sync/atomic"
 
 	"github.com/Masterminds/semver"
 	"github.com/Masterminds/vcs"
@@ -77,13 +78,16 @@ type SourceMgr struct {
 	cachedir string
 	pms      map[string]*pmState
 	pmut     sync.RWMutex
+	srcs     map[string]source
+	srcmut   sync.RWMutex
 	rr       map[string]struct {
 		rr  *remoteRepo
 		err error
 	}
-	rmut sync.RWMutex
-	an   ProjectAnalyzer
-	dxt  deducerTrie
+	rmut   sync.RWMutex
+	an     ProjectAnalyzer
+	dxt    deducerTrie
+	rootxt prTrie
 }
 
 var _ SourceManager = &SourceMgr{}
@@ -137,12 +141,14 @@ func NewSourceManager(an ProjectAnalyzer, cachedir string, force bool) (*SourceM
 	return &SourceMgr{
 		cachedir: cachedir,
 		pms:      make(map[string]*pmState),
+		srcs:     make(map[string]source),
 		rr: make(map[string]struct {
 			rr  *remoteRepo
 			err error
 		}),
-		an:  an,
-		dxt: pathDeducerTrie(),
+		an:     an,
+		dxt:    pathDeducerTrie(),
+		rootxt: newProjectRootTrie(),
 	}, nil
 }
 
@@ -239,6 +245,174 @@ func (sm *SourceMgr) ExportProject(id ProjectIdentifier, v Version, to string) e
 	return pms.pm.ExportVersionTo(v, to)
 }
 
+// DeduceRootProject takes an import path and deduces the
+//
+// Note that some import paths may require network activity to correctly
+// determine the root of the path, such as, but not limited to, vanity import
+// paths. (A special exception is written for gopkg.in to minimize network
+// activity, as its behavior is well-structured)
+func (sm *SourceMgr) DeduceProjectRoot(ip string) (ProjectRoot, error) {
+	if prefix, root, has := sm.rootxt.LongestPrefix(ip); has {
+		// The non-matching tail of the import path could still be malformed.
+		// Validate just that part, if it exists
+		if prefix != ip {
+			if !pathvld.MatchString(strings.TrimPrefix(ip, prefix)) {
+				return "", fmt.Errorf("%q is not a valid import path", ip)
+			}
+			// There was one, and it validated fine - add it so we don't have to
+			// revalidate it later
+			sm.rootxt.Insert(ip, root)
+		}
+		return root, nil
+	}
+
+	rootf, _, err := sm.deducePathAndProcess(ip)
+	if err != nil {
+		return "", err
+	}
+
+	r, err := rootf()
+	return ProjectRoot(r), err
+}
+
+func (sm *SourceMgr) getSourceFor(id ProjectIdentifier) (source, error) {
+	nn := id.netName()
+
+	sm.srcmut.RLock()
+	src, has := sm.srcs[nn]
+	sm.srcmut.RUnlock()
+	if has {
+		return src, nil
+	}
+
+	_, srcf, err := sm.deducePathAndProcess(nn)
+	if err != nil {
+		return nil, err
+	}
+
+	// we don't care about the ident here
+	src, _, err = srcf()
+	return src, err
+}
+
+func (sm *SourceMgr) deducePathAndProcess(path string) (stringFuture, sourceFuture, error) {
+	df, err := sm.deduceFromPath(path)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	var rstart, sstart int32
+	rc := make(chan struct{}, 1)
+	sc := make(chan struct{}, 1)
+
+	// Rewrap in a deferred future, so the caller can decide when to trigger it
+	rootf := func() (pr string, err error) {
+		// CAS because a bad interleaving here would panic on double-closing rc
+		if atomic.CompareAndSwapInt32(&rstart, 0, 1) {
+			go func() {
+				defer close(rc)
+				pr, err = df.root()
+				if err != nil {
+					// Don't cache errs. This doesn't really hurt the solver, and is
+					// beneficial for other use cases because it means we don't have to
+					// expose any kind of controls for clearing caches.
+					return
+				}
+
+				tpr := ProjectRoot(pr)
+				sm.rootxt.Insert(pr, tpr)
+				// It's not harmful if the netname was a URL rather than an
+				// import path
+				if pr != path {
+					// Insert the result into the rootxt twice - once at the
+					// root itself, so as to catch siblings/relatives, and again
+					// at the exact provided import path (assuming they were
+					// different), so that on subsequent calls, exact matches
+					// can skip the regex above.
+					sm.rootxt.Insert(path, tpr)
+				}
+			}()
+		}
+
+		<-rc
+		return pr, err
+	}
+
+	// Now, handle the source
+	fut := df.psf(sm.cachedir, sm.an)
+
+	// Rewrap in a deferred future, so the caller can decide when to trigger it
+	srcf := func() (src source, ident string, err error) {
+		// CAS because a bad interleaving here would panic on double-closing sc
+		if atomic.CompareAndSwapInt32(&sstart, 0, 1) {
+			go func() {
+				defer close(sc)
+				src, ident, err = fut()
+				if err != nil {
+					// Don't cache errs. This doesn't really hurt the solver, and is
+					// beneficial for other use cases because it means we don't have
+					// to expose any kind of controls for clearing caches.
+					return
+				}
+
+				sm.srcmut.Lock()
+				defer sm.srcmut.Unlock()
+
+				// Check to make sure a source hasn't shown up in the meantime, or that
+				// there wasn't already one at the ident.
+				var hasi, hasp bool
+				var srci, srcp source
+				if ident != "" {
+					srci, hasi = sm.srcs[ident]
+				}
+				srcp, hasp = sm.srcs[path]
+
+				// if neither the ident nor the input path have an entry for this src,
+				// we're in the simple case - write them both in and we're done
+				if !hasi && !hasp {
+					sm.srcs[path] = src
+					if ident != path && ident != "" {
+						sm.srcs[ident] = src
+					}
+					return
+				}
+
+				// Now, the xors.
+				//
+				// If already present for ident but not for path, copy ident's src
+				// to path. This covers cases like a gopkg.in path referring back
+				// onto a github repository, where something else already explicitly
+				// looked up that same gh repo.
+				if hasi && !hasp {
+					sm.srcs[path] = srci
+					src = srci
+				}
+				// If already present for path but not for ident, do NOT copy path's
+				// src to ident, but use the returned one instead. Really, this case
+				// shouldn't occur at all...? But the crucial thing is that the
+				// path-based one has already discovered what actual ident of source
+				// they want to use, and changing that arbitrarily would have
+				// undefined effects.
+				if hasp && !hasi && ident != "" {
+					sm.srcs[ident] = src
+				}
+
+				// If both are present, then assume we're good, and use the path one
+				if hasp && hasi {
+					// TODO(sdboyer) compare these (somehow? reflect? pointer?) and if they're not the
+					// same object, panic
+					src = srcp
+				}
+			}()
+		}
+
+		<-sc
+		return
+	}
+
+	return rootf, srcf, nil
+}
+
 // getProjectManager gets the project manager for the given ProjectIdentifier.
 //
 // If no such manager yet exists, it attempts to create one.

From 0f0690fb44fea0db88d4e08a12ca50135a29a790 Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Mon, 15 Aug 2016 01:11:35 -0400
Subject: [PATCH 58/71] Convert old version list test to source-getting

---
 manager_test.go   | 126 ++++++++++++++++++----------------------------
 maybe_source.go   |   2 +-
 remote.go         |  21 +++++---
 source_manager.go |   6 +--
 4 files changed, 65 insertions(+), 90 deletions(-)

diff --git a/manager_test.go b/manager_test.go
index ba946e4..0081964 100644
--- a/manager_test.go
+++ b/manager_test.go
@@ -193,14 +193,14 @@ func TestProjectManagerInit(t *testing.T) {
 		//t.Error("Metadata cache json file does not exist in expected location")
 	}
 
-	// Ensure project existence values are what we expect
+	// Ensure source existence values are what we expect
 	var exists bool
 	exists, err = sm.SourceExists(id)
 	if err != nil {
 		t.Errorf("Error on checking SourceExists: %s", err)
 	}
 	if !exists {
-		t.Error("Repo should exist after non-erroring call to ListVersions")
+		t.Error("Source should exist after non-erroring call to ListVersions")
 	}
 
 	// Now reach inside the black box
@@ -216,99 +216,69 @@ func TestProjectManagerInit(t *testing.T) {
 	}
 }
 
-func TestRepoVersionFetching(t *testing.T) {
-	// This test is quite slow, skip it on -short
+func TestGetSources(t *testing.T) {
+	// This test is a tad slow, skip it on -short
 	if testing.Short() {
-		t.Skip("Skipping repo version fetching test in short mode")
+		t.Skip("Skipping source setup test in short mode")
 	}
 
 	sm, clean := mkNaiveSM(t)
 
-	upstreams := []ProjectIdentifier{
+	pil := []ProjectIdentifier{
 		mkPI("github.com/Masterminds/VCSTestRepo"),
 		mkPI("bitbucket.org/mattfarina/testhgrepo"),
 		mkPI("launchpad.net/govcstestbzrrepo"),
 	}
 
-	pms := make([]*projectManager, len(upstreams))
-	for k, u := range upstreams {
-		pmi, err := sm.getProjectManager(u)
-		if err != nil {
-			clean()
-			t.Errorf("Unexpected error on ProjectManager creation: %s", err)
-			t.FailNow()
-		}
-		pms[k] = pmi.pm
-	}
-
-	defer clean()
+	wg := &sync.WaitGroup{}
+	wg.Add(3)
+	for _, pi := range pil {
+		go func(lpi ProjectIdentifier) {
+			nn := lpi.netName()
+			src, err := sm.getSourceFor(lpi)
+			if err != nil {
+				t.Errorf("(src %q) unexpected error setting up source: %s", nn, err)
+				return
+			}
 
-	// test git first
-	vlist, exbits, err := pms[0].crepo.getCurrentVersionPairs()
-	if err != nil {
-		t.Errorf("Unexpected error getting version pairs from git repo: %s", err)
-	}
-	if exbits != existsUpstream {
-		t.Errorf("git pair fetch should only set upstream existence bits, but got %v", exbits)
-	}
-	if len(vlist) != 3 {
-		t.Errorf("git test repo should've produced three versions, got %v", len(vlist))
-	} else {
-		v := NewBranch("master").Is(Revision("30605f6ac35fcb075ad0bfa9296f90a7d891523e"))
-		if vlist[0] != v {
-			t.Errorf("git pair fetch reported incorrect first version, got %s", vlist[0])
-		}
+			// Re-get the same, make sure they are the same
+			src2, err := sm.getSourceFor(lpi)
+			if err != nil {
+				t.Errorf("(src %q) unexpected error re-getting source: %s", nn, err)
+			} else if src != src2 {
+				t.Errorf("(src %q) first and second sources are not eq", nn)
+			}
 
-		v = NewBranch("test").Is(Revision("30605f6ac35fcb075ad0bfa9296f90a7d891523e"))
-		if vlist[1] != v {
-			t.Errorf("git pair fetch reported incorrect second version, got %s", vlist[1])
-		}
+			// All of them _should_ select https, so this should work
+			lpi.NetworkName = "https://" + lpi.NetworkName
+			src3, err := sm.getSourceFor(lpi)
+			if err != nil {
+				t.Errorf("(src %q) unexpected error getting explicit https source: %s", nn, err)
+			} else if src != src3 {
+				t.Errorf("(src %q) explicit https source should reuse autodetected https source", nn)
+			}
 
-		v = NewVersion("1.0.0").Is(Revision("30605f6ac35fcb075ad0bfa9296f90a7d891523e"))
-		if vlist[2] != v {
-			t.Errorf("git pair fetch reported incorrect third version, got %s", vlist[2])
-		}
-	}
+			// Now put in http, and they should differ
+			lpi.NetworkName = "http://" + string(lpi.ProjectRoot)
+			src4, err := sm.getSourceFor(lpi)
+			if err != nil {
+				t.Errorf("(src %q) unexpected error getting explicit http source: %s", nn, err)
+			} else if src == src4 {
+				t.Errorf("(src %q) explicit http source should create a new src", nn)
+			}
 
-	// now hg
-	vlist, exbits, err = pms[1].crepo.getCurrentVersionPairs()
-	if err != nil {
-		t.Errorf("Unexpected error getting version pairs from hg repo: %s", err)
+			wg.Done()
+		}(pi)
 	}
-	if exbits != existsUpstream|existsInCache {
-		t.Errorf("hg pair fetch should set upstream and cache existence bits, but got %v", exbits)
-	}
-	if len(vlist) != 2 {
-		t.Errorf("hg test repo should've produced two versions, got %v", len(vlist))
-	} else {
-		v := NewVersion("1.0.0").Is(Revision("d680e82228d206935ab2eaa88612587abe68db07"))
-		if vlist[0] != v {
-			t.Errorf("hg pair fetch reported incorrect first version, got %s", vlist[0])
-		}
 
-		v = NewBranch("test").Is(Revision("6c44ee3fe5d87763616c19bf7dbcadb24ff5a5ce"))
-		if vlist[1] != v {
-			t.Errorf("hg pair fetch reported incorrect second version, got %s", vlist[1])
-		}
-	}
+	wg.Wait()
 
-	// bzr last
-	vlist, exbits, err = pms[2].crepo.getCurrentVersionPairs()
-	if err != nil {
-		t.Errorf("Unexpected error getting version pairs from bzr repo: %s", err)
-	}
-	if exbits != existsUpstream|existsInCache {
-		t.Errorf("bzr pair fetch should set upstream and cache existence bits, but got %v", exbits)
-	}
-	if len(vlist) != 1 {
-		t.Errorf("bzr test repo should've produced one version, got %v", len(vlist))
-	} else {
-		v := NewVersion("1.0.0").Is(Revision("matt@mattfarina.com-20150731135137-pbphasfppmygpl68"))
-		if vlist[0] != v {
-			t.Errorf("bzr pair fetch reported incorrect first version, got %s", vlist[0])
-		}
+	// nine entries (of which three are dupes): for each vcs, raw import path,
+	// the https url, and the http url
+	if len(sm.srcs) != 9 {
+		t.Errorf("Should have nine discrete entries in the srcs map, got %v", len(sm.srcs))
 	}
-	// no svn for now, because...svn
+	clean()
 }
 
 // Regression test for #32
@@ -363,7 +333,7 @@ func TestDeduceProjectRoot(t *testing.T) {
 		t.Errorf("Wrong project root was deduced;\n\t(GOT) %s\n\t(WNT) %s", pr, in)
 	}
 	if sm.rootxt.Len() != 1 {
-		t.Errorf("Root path trie should have one element after performing the same deduction twice; has %v", sm.rootxt.Len())
+		t.Errorf("Root path trie should still have one element after performing the same deduction twice; has %v", sm.rootxt.Len())
 	}
 
 	// Now do a subpath
diff --git a/maybe_source.go b/maybe_source.go
index 8d4cf72..5565ba4 100644
--- a/maybe_source.go
+++ b/maybe_source.go
@@ -118,7 +118,7 @@ type maybeHgSource struct {
 func (m maybeHgSource) try(cachedir string, an ProjectAnalyzer) (source, string, error) {
 	ustr := m.url.String()
 	path := filepath.Join(cachedir, "sources", sanitizer.Replace(ustr))
-	r, err := vcs.NewBzrRepo(ustr, path)
+	r, err := vcs.NewHgRepo(ustr, path)
 	if err != nil {
 		return nil, "", err
 	}
diff --git a/remote.go b/remote.go
index 17fcf3a..14ff9e8 100644
--- a/remote.go
+++ b/remote.go
@@ -202,25 +202,30 @@ func (m bitbucketDeducer) deduceSource(path string, u *url.URL) (maybeSource, er
 	}
 
 	mb := make(maybeSources, 0)
-	if !ishg {
-		for _, scheme := range gitSchemes {
+	// git is probably more common, even on bitbucket. however, bitbucket
+	// appears to fail _extremely_ slowly on git pings (ls-remote) when the
+	// underlying repository is actually an hg repository, so it's better
+	// to try hg first.
+	// TODO(sdboyer) resolve the ambiguity by querying bitbucket's REST API.
+	if !isgit {
+		for _, scheme := range hgSchemes {
 			u2 := *u
 			if scheme == "ssh" {
-				u2.User = url.User("git")
+				u2.User = url.User("hg")
 			}
 			u2.Scheme = scheme
-			mb = append(mb, maybeGitSource{url: &u2})
+			mb = append(mb, maybeHgSource{url: &u2})
 		}
 	}
 
-	if !isgit {
-		for _, scheme := range hgSchemes {
+	if !ishg {
+		for _, scheme := range gitSchemes {
 			u2 := *u
 			if scheme == "ssh" {
-				u2.User = url.User("hg")
+				u2.User = url.User("git")
 			}
 			u2.Scheme = scheme
-			mb = append(mb, maybeHgSource{url: &u2})
+			mb = append(mb, maybeGitSource{url: &u2})
 		}
 	}
 
diff --git a/source_manager.go b/source_manager.go
index 027ac66..c6d8e53 100644
--- a/source_manager.go
+++ b/source_manager.go
@@ -290,7 +290,8 @@ func (sm *SourceMgr) getSourceFor(id ProjectIdentifier) (source, error) {
 		return nil, err
 	}
 
-	// we don't care about the ident here
+	// we don't care about the ident here, and the future produced by
+	// deducePathAndProcess will dedupe with what's in the sm.srcs map
 	src, _, err = srcf()
 	return src, err
 }
@@ -302,8 +303,7 @@ func (sm *SourceMgr) deducePathAndProcess(path string) (stringFuture, sourceFutu
 	}
 
 	var rstart, sstart int32
-	rc := make(chan struct{}, 1)
-	sc := make(chan struct{}, 1)
+	rc, sc := make(chan struct{}, 1), make(chan struct{}, 1)
 
 	// Rewrap in a deferred future, so the caller can decide when to trigger it
 	rootf := func() (pr string, err error) {

From 319a07ee8179835c5a89a926a1580ee96982d1d9 Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Mon, 15 Aug 2016 01:16:57 -0400
Subject: [PATCH 59/71] Rename remote*.go to deduce*.go

---
 remote.go => deduce.go           | 0
 remote_test.go => deduce_test.go | 0
 2 files changed, 0 insertions(+), 0 deletions(-)
 rename remote.go => deduce.go (100%)
 rename remote_test.go => deduce_test.go (100%)

diff --git a/remote.go b/deduce.go
similarity index 100%
rename from remote.go
rename to deduce.go
diff --git a/remote_test.go b/deduce_test.go
similarity index 100%
rename from remote_test.go
rename to deduce_test.go

From bc1bae3b1287ede1fbfe24a270063ce91d2bd777 Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Mon, 15 Aug 2016 01:19:45 -0400
Subject: [PATCH 60/71] Fix deduction tests wrt bitbucket reordering

Forgot to include this on the earlier commit that privileged hg over
git.
---
 deduce.go      |  5 ++---
 deduce_test.go | 14 +++++++-------
 2 files changed, 9 insertions(+), 10 deletions(-)

diff --git a/deduce.go b/deduce.go
index 14ff9e8..02afb6e 100644
--- a/deduce.go
+++ b/deduce.go
@@ -175,6 +175,7 @@ func (m bitbucketDeducer) deduceSource(path string, u *url.URL) (maybeSource, er
 	isgit := strings.HasSuffix(u.Path, ".git") || (u.User != nil && u.User.Username() == "git")
 	ishg := strings.HasSuffix(u.Path, ".hg") || (u.User != nil && u.User.Username() == "hg")
 
+	// TODO(sdboyer) resolve scm ambiguity if needed by querying bitbucket's REST API
 	if u.Scheme != "" {
 		validgit, validhg := validateVCSScheme(u.Scheme, "git"), validateVCSScheme(u.Scheme, "hg")
 		if isgit {
@@ -195,9 +196,8 @@ func (m bitbucketDeducer) deduceSource(path string, u *url.URL) (maybeSource, er
 
 		// No other choice, make an option for both git and hg
 		return maybeSources{
-			// Git first, because it's a) faster and b) git
-			maybeGitSource{url: u},
 			maybeHgSource{url: u},
+			maybeGitSource{url: u},
 		}, nil
 	}
 
@@ -206,7 +206,6 @@ func (m bitbucketDeducer) deduceSource(path string, u *url.URL) (maybeSource, er
 	// appears to fail _extremely_ slowly on git pings (ls-remote) when the
 	// underlying repository is actually an hg repository, so it's better
 	// to try hg first.
-	// TODO(sdboyer) resolve the ambiguity by querying bitbucket's REST API.
 	if !isgit {
 		for _, scheme := range hgSchemes {
 			u2 := *u
diff --git a/deduce_test.go b/deduce_test.go
index bb18a76..b7647f9 100644
--- a/deduce_test.go
+++ b/deduce_test.go
@@ -212,34 +212,34 @@ var pathDeductionFixtures = map[string][]pathDeductionFixture{
 			in:   "bitbucket.org/sdboyer/reporoot",
 			root: "bitbucket.org/sdboyer/reporoot",
 			mb: maybeSources{
+				maybeHgSource{url: mkurl("https://bitbucket.org/sdboyer/reporoot")},
+				maybeHgSource{url: mkurl("ssh://hg@bitbucket.org/sdboyer/reporoot")},
+				maybeHgSource{url: mkurl("http://bitbucket.org/sdboyer/reporoot")},
 				maybeGitSource{url: mkurl("https://bitbucket.org/sdboyer/reporoot")},
 				maybeGitSource{url: mkurl("ssh://git@bitbucket.org/sdboyer/reporoot")},
 				maybeGitSource{url: mkurl("git://bitbucket.org/sdboyer/reporoot")},
 				maybeGitSource{url: mkurl("http://bitbucket.org/sdboyer/reporoot")},
-				maybeHgSource{url: mkurl("https://bitbucket.org/sdboyer/reporoot")},
-				maybeHgSource{url: mkurl("ssh://hg@bitbucket.org/sdboyer/reporoot")},
-				maybeHgSource{url: mkurl("http://bitbucket.org/sdboyer/reporoot")},
 			},
 		},
 		{
 			in:   "bitbucket.org/sdboyer/reporoot/foo/bar",
 			root: "bitbucket.org/sdboyer/reporoot",
 			mb: maybeSources{
+				maybeHgSource{url: mkurl("https://bitbucket.org/sdboyer/reporoot")},
+				maybeHgSource{url: mkurl("ssh://hg@bitbucket.org/sdboyer/reporoot")},
+				maybeHgSource{url: mkurl("http://bitbucket.org/sdboyer/reporoot")},
 				maybeGitSource{url: mkurl("https://bitbucket.org/sdboyer/reporoot")},
 				maybeGitSource{url: mkurl("ssh://git@bitbucket.org/sdboyer/reporoot")},
 				maybeGitSource{url: mkurl("git://bitbucket.org/sdboyer/reporoot")},
 				maybeGitSource{url: mkurl("http://bitbucket.org/sdboyer/reporoot")},
-				maybeHgSource{url: mkurl("https://bitbucket.org/sdboyer/reporoot")},
-				maybeHgSource{url: mkurl("ssh://hg@bitbucket.org/sdboyer/reporoot")},
-				maybeHgSource{url: mkurl("http://bitbucket.org/sdboyer/reporoot")},
 			},
 		},
 		{
 			in:   "https://bitbucket.org/sdboyer/reporoot/foo/bar",
 			root: "bitbucket.org/sdboyer/reporoot",
 			mb: maybeSources{
-				maybeGitSource{url: mkurl("https://bitbucket.org/sdboyer/reporoot")},
 				maybeHgSource{url: mkurl("https://bitbucket.org/sdboyer/reporoot")},
+				maybeGitSource{url: mkurl("https://bitbucket.org/sdboyer/reporoot")},
 			},
 		},
 		// Less standard behaviors possible due to the hg/git ambiguity

From 1006147e1f4a973a9ef1e41cb2639c127e6588fc Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Mon, 15 Aug 2016 01:48:11 -0400
Subject: [PATCH 61/71] Remove project_manager.go entirely; tests passing

This is a major milestone. The old way of handling source information,
the projectManager, is totally gone, replaced by the new source-based
system. There's still work to be done there, but with tests passing,
we're more or less at parity with what we had before.
---
 manager_test.go    |  21 ++--
 project_manager.go | 305 ---------------------------------------------
 source.go          |  14 +++
 source_manager.go  | 210 +++----------------------------
 vcs_source.go      |   6 +
 5 files changed, 41 insertions(+), 515 deletions(-)
 delete mode 100644 project_manager.go

diff --git a/manager_test.go b/manager_test.go
index 0081964..ebe9a87 100644
--- a/manager_test.go
+++ b/manager_test.go
@@ -5,6 +5,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path"
+	"path/filepath"
 	"runtime"
 	"sort"
 	"sync"
@@ -181,13 +182,17 @@ func TestProjectManagerInit(t *testing.T) {
 		}
 	}
 
+	// use ListPackages to ensure the repo is actually on disk
+	// TODO(sdboyer) ugh, maybe we do need an explicit prefetch method
+	smc.ListPackages(id, NewVersion("1.0.0"))
+
 	// Ensure that the appropriate cache dirs and files exist
-	_, err = os.Stat(path.Join(cpath, "sources", "https---github.com-Masterminds-VCSTestRepo", ".git"))
+	_, err = os.Stat(filepath.Join(cpath, "sources", "https---github.com-Masterminds-VCSTestRepo", ".git"))
 	if err != nil {
 		t.Error("Cache repo does not exist in expected location")
 	}
 
-	_, err = os.Stat(path.Join(cpath, "metadata", "github.com", "Masterminds", "VCSTestRepo", "cache.json"))
+	_, err = os.Stat(filepath.Join(cpath, "metadata", "github.com", "Masterminds", "VCSTestRepo", "cache.json"))
 	if err != nil {
 		// TODO(sdboyer) disabled until we get caching working
 		//t.Error("Metadata cache json file does not exist in expected location")
@@ -202,18 +207,6 @@ func TestProjectManagerInit(t *testing.T) {
 	if !exists {
 		t.Error("Source should exist after non-erroring call to ListVersions")
 	}
-
-	// Now reach inside the black box
-	pms, err := sm.getProjectManager(id)
-	if err != nil {
-		t.Errorf("Error on grabbing project manager obj: %s", err)
-		t.FailNow()
-	}
-
-	// Check upstream existence flag
-	if !pms.pm.CheckExistence(existsUpstream) {
-		t.Errorf("ExistsUpstream flag not being correctly set the project")
-	}
 }
 
 func TestGetSources(t *testing.T) {
diff --git a/project_manager.go b/project_manager.go
deleted file mode 100644
index 992f6f3..0000000
--- a/project_manager.go
+++ /dev/null
@@ -1,305 +0,0 @@
-package gps
-
-import (
-	"fmt"
-	"go/build"
-)
-
-type projectManager struct {
-	// The upstream URL from which the project is sourced.
-	n string
-
-	// build.Context to use in any analysis, and to pass to the analyzer
-	ctx build.Context
-
-	// Object for the cache repository
-	crepo *repo
-
-	// Indicates the extent to which we have searched for, and verified, the
-	// existence of the project/repo.
-	ex existence
-
-	// Analyzer, injected by way of the SourceManager and originally from the
-	// sm's creator
-	an ProjectAnalyzer
-
-	// Whether the cache has the latest info on versions
-	cvsync bool
-
-	// The project metadata cache. This is persisted to disk, for reuse across
-	// solver runs.
-	// TODO(sdboyer) protect with mutex
-	dc *sourceMetaCache
-}
-
-type existence struct {
-	// The existence levels for which a search/check has been performed
-	s sourceExistence
-
-	// The existence levels verified to be present through searching
-	f sourceExistence
-}
-
-// projectInfo holds manifest and lock
-type projectInfo struct {
-	Manifest
-	Lock
-}
-
-func (pm *projectManager) GetManifestAndLock(r ProjectRoot, v Version) (Manifest, Lock, error) {
-	if err := pm.ensureCacheExistence(); err != nil {
-		return nil, nil, err
-	}
-
-	rev, err := pm.toRevOrErr(v)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	// Return the info from the cache, if we already have it
-	if pi, exists := pm.dc.infos[rev]; exists {
-		return pi.Manifest, pi.Lock, nil
-	}
-
-	pm.crepo.mut.Lock()
-	if !pm.crepo.synced {
-		err = pm.crepo.r.Update()
-		if err != nil {
-			return nil, nil, fmt.Errorf("could not fetch latest updates into repository")
-		}
-		pm.crepo.synced = true
-	}
-
-	// Always prefer a rev, if it's available
-	if pv, ok := v.(PairedVersion); ok {
-		err = pm.crepo.r.UpdateVersion(pv.Underlying().String())
-	} else {
-		err = pm.crepo.r.UpdateVersion(v.String())
-	}
-	pm.crepo.mut.Unlock()
-	if err != nil {
-		// TODO(sdboyer) More-er proper-er error
-		panic(fmt.Sprintf("canary - why is checkout/whatever failing: %s %s %s", pm.n, v.String(), err))
-	}
-
-	pm.crepo.mut.RLock()
-	m, l, err := pm.an.DeriveManifestAndLock(pm.crepo.rpath, r)
-	// TODO(sdboyer) cache results
-	pm.crepo.mut.RUnlock()
-
-	if err == nil {
-		if l != nil {
-			l = prepLock(l)
-		}
-
-		// If m is nil, prepManifest will provide an empty one.
-		pi := projectInfo{
-			Manifest: prepManifest(m),
-			Lock:     l,
-		}
-
-		// TODO(sdboyer) this just clobbers all over and ignores the paired/unpaired
-		// distinction; serious fix is needed
-		pm.dc.infos[rev] = pi
-
-		return pi.Manifest, pi.Lock, nil
-	}
-
-	return nil, nil, err
-}
-
-func (pm *projectManager) ListPackages(pr ProjectRoot, v Version) (ptree PackageTree, err error) {
-	if err = pm.ensureCacheExistence(); err != nil {
-		return
-	}
-
-	var r Revision
-	if r, err = pm.toRevOrErr(v); err != nil {
-		return
-	}
-
-	// Return the ptree from the cache, if we already have it
-	var exists bool
-	if ptree, exists = pm.dc.ptrees[r]; exists {
-		return
-	}
-
-	// Not in the cache; check out the version and do the analysis
-	pm.crepo.mut.Lock()
-	// Check out the desired version for analysis
-	if r != "" {
-		// Always prefer a rev, if it's available
-		err = pm.crepo.r.UpdateVersion(string(r))
-	} else {
-		// If we don't have a rev, ensure the repo is up to date, otherwise we
-		// could have a desync issue
-		if !pm.crepo.synced {
-			err = pm.crepo.r.Update()
-			if err != nil {
-				return PackageTree{}, fmt.Errorf("could not fetch latest updates into repository: %s", err)
-			}
-			pm.crepo.synced = true
-		}
-		err = pm.crepo.r.UpdateVersion(v.String())
-	}
-
-	ptree, err = listPackages(pm.crepo.rpath, string(pr))
-	pm.crepo.mut.Unlock()
-
-	// TODO(sdboyer) cache errs?
-	if err != nil {
-		pm.dc.ptrees[r] = ptree
-	}
-
-	return
-}
-
-func (pm *projectManager) ensureCacheExistence() error {
-	// Technically, methods could could attempt to return straight from the
-	// metadata cache even if the repo cache doesn't exist on disk. But that
-	// would allow weird state inconsistencies (cache exists, but no repo...how
-	// does that even happen?) that it'd be better to just not allow so that we
-	// don't have to think about it elsewhere
-	if !pm.CheckExistence(existsInCache) {
-		if pm.CheckExistence(existsUpstream) {
-			pm.crepo.mut.Lock()
-			err := pm.crepo.r.Get()
-			pm.crepo.mut.Unlock()
-
-			if err != nil {
-				return fmt.Errorf("failed to create repository cache for %s", pm.n)
-			}
-			pm.ex.s |= existsInCache
-			pm.ex.f |= existsInCache
-		} else {
-			return fmt.Errorf("project %s does not exist upstream", pm.n)
-		}
-	}
-
-	return nil
-}
-
-func (pm *projectManager) ListVersions() (vlist []Version, err error) {
-	if !pm.cvsync {
-		// This check only guarantees that the upstream exists, not the cache
-		pm.ex.s |= existsUpstream
-		vpairs, exbits, err := pm.crepo.getCurrentVersionPairs()
-		// But it *may* also check the local existence
-		pm.ex.s |= exbits
-		pm.ex.f |= exbits
-
-		if err != nil {
-			// TODO(sdboyer) More-er proper-er error
-			return nil, err
-		}
-
-		vlist = make([]Version, len(vpairs))
-		// mark our cache as synced if we got ExistsUpstream back
-		if exbits&existsUpstream == existsUpstream {
-			pm.cvsync = true
-		}
-
-		// Process the version data into the cache
-		// TODO(sdboyer) detect out-of-sync data as we do this?
-		for k, v := range vpairs {
-			u, r := v.Unpair(), v.Underlying()
-			pm.dc.vMap[u] = r
-			pm.dc.rMap[r] = append(pm.dc.rMap[r], u)
-			vlist[k] = v
-		}
-	} else {
-		vlist = make([]Version, len(pm.dc.vMap))
-		k := 0
-		for v, r := range pm.dc.vMap {
-			vlist[k] = v.Is(r)
-			k++
-		}
-	}
-
-	return
-}
-
-// toRevOrErr makes all efforts to convert a Version into a rev, including
-// updating the cache repo (if needed). It does not guarantee that the returned
-// Revision actually exists in the repository (as one of the cheaper methods may
-// have had bad data).
-func (pm *projectManager) toRevOrErr(v Version) (r Revision, err error) {
-	r = pm.dc.toRevision(v)
-	if r == "" {
-		// Rev can be empty if:
-		//  - The cache is unsynced
-		//  - A version was passed that used to exist, but no longer does
-		//  - A garbage version was passed. (Functionally indistinguishable from
-		//  the previous)
-		if !pm.cvsync {
-			_, err = pm.ListVersions()
-			if err != nil {
-				return
-			}
-		}
-
-		r = pm.dc.toRevision(v)
-		// If we still don't have a rev, then the version's no good
-		if r == "" {
-			err = fmt.Errorf("version %s does not exist in source %s", v, pm.crepo.r.Remote())
-		}
-	}
-
-	return
-}
-
-func (pm *projectManager) RevisionPresentIn(pr ProjectRoot, r Revision) (bool, error) {
-	// First and fastest path is to check the data cache to see if the rev is
-	// present. This could give us false positives, but the cases where that can
-	// occur would require a type of cache staleness that seems *exceedingly*
-	// unlikely to occur.
-	if _, has := pm.dc.infos[r]; has {
-		return true, nil
-	} else if _, has := pm.dc.rMap[r]; has {
-		return true, nil
-	}
-
-	// For now at least, just run GetInfoAt(); it basically accomplishes the
-	// same thing.
-	if _, _, err := pm.GetManifestAndLock(pr, r); err != nil {
-		return false, err
-	}
-	return true, nil
-}
-
-// CheckExistence provides a direct method for querying existence levels of the
-// project. It will only perform actual searching (local fs or over the network)
-// if no previous attempt at that search has been made.
-//
-// Note that this may perform read-ish operations on the cache repo, and it
-// takes a lock accordingly. Deadlock may result from calling it during a
-// segment where the cache repo mutex is already write-locked.
-func (pm *projectManager) CheckExistence(ex sourceExistence) bool {
-	if pm.ex.s&ex != ex {
-		if ex&existsInVendorRoot != 0 && pm.ex.s&existsInVendorRoot == 0 {
-			panic("should now be implemented in bridge")
-		}
-		if ex&existsInCache != 0 && pm.ex.s&existsInCache == 0 {
-			pm.crepo.mut.RLock()
-			pm.ex.s |= existsInCache
-			if pm.crepo.r.CheckLocal() {
-				pm.ex.f |= existsInCache
-			}
-			pm.crepo.mut.RUnlock()
-		}
-		if ex&existsUpstream != 0 && pm.ex.s&existsUpstream == 0 {
-			pm.crepo.mut.RLock()
-			pm.ex.s |= existsUpstream
-			if pm.crepo.r.Ping() {
-				pm.ex.f |= existsUpstream
-			}
-			pm.crepo.mut.RUnlock()
-		}
-	}
-
-	return ex&pm.ex.f == ex
-}
-
-func (pm *projectManager) ExportVersionTo(v Version, to string) error {
-	return pm.crepo.exportVersionTo(v, to)
-}
diff --git a/source.go b/source.go
index 1d431bc..515ce94 100644
--- a/source.go
+++ b/source.go
@@ -20,6 +20,20 @@ type sourceMetaCache struct {
 	// TODO(sdboyer) mutexes. actually probably just one, b/c complexity
 }
 
+// projectInfo holds manifest and lock
+type projectInfo struct {
+	Manifest
+	Lock
+}
+
+type existence struct {
+	// The existence levels for which a search/check has been performed
+	s sourceExistence
+
+	// The existence levels verified to be present through searching
+	f sourceExistence
+}
+
 func newMetaCache() *sourceMetaCache {
 	return &sourceMetaCache{
 		infos:  make(map[Revision]projectInfo),
diff --git a/source_manager.go b/source_manager.go
index c6d8e53..89b5dfa 100644
--- a/source_manager.go
+++ b/source_manager.go
@@ -1,7 +1,6 @@
 package gps
 
 import (
-	"encoding/json"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -10,7 +9,6 @@ import (
 	"sync/atomic"
 
 	"github.com/Masterminds/semver"
-	"github.com/Masterminds/vcs"
 )
 
 // Used to compute a friendly filepath from a URL-shaped input
@@ -76,8 +74,6 @@ type ProjectAnalyzer interface {
 // tools; control via dependency injection is intended to be sufficient.
 type SourceMgr struct {
 	cachedir string
-	pms      map[string]*pmState
-	pmut     sync.RWMutex
 	srcs     map[string]source
 	srcmut   sync.RWMutex
 	rr       map[string]struct {
@@ -92,14 +88,6 @@ type SourceMgr struct {
 
 var _ SourceManager = &SourceMgr{}
 
-// Holds a projectManager, caches of the managed project's data, and information
-// about the freshness of those caches
-type pmState struct {
-	pm   *projectManager
-	cf   *os.File // handle for the cache file
-	vcur bool     // indicates that we've called ListVersions()
-}
-
 // NewSourceManager produces an instance of gps's built-in SourceManager. It
 // takes a cache directory (where local instances of upstream repositories are
 // stored), a vendor directory for the project currently being worked on, and a
@@ -140,7 +128,6 @@ func NewSourceManager(an ProjectAnalyzer, cachedir string, force bool) (*SourceM
 
 	return &SourceMgr{
 		cachedir: cachedir,
-		pms:      make(map[string]*pmState),
 		srcs:     make(map[string]source),
 		rr: make(map[string]struct {
 			rr  *remoteRepo
@@ -170,23 +157,23 @@ func (sm *SourceMgr) AnalyzerInfo() (name string, version *semver.Version) {
 // The work of producing the manifest and lock is delegated to the injected
 // ProjectAnalyzer's DeriveManifestAndLock() method.
 func (sm *SourceMgr) GetManifestAndLock(id ProjectIdentifier, v Version) (Manifest, Lock, error) {
-	pmc, err := sm.getProjectManager(id)
+	src, err := sm.getSourceFor(id)
 	if err != nil {
 		return nil, nil, err
 	}
 
-	return pmc.pm.GetManifestAndLock(id.ProjectRoot, v)
+	return src.getManifestAndLock(id.ProjectRoot, v)
 }
 
 // ListPackages parses the tree of the Go packages at and below the ProjectRoot
 // of the given ProjectIdentifier, at the given version.
 func (sm *SourceMgr) ListPackages(id ProjectIdentifier, v Version) (PackageTree, error) {
-	pmc, err := sm.getProjectManager(id)
+	src, err := sm.getSourceFor(id)
 	if err != nil {
 		return PackageTree{}, err
 	}
 
-	return pmc.pm.ListPackages(id.ProjectRoot, v)
+	return src.listPackages(id.ProjectRoot, v)
 }
 
 // ListVersions retrieves a list of the available versions for a given
@@ -202,50 +189,51 @@ func (sm *SourceMgr) ListPackages(id ProjectIdentifier, v Version) (PackageTree,
 // is not accessible (network outage, access issues, or the resource actually
 // went away), an error will be returned.
 func (sm *SourceMgr) ListVersions(id ProjectIdentifier) ([]Version, error) {
-	pmc, err := sm.getProjectManager(id)
+	src, err := sm.getSourceFor(id)
 	if err != nil {
 		// TODO(sdboyer) More-er proper-er errors
 		return nil, err
 	}
 
-	return pmc.pm.ListVersions()
+	return src.listVersions()
 }
 
 // RevisionPresentIn indicates whether the provided Revision is present in the given
 // repository.
 func (sm *SourceMgr) RevisionPresentIn(id ProjectIdentifier, r Revision) (bool, error) {
-	pmc, err := sm.getProjectManager(id)
+	src, err := sm.getSourceFor(id)
 	if err != nil {
 		// TODO(sdboyer) More-er proper-er errors
 		return false, err
 	}
 
-	return pmc.pm.RevisionPresentIn(id.ProjectRoot, r)
+	return src.revisionPresentIn(r)
 }
 
 // SourceExists checks if a repository exists, either upstream or in the cache,
 // for the provided ProjectIdentifier.
 func (sm *SourceMgr) SourceExists(id ProjectIdentifier) (bool, error) {
-	pms, err := sm.getProjectManager(id)
+	src, err := sm.getSourceFor(id)
 	if err != nil {
 		return false, err
 	}
 
-	return pms.pm.CheckExistence(existsInCache) || pms.pm.CheckExistence(existsUpstream), nil
+	return src.checkExistence(existsInCache) || src.checkExistence(existsUpstream), nil
 }
 
 // ExportProject writes out the tree of the provided ProjectIdentifier's
 // ProjectRoot, at the provided version, to the provided directory.
 func (sm *SourceMgr) ExportProject(id ProjectIdentifier, v Version, to string) error {
-	pms, err := sm.getProjectManager(id)
+	src, err := sm.getSourceFor(id)
 	if err != nil {
 		return err
 	}
 
-	return pms.pm.ExportVersionTo(v, to)
+	return src.exportVersionTo(v, to)
 }
 
-// DeduceRootProject takes an import path and deduces the
+// DeduceRootProject takes an import path and deduces the corresponding
+// project/source root.
 //
 // Note that some import paths may require network activity to correctly
 // determine the root of the path, such as, but not limited to, vanity import
@@ -412,173 +400,3 @@ func (sm *SourceMgr) deducePathAndProcess(path string) (stringFuture, sourceFutu
 
 	return rootf, srcf, nil
 }
-
-// getProjectManager gets the project manager for the given ProjectIdentifier.
-//
-// If no such manager yet exists, it attempts to create one.
-func (sm *SourceMgr) getProjectManager(id ProjectIdentifier) (*pmState, error) {
-	// TODO(sdboyer) finish this, it's not sufficient (?)
-	n := id.netName()
-	var rpath string
-
-	// Early check to see if we already have a pm in the cache for this net name
-	if pm, exists := sm.pms[n]; exists {
-		return pm, nil
-	}
-
-	// Figure out the remote repo path
-	rr, err := deduceRemoteRepo(n)
-	if err != nil {
-		// Not a valid import path, must reject
-		// TODO(sdboyer) wrap error
-		return nil, err
-	}
-
-	// Check the cache again, see if exact resulting clone url is in there
-	if pm, exists := sm.pms[rr.CloneURL.String()]; exists {
-		// Found it - re-register this PM at the original netname so that it
-		// doesn't need to deduce next time
-		// TODO(sdboyer) is this OK to do? are there consistency side effects?
-		sm.pms[n] = pm
-		return pm, nil
-	}
-
-	// No luck again. Now, walk through the scheme options the deducer returned,
-	// checking if each is in the cache
-	for _, scheme := range rr.Schemes {
-		rr.CloneURL.Scheme = scheme
-		// See if THIS scheme has a match, now
-		if pm, exists := sm.pms[rr.CloneURL.String()]; exists {
-			// Yep - again, re-register this PM at the original netname so that it
-			// doesn't need to deduce next time
-			// TODO(sdboyer) is this OK to do? are there consistency side effects?
-			sm.pms[n] = pm
-			return pm, nil
-		}
-	}
-
-	// Definitively no match for anything in the cache, so we know we have to
-	// create the entry. Next question is whether there's already a repo on disk
-	// for any of the schemes, or if we need to create that, too.
-
-	// TODO(sdboyer) this strategy kinda locks in the scheme to use over
-	// multiple invocations in a way that maybe isn't the best.
-	var r vcs.Repo
-	for _, scheme := range rr.Schemes {
-		rr.CloneURL.Scheme = scheme
-		url := rr.CloneURL.String()
-		sn := sanitizer.Replace(url)
-		rpath = filepath.Join(sm.cachedir, "sources", sn)
-
-		if fi, err := os.Stat(rpath); err == nil && fi.IsDir() {
-			// This one exists, so set up here
-			r, err = vcs.NewRepo(url, rpath)
-			if err != nil {
-				return nil, err
-			}
-			goto decided
-		}
-	}
-
-	// Nothing on disk, either. Iterate through the schemes, trying each and
-	// failing out only if none resulted in successfully setting up the local.
-	for _, scheme := range rr.Schemes {
-		rr.CloneURL.Scheme = scheme
-		url := rr.CloneURL.String()
-		sn := sanitizer.Replace(url)
-		rpath = filepath.Join(sm.cachedir, "sources", sn)
-
-		r, err = vcs.NewRepo(url, rpath)
-		if err != nil {
-			continue
-		}
-
-		// FIXME(sdboyer) cloning the repo here puts it on a blocking path. that
-		// aspect of state management needs to be deferred into the
-		// projectManager
-		err = r.Get()
-		if err != nil {
-			continue
-		}
-		goto decided
-	}
-
-	// If we've gotten this far, we got some brokeass input.
-	return nil, fmt.Errorf("Could not reach source repository for %s", n)
-
-decided:
-	// Ensure cache dir exists
-	metadir := filepath.Join(sm.cachedir, "metadata", string(n))
-	err = os.MkdirAll(metadir, 0777)
-	if err != nil {
-		// TODO(sdboyer) be better
-		return nil, err
-	}
-
-	pms := &pmState{}
-	cpath := filepath.Join(metadir, "cache.json")
-	fi, err := os.Stat(cpath)
-	var dc *sourceMetaCache
-	if fi != nil {
-		pms.cf, err = os.OpenFile(cpath, os.O_RDWR, 0777)
-		if err != nil {
-			// TODO(sdboyer) be better
-			return nil, fmt.Errorf("Err on opening metadata cache file: %s", err)
-		}
-
-		err = json.NewDecoder(pms.cf).Decode(dc)
-		if err != nil {
-			// TODO(sdboyer) be better
-			return nil, fmt.Errorf("Err on JSON decoding metadata cache file: %s", err)
-		}
-	} else {
-		// TODO(sdboyer) commented this out for now, until we manage it correctly
-		//pms.cf, err = os.Create(cpath)
-		//if err != nil {
-		//// TODO(sdboyer) be better
-		//return nil, fmt.Errorf("Err on creating metadata cache file: %s", err)
-		//}
-
-		dc = newMetaCache()
-	}
-
-	pm := &projectManager{
-		n:  n,
-		an: sm.an,
-		dc: dc,
-		crepo: &repo{
-			rpath: rpath,
-			r:     r,
-		},
-	}
-
-	pms.pm = pm
-	sm.pms[n] = pms
-	return pms, nil
-}
-
-func (sm *SourceMgr) whatsInAName(nn string) (*remoteRepo, error) {
-	sm.rmut.RLock()
-	tuple, exists := sm.rr[nn]
-	sm.rmut.RUnlock()
-	if exists {
-		return tuple.rr, tuple.err
-	}
-
-	// Don't lock around the deduceRemoteRepo call, because that itself can be
-	// slow. The tradeoff is that it's possible we might duplicate work if two
-	// calls for the same id were to made simultaneously, but as those results
-	// would be the same, clobbering is OK, and better than the alternative of
-	// serializing all calls.
-	rr, err := deduceRemoteRepo(nn)
-	sm.rmut.Lock()
-	sm.rr[nn] = struct {
-		rr  *remoteRepo
-		err error
-	}{
-		rr:  rr,
-		err: err,
-	}
-	sm.rmut.Unlock()
-	return rr, err
-}
diff --git a/vcs_source.go b/vcs_source.go
index 3591c0d..dff90d0 100644
--- a/vcs_source.go
+++ b/vcs_source.go
@@ -34,6 +34,12 @@ func (s *gitSource) exportVersionTo(v Version, to string) error {
 	defer s.crepo.mut.Unlock()
 
 	r := s.crepo.r
+	if !r.CheckLocal() {
+		err := r.Get()
+		if err != nil {
+			return fmt.Errorf("failed to clone repo from %s", r.Remote())
+		}
+	}
 	// Back up original index
 	idx, bak := filepath.Join(r.LocalPath(), ".git", "index"), filepath.Join(r.LocalPath(), ".git", "origindex")
 	err := os.Rename(idx, bak)

From 264cd82aa18b4274966d495e51183bacd7fd05e0 Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Mon, 15 Aug 2016 10:58:20 -0400
Subject: [PATCH 62/71] Add DeduceProjectRoot() to SourceManager

This entails updating the bridge, which entails updating the solver. It
also entails updating the testing depspec source manager. All are in
alignment now, and tests are passing, so we're fully converted to the
new source-based system for gathering information (woohoo!).
---
 bridge.go           | 14 ++------------
 solve_basic_test.go | 10 ++++++++++
 solver.go           | 10 +++++-----
 source_manager.go   |  5 +++++
 4 files changed, 22 insertions(+), 17 deletions(-)

diff --git a/bridge.go b/bridge.go
index 0591ad5..2aae74b 100644
--- a/bridge.go
+++ b/bridge.go
@@ -21,19 +21,11 @@ type sourceBridge interface {
 	matches(id ProjectIdentifier, c Constraint, v Version) bool
 	matchesAny(id ProjectIdentifier, c1, c2 Constraint) bool
 	intersect(id ProjectIdentifier, c1, c2 Constraint) Constraint
-	deduceRemoteRepo(path string) (*remoteRepo, error)
 }
 
 // bridge is an adapter around a proper SourceManager. It provides localized
 // caching that's tailored to the requirements of a particular solve run.
 //
-// It also performs transformations between ProjectIdentifiers, which is what
-// the solver primarily deals in, and ProjectRoot, which is what the
-// SourceManager primarily deals in. This separation is helpful because it keeps
-// the complexities of deciding what a particular name "means" entirely within
-// the solver, while the SourceManager can traffic exclusively in
-// globally-unique network names.
-//
 // Finally, it provides authoritative version/constraint operations, ensuring
 // that any possible approach to a match - even those not literally encoded in
 // the inputs - is achieved.
@@ -417,10 +409,8 @@ func (b *bridge) verifyRootDir(path string) error {
 	return nil
 }
 
-// deduceRemoteRepo deduces certain network-oriented properties about an import
-// path.
-func (b *bridge) deduceRemoteRepo(path string) (*remoteRepo, error) {
-	return deduceRemoteRepo(path)
+func (b *bridge) DeduceProjectRoot(ip string) (ProjectRoot, error) {
+	return b.sm.DeduceProjectRoot(ip)
 }
 
 // versionTypeUnion represents a set of versions that are, within the scope of
diff --git a/solve_basic_test.go b/solve_basic_test.go
index b4b6fac..8e96687 100644
--- a/solve_basic_test.go
+++ b/solve_basic_test.go
@@ -1303,6 +1303,16 @@ func (sm *depspecSourceManager) ExportProject(id ProjectIdentifier, v Version, t
 	return fmt.Errorf("dummy sm doesn't support exporting")
 }
 
+func (sm *depspecSourceManager) DeduceProjectRoot(ip string) (ProjectRoot, error) {
+	for _, ds := range sm.allSpecs() {
+		n := string(ds.n)
+		if ip == n || strings.HasPrefix(ip, n+"/") {
+			return ProjectRoot(n), nil
+		}
+	}
+	return "", fmt.Errorf("Could not find %s, or any parent, in list of known fixtures", ip)
+}
+
 func (sm *depspecSourceManager) rootSpec() depspec {
 	return sm.specs[0]
 }
diff --git a/solver.go b/solver.go
index e11f69c..d82a40c 100644
--- a/solver.go
+++ b/solver.go
@@ -596,7 +596,7 @@ func (s *solver) intersectConstraintsWithImports(deps []workingConstraint, reach
 		}
 
 		// No match. Let the SourceManager try to figure out the root
-		root, err := s.b.deduceRemoteRepo(rp)
+		root, err := s.b.DeduceProjectRoot(rp)
 		if err != nil {
 			// Nothing we can do if we can't suss out a root
 			return nil, err
@@ -605,17 +605,17 @@ func (s *solver) intersectConstraintsWithImports(deps []workingConstraint, reach
 		// Make a new completeDep with an open constraint, respecting overrides
 		pd := s.ovr.override(ProjectConstraint{
 			Ident: ProjectIdentifier{
-				ProjectRoot: ProjectRoot(root.Base),
-				NetworkName: root.Base,
+				ProjectRoot: root,
+				NetworkName: string(root),
 			},
 			Constraint: Any(),
 		})
 
 		// Insert the pd into the trie so that further deps from this
 		// project get caught by the prefix search
-		xt.Insert(root.Base, pd)
+		xt.Insert(string(root), pd)
 		// And also put the complete dep into the dmap
-		dmap[ProjectRoot(root.Base)] = completeDep{
+		dmap[root] = completeDep{
 			workingConstraint: pd,
 			pl:                []string{rp},
 		}
diff --git a/source_manager.go b/source_manager.go
index 89b5dfa..c32d9ec 100644
--- a/source_manager.go
+++ b/source_manager.go
@@ -55,6 +55,10 @@ type SourceManager interface {
 	// AnalyzerInfo reports the name and version of the logic used to service
 	// GetManifestAndLock().
 	AnalyzerInfo() (name string, version *semver.Version)
+
+	// DeduceRootProject takes an import path and deduces the corresponding
+	// project/source root.
+	DeduceProjectRoot(ip string) (ProjectRoot, error)
 }
 
 // A ProjectAnalyzer is responsible for analyzing a given path for Manifest and
@@ -64,6 +68,7 @@ type ProjectAnalyzer interface {
 	// root import path importRoot, to determine the project's constraints, as
 	// indicated by a Manifest and Lock.
 	DeriveManifestAndLock(path string, importRoot ProjectRoot) (Manifest, Lock, error)
+
 	// Report the name and version of this ProjectAnalyzer.
 	Info() (name string, version *semver.Version)
 }

From 185880be514783093d619b53a21ad563bd849dc0 Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Mon, 15 Aug 2016 11:00:03 -0400
Subject: [PATCH 63/71] Remove remoteRepo and all related crufty code

---
 deduce.go           | 70 ---------------------------------------------
 solve_basic_test.go | 15 ----------
 source_manager.go   | 21 ++++----------
 3 files changed, 6 insertions(+), 100 deletions(-)

diff --git a/deduce.go b/deduce.go
index 02afb6e..3aa2d05 100644
--- a/deduce.go
+++ b/deduce.go
@@ -10,19 +10,6 @@ import (
 	"strings"
 )
 
-// A remoteRepo represents a potential remote repository resource.
-//
-// RemoteRepos are based purely on lexical analysis; successfully constructing
-// one is not a guarantee that the resource it identifies actually exists or is
-// accessible.
-type remoteRepo struct {
-	Base     string
-	RelPkg   string
-	CloneURL *url.URL
-	Schemes  []string
-	VCS      []string
-}
-
 var (
 	gitSchemes = []string{"https", "ssh", "git", "http"}
 	bzrSchemes = []string{"https", "bzr+ssh", "bzr", "http"}
@@ -737,63 +724,6 @@ func normalizeURI(p string) (u *url.URL, newpath string, err error) {
 	return
 }
 
-// deduceRemoteRepo takes a potential import path and returns a RemoteRepo
-// representing the remote location of the source of an import path. Remote
-// repositories can be bare import paths, or urls including a checkout scheme.
-func deduceRemoteRepo(path string) (rr *remoteRepo, err error) {
-	rr = &remoteRepo{}
-	if m := scpSyntaxRe.FindStringSubmatch(path); m != nil {
-		// Match SCP-like syntax and convert it to a URL.
-		// Eg, "git@github.com:user/repo" becomes
-		// "ssh://git@github.com/user/repo".
-		rr.CloneURL = &url.URL{
-			Scheme: "ssh",
-			User:   url.User(m[1]),
-			Host:   m[2],
-			Path:   "/" + m[3],
-			// TODO(sdboyer) This is what stdlib sets; grok why better
-			//RawPath: m[3],
-		}
-	} else {
-		rr.CloneURL, err = url.Parse(path)
-		if err != nil {
-			return nil, fmt.Errorf("%q is not a valid import path", path)
-		}
-	}
-
-	if rr.CloneURL.Host != "" {
-		path = rr.CloneURL.Host + "/" + strings.TrimPrefix(rr.CloneURL.Path, "/")
-	} else {
-		path = rr.CloneURL.Path
-	}
-
-	if !pathvld.MatchString(path) {
-		return nil, fmt.Errorf("%q is not a valid import path", path)
-	}
-
-	if rr.CloneURL.Scheme != "" {
-		rr.Schemes = []string{rr.CloneURL.Scheme}
-	}
-
-	// TODO(sdboyer) instead of a switch, encode base domain in radix tree and pick
-	// detector from there; if failure, then fall back on metadata work
-
-	// No luck so far. maybe it's one of them vanity imports?
-	// We have to get a little fancier for the metadata lookup - wrap a future
-	// around a future
-	var importroot, vcs string
-	// We have a real URL. Set the other values and return.
-	rr.Base = importroot
-	rr.RelPkg = strings.TrimPrefix(path[len(importroot):], "/")
-
-	rr.VCS = []string{vcs}
-	if rr.CloneURL.Scheme != "" {
-		rr.Schemes = []string{rr.CloneURL.Scheme}
-	}
-
-	return rr, nil
-}
-
 // fetchMetadata fetches the remote metadata for path.
 func fetchMetadata(path string) (rc io.ReadCloser, err error) {
 	defer func() {
diff --git a/solve_basic_test.go b/solve_basic_test.go
index 8e96687..6b6a092 100644
--- a/solve_basic_test.go
+++ b/solve_basic_test.go
@@ -1358,21 +1358,6 @@ func (b *depspecBridge) ListPackages(id ProjectIdentifier, v Version) (PackageTr
 	return b.sm.(fixSM).ListPackages(id, v)
 }
 
-// override deduceRemoteRepo on bridge to make all our pkg/project mappings work
-// as expected
-func (b *depspecBridge) deduceRemoteRepo(path string) (*remoteRepo, error) {
-	for _, ds := range b.sm.(fixSM).allSpecs() {
-		n := string(ds.n)
-		if path == n || strings.HasPrefix(path, n+"/") {
-			return &remoteRepo{
-				Base:   n,
-				RelPkg: strings.TrimPrefix(path, n+"/"),
-			}, nil
-		}
-	}
-	return nil, fmt.Errorf("Could not find %s, or any parent, in list of known fixtures", path)
-}
-
 // enforce interfaces
 var _ Manifest = depspec{}
 var _ Lock = dummyLock{}
diff --git a/source_manager.go b/source_manager.go
index c32d9ec..11ec567 100644
--- a/source_manager.go
+++ b/source_manager.go
@@ -81,14 +81,9 @@ type SourceMgr struct {
 	cachedir string
 	srcs     map[string]source
 	srcmut   sync.RWMutex
-	rr       map[string]struct {
-		rr  *remoteRepo
-		err error
-	}
-	rmut   sync.RWMutex
-	an     ProjectAnalyzer
-	dxt    deducerTrie
-	rootxt prTrie
+	an       ProjectAnalyzer
+	dxt      deducerTrie
+	rootxt   prTrie
 }
 
 var _ SourceManager = &SourceMgr{}
@@ -134,13 +129,9 @@ func NewSourceManager(an ProjectAnalyzer, cachedir string, force bool) (*SourceM
 	return &SourceMgr{
 		cachedir: cachedir,
 		srcs:     make(map[string]source),
-		rr: make(map[string]struct {
-			rr  *remoteRepo
-			err error
-		}),
-		an:     an,
-		dxt:    pathDeducerTrie(),
-		rootxt: newProjectRootTrie(),
+		an:       an,
+		dxt:      pathDeducerTrie(),
+		rootxt:   newProjectRootTrie(),
 	}, nil
 }
 

From 4e1f643312decce0b3022bd1e40c755803964d47 Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Mon, 15 Aug 2016 15:20:06 -0400
Subject: [PATCH 64/71] hg should already be available on appveyor

---
 appveyor.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/appveyor.yml b/appveyor.yml
index 8f25b03..8c6b1fd 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -12,7 +12,7 @@ platform:
 install:
   - go version
   - go env
-  - choco install bzr hg
+  - choco install bzr
   - set PATH=C:\Program Files (x86)\Bazaar\;C:\Program Files\Mercurial\;%PATH%
 build_script:
   - go get github.com/Masterminds/glide

From cdd194ebf9bd7fff51d08eb184bfd2564b954332 Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Mon, 15 Aug 2016 16:09:53 -0400
Subject: [PATCH 65/71] Fix waitgroup race conditions in tests

---
 manager_test.go | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/manager_test.go b/manager_test.go
index ebe9a87..2d2046c 100644
--- a/manager_test.go
+++ b/manager_test.go
@@ -389,8 +389,8 @@ func TestMultiDeduceThreadsafe(t *testing.T) {
 	// Set up channel for everything else to block on
 	c := make(chan struct{}, 1)
 	f := func(rnum int) {
-		wg.Add(1)
 		defer func() {
+			wg.Done()
 			if e := recover(); e != nil {
 				t.Errorf("goroutine number %v panicked with err: %s", rnum, e)
 			}
@@ -400,10 +400,10 @@ func TestMultiDeduceThreadsafe(t *testing.T) {
 		if err != nil {
 			t.Errorf("err was non-nil on root detection in goroutine number %v: %s", rnum, err)
 		}
-		wg.Done()
 	}
 
 	for k := range make([]struct{}, cnum) {
+		wg.Add(1)
 		go f(k)
 		runtime.Gosched()
 	}
@@ -414,10 +414,11 @@ func TestMultiDeduceThreadsafe(t *testing.T) {
 	}
 
 	// repeat for srcf
+	wg2 := &sync.WaitGroup{}
 	c = make(chan struct{}, 1)
 	f = func(rnum int) {
-		wg.Add(1)
 		defer func() {
+			wg2.Done()
 			if e := recover(); e != nil {
 				t.Errorf("goroutine number %v panicked with err: %s", rnum, e)
 			}
@@ -427,15 +428,15 @@ func TestMultiDeduceThreadsafe(t *testing.T) {
 		if err != nil {
 			t.Errorf("err was non-nil on root detection in goroutine number %v: %s", rnum, err)
 		}
-		wg.Done()
 	}
 
 	for k := range make([]struct{}, cnum) {
+		wg2.Add(1)
 		go f(k)
 		runtime.Gosched()
 	}
 	close(c)
-	wg.Wait()
+	wg2.Wait()
 	if len(sm.srcs) != 2 {
 		t.Errorf("Sources map should have just two elements, but has %v", len(sm.srcs))
 	}

From ec580d99eb05567a072c1ff3d78b8fe554407013 Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Mon, 15 Aug 2016 20:42:58 -0400
Subject: [PATCH 66/71] Fix gopkg.in deducer implementation

I totally misread the docs, interpreting the "pkg" as a literal for the
shortened form. Not so, not so at all.
---
 deduce.go      | 12 ++----------
 deduce_test.go | 10 ++++++++++
 2 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/deduce.go b/deduce.go
index 3aa2d05..25dc93d 100644
--- a/deduce.go
+++ b/deduce.go
@@ -261,17 +261,9 @@ func (m gopkginDeducer) deduceSource(p string, u *url.URL) (maybeSource, error)
 
 	// gopkg.in is always backed by github
 	u.Host = "github.com"
-	// If the third position is empty, it's the shortened form that expands
-	// to the go-pkg github user
 	if v[2] == "" {
-		// Apparently gopkg.in special-cases gopkg.in/yaml, violating its own rules?
-		// If we find one more exception, chuck this and just rely on vanity
-		// metadata resolving.
-		if v[3] == "/yaml" {
-			u.Path = "/go-yaml/yaml"
-		} else {
-			u.Path = path.Join("/go-pkg", v[3])
-		}
+		elem := v[3][1:]
+		u.Path = path.Join("/go-"+elem, elem)
 	} else {
 		u.Path = path.Join(v[2], v[3])
 	}
diff --git a/deduce_test.go b/deduce_test.go
index b7647f9..bbe6490 100644
--- a/deduce_test.go
+++ b/deduce_test.go
@@ -156,6 +156,16 @@ var pathDeductionFixtures = map[string][]pathDeductionFixture{
 				maybeGitSource{url: mkurl("http://github.com/go-yaml/yaml")},
 			},
 		},
+		{
+			in:   "gopkg.in/inf.v0",
+			root: "gopkg.in/inf.v0",
+			mb: maybeSources{
+				maybeGitSource{url: mkurl("https://github.com/go-inf/inf")},
+				maybeGitSource{url: mkurl("ssh://git@github.com/go-inf/inf")},
+				maybeGitSource{url: mkurl("git://github.com/go-inf/inf")},
+				maybeGitSource{url: mkurl("http://github.com/go-inf/inf")},
+			},
+		},
 		{
 			// gopkg.in only allows specifying major version in import path
 			in:   "gopkg.in/yaml.v1.2",

From 92406eafb773b4556fbee4be3281a8b4b89b703a Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Mon, 15 Aug 2016 20:44:39 -0400
Subject: [PATCH 67/71] Remove a truckload of dead code

---
 source.go     |  40 ------------
 vcs_source.go | 175 +-------------------------------------------------
 2 files changed, 3 insertions(+), 212 deletions(-)

diff --git a/source.go b/source.go
index 515ce94..feaba15 100644
--- a/source.go
+++ b/source.go
@@ -167,46 +167,6 @@ func (dc *sourceMetaCache) toUnpaired(v Version) UnpairedVersion {
 	}
 }
 
-func (bs *baseVCSSource) listVersions() (vlist []Version, err error) {
-	if !bs.cvsync {
-		// This check only guarantees that the upstream exists, not the cache
-		bs.ex.s |= existsUpstream
-		vpairs, exbits, err := bs.crepo.getCurrentVersionPairs()
-		// But it *may* also check the local existence
-		bs.ex.s |= exbits
-		bs.ex.f |= exbits
-
-		if err != nil {
-			// TODO(sdboyer) More-er proper-er error
-			return nil, err
-		}
-
-		vlist = make([]Version, len(vpairs))
-		// mark our cache as synced if we got ExistsUpstream back
-		if exbits&existsUpstream == existsUpstream {
-			bs.cvsync = true
-		}
-
-		// Process the version data into the cache
-		// TODO(sdboyer) detect out-of-sync data as we do this?
-		for k, v := range vpairs {
-			u, r := v.Unpair(), v.Underlying()
-			bs.dc.vMap[u] = r
-			bs.dc.rMap[r] = append(bs.dc.rMap[r], u)
-			vlist[k] = v
-		}
-	} else {
-		vlist = make([]Version, len(bs.dc.vMap))
-		k := 0
-		for v := range bs.dc.vMap {
-			vlist[k] = v
-			k++
-		}
-	}
-
-	return
-}
-
 func (bs *baseVCSSource) revisionPresentIn(r Revision) (bool, error) {
 	// First and fastest path is to check the data cache to see if the rev is
 	// present. This could give us false positives, but the cases where that can
diff --git a/vcs_source.go b/vcs_source.go
index dff90d0..2036ea5 100644
--- a/vcs_source.go
+++ b/vcs_source.go
@@ -15,12 +15,12 @@ import (
 
 type vcsSource interface {
 	syncLocal() error
+	ensureLocal() error
 	listLocalVersionPairs() ([]PairedVersion, sourceExistence, error)
 	listUpstreamVersionPairs() ([]PairedVersion, sourceExistence, error)
-	revisionPresentIn(Revision) (bool, error)
+	hasRevision(Revision) (bool, error)
 	checkout(Version) error
-	ping() bool
-	ensureCacheExistence() error
+	exportVersionTo(Version, string) error
 }
 
 // gitSource is a generic git repository implementation that should work with
@@ -379,175 +379,6 @@ type repo struct {
 	synced bool
 }
 
-func (r *repo) getCurrentVersionPairs() (vlist []PairedVersion, exbits sourceExistence, err error) {
-	r.mut.Lock()
-	defer r.mut.Unlock()
-
-	switch r.r.(type) {
-	case *vcs.GitRepo:
-		var out []byte
-		c := exec.Command("git", "ls-remote", r.r.Remote())
-		// Ensure no terminal prompting for PWs
-		c.Env = mergeEnvLists([]string{"GIT_TERMINAL_PROMPT=0"}, os.Environ())
-		out, err = c.CombinedOutput()
-
-		all := bytes.Split(bytes.TrimSpace(out), []byte("\n"))
-		if err != nil || len(all) == 0 {
-			// TODO(sdboyer) remove this path? it really just complicates things, for
-			// probably not much benefit
-
-			// ls-remote failed, probably due to bad communication or a faulty
-			// upstream implementation. So fetch updates, then build the list
-			// locally
-			err = r.r.Update()
-			if err != nil {
-				// Definitely have a problem, now - bail out
-				return
-			}
-
-			// Upstream and cache must exist, so add that to exbits
-			exbits |= existsUpstream | existsInCache
-			// Also, local is definitely now synced
-			r.synced = true
-
-			out, err = r.r.RunFromDir("git", "show-ref", "--dereference")
-			if err != nil {
-				return
-			}
-
-			all = bytes.Split(bytes.TrimSpace(out), []byte("\n"))
-		}
-		// Local cache may not actually exist here, but upstream definitely does
-		exbits |= existsUpstream
-
-		tmap := make(map[string]PairedVersion)
-		for _, pair := range all {
-			var v PairedVersion
-			if string(pair[46:51]) == "heads" {
-				v = NewBranch(string(pair[52:])).Is(Revision(pair[:40])).(PairedVersion)
-				vlist = append(vlist, v)
-			} else if string(pair[46:50]) == "tags" {
-				vstr := string(pair[51:])
-				if strings.HasSuffix(vstr, "^{}") {
-					// If the suffix is there, then we *know* this is the rev of
-					// the underlying commit object that we actually want
-					vstr = strings.TrimSuffix(vstr, "^{}")
-				} else if _, exists := tmap[vstr]; exists {
-					// Already saw the deref'd version of this tag, if one
-					// exists, so skip this.
-					continue
-					// Can only hit this branch if we somehow got the deref'd
-					// version first. Which should be impossible, but this
-					// covers us in case of weirdness, anyway.
-				}
-				v = NewVersion(vstr).Is(Revision(pair[:40])).(PairedVersion)
-				tmap[vstr] = v
-			}
-		}
-
-		// Append all the deref'd (if applicable) tags into the list
-		for _, v := range tmap {
-			vlist = append(vlist, v)
-		}
-	case *vcs.BzrRepo:
-		var out []byte
-		// Update the local first
-		err = r.r.Update()
-		if err != nil {
-			return
-		}
-		// Upstream and cache must exist, so add that to exbits
-		exbits |= existsUpstream | existsInCache
-		// Also, local is definitely now synced
-		r.synced = true
-
-		// Now, list all the tags
-		out, err = r.r.RunFromDir("bzr", "tags", "--show-ids", "-v")
-		if err != nil {
-			return
-		}
-
-		all := bytes.Split(bytes.TrimSpace(out), []byte("\n"))
-		for _, line := range all {
-			idx := bytes.IndexByte(line, 32) // space
-			v := NewVersion(string(line[:idx])).Is(Revision(bytes.TrimSpace(line[idx:]))).(PairedVersion)
-			vlist = append(vlist, v)
-		}
-
-	case *vcs.HgRepo:
-		var out []byte
-		err = r.r.Update()
-		if err != nil {
-			return
-		}
-
-		// Upstream and cache must exist, so add that to exbits
-		exbits |= existsUpstream | existsInCache
-		// Also, local is definitely now synced
-		r.synced = true
-
-		out, err = r.r.RunFromDir("hg", "tags", "--debug", "--verbose")
-		if err != nil {
-			return
-		}
-
-		all := bytes.Split(bytes.TrimSpace(out), []byte("\n"))
-		lbyt := []byte("local")
-		nulrev := []byte("0000000000000000000000000000000000000000")
-		for _, line := range all {
-			if bytes.Equal(lbyt, line[len(line)-len(lbyt):]) {
-				// Skip local tags
-				continue
-			}
-
-			// tip is magic, don't include it
-			if bytes.HasPrefix(line, []byte("tip")) {
-				continue
-			}
-
-			// Split on colon; this gets us the rev and the tag plus local revno
-			pair := bytes.Split(line, []byte(":"))
-			if bytes.Equal(nulrev, pair[1]) {
-				// null rev indicates this tag is marked for deletion
-				continue
-			}
-
-			idx := bytes.IndexByte(pair[0], 32) // space
-			v := NewVersion(string(pair[0][:idx])).Is(Revision(pair[1])).(PairedVersion)
-			vlist = append(vlist, v)
-		}
-
-		out, err = r.r.RunFromDir("hg", "branches", "--debug", "--verbose")
-		if err != nil {
-			// better nothing than incomplete
-			vlist = nil
-			return
-		}
-
-		all = bytes.Split(bytes.TrimSpace(out), []byte("\n"))
-		lbyt = []byte("(inactive)")
-		for _, line := range all {
-			if bytes.Equal(lbyt, line[len(line)-len(lbyt):]) {
-				// Skip inactive branches
-				continue
-			}
-
-			// Split on colon; this gets us the rev and the branch plus local revno
-			pair := bytes.Split(line, []byte(":"))
-			idx := bytes.IndexByte(pair[0], 32) // space
-			v := NewBranch(string(pair[0][:idx])).Is(Revision(pair[1])).(PairedVersion)
-			vlist = append(vlist, v)
-		}
-	case *vcs.SvnRepo:
-		// TODO(sdboyer) is it ok to return empty vlist and no error?
-		// TODO(sdboyer) ...gotta do something for svn, right?
-	default:
-		panic("unknown repo type")
-	}
-
-	return
-}
-
 func (r *repo) exportVersionTo(v Version, to string) error {
 	r.mut.Lock()
 	defer r.mut.Unlock()

From 5d88145c7bb45758bd63822bfec304d5c14565c7 Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Mon, 15 Aug 2016 20:44:51 -0400
Subject: [PATCH 68/71] Populate baseVCSSource.lvfunc

Hopefully this ends up being a temporary measure, but in the meantime...
---
 maybe_source.go | 26 ++++++++++++++++++++------
 1 file changed, 20 insertions(+), 6 deletions(-)

diff --git a/maybe_source.go b/maybe_source.go
index 5565ba4..34fd5d5 100644
--- a/maybe_source.go
+++ b/maybe_source.go
@@ -74,11 +74,11 @@ func (m maybeGitSource) try(cachedir string, an ProjectAnalyzer) (source, string
 		},
 	}
 
+	src.baseVCSSource.lvfunc = src.listVersions
+
 	_, err = src.listVersions()
 	if err != nil {
 		return nil, "", err
-		//} else if pm.ex.f&existsUpstream == existsUpstream {
-		//return pm, nil
 	}
 
 	return src, ustr, nil
@@ -99,16 +99,23 @@ func (m maybeBzrSource) try(cachedir string, an ProjectAnalyzer) (source, string
 		return nil, "", fmt.Errorf("Remote repository at %s does not exist, or is inaccessible", ustr)
 	}
 
-	return &bzrSource{
+	src := &bzrSource{
 		baseVCSSource: baseVCSSource{
 			an: an,
 			dc: newMetaCache(),
+			ex: existence{
+				s: existsUpstream,
+				f: existsUpstream,
+			},
 			crepo: &repo{
 				r:     r,
 				rpath: path,
 			},
 		},
-	}, ustr, nil
+	}
+	src.baseVCSSource.lvfunc = src.listVersions
+
+	return src, ustr, nil
 }
 
 type maybeHgSource struct {
@@ -126,14 +133,21 @@ func (m maybeHgSource) try(cachedir string, an ProjectAnalyzer) (source, string,
 		return nil, "", fmt.Errorf("Remote repository at %s does not exist, or is inaccessible", ustr)
 	}
 
-	return &hgSource{
+	src := &hgSource{
 		baseVCSSource: baseVCSSource{
 			an: an,
 			dc: newMetaCache(),
+			ex: existence{
+				s: existsUpstream,
+				f: existsUpstream,
+			},
 			crepo: &repo{
 				r:     r,
 				rpath: path,
 			},
 		},
-	}, ustr, nil
+	}
+	src.baseVCSSource.lvfunc = src.listVersions
+
+	return src, ustr, nil
 }

From 8fea82442035f793c17b45e1d872a12a460c2a79 Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Mon, 15 Aug 2016 21:45:27 -0400
Subject: [PATCH 69/71] Vanity import deduction tests

---
 deduce_test.go  | 46 ++++++++++++++++++++++++++++++++++++++++++++++
 manager_test.go | 12 ++++++++++++
 2 files changed, 58 insertions(+)

diff --git a/deduce_test.go b/deduce_test.go
index bbe6490..23ffe38 100644
--- a/deduce_test.go
+++ b/deduce_test.go
@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"net/url"
 	"reflect"
+	"sync"
 	"testing"
 )
 
@@ -558,6 +559,51 @@ func TestDeduceFromPath(t *testing.T) {
 	}
 }
 
+func TestVanityDeduction(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping slow test in short mode")
+	}
+
+	sm, clean := mkNaiveSM(t)
+	defer clean()
+
+	vanities := pathDeductionFixtures["vanity"]
+	wg := &sync.WaitGroup{}
+	wg.Add(len(vanities))
+
+	for _, fix := range vanities {
+		go func(fix pathDeductionFixture) {
+			defer wg.Done()
+			pr, err := sm.DeduceProjectRoot(fix.in)
+			if err != nil {
+				t.Errorf("(in: %s) Unexpected err on deducing project root: %s", fix.in, err)
+				return
+			} else if string(pr) != fix.root {
+				t.Errorf("(in: %s) Deducer did not return expected root:\n\t(GOT) %s\n\t(WNT) %s", fix.in, pr, fix.root)
+			}
+
+			_, srcf, err := sm.deducePathAndProcess(fix.in)
+			if err != nil {
+				t.Errorf("(in: %s) Unexpected err on deducing source: %s", fix.in, err)
+				return
+			}
+
+			_, ident, err := srcf()
+			if err != nil {
+				t.Errorf("(in: %s) Unexpected err on executing source future: %s", fix.in, err)
+				return
+			}
+
+			ustr := fix.mb.(maybeGitSource).url.String()
+			if ident != ustr {
+				t.Errorf("(in: %s) Deduced repo ident does not match fixture:\n\t(GOT) %s\n\t(WNT) %s", fix.in, ident, ustr)
+			}
+		}(fix)
+	}
+
+	wg.Wait()
+}
+
 // borrow from stdlib
 // more useful string for debugging than fmt's struct printer
 func ufmt(u *url.URL) string {
diff --git a/manager_test.go b/manager_test.go
index 2d2046c..439d8b4 100644
--- a/manager_test.go
+++ b/manager_test.go
@@ -365,6 +365,18 @@ func TestDeduceProjectRoot(t *testing.T) {
 	if sm.rootxt.Len() != 5 {
 		t.Errorf("Root path trie should have five elements, one for each unique root and subpath; has %v", sm.rootxt.Len())
 	}
+
+	// Ensure that vcs extension-based matching comes through
+	in5 := "ffffrrrraaaaaapppppdoesnotresolve.com/baz.git"
+	pr, err = sm.DeduceProjectRoot(in5)
+	if err != nil {
+		t.Errorf("Problem while detecting root of %q %s", in5, err)
+	} else if string(pr) != in5 {
+		t.Errorf("Wrong project root was deduced;\n\t(GOT) %s\n\t(WNT) %s", pr, in)
+	}
+	if sm.rootxt.Len() != 6 {
+		t.Errorf("Root path trie should have six elements, one for each unique root and subpath; has %v", sm.rootxt.Len())
+	}
 }
 
 // Test that the future returned from SourceMgr.deducePathAndProcess() is safe

From cff2ef6370297e7defbe97bac4445805b97289e8 Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Mon, 15 Aug 2016 22:25:46 -0400
Subject: [PATCH 70/71] Flesh out the source testing a little more

---
 source_test.go | 54 +++++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 51 insertions(+), 3 deletions(-)

diff --git a/source_test.go b/source_test.go
index 33d2acb..907d9c3 100644
--- a/source_test.go
+++ b/source_test.go
@@ -8,7 +8,7 @@ import (
 	"testing"
 )
 
-func TestGitVersionFetching(t *testing.T) {
+func TestGitSourceInteractions(t *testing.T) {
 	// This test is slowish, skip it on -short
 	if testing.Short() {
 		t.Skip("Skipping git source version fetching test in short mode")
@@ -73,6 +73,14 @@ func TestGitVersionFetching(t *testing.T) {
 		t.Errorf("gitSource.listVersions() should not have set the cache existence bit for found")
 	}
 
+	// check that an expected rev is present
+	is, err := src.revisionPresentIn(Revision("30605f6ac35fcb075ad0bfa9296f90a7d891523e"))
+	if err != nil {
+		t.Errorf("Unexpected error while checking revision presence: %s", err)
+	} else if !is {
+		t.Errorf("Revision that should exist was not present")
+	}
+
 	if len(vlist) != 3 {
 		t.Errorf("git test repo should've produced three versions, got %v: vlist was %s", len(vlist), vlist)
 	} else {
@@ -86,9 +94,17 @@ func TestGitVersionFetching(t *testing.T) {
 			t.Errorf("Version list was not what we expected:\n\t(GOT): %s\n\t(WNT): %s", vlist, evl)
 		}
 	}
+
+	// recheck that rev is present, this time interacting with cache differently
+	is, err = src.revisionPresentIn(Revision("30605f6ac35fcb075ad0bfa9296f90a7d891523e"))
+	if err != nil {
+		t.Errorf("Unexpected error while re-checking revision presence: %s", err)
+	} else if !is {
+		t.Errorf("Revision that should exist was not present on re-check")
+	}
 }
 
-func TestBzrVersionFetching(t *testing.T) {
+func TestBzrSourceInteractions(t *testing.T) {
 	// This test is quite slow (ugh bzr), so skip it on -short
 	if testing.Short() {
 		t.Skip("Skipping bzr source version fetching test in short mode")
@@ -133,6 +149,14 @@ func TestBzrVersionFetching(t *testing.T) {
 		t.Errorf("Expected %s as source ident, got %s", un, ident)
 	}
 
+	// check that an expected rev is present
+	is, err := src.revisionPresentIn(Revision("matt@mattfarina.com-20150731135137-pbphasfppmygpl68"))
+	if err != nil {
+		t.Errorf("Unexpected error while checking revision presence: %s", err)
+	} else if !is {
+		t.Errorf("Revision that should exist was not present")
+	}
+
 	vlist, err := src.listVersions()
 	if err != nil {
 		t.Errorf("Unexpected error getting version pairs from bzr repo: %s", err)
@@ -175,9 +199,17 @@ func TestBzrVersionFetching(t *testing.T) {
 			t.Errorf("bzr pair fetch reported incorrect first version, got %s", vlist[0])
 		}
 	}
+
+	// recheck that rev is present, this time interacting with cache differently
+	is, err = src.revisionPresentIn(Revision("matt@mattfarina.com-20150731135137-pbphasfppmygpl68"))
+	if err != nil {
+		t.Errorf("Unexpected error while re-checking revision presence: %s", err)
+	} else if !is {
+		t.Errorf("Revision that should exist was not present on re-check")
+	}
 }
 
-func TestHgVersionFetching(t *testing.T) {
+func TestHgSourceInteractions(t *testing.T) {
 	// This test is slow, so skip it on -short
 	if testing.Short() {
 		t.Skip("Skipping hg source version fetching test in short mode")
@@ -222,6 +254,14 @@ func TestHgVersionFetching(t *testing.T) {
 		t.Errorf("Expected %s as source ident, got %s", un, ident)
 	}
 
+	// check that an expected rev is present
+	is, err := src.revisionPresentIn(Revision("d680e82228d206935ab2eaa88612587abe68db07"))
+	if err != nil {
+		t.Errorf("Unexpected error while checking revision presence: %s", err)
+	} else if !is {
+		t.Errorf("Revision that should exist was not present")
+	}
+
 	vlist, err := src.listVersions()
 	if err != nil {
 		t.Errorf("Unexpected error getting version pairs from hg repo: %s", err)
@@ -268,4 +308,12 @@ func TestHgVersionFetching(t *testing.T) {
 			t.Errorf("Version list was not what we expected:\n\t(GOT): %s\n\t(WNT): %s", vlist, evl)
 		}
 	}
+
+	// recheck that rev is present, this time interacting with cache differently
+	is, err = src.revisionPresentIn(Revision("d680e82228d206935ab2eaa88612587abe68db07"))
+	if err != nil {
+		t.Errorf("Unexpected error while re-checking revision presence: %s", err)
+	} else if !is {
+		t.Errorf("Revision that should exist was not present on re-check")
+	}
 }

From 737841a10cecd7dd71ac5c84054d1e728fd78934 Mon Sep 17 00:00:00 2001
From: sam boyer <tech@samboyer.org>
Date: Mon, 15 Aug 2016 22:26:01 -0400
Subject: [PATCH 71/71] More crufty code removal

---
 vcs_source.go | 77 +++++++++++++++------------------------------------
 1 file changed, 22 insertions(+), 55 deletions(-)

diff --git a/vcs_source.go b/vcs_source.go
index 2036ea5..277b1db 100644
--- a/vcs_source.go
+++ b/vcs_source.go
@@ -383,64 +383,31 @@ func (r *repo) exportVersionTo(v Version, to string) error {
 	r.mut.Lock()
 	defer r.mut.Unlock()
 
-	switch r.r.(type) {
-	case *vcs.GitRepo:
-		// Back up original index
-		idx, bak := filepath.Join(r.rpath, ".git", "index"), filepath.Join(r.rpath, ".git", "origindex")
-		err := os.Rename(idx, bak)
-		if err != nil {
-			return err
-		}
-
-		// TODO(sdboyer) could have an err here
-		defer os.Rename(bak, idx)
-
-		vstr := v.String()
-		if rv, ok := v.(PairedVersion); ok {
-			vstr = rv.Underlying().String()
-		}
-		_, err = r.r.RunFromDir("git", "read-tree", vstr)
-		if err != nil {
-			return err
-		}
-
-		// Ensure we have exactly one trailing slash
-		to = strings.TrimSuffix(to, string(os.PathSeparator)) + string(os.PathSeparator)
-		// Checkout from our temporary index to the desired target location on disk;
-		// now it's git's job to make it fast. Sadly, this approach *does* also
-		// write out vendor dirs. There doesn't appear to be a way to make
-		// checkout-index respect sparse checkout rules (-a supercedes it);
-		// the alternative is using plain checkout, though we have a bunch of
-		// housekeeping to do to set up, then tear down, the sparse checkout
-		// controls, as well as restore the original index and HEAD.
-		_, err = r.r.RunFromDir("git", "checkout-index", "-a", "--prefix="+to)
-		return err
-	default:
-		// TODO(sdboyer) This is a dumb, slow approach, but we're punting on making these
-		// fast for now because git is the OVERWHELMING case
-		r.r.UpdateVersion(v.String())
-
-		cfg := &shutil.CopyTreeOptions{
-			Symlinks:     true,
-			CopyFunction: shutil.Copy,
-			Ignore: func(src string, contents []os.FileInfo) (ignore []string) {
-				for _, fi := range contents {
-					if !fi.IsDir() {
-						continue
-					}
-					n := fi.Name()
-					switch n {
-					case "vendor", ".bzr", ".svn", ".hg":
-						ignore = append(ignore, n)
-					}
+	// TODO(sdboyer) This is a dumb, slow approach, but we're punting on making
+	// these fast for now because git is the OVERWHELMING case (it's handled in
+	// its own method)
+	r.r.UpdateVersion(v.String())
+
+	cfg := &shutil.CopyTreeOptions{
+		Symlinks:     true,
+		CopyFunction: shutil.Copy,
+		Ignore: func(src string, contents []os.FileInfo) (ignore []string) {
+			for _, fi := range contents {
+				if !fi.IsDir() {
+					continue
 				}
+				n := fi.Name()
+				switch n {
+				case "vendor", ".bzr", ".svn", ".hg":
+					ignore = append(ignore, n)
+				}
+			}
 
-				return
-			},
-		}
-
-		return shutil.CopyTree(r.rpath, to, cfg)
+			return
+		},
 	}
+
+	return shutil.CopyTree(r.rpath, to, cfg)
 }
 
 // This func copied from Masterminds/vcs so we can exec our own commands