diff --git a/manager_test.go b/manager_test.go index 0081964..ebe9a87 100644 --- a/manager_test.go +++ b/manager_test.go @@ -5,6 +5,7 @@ import ( "io/ioutil" "os" "path" + "path/filepath" "runtime" "sort" "sync" @@ -181,13 +182,17 @@ func TestProjectManagerInit(t *testing.T) { } } + // use ListPackages to ensure the repo is actually on disk + // TODO(sdboyer) ugh, maybe we do need an explicit prefetch method + smc.ListPackages(id, NewVersion("1.0.0")) + // Ensure that the appropriate cache dirs and files exist - _, err = os.Stat(path.Join(cpath, "sources", "https---github.com-Masterminds-VCSTestRepo", ".git")) + _, err = os.Stat(filepath.Join(cpath, "sources", "https---github.com-Masterminds-VCSTestRepo", ".git")) if err != nil { t.Error("Cache repo does not exist in expected location") } - _, err = os.Stat(path.Join(cpath, "metadata", "github.com", "Masterminds", "VCSTestRepo", "cache.json")) + _, err = os.Stat(filepath.Join(cpath, "metadata", "github.com", "Masterminds", "VCSTestRepo", "cache.json")) if err != nil { // TODO(sdboyer) disabled until we get caching working //t.Error("Metadata cache json file does not exist in expected location") @@ -202,18 +207,6 @@ func TestProjectManagerInit(t *testing.T) { if !exists { t.Error("Source should exist after non-erroring call to ListVersions") } - - // Now reach inside the black box - pms, err := sm.getProjectManager(id) - if err != nil { - t.Errorf("Error on grabbing project manager obj: %s", err) - t.FailNow() - } - - // Check upstream existence flag - if !pms.pm.CheckExistence(existsUpstream) { - t.Errorf("ExistsUpstream flag not being correctly set the project") - } } func TestGetSources(t *testing.T) { diff --git a/project_manager.go b/project_manager.go deleted file mode 100644 index 992f6f3..0000000 --- a/project_manager.go +++ /dev/null @@ -1,305 +0,0 @@ -package gps - -import ( - "fmt" - "go/build" -) - -type projectManager struct { - // The upstream URL from which the project is sourced. - n string - - // build.Context to use in any analysis, and to pass to the analyzer - ctx build.Context - - // Object for the cache repository - crepo *repo - - // Indicates the extent to which we have searched for, and verified, the - // existence of the project/repo. - ex existence - - // Analyzer, injected by way of the SourceManager and originally from the - // sm's creator - an ProjectAnalyzer - - // Whether the cache has the latest info on versions - cvsync bool - - // The project metadata cache. This is persisted to disk, for reuse across - // solver runs. - // TODO(sdboyer) protect with mutex - dc *sourceMetaCache -} - -type existence struct { - // The existence levels for which a search/check has been performed - s sourceExistence - - // The existence levels verified to be present through searching - f sourceExistence -} - -// projectInfo holds manifest and lock -type projectInfo struct { - Manifest - Lock -} - -func (pm *projectManager) GetManifestAndLock(r ProjectRoot, v Version) (Manifest, Lock, error) { - if err := pm.ensureCacheExistence(); err != nil { - return nil, nil, err - } - - rev, err := pm.toRevOrErr(v) - if err != nil { - return nil, nil, err - } - - // Return the info from the cache, if we already have it - if pi, exists := pm.dc.infos[rev]; exists { - return pi.Manifest, pi.Lock, nil - } - - pm.crepo.mut.Lock() - if !pm.crepo.synced { - err = pm.crepo.r.Update() - if err != nil { - return nil, nil, fmt.Errorf("could not fetch latest updates into repository") - } - pm.crepo.synced = true - } - - // Always prefer a rev, if it's available - if pv, ok := v.(PairedVersion); ok { - err = pm.crepo.r.UpdateVersion(pv.Underlying().String()) - } else { - err = pm.crepo.r.UpdateVersion(v.String()) - } - pm.crepo.mut.Unlock() - if err != nil { - // TODO(sdboyer) More-er proper-er error - panic(fmt.Sprintf("canary - why is checkout/whatever failing: %s %s %s", pm.n, v.String(), err)) - } - - pm.crepo.mut.RLock() - m, l, err := pm.an.DeriveManifestAndLock(pm.crepo.rpath, r) - // TODO(sdboyer) cache results - pm.crepo.mut.RUnlock() - - if err == nil { - if l != nil { - l = prepLock(l) - } - - // If m is nil, prepManifest will provide an empty one. - pi := projectInfo{ - Manifest: prepManifest(m), - Lock: l, - } - - // TODO(sdboyer) this just clobbers all over and ignores the paired/unpaired - // distinction; serious fix is needed - pm.dc.infos[rev] = pi - - return pi.Manifest, pi.Lock, nil - } - - return nil, nil, err -} - -func (pm *projectManager) ListPackages(pr ProjectRoot, v Version) (ptree PackageTree, err error) { - if err = pm.ensureCacheExistence(); err != nil { - return - } - - var r Revision - if r, err = pm.toRevOrErr(v); err != nil { - return - } - - // Return the ptree from the cache, if we already have it - var exists bool - if ptree, exists = pm.dc.ptrees[r]; exists { - return - } - - // Not in the cache; check out the version and do the analysis - pm.crepo.mut.Lock() - // Check out the desired version for analysis - if r != "" { - // Always prefer a rev, if it's available - err = pm.crepo.r.UpdateVersion(string(r)) - } else { - // If we don't have a rev, ensure the repo is up to date, otherwise we - // could have a desync issue - if !pm.crepo.synced { - err = pm.crepo.r.Update() - if err != nil { - return PackageTree{}, fmt.Errorf("could not fetch latest updates into repository: %s", err) - } - pm.crepo.synced = true - } - err = pm.crepo.r.UpdateVersion(v.String()) - } - - ptree, err = listPackages(pm.crepo.rpath, string(pr)) - pm.crepo.mut.Unlock() - - // TODO(sdboyer) cache errs? - if err != nil { - pm.dc.ptrees[r] = ptree - } - - return -} - -func (pm *projectManager) ensureCacheExistence() error { - // Technically, methods could could attempt to return straight from the - // metadata cache even if the repo cache doesn't exist on disk. But that - // would allow weird state inconsistencies (cache exists, but no repo...how - // does that even happen?) that it'd be better to just not allow so that we - // don't have to think about it elsewhere - if !pm.CheckExistence(existsInCache) { - if pm.CheckExistence(existsUpstream) { - pm.crepo.mut.Lock() - err := pm.crepo.r.Get() - pm.crepo.mut.Unlock() - - if err != nil { - return fmt.Errorf("failed to create repository cache for %s", pm.n) - } - pm.ex.s |= existsInCache - pm.ex.f |= existsInCache - } else { - return fmt.Errorf("project %s does not exist upstream", pm.n) - } - } - - return nil -} - -func (pm *projectManager) ListVersions() (vlist []Version, err error) { - if !pm.cvsync { - // This check only guarantees that the upstream exists, not the cache - pm.ex.s |= existsUpstream - vpairs, exbits, err := pm.crepo.getCurrentVersionPairs() - // But it *may* also check the local existence - pm.ex.s |= exbits - pm.ex.f |= exbits - - if err != nil { - // TODO(sdboyer) More-er proper-er error - return nil, err - } - - vlist = make([]Version, len(vpairs)) - // mark our cache as synced if we got ExistsUpstream back - if exbits&existsUpstream == existsUpstream { - pm.cvsync = true - } - - // Process the version data into the cache - // TODO(sdboyer) detect out-of-sync data as we do this? - for k, v := range vpairs { - u, r := v.Unpair(), v.Underlying() - pm.dc.vMap[u] = r - pm.dc.rMap[r] = append(pm.dc.rMap[r], u) - vlist[k] = v - } - } else { - vlist = make([]Version, len(pm.dc.vMap)) - k := 0 - for v, r := range pm.dc.vMap { - vlist[k] = v.Is(r) - k++ - } - } - - return -} - -// toRevOrErr makes all efforts to convert a Version into a rev, including -// updating the cache repo (if needed). It does not guarantee that the returned -// Revision actually exists in the repository (as one of the cheaper methods may -// have had bad data). -func (pm *projectManager) toRevOrErr(v Version) (r Revision, err error) { - r = pm.dc.toRevision(v) - if r == "" { - // Rev can be empty if: - // - The cache is unsynced - // - A version was passed that used to exist, but no longer does - // - A garbage version was passed. (Functionally indistinguishable from - // the previous) - if !pm.cvsync { - _, err = pm.ListVersions() - if err != nil { - return - } - } - - r = pm.dc.toRevision(v) - // If we still don't have a rev, then the version's no good - if r == "" { - err = fmt.Errorf("version %s does not exist in source %s", v, pm.crepo.r.Remote()) - } - } - - return -} - -func (pm *projectManager) RevisionPresentIn(pr ProjectRoot, r Revision) (bool, error) { - // First and fastest path is to check the data cache to see if the rev is - // present. This could give us false positives, but the cases where that can - // occur would require a type of cache staleness that seems *exceedingly* - // unlikely to occur. - if _, has := pm.dc.infos[r]; has { - return true, nil - } else if _, has := pm.dc.rMap[r]; has { - return true, nil - } - - // For now at least, just run GetInfoAt(); it basically accomplishes the - // same thing. - if _, _, err := pm.GetManifestAndLock(pr, r); err != nil { - return false, err - } - return true, nil -} - -// CheckExistence provides a direct method for querying existence levels of the -// project. It will only perform actual searching (local fs or over the network) -// if no previous attempt at that search has been made. -// -// Note that this may perform read-ish operations on the cache repo, and it -// takes a lock accordingly. Deadlock may result from calling it during a -// segment where the cache repo mutex is already write-locked. -func (pm *projectManager) CheckExistence(ex sourceExistence) bool { - if pm.ex.s&ex != ex { - if ex&existsInVendorRoot != 0 && pm.ex.s&existsInVendorRoot == 0 { - panic("should now be implemented in bridge") - } - if ex&existsInCache != 0 && pm.ex.s&existsInCache == 0 { - pm.crepo.mut.RLock() - pm.ex.s |= existsInCache - if pm.crepo.r.CheckLocal() { - pm.ex.f |= existsInCache - } - pm.crepo.mut.RUnlock() - } - if ex&existsUpstream != 0 && pm.ex.s&existsUpstream == 0 { - pm.crepo.mut.RLock() - pm.ex.s |= existsUpstream - if pm.crepo.r.Ping() { - pm.ex.f |= existsUpstream - } - pm.crepo.mut.RUnlock() - } - } - - return ex&pm.ex.f == ex -} - -func (pm *projectManager) ExportVersionTo(v Version, to string) error { - return pm.crepo.exportVersionTo(v, to) -} diff --git a/source.go b/source.go index 1d431bc..515ce94 100644 --- a/source.go +++ b/source.go @@ -20,6 +20,20 @@ type sourceMetaCache struct { // TODO(sdboyer) mutexes. actually probably just one, b/c complexity } +// projectInfo holds manifest and lock +type projectInfo struct { + Manifest + Lock +} + +type existence struct { + // The existence levels for which a search/check has been performed + s sourceExistence + + // The existence levels verified to be present through searching + f sourceExistence +} + func newMetaCache() *sourceMetaCache { return &sourceMetaCache{ infos: make(map[Revision]projectInfo), diff --git a/source_manager.go b/source_manager.go index c6d8e53..89b5dfa 100644 --- a/source_manager.go +++ b/source_manager.go @@ -1,7 +1,6 @@ package gps import ( - "encoding/json" "fmt" "os" "path/filepath" @@ -10,7 +9,6 @@ import ( "sync/atomic" "github.com/Masterminds/semver" - "github.com/Masterminds/vcs" ) // Used to compute a friendly filepath from a URL-shaped input @@ -76,8 +74,6 @@ type ProjectAnalyzer interface { // tools; control via dependency injection is intended to be sufficient. type SourceMgr struct { cachedir string - pms map[string]*pmState - pmut sync.RWMutex srcs map[string]source srcmut sync.RWMutex rr map[string]struct { @@ -92,14 +88,6 @@ type SourceMgr struct { var _ SourceManager = &SourceMgr{} -// Holds a projectManager, caches of the managed project's data, and information -// about the freshness of those caches -type pmState struct { - pm *projectManager - cf *os.File // handle for the cache file - vcur bool // indicates that we've called ListVersions() -} - // NewSourceManager produces an instance of gps's built-in SourceManager. It // takes a cache directory (where local instances of upstream repositories are // stored), a vendor directory for the project currently being worked on, and a @@ -140,7 +128,6 @@ func NewSourceManager(an ProjectAnalyzer, cachedir string, force bool) (*SourceM return &SourceMgr{ cachedir: cachedir, - pms: make(map[string]*pmState), srcs: make(map[string]source), rr: make(map[string]struct { rr *remoteRepo @@ -170,23 +157,23 @@ func (sm *SourceMgr) AnalyzerInfo() (name string, version *semver.Version) { // The work of producing the manifest and lock is delegated to the injected // ProjectAnalyzer's DeriveManifestAndLock() method. func (sm *SourceMgr) GetManifestAndLock(id ProjectIdentifier, v Version) (Manifest, Lock, error) { - pmc, err := sm.getProjectManager(id) + src, err := sm.getSourceFor(id) if err != nil { return nil, nil, err } - return pmc.pm.GetManifestAndLock(id.ProjectRoot, v) + return src.getManifestAndLock(id.ProjectRoot, v) } // ListPackages parses the tree of the Go packages at and below the ProjectRoot // of the given ProjectIdentifier, at the given version. func (sm *SourceMgr) ListPackages(id ProjectIdentifier, v Version) (PackageTree, error) { - pmc, err := sm.getProjectManager(id) + src, err := sm.getSourceFor(id) if err != nil { return PackageTree{}, err } - return pmc.pm.ListPackages(id.ProjectRoot, v) + return src.listPackages(id.ProjectRoot, v) } // ListVersions retrieves a list of the available versions for a given @@ -202,50 +189,51 @@ func (sm *SourceMgr) ListPackages(id ProjectIdentifier, v Version) (PackageTree, // is not accessible (network outage, access issues, or the resource actually // went away), an error will be returned. func (sm *SourceMgr) ListVersions(id ProjectIdentifier) ([]Version, error) { - pmc, err := sm.getProjectManager(id) + src, err := sm.getSourceFor(id) if err != nil { // TODO(sdboyer) More-er proper-er errors return nil, err } - return pmc.pm.ListVersions() + return src.listVersions() } // RevisionPresentIn indicates whether the provided Revision is present in the given // repository. func (sm *SourceMgr) RevisionPresentIn(id ProjectIdentifier, r Revision) (bool, error) { - pmc, err := sm.getProjectManager(id) + src, err := sm.getSourceFor(id) if err != nil { // TODO(sdboyer) More-er proper-er errors return false, err } - return pmc.pm.RevisionPresentIn(id.ProjectRoot, r) + return src.revisionPresentIn(r) } // SourceExists checks if a repository exists, either upstream or in the cache, // for the provided ProjectIdentifier. func (sm *SourceMgr) SourceExists(id ProjectIdentifier) (bool, error) { - pms, err := sm.getProjectManager(id) + src, err := sm.getSourceFor(id) if err != nil { return false, err } - return pms.pm.CheckExistence(existsInCache) || pms.pm.CheckExistence(existsUpstream), nil + return src.checkExistence(existsInCache) || src.checkExistence(existsUpstream), nil } // ExportProject writes out the tree of the provided ProjectIdentifier's // ProjectRoot, at the provided version, to the provided directory. func (sm *SourceMgr) ExportProject(id ProjectIdentifier, v Version, to string) error { - pms, err := sm.getProjectManager(id) + src, err := sm.getSourceFor(id) if err != nil { return err } - return pms.pm.ExportVersionTo(v, to) + return src.exportVersionTo(v, to) } -// DeduceRootProject takes an import path and deduces the +// DeduceRootProject takes an import path and deduces the corresponding +// project/source root. // // Note that some import paths may require network activity to correctly // determine the root of the path, such as, but not limited to, vanity import @@ -412,173 +400,3 @@ func (sm *SourceMgr) deducePathAndProcess(path string) (stringFuture, sourceFutu return rootf, srcf, nil } - -// getProjectManager gets the project manager for the given ProjectIdentifier. -// -// If no such manager yet exists, it attempts to create one. -func (sm *SourceMgr) getProjectManager(id ProjectIdentifier) (*pmState, error) { - // TODO(sdboyer) finish this, it's not sufficient (?) - n := id.netName() - var rpath string - - // Early check to see if we already have a pm in the cache for this net name - if pm, exists := sm.pms[n]; exists { - return pm, nil - } - - // Figure out the remote repo path - rr, err := deduceRemoteRepo(n) - if err != nil { - // Not a valid import path, must reject - // TODO(sdboyer) wrap error - return nil, err - } - - // Check the cache again, see if exact resulting clone url is in there - if pm, exists := sm.pms[rr.CloneURL.String()]; exists { - // Found it - re-register this PM at the original netname so that it - // doesn't need to deduce next time - // TODO(sdboyer) is this OK to do? are there consistency side effects? - sm.pms[n] = pm - return pm, nil - } - - // No luck again. Now, walk through the scheme options the deducer returned, - // checking if each is in the cache - for _, scheme := range rr.Schemes { - rr.CloneURL.Scheme = scheme - // See if THIS scheme has a match, now - if pm, exists := sm.pms[rr.CloneURL.String()]; exists { - // Yep - again, re-register this PM at the original netname so that it - // doesn't need to deduce next time - // TODO(sdboyer) is this OK to do? are there consistency side effects? - sm.pms[n] = pm - return pm, nil - } - } - - // Definitively no match for anything in the cache, so we know we have to - // create the entry. Next question is whether there's already a repo on disk - // for any of the schemes, or if we need to create that, too. - - // TODO(sdboyer) this strategy kinda locks in the scheme to use over - // multiple invocations in a way that maybe isn't the best. - var r vcs.Repo - for _, scheme := range rr.Schemes { - rr.CloneURL.Scheme = scheme - url := rr.CloneURL.String() - sn := sanitizer.Replace(url) - rpath = filepath.Join(sm.cachedir, "sources", sn) - - if fi, err := os.Stat(rpath); err == nil && fi.IsDir() { - // This one exists, so set up here - r, err = vcs.NewRepo(url, rpath) - if err != nil { - return nil, err - } - goto decided - } - } - - // Nothing on disk, either. Iterate through the schemes, trying each and - // failing out only if none resulted in successfully setting up the local. - for _, scheme := range rr.Schemes { - rr.CloneURL.Scheme = scheme - url := rr.CloneURL.String() - sn := sanitizer.Replace(url) - rpath = filepath.Join(sm.cachedir, "sources", sn) - - r, err = vcs.NewRepo(url, rpath) - if err != nil { - continue - } - - // FIXME(sdboyer) cloning the repo here puts it on a blocking path. that - // aspect of state management needs to be deferred into the - // projectManager - err = r.Get() - if err != nil { - continue - } - goto decided - } - - // If we've gotten this far, we got some brokeass input. - return nil, fmt.Errorf("Could not reach source repository for %s", n) - -decided: - // Ensure cache dir exists - metadir := filepath.Join(sm.cachedir, "metadata", string(n)) - err = os.MkdirAll(metadir, 0777) - if err != nil { - // TODO(sdboyer) be better - return nil, err - } - - pms := &pmState{} - cpath := filepath.Join(metadir, "cache.json") - fi, err := os.Stat(cpath) - var dc *sourceMetaCache - if fi != nil { - pms.cf, err = os.OpenFile(cpath, os.O_RDWR, 0777) - if err != nil { - // TODO(sdboyer) be better - return nil, fmt.Errorf("Err on opening metadata cache file: %s", err) - } - - err = json.NewDecoder(pms.cf).Decode(dc) - if err != nil { - // TODO(sdboyer) be better - return nil, fmt.Errorf("Err on JSON decoding metadata cache file: %s", err) - } - } else { - // TODO(sdboyer) commented this out for now, until we manage it correctly - //pms.cf, err = os.Create(cpath) - //if err != nil { - //// TODO(sdboyer) be better - //return nil, fmt.Errorf("Err on creating metadata cache file: %s", err) - //} - - dc = newMetaCache() - } - - pm := &projectManager{ - n: n, - an: sm.an, - dc: dc, - crepo: &repo{ - rpath: rpath, - r: r, - }, - } - - pms.pm = pm - sm.pms[n] = pms - return pms, nil -} - -func (sm *SourceMgr) whatsInAName(nn string) (*remoteRepo, error) { - sm.rmut.RLock() - tuple, exists := sm.rr[nn] - sm.rmut.RUnlock() - if exists { - return tuple.rr, tuple.err - } - - // Don't lock around the deduceRemoteRepo call, because that itself can be - // slow. The tradeoff is that it's possible we might duplicate work if two - // calls for the same id were to made simultaneously, but as those results - // would be the same, clobbering is OK, and better than the alternative of - // serializing all calls. - rr, err := deduceRemoteRepo(nn) - sm.rmut.Lock() - sm.rr[nn] = struct { - rr *remoteRepo - err error - }{ - rr: rr, - err: err, - } - sm.rmut.Unlock() - return rr, err -} diff --git a/vcs_source.go b/vcs_source.go index 3591c0d..dff90d0 100644 --- a/vcs_source.go +++ b/vcs_source.go @@ -34,6 +34,12 @@ func (s *gitSource) exportVersionTo(v Version, to string) error { defer s.crepo.mut.Unlock() r := s.crepo.r + if !r.CheckLocal() { + err := r.Get() + if err != nil { + return fmt.Errorf("failed to clone repo from %s", r.Remote()) + } + } // Back up original index idx, bak := filepath.Join(r.LocalPath(), ".git", "index"), filepath.Join(r.LocalPath(), ".git", "origindex") err := os.Rename(idx, bak)