Skip to content
This repository was archived by the owner on Sep 9, 2020. It is now read-only.

status: make collectConstraints() concurrent #1413

Merged
merged 3 commits into from
Dec 18, 2017
Merged

status: make collectConstraints() concurrent #1413

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
5642a3b
status: make collectConstraints() concurrent
darkowlzz Nov 29, 2017
73ca5d9
status: add projectConstraints sorting by Project
darkowlzz Nov 29, 2017
e8a2617
status: return slice of error from collectConstraints()
darkowlzz Nov 30, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
96 changes: 76 additions & 20 deletions cmd/dep/status.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -417,7 +417,12 @@ func runStatusAll(ctx *dep.Ctx, out outputter, p *dep.Project, sm gps.SourceMana
return false, 0, errors.Wrapf(err, "could not set up solver for input hashing")
}

cm := collectConstraints(ctx, p, sm)
// Errors while collecting constraints should not fail the whole status run.
// It should count the error and tell the user about incomplete results.
cm, ccerrs := collectConstraints(ctx, p, sm)
if len(ccerrs) > 0 {
errCount += len(ccerrs)
}

// Get the project list and sort it so that the printed output users see is
// deterministically ordered. (This may be superfluous if the lock is always
Expand Down Expand Up @@ -566,7 +571,7 @@ func runStatusAll(ctx *dep.Ctx, out outputter, p *dep.Project, sm gps.SourceMana

// Count ListVersions error because we get partial results when
// this happens.
errCount = len(errListVerCh)
errCount += len(errListVerCh)
if ctx.Verbose {
for err := range errListVerCh {
ctx.Err.Println(err.Error())
Expand Down Expand Up @@ -684,37 +689,88 @@ type projectConstraint struct {
type constraintsCollection map[string][]projectConstraint

// collectConstraints collects constraints declared by all the dependencies.
func collectConstraints(ctx *dep.Ctx, p *dep.Project, sm gps.SourceManager) constraintsCollection {
// It returns constraintsCollection and a slice of errors encountered while
// collecting the constraints, if any.
func collectConstraints(ctx *dep.Ctx, p *dep.Project, sm gps.SourceManager) (constraintsCollection, []error) {
logger := ctx.Err
if !ctx.Verbose {
logger = log.New(ioutil.Discard, "", 0)
}

logger.Println("Collecting project constraints:")

var mutex sync.Mutex
constraintCollection := make(constraintsCollection)

// Get direct deps of the root project.
_, directDeps, err := getDirectDependencies(sm, p)
if err != nil {
ctx.Err.Println("Error getting direct deps:", err)
// Return empty collection, not nil, if we fail here.
return constraintCollection, []error{errors.Wrap(err, "failed to get direct dependencies")}
}

// Create a root analyzer.
rootAnalyzer := newRootAnalyzer(true, ctx, directDeps, sm)

lp := p.Lock.Projects()

// Channel for receiving all the errors.
errCh := make(chan error, len(lp))

var wg sync.WaitGroup

// Iterate through the locked projects and collect constraints of all the projects.
for _, proj := range p.Lock.Projects() {
manifest, _, err := sm.GetManifestAndLock(proj.Ident(), proj.Version(), rootAnalyzer)
if err != nil {
ctx.Err.Println("Error getting manifest and lock:", err)
continue
}
for i, proj := range lp {
wg.Add(1)
logger.Printf("(%d/%d) %s\n", i+1, len(lp), proj.Ident().ProjectRoot)

go func(proj gps.LockedProject) {
defer wg.Done()

manifest, _, err := sm.GetManifestAndLock(proj.Ident(), proj.Version(), rootAnalyzer)
if err != nil {
errCh <- errors.Wrap(err, "error getting manifest and lock")
return
}

// Get project constraints.
pc := manifest.DependencyConstraints()
// Get project constraints.
pc := manifest.DependencyConstraints()

// Obtain a lock for constraintCollection.
mutex.Lock()
defer mutex.Unlock()
// Iterate through the project constraints to get individual dependency
// project and constraint values.
for pr, pp := range pc {
tempCC := append(
constraintCollection[string(pr)],
projectConstraint{proj.Ident().ProjectRoot, pp.Constraint},
)

// Sort the inner projectConstraint slice by Project string.
// Required for consistent returned value.
sort.Sort(byProject(tempCC))
constraintCollection[string(pr)] = tempCC
}
}(proj)
}

wg.Wait()
close(errCh)

// Iterate through the project constraints to get individual dependency
// project and constraint values.
for pr, pp := range pc {
constraintCollection[string(pr)] = append(
constraintCollection[string(pr)],
projectConstraint{proj.Ident().ProjectRoot, pp.Constraint},
)
var errs []error
if len(errCh) > 0 {
for e := range errCh {
errs = append(errs, e)
logger.Println(e.Error())
}
}

return constraintCollection
return constraintCollection, errs
}

type byProject []projectConstraint

func (p byProject) Len() int { return len(p) }
func (p byProject) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p byProject) Less(i, j int) bool { return p[i].Project > p[j].Project }
106 changes: 57 additions & 49 deletions cmd/dep/status_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"bytes"
"io/ioutil"
"log"
"path/filepath"
"reflect"
"testing"
"text/tabwriter"
Expand Down Expand Up @@ -299,83 +300,79 @@ func TestCollectConstraints(t *testing.T) {

cases := []struct {
name string
project dep.Project
lock dep.Lock
wantConstraints constraintsCollection
wantErr bool
}{
{
name: "without any constraints",
project: dep.Project{
Lock: &dep.Lock{
P: []gps.LockedProject{
gps.NewLockedProject(
gps.ProjectIdentifier{ProjectRoot: gps.ProjectRoot("github.com/sdboyer/deptest")},
gps.NewVersion("v1.0.0"),
[]string{"."},
),
},
lock: dep.Lock{
P: []gps.LockedProject{
gps.NewLockedProject(
gps.ProjectIdentifier{ProjectRoot: gps.ProjectRoot("github.com/sdboyer/deptest")},
gps.NewVersion("v1.0.0"),
[]string{"."},
),
},
},
wantConstraints: constraintsCollection{},
},
{
name: "with multiple constraints",
project: dep.Project{
Lock: &dep.Lock{
P: []gps.LockedProject{
gps.NewLockedProject(
gps.ProjectIdentifier{ProjectRoot: gps.ProjectRoot("github.com/sdboyer/deptest")},
gps.NewVersion("v1.0.0"),
[]string{"."},
),
gps.NewLockedProject(
gps.ProjectIdentifier{ProjectRoot: gps.ProjectRoot("github.com/darkowlzz/deptest-project-1")},
gps.NewVersion("v0.1.0"),
[]string{"."},
),
gps.NewLockedProject(
gps.ProjectIdentifier{ProjectRoot: gps.ProjectRoot("github.com/darkowlzz/deptest-project-2")},
gps.NewBranch("master").Pair(gps.Revision("824a8d56a4c6b2f4718824a98cd6d70d3dbd4c3e")),
[]string{"."},
),
},
lock: dep.Lock{
P: []gps.LockedProject{
gps.NewLockedProject(
gps.ProjectIdentifier{ProjectRoot: gps.ProjectRoot("github.com/sdboyer/deptest")},
gps.NewVersion("v1.0.0"),
[]string{"."},
),
gps.NewLockedProject(
gps.ProjectIdentifier{ProjectRoot: gps.ProjectRoot("github.com/darkowlzz/deptest-project-1")},
gps.NewVersion("v0.1.0"),
[]string{"."},
),
gps.NewLockedProject(
gps.ProjectIdentifier{ProjectRoot: gps.ProjectRoot("github.com/darkowlzz/deptest-project-2")},
gps.NewBranch("master").Pair(gps.Revision("824a8d56a4c6b2f4718824a98cd6d70d3dbd4c3e")),
[]string{"."},
),
},
},
wantConstraints: constraintsCollection{
"github.com/sdboyer/deptest": []projectConstraint{
{"github.com/darkowlzz/deptest-project-1", ver1},
{"github.com/darkowlzz/deptest-project-2", ver08},
},
"github.com/sdboyer/deptestdos": []projectConstraint{
{"github.com/darkowlzz/deptest-project-2", ver2},
},
"github.com/sdboyer/dep-test": []projectConstraint{
{"github.com/darkowlzz/deptest-project-2", ver1},
},
"github.com/sdboyer/deptest": []projectConstraint{
{"github.com/darkowlzz/deptest-project-2", ver08},
{"github.com/darkowlzz/deptest-project-1", ver1},
},
},
},
{
name: "skip projects with invalid versions",
project: dep.Project{
Lock: &dep.Lock{
P: []gps.LockedProject{
gps.NewLockedProject(
gps.ProjectIdentifier{ProjectRoot: gps.ProjectRoot("github.com/darkowlzz/deptest-project-1")},
gps.NewVersion("v0.1.0"),
[]string{"."},
),
gps.NewLockedProject(
gps.ProjectIdentifier{ProjectRoot: gps.ProjectRoot("github.com/darkowlzz/deptest-project-2")},
gps.NewVersion("v1.0.0"),
[]string{"."},
),
},
lock: dep.Lock{
P: []gps.LockedProject{
gps.NewLockedProject(
gps.ProjectIdentifier{ProjectRoot: gps.ProjectRoot("github.com/darkowlzz/deptest-project-1")},
gps.NewVersion("v0.1.0"),
[]string{"."},
),
gps.NewLockedProject(
gps.ProjectIdentifier{ProjectRoot: gps.ProjectRoot("github.com/darkowlzz/deptest-project-2")},
gps.NewVersion("v1.0.0"),
[]string{"."},
),
},
},
wantConstraints: constraintsCollection{
"github.com/sdboyer/deptest": []projectConstraint{
{"github.com/darkowlzz/deptest-project-1", ver1},
},
},
wantErr: true,
},
}

Expand All @@ -396,12 +393,23 @@ func TestCollectConstraints(t *testing.T) {
h.Must(err)
defer sm.Release()

// Create new project and set root. Setting root is required for PackageList
// to run properly.
p := new(dep.Project)
p.SetRoot(filepath.Join(pwd, "src"))

for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
gotConstraints := collectConstraints(ctx, &c.project, sm)
p.Lock = &c.lock
gotConstraints, err := collectConstraints(ctx, p, sm)
if len(err) > 0 && !c.wantErr {
t.Fatalf("unexpected errors while collecting constraints: %v", err)
} else if len(err) == 0 && c.wantErr {
t.Fatalf("expected errors while collecting constraints, but got none")
}

if !reflect.DeepEqual(gotConstraints, c.wantConstraints) {
t.Fatalf("Unexpected collected constraints: \n\t(GOT): %v\n\t(WNT): %v", gotConstraints, c.wantConstraints)
t.Fatalf("unexpected collected constraints: \n\t(GOT): %v\n\t(WNT): %v", gotConstraints, c.wantConstraints)
}
})
}
Expand Down