Skip to content

Commit

Permalink
add clone_git_modules config setting, fixes #151
Browse files Browse the repository at this point in the history
  • Loading branch information
@xorpaul
xorpaul committed Dec 20, 2019
1 parent cc63c86 commit 4cfc458
Show file tree
Hide file tree
Showing 3 changed files with 53 additions and 33 deletions.
2 changes: 2 additions & 0 deletions g10k.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,7 @@ type ConfigSettings struct {
GenerateTypes bool `yaml:"generate_types"`
PuppetPath string `yaml:"puppet_path"`
PurgeBlacklist []string `yaml:"purge_blacklist"`
CloneGitModules bool `yaml:"clone_git_modules"`
}

// DeploySettings is a struct for settings for controlling how g10k deploys behave.
Expand Down Expand Up @@ -168,6 +169,7 @@ type GitModule struct {
tag string
commit string
ref string
tree string
link bool
ignoreUnreachable bool
fallback []string
Expand Down
64 changes: 37 additions & 27 deletions git.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ func resolveGitRepositories(uniqueGitModules map[string]GitModule) {
for url, gm := range uniqueGitModules {
Debugf("git repo url " + url)
privateKey := gm.privateKey
go func(url string, privateKey string, gm GitModule, bar *uiprogress.Bar) {
go func(url string, gm GitModule, bar *uiprogress.Bar) {
// Try to receive from the concurrentGoroutines channel. When we have something,
// it means we can start a new goroutine because another one finished.
// Otherwise, it will block the execution until an execution
Expand All @@ -79,55 +79,61 @@ func resolveGitRepositories(uniqueGitModules map[string]GitModule) {
repoDir := strings.Replace(strings.Replace(url, "/", "_", -1), ":", "-", -1)
workDir := filepath.Join(config.ModulesCacheDir, repoDir)

success := doMirrorOrUpdate(url, workDir, privateKey, gm.ignoreUnreachable, 1)
success := doMirrorOrUpdate(gm, workDir, 0)
if !success && config.UseCacheFallback == false {
Fatalf("Fatal: Could not reach git repository " + url)
}
// doCloneOrPull(source, workDir, targetDir, sa.Remote, branch, sa.PrivateKey)
done <- true
}(url, privateKey, gm, bar)
}(url, gm, bar)
}

// Wait for all jobs to finish
<-waitForAllJobs
wg.Wait()
}

func doMirrorOrUpdate(url string, workDir string, sshPrivateKey string, allowFail bool, retryCount int) bool {
func doMirrorOrUpdate(gitModule GitModule, workDir string, retryCount int) bool {
needSSHKey := true
if strings.Contains(url, "github.com") || len(sshPrivateKey) == 0 {
if strings.Contains(gitModule.git, "github.com") || len(gitModule.privateKey) == 0 {
needSSHKey = false
}
isControlRepo := strings.HasPrefix(workDir, config.EnvCacheDir)
isInModulesCacheDir := strings.HasPrefix(workDir, config.ModulesCacheDir)

er := ExecResult{}
gitCmd := "git clone --mirror " + url + " " + workDir
gitCmd := "git clone --mirror " + gitModule.git + " " + workDir
if config.CloneGitModules && !isControlRepo && !isInModulesCacheDir {
fmt.Printf("%+v\n", gitModule)
gitCmd = "git clone --single-branch --branch " + gitModule.tree + " " + gitModule.git + " " + workDir
}
if isDir(workDir) {
gitCmd = "git --git-dir " + workDir + " remote update --prune"
}

if needSSHKey {
er = executeCommand("ssh-agent bash -c 'ssh-add "+sshPrivateKey+"; "+gitCmd+"'", config.Timeout, allowFail)
er = executeCommand("ssh-agent bash -c 'ssh-add "+gitModule.privateKey+"; "+gitCmd+"'", config.Timeout, gitModule.ignoreUnreachable)
} else {
er = executeCommand(gitCmd, config.Timeout, allowFail)
er = executeCommand(gitCmd, config.Timeout, gitModule.ignoreUnreachable)
}

if er.returnCode != 0 {
if config.UseCacheFallback {
Warnf("WARN: git repository " + url + " does not exist or is unreachable at this moment!")
Warnf("WARN: Trying to use cache for " + url + " git repository")
Warnf("WARN: git repository " + gitModule.git + " does not exist or is unreachable at this moment!")
Warnf("WARN: Trying to use cache for " + gitModule.git + " git repository")
return false
} else if config.RetryGitCommands && retryCount > 0 {
} else if config.RetryGitCommands && retryCount > -1 {
Warnf("WARN: git command failed: " + gitCmd + " deleting local cached repository and retrying...")
purgeDir(workDir, "doMirrorOrUpdate, because git command failed, retrying")
return doMirrorOrUpdate(url, workDir, sshPrivateKey, false, retryCount-1)
return doMirrorOrUpdate(gitModule, workDir, retryCount-1)
}
Warnf("WARN: git repository " + url + " does not exist or is unreachable at this moment!")
Warnf("WARN: git repository " + gitModule.git + " does not exist or is unreachable at this moment!")
return false
}
return true
}

func syncToModuleDir(srcDir string, targetDir string, tree string, allowFail bool, ignoreUnreachable bool, correspondingPuppetEnvironment string) bool {
func syncToModuleDir(gitModule GitModule, srcDir string, targetDir string, correspondingPuppetEnvironment string) bool {
startedAt := time.Now()
mutex.Lock()
syncGitCount++
Expand All @@ -137,19 +143,21 @@ func syncToModuleDir(srcDir string, targetDir string, tree string, allowFail boo
Fatalf("Could not find cached git module " + srcDir)
}
}
logCmd := "git --git-dir " + srcDir + " rev-parse --verify '" + tree
logCmd := "git --git-dir " + srcDir + " rev-parse --verify '" + gitModule.tree
if config.GitObjectSyntaxNotSupported != true {
logCmd = logCmd + "^{object}'"
} else {
logCmd = logCmd + "'"
}

er := executeCommand(logCmd, config.Timeout, allowFail)
isControlRepo := strings.HasPrefix(srcDir, config.EnvCacheDir)

er := executeCommand(logCmd, config.Timeout, gitModule.ignoreUnreachable)
hashFile := filepath.Join(targetDir, ".latest_commit")
deployFile := filepath.Join(targetDir, ".g10k-deploy.json")
needToSync := true
if er.returnCode != 0 {
if allowFail && ignoreUnreachable {
if gitModule.ignoreUnreachable {
Debugf("Failed to populate module " + targetDir + " but ignore-unreachable is set. Continuing...")
purgeDir(targetDir, "syncToModuleDir, because ignore-unreachable is set for this module")
}
Expand All @@ -166,7 +174,7 @@ func syncToModuleDir(srcDir string, targetDir string, tree string, allowFail boo
if dr.Signature == strings.TrimSuffix(er.output, "\n") {
needToSync = false
// need to get the content of the git repository to detect and purge unmanaged files
addDesiredContent(srcDir, tree, targetDir)
addDesiredContent(srcDir, gitModule.tree, targetDir)
}
}
} else {
Expand Down Expand Up @@ -197,19 +205,19 @@ func syncToModuleDir(srcDir string, targetDir string, tree string, allowFail boo
needSyncGitCount++
mutex.Unlock()

if !dryRun {
if !dryRun && !config.CloneGitModules || isControlRepo {
checkDirAndCreate(targetDir, "git dir")
gitArchiveArgs := []string{"--git-dir", srcDir, "archive", tree}
gitArchiveArgs := []string{"--git-dir", srcDir, "archive", gitModule.tree}
cmd := exec.Command("git", gitArchiveArgs...)
Debugf("Executing git --git-dir " + srcDir + " archive " + tree)
Debugf("Executing git --git-dir " + srcDir + " archive " + gitModule.tree)
cmdOut, err := cmd.StdoutPipe()
if err != nil {
if !allowFail {
if !gitModule.ignoreUnreachable {
Infof("Failed to populate module " + targetDir + " but ignore-unreachable is set. Continuing...")
} else {
return false
}
Fatalf("syncToModuleDir(): Failed to execute command: git --git-dir " + srcDir + " archive " + tree + " Error: " + err.Error())
Fatalf("syncToModuleDir(): Failed to execute command: git --git-dir " + srcDir + " archive " + gitModule.tree + " Error: " + err.Error())
}
cmd.Start()

Expand All @@ -222,23 +230,23 @@ func syncToModuleDir(srcDir string, targetDir string, tree string, allowFail boo

err = cmd.Wait()
if err != nil {
Fatalf("syncToModuleDir(): Failed to execute command: git --git-dir " + srcDir + " archive " + tree + " Error: " + err.Error())
Fatalf("syncToModuleDir(): Failed to execute command: git --git-dir " + srcDir + " archive " + gitModule.tree + " Error: " + err.Error())
//"\nIf you are using GitLab please ensure that you've added your deploy key to your repository." +
//"\nThe Puppet environment which is using this unresolveable repository is " + correspondingPuppetEnvironment)
}

Verbosef("syncToModuleDir(): Executing git --git-dir " + srcDir + " archive " + tree + " took " + strconv.FormatFloat(duration, 'f', 5, 64) + "s")
Verbosef("syncToModuleDir(): Executing git --git-dir " + srcDir + " archive " + gitModule.tree + " took " + strconv.FormatFloat(duration, 'f', 5, 64) + "s")

er = executeCommand(logCmd, config.Timeout, false)
if er.returnCode != 0 {
Fatalf("executeCommand(): git command failed: " + logCmd + " " + err.Error() + "\nOutput: " + er.output)
}
if len(er.output) > 0 {
commitHash := strings.TrimSuffix(er.output, "\n")
if strings.HasPrefix(srcDir, config.EnvCacheDir) {
if isControlRepo {
Debugf("Writing to deploy file " + deployFile)
dr := DeployResult{
Name: tree,
Name: gitModule.tree,
Signature: commitHash,
StartedAt: startedAt,
}
Expand All @@ -252,6 +260,8 @@ func syncToModuleDir(srcDir string, targetDir string, tree string, allowFail boo
}

}
} else if config.CloneGitModules {
return doMirrorOrUpdate(gitModule, targetDir, 0)
}
}
return true
Expand Down
20 changes: 14 additions & 6 deletions puppetfile.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,10 @@ func resolvePuppetEnvironment(tags bool, outputNameTag string) {
// check if sa.Basedir exists
checkDirAndCreate(sa.Basedir, "basedir")

if success := doMirrorOrUpdate(sa.Remote, workDir, sa.PrivateKey, true, 1); success {
controlRepoGit := GitModule{}
controlRepoGit.git = sa.Remote
controlRepoGit.privateKey = sa.PrivateKey
if success := doMirrorOrUpdate(controlRepoGit, workDir, 0); success {

// get all branches
er := executeCommand("git --git-dir "+workDir+" branch", config.Timeout, false)
Expand Down Expand Up @@ -131,10 +134,12 @@ func resolvePuppetEnvironment(tags bool, outputNameTag string) {
targetDir = normalizeDir(targetDir)

env := strings.Replace(strings.Replace(targetDir, sa.Basedir, "", 1), "/", "", -1)
syncToModuleDir(workDir, targetDir, branch, false, false, env)
gitModule := GitModule{}
gitModule.tree = branch
syncToModuleDir(gitModule, workDir, targetDir, env)
pf := filepath.Join(targetDir, "Puppetfile")
if !fileExists(pf) {
Debugf("Skipping branch " + source + "_" + branch + " because " + targetDir + "Puppetfile does not exist")
Debugf("resolvePuppetEnvironment(): Skipping branch " + source + "_" + branch + " because " + pf + " does not exist")
} else {
puppetfile := readPuppetfile(pf, sa.PrivateKey, source, branch, sa.ForceForgeVersions, false)
puppetfile.workDir = normalizeDir(targetDir)
Expand Down Expand Up @@ -337,7 +342,8 @@ func resolvePuppetfile(allPuppetfiles map[string]Puppetfile) {

if gitModule.link {
Debugf("Trying to resolve " + moduleCacheDir + " with branch " + tree)
success = syncToModuleDir(moduleCacheDir, targetDir, tree, true, gitModule.ignoreUnreachable, env)
gitModule.tree = tree
success = syncToModuleDir(gitModule, moduleCacheDir, targetDir, env)
}

if len(gitModule.fallback) > 0 {
Expand All @@ -348,15 +354,17 @@ func resolvePuppetfile(allPuppetfiles map[string]Puppetfile) {
gitModule.ignoreUnreachable = true
}
Debugf("Trying to resolve " + moduleCacheDir + " with branch " + fallbackBranch)
success = syncToModuleDir(moduleCacheDir, targetDir, fallbackBranch, true, gitModule.ignoreUnreachable, env)
gitModule.tree = fallbackBranch
success = syncToModuleDir(gitModule, moduleCacheDir, targetDir, env)
if success {
break
}
}
// possible TODO: shouldn't this fail if all fallback branches fail?
}
} else {
success = syncToModuleDir(moduleCacheDir, targetDir, tree, gitModule.ignoreUnreachable, gitModule.ignoreUnreachable, env)
gitModule.tree = tree
success = syncToModuleDir(gitModule, moduleCacheDir, targetDir, env)
if !success && !config.IgnoreUnreachableModules {
Fatalf("Failed to resolve git module '" + gitName + "' with repository " + gitModule.git + " and branch/reference '" + tree + "' used in control repository branch '" + pf.sourceBranch + "' or Puppet environment '" + env + "'")
}
Expand Down

0 comments on commit 4cfc458

Please sign in to comment.