diff --git a/config.go b/config.go index b33ea77..34b0a83 100644 --- a/config.go +++ b/config.go @@ -98,7 +98,8 @@ func readPuppetfile(pf string, sshKey string, source string) Puppetfile { n := preparePuppetfile(pf) - reModuledir := regexp.MustCompile("^\\s*(?:moduledir)\\s*['\"]?([^'\"]+)['\"]") + reModuledir := regexp.MustCompile("^\\s*(?:moduledir)\\s*['\"]?([^'\"]+)['\"]?") + reForgeBaseURL := regexp.MustCompile("^\\s*(?:forge.baseUrl)\\s*['\"]?([^'\"]+)['\"]?") reForgeModule := regexp.MustCompile("^\\s*(?:mod)\\s*['\"]?([^'\"]+/[^'\"]+)['\"](?:\\s*(,)\\s*['\"]?([^'\"]*))?") reGitModule := regexp.MustCompile("^\\s*(?:mod)\\s*['\"]?([^'\"/]+)['\"]\\s*,(.*)") reGitAttribute := regexp.MustCompile("\\s*:(git|commit|tag|branch|ref|link|ignore[-_]unreachable)\\s*=>\\s*['\"]?([^'\"]+)['\"]?") @@ -113,6 +114,9 @@ func readPuppetfile(pf string, sshKey string, source string) Puppetfile { } if m := reModuledir.FindStringSubmatch(line); len(m) > 1 { puppetFile.moduleDir = m[1] + } else if m := reForgeBaseURL.FindStringSubmatch(line); len(m) > 1 { + puppetFile.forgeBaseURL = m[1] + //fmt.Println("found forge base URL parameter ---> ", m[1]) } else if m := reForgeModule.FindStringSubmatch(line); len(m) > 1 { //fmt.Println("found forge mod name ---> ", m[1]) comp := strings.Split(m[1], "/") diff --git a/forge.go b/forge.go index 330f303..7f8a63a 100644 --- a/forge.go +++ b/forge.go @@ -12,14 +12,13 @@ import ( "net/http" "os" "os/exec" - "regexp" "strconv" "strings" "sync" "time" ) -func doModuleInstallOrNothing(m string) { +func doModuleInstallOrNothing(m string, fm ForgeModule) { ma := strings.Split(m, "-") moduleName := ma[0] + "-" + ma[1] moduleVersion := ma[2] @@ -32,7 +31,7 @@ func doModuleInstallOrNothing(m string) { if _, err := os.Stat(workDir); os.IsNotExist(err) { Debugf("doModuleInstallOrNothing(): " + workDir + " does not exist, fetching module") // check forge API what the latest version is - fr = queryForgeAPI(moduleName, "false") + fr = queryForgeAPI(moduleName, "false", fm) if fr.needToGet { if _, ok := uniqueForgeModules[moduleName+"-"+fr.versionNumber]; ok { Debugf("doModuleInstallOrNothing(): no need to fetch Forge module " + moduleName + " in latest, because latest is " + fr.versionNumber + " and that will already be fetched") @@ -55,7 +54,7 @@ func doModuleInstallOrNothing(m string) { // XXX: disable adding If-Modified-Since header for now // because then the latestForgeModules does not get set with the actual module version for latest // maybe if received 304 get the actual version from the -latest symlink - fr = queryForgeAPI(moduleName, "false") + fr = queryForgeAPI(moduleName, "false", fm) //fmt.Println(needToGet) } @@ -68,7 +67,7 @@ func doModuleInstallOrNothing(m string) { return } Debugf("doModuleInstallOrNothing(): we got " + m + ", but no " + latestDir + " to use. Getting -latest") - doModuleInstallOrNothing(moduleName + "-latest") + doModuleInstallOrNothing(moduleName+"-latest", fm) return } Debugf("doModuleInstallOrNothing(): Nothing to do for module " + m + ", because " + latestDir + " exists") @@ -102,13 +101,17 @@ func doModuleInstallOrNothing(m string) { } } } - downloadForgeModule(moduleName, fr.versionNumber) + downloadForgeModule(moduleName, fr.versionNumber, fm) } } -func queryForgeAPI(name string, file string) ForgeResult { +func queryForgeAPI(name string, file string, fm ForgeModule) ForgeResult { //url := "https://forgeapi.puppetlabs.com:443/v3/modules/" + strings.Replace(name, "/", "-", -1) - url := config.Forge.Baseurl+"/v3/modules?query=" + name + baseUrl := config.Forge.Baseurl + if len(fm.baseUrl) > 0 { + baseUrl = fm.baseUrl + } + url := baseUrl + "/v3/modules?query=" + name req, err := http.NewRequest("GET", url, nil) if err != nil { log.Fatal("queryForgeAPI(): Error creating GET request for Puppetlabs forge API", err) @@ -123,8 +126,7 @@ func queryForgeAPI(name string, file string) ForgeResult { proxyURL, err := http.ProxyFromEnvironment(req) if err != nil { - log.Fatal("queryForgeAPI(): Error while getting http proxy with golang http.ProxyFromEnvironment()", err) - os.Exit(1) + Fatalf("queryForgeAPI(): Error while getting http proxy with golang http.ProxyFromEnvironment()" + err.Error()) } client := &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxyURL)}} before := time.Now() @@ -143,24 +145,47 @@ func queryForgeAPI(name string, file string) ForgeResult { // need to get latest version body, err := ioutil.ReadAll(resp.Body) - //fmt.Println(string(body)) - reCurrent := regexp.MustCompile("\\s*\"current_release\": {\n\\s*\"uri\": \"([^\"]+)\",") - if m := reCurrent.FindStringSubmatch(string(body)); len(m) > 1 { - //fmt.Println(m[1]) - if strings.Count(m[1], "-") < 2 { - log.Fatal("queryForgeAPI(): Error: Something went wrong while trying to figure out what version is current for Forge module ", name, " ", m[1], " should contain three '-' characters") - os.Exit(1) - } else { - // modified the split because I found a module with version 4.0.0-beta1 mayflower-php - version := strings.Split(m[1], name+"-")[1] - Debugf("queryForgeAPI(): found version " + version + " for " + name + "-latest") - mutex.Lock() - latestForgeModules[name] = version - mutex.Unlock() - return ForgeResult{true, version} + var f interface{} + if err := json.Unmarshal(body, &f); err != nil { + Fatalf("queryForgeAPI(): Error while decoding JSON from URL " + url + " Error: " + err.Error()) + } + currentUri := "" + m := f.(map[string]interface{}) + for _, v := range m { + switch vv := v.(type) { + case []interface{}: + if len(vv) >= 1 { + if curRel, ok := vv[0].(map[string]interface{})["current_release"]; ok { + if val, ok := curRel.(map[string]interface{})["uri"]; ok { + //fmt.Println("uri --> ", val) + currentUri = val.(string) + } else { + Fatalf("queryForgeAPI(): Error: Unexpected JSON response while trying to figure out what version is current for Forge module " + name + " using " + url) + } + } else { + Fatalf("queryForgeAPI(): Error: Unexpected JSON response while trying to figure out what version is current for Forge module " + name + " using " + url) + } + } else { + Fatalf("queryForgeAPI(): Error: Unexpected JSON response while trying to figure out what version is current for Forge module " + name + " using " + url) + } + default: + // skip, we'll do a sanity for the currentUri value later anyway } } + //fmt.Println(string(body)) + if strings.Count(currentUri, "-") < 2 { + log.Fatal("queryForgeAPI(): Error: Something went wrong while trying to figure out what version is current for Forge module ", name, " ", currentUri, " should contain three '-' characters") + } else { + // modified the split because I found a module with version 4.0.0-beta1 mayflower-php + version := strings.Split(currentUri, name+"-")[1] + Debugf("queryForgeAPI(): found version " + version + " for " + name + "-latest") + mutex.Lock() + latestForgeModules[name] = version + mutex.Unlock() + return ForgeResult{true, version} + } + if err != nil { panic(err) } @@ -174,11 +199,15 @@ func queryForgeAPI(name string, file string) ForgeResult { } } -func downloadForgeModule(name string, version string) { +func downloadForgeModule(name string, version string, fm ForgeModule) { //url := "https://forgeapi.puppetlabs.com/v3/files/puppetlabs-apt-2.1.1.tar.gz" fileName := name + "-" + version + ".tar.gz" if _, err := os.Stat(config.ForgeCacheDir + name + "-" + version); os.IsNotExist(err) { - url := "https://forgeapi.puppetlabs.com/v3/files/" + fileName + baseUrl := config.Forge.Baseurl + if len(fm.baseUrl) > 0 { + baseUrl = fm.baseUrl + } + url := baseUrl + "/v3/files/" + fileName req, err := http.NewRequest("GET", url, nil) req.Header.Set("User-Agent", "https://github.com/xorpaul/g10k/") req.Header.Set("Connection", "close") @@ -307,15 +336,15 @@ func readModuleMetadata(file string) ForgeModule { return ForgeModule{name: strings.Split(m["name"].(string), "-")[1], version: m["version"].(string), author: strings.ToLower(m["author"].(string))} } -func resolveForgeModules(modules map[string]struct{}) { +func resolveForgeModules(modules map[string]ForgeModule) { var wgForge sync.WaitGroup - for m := range modules { + for m, fm := range modules { wgForge.Add(1) - go func(m string) { + go func(m string, fm ForgeModule) { defer wgForge.Done() - Debugf("Trying to get forge module " + m) - doModuleInstallOrNothing(m) - }(m) + Debugf("Trying to get forge module " + m + " with Forge base url " + fm.baseUrl) + doModuleInstallOrNothing(m, fm) + }(m, fm) } wgForge.Wait() } @@ -371,8 +400,7 @@ func syncForgeToModuleDir(name string, m ForgeModule, moduleDir string) { } workDir := config.ForgeCacheDir + moduleName + "-" + m.version + "/" if _, err := os.Stat(workDir); os.IsNotExist(err) { - log.Print("syncForgeToModuleDir(): Forge module not found in dir: ", workDir) - os.Exit(1) + Fatalf("syncForgeToModuleDir(): Forge module not found in dir: " + workDir) } else { Infof("Need to sync " + targetDir) cmd := "cp --link --archive " + workDir + "* " + targetDir diff --git a/g10k.go b/g10k.go index f04eea8..7c04fff 100644 --- a/g10k.go +++ b/g10k.go @@ -32,7 +32,7 @@ var ( cpGitTime float64 cpForgeTime float64 buildtime string - uniqueForgeModules map[string]struct{} + uniqueForgeModules map[string]ForgeModule latestForgeModules map[string]string ) @@ -46,13 +46,15 @@ type ConfigSettings struct { privateKey string `yaml:"private_key"` username string } - Forge struct { - Baseurl string `yaml:"baseurl"` - } + Forge Forge Sources map[string]Source Timeout int `yaml:"timeout"` } +type Forge struct { + Baseurl string `yaml:"baseurl"` +} + // Source contains basic information about a Puppet environment repository type Source struct { Remote string @@ -64,17 +66,19 @@ type Source struct { // Puppetfile contains the key value pairs from the Puppetfile type Puppetfile struct { moduleDir string + forgeBaseURL string forgeModules map[string]ForgeModule gitModules map[string]GitModule privateKey string source string } -// ForgeModule contains information (Version, Name, Author) about a Puppetlabs Forge module +// ForgeModule contains information (Version, Name, Author, Forge BaseURL if custom) about a Puppetlabs Forge module type ForgeModule struct { version string name string author string + baseUrl string } // GitModule contains information about a Git Puppet module @@ -146,7 +150,10 @@ func main() { before := time.Now() if len(*configFile) > 0 { if usemove { - log.Fatalln("Error: -usemove parameter is only allowed in -puppetfile mode!") + Fatalf("Error: -usemove parameter is only allowed in -puppetfile mode!") + } + if pfMode { + Fatalf("Error: -puppetfile parameter is not allowed with -config parameter!") } Debugf("Using as config file: " + *configFile) config = readConfigfile(*configFile) @@ -170,7 +177,9 @@ func main() { } else { cachedir = checkDirAndCreate(cachedir, "cachedir default value") } - config = ConfigSettings{CacheDir: cachedir, ForgeCacheDir: cachedir, ModulesCacheDir: cachedir, EnvCacheDir: cachedir, Sources: sm} + //config = ConfigSettings{CacheDir: cachedir, ForgeCacheDir: cachedir, ModulesCacheDir: cachedir, EnvCacheDir: cachedir, Forge:{Baseurl: "https://forgeapi.puppetlabs.com"}, Sources: sm} + forgeDefaultSettings := Forge{Baseurl: "https://forgeapi.puppetlabs.com"} + config = ConfigSettings{CacheDir: cachedir, ForgeCacheDir: cachedir, ModulesCacheDir: cachedir, EnvCacheDir: cachedir, Sources: sm, Forge: forgeDefaultSettings} target = "./Puppetfile" puppetfile := readPuppetfile("./Puppetfile", "", "cmdlineparam") pfm := make(map[string]Puppetfile) diff --git a/helper.go b/helper.go index 7d51db5..88bb4d5 100644 --- a/helper.go +++ b/helper.go @@ -20,14 +20,14 @@ func Debugf(s string) { } } -// Verbosef is a helper function for debug logging if global variable verbose is set to true +// Verbosef is a helper function for verbose logging if global variable verbose is set to true func Verbosef(s string) { if debug != false || verbose != false { log.Print(fmt.Sprint(s)) } } -// Infof is a helper function for debug logging if global variable info is set to true +// Infof is a helper function for info logging if global variable info is set to true func Infof(s string) { if debug != false || verbose != false || info != false { color.Set(color.FgGreen) @@ -36,6 +36,13 @@ func Infof(s string) { } } +// Fatalf is a helper function for fatal logging +func Fatalf(s string) { + color.Set(color.FgRed) + log.Fatal(s) + color.Unset() +} + // fileExists checks if the given file exists and return a bool func fileExists(file string) bool { if _, err := os.Stat(file); os.IsNotExist(err) { @@ -51,14 +58,12 @@ func checkDirAndCreate(dir string, name string) string { if _, err := os.Stat(dir); os.IsNotExist(err) { //log.Printf("checkDirAndCreate(): trying to create dir '%s' as %s", dir, name) if err := os.MkdirAll(dir, 0777); err != nil { - log.Print("checkDirAndCreate(): Error: failed to create directory: ", dir) - os.Exit(1) + Fatalf("checkDirAndCreate(): Error: failed to create directory: " + dir) } } } else { // TODO make dir optional - log.Print("dir setting '" + name + "' missing! Exiting!") - os.Exit(1) + Fatalf("checkDirAndCreate(): Error: dir setting '" + name + "' missing! Exiting!") } } if !strings.HasSuffix(dir, "/") { diff --git a/puppetfile.go b/puppetfile.go index 9c282ad..03bdf83 100644 --- a/puppetfile.go +++ b/puppetfile.go @@ -96,7 +96,7 @@ func resolvePuppetEnvironment(envBranch string) { func resolvePuppetfile(allPuppetfiles map[string]Puppetfile) { var wg sync.WaitGroup uniqueGitModules := make(map[string]GitModule) - uniqueForgeModules = make(map[string]struct{}) + uniqueForgeModules = make(map[string]ForgeModule) latestForgeModules = make(map[string]string) exisitingModuleDirs := make(map[string]struct{}) for env, pf := range allPuppetfiles { @@ -112,10 +112,15 @@ func resolvePuppetfile(allPuppetfiles map[string]Puppetfile) { } for forgeModuleName, fm := range pf.forgeModules { //fmt.Println("Found Forge module ", forgeModuleName, " with version", fm.version) + if len(pf.forgeBaseURL) > 0 { + fm.baseUrl = pf.forgeBaseURL + } else { + fm.baseUrl = "" + } mutex.Lock() forgeModuleName = strings.Replace(forgeModuleName, "/", "-", -1) if _, ok := uniqueForgeModules[forgeModuleName+"-"+fm.version]; !ok { - uniqueForgeModules[forgeModuleName+"-"+fm.version] = empty + uniqueForgeModules[forgeModuleName+"-"+fm.version] = fm } mutex.Unlock() }