Skip to content

Commit

Permalink
Merge pull request #21 from xorpaul/forge_baseurl
Browse files Browse the repository at this point in the history 
add custom Forge base URL support in Puppetfile
  • Loading branch information
@xorpaul
xorpaul authored Aug 11, 2016
2 parents 8d197f6 + 287e6b1 commit 7f67550
Show file tree
Hide file tree
Showing 5 changed files with 102 additions and 51 deletions.
6 changes: 5 additions & 1 deletion config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,8 @@ func readPuppetfile(pf string, sshKey string, source string) Puppetfile {

n := preparePuppetfile(pf)

reModuledir := regexp.MustCompile("^\\s*(?:moduledir)\\s*['\"]?([^'\"]+)['\"]")
reModuledir := regexp.MustCompile("^\\s*(?:moduledir)\\s*['\"]?([^'\"]+)['\"]?")
reForgeBaseURL := regexp.MustCompile("^\\s*(?:forge.baseUrl)\\s*['\"]?([^'\"]+)['\"]?")
reForgeModule := regexp.MustCompile("^\\s*(?:mod)\\s*['\"]?([^'\"]+/[^'\"]+)['\"](?:\\s*(,)\\s*['\"]?([^'\"]*))?")
reGitModule := regexp.MustCompile("^\\s*(?:mod)\\s*['\"]?([^'\"/]+)['\"]\\s*,(.*)")
reGitAttribute := regexp.MustCompile("\\s*:(git|commit|tag|branch|ref|link|ignore[-_]unreachable)\\s*=>\\s*['\"]?([^'\"]+)['\"]?")
Expand All @@ -113,6 +114,9 @@ func readPuppetfile(pf string, sshKey string, source string) Puppetfile {
}
if m := reModuledir.FindStringSubmatch(line); len(m) > 1 {
puppetFile.moduleDir = m[1]
} else if m := reForgeBaseURL.FindStringSubmatch(line); len(m) > 1 {
puppetFile.forgeBaseURL = m[1]
//fmt.Println("found forge base URL parameter ---> ", m[1])
} else if m := reForgeModule.FindStringSubmatch(line); len(m) > 1 {
//fmt.Println("found forge mod name ---> ", m[1])
comp := strings.Split(m[1], "/")
Expand Down
98 changes: 63 additions & 35 deletions forge.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,14 +12,13 @@ import (
"net/http"
"os"
"os/exec"
"regexp"
"strconv"
"strings"
"sync"
"time"
)

func doModuleInstallOrNothing(m string) {
func doModuleInstallOrNothing(m string, fm ForgeModule) {
ma := strings.Split(m, "-")
moduleName := ma[0] + "-" + ma[1]
moduleVersion := ma[2]
Expand All @@ -32,7 +31,7 @@ func doModuleInstallOrNothing(m string) {
if _, err := os.Stat(workDir); os.IsNotExist(err) {
Debugf("doModuleInstallOrNothing(): " + workDir + " does not exist, fetching module")
// check forge API what the latest version is
fr = queryForgeAPI(moduleName, "false")
fr = queryForgeAPI(moduleName, "false", fm)
if fr.needToGet {
if _, ok := uniqueForgeModules[moduleName+"-"+fr.versionNumber]; ok {
Debugf("doModuleInstallOrNothing(): no need to fetch Forge module " + moduleName + " in latest, because latest is " + fr.versionNumber + " and that will already be fetched")
Expand All @@ -55,7 +54,7 @@ func doModuleInstallOrNothing(m string) {
// XXX: disable adding If-Modified-Since header for now
// because then the latestForgeModules does not get set with the actual module version for latest
// maybe if received 304 get the actual version from the -latest symlink
fr = queryForgeAPI(moduleName, "false")
fr = queryForgeAPI(moduleName, "false", fm)
//fmt.Println(needToGet)
}

Expand All @@ -68,7 +67,7 @@ func doModuleInstallOrNothing(m string) {
return
}
Debugf("doModuleInstallOrNothing(): we got " + m + ", but no " + latestDir + " to use. Getting -latest")
doModuleInstallOrNothing(moduleName + "-latest")
doModuleInstallOrNothing(moduleName+"-latest", fm)
return
}
Debugf("doModuleInstallOrNothing(): Nothing to do for module " + m + ", because " + latestDir + " exists")
Expand Down Expand Up @@ -102,13 +101,17 @@ func doModuleInstallOrNothing(m string) {
}
}
}
downloadForgeModule(moduleName, fr.versionNumber)
downloadForgeModule(moduleName, fr.versionNumber, fm)
}
}

func queryForgeAPI(name string, file string) ForgeResult {
func queryForgeAPI(name string, file string, fm ForgeModule) ForgeResult {
//url := "https://forgeapi.puppetlabs.com:443/v3/modules/" + strings.Replace(name, "/", "-", -1)
url := config.Forge.Baseurl+"/v3/modules?query=" + name
baseUrl := config.Forge.Baseurl
if len(fm.baseUrl) > 0 {
baseUrl = fm.baseUrl
}
url := baseUrl + "/v3/modules?query=" + name
req, err := http.NewRequest("GET", url, nil)
if err != nil {
log.Fatal("queryForgeAPI(): Error creating GET request for Puppetlabs forge API", err)
Expand All @@ -123,8 +126,7 @@ func queryForgeAPI(name string, file string) ForgeResult {

proxyURL, err := http.ProxyFromEnvironment(req)
if err != nil {
log.Fatal("queryForgeAPI(): Error while getting http proxy with golang http.ProxyFromEnvironment()", err)
os.Exit(1)
Fatalf("queryForgeAPI(): Error while getting http proxy with golang http.ProxyFromEnvironment()" + err.Error())
}
client := &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxyURL)}}
before := time.Now()
Expand All @@ -143,24 +145,47 @@ func queryForgeAPI(name string, file string) ForgeResult {
// need to get latest version
body, err := ioutil.ReadAll(resp.Body)

//fmt.Println(string(body))
reCurrent := regexp.MustCompile("\\s*\"current_release\": {\n\\s*\"uri\": \"([^\"]+)\",")
if m := reCurrent.FindStringSubmatch(string(body)); len(m) > 1 {
//fmt.Println(m[1])
if strings.Count(m[1], "-") < 2 {
log.Fatal("queryForgeAPI(): Error: Something went wrong while trying to figure out what version is current for Forge module ", name, " ", m[1], " should contain three '-' characters")
os.Exit(1)
} else {
// modified the split because I found a module with version 4.0.0-beta1 mayflower-php
version := strings.Split(m[1], name+"-")[1]
Debugf("queryForgeAPI(): found version " + version + " for " + name + "-latest")
mutex.Lock()
latestForgeModules[name] = version
mutex.Unlock()
return ForgeResult{true, version}
var f interface{}
if err := json.Unmarshal(body, &f); err != nil {
Fatalf("queryForgeAPI(): Error while decoding JSON from URL " + url + " Error: " + err.Error())
}
currentUri := ""
m := f.(map[string]interface{})
for _, v := range m {
switch vv := v.(type) {
case []interface{}:
if len(vv) >= 1 {
if curRel, ok := vv[0].(map[string]interface{})["current_release"]; ok {
if val, ok := curRel.(map[string]interface{})["uri"]; ok {
//fmt.Println("uri --> ", val)
currentUri = val.(string)
} else {
Fatalf("queryForgeAPI(): Error: Unexpected JSON response while trying to figure out what version is current for Forge module " + name + " using " + url)
}
} else {
Fatalf("queryForgeAPI(): Error: Unexpected JSON response while trying to figure out what version is current for Forge module " + name + " using " + url)
}
} else {
Fatalf("queryForgeAPI(): Error: Unexpected JSON response while trying to figure out what version is current for Forge module " + name + " using " + url)
}
default:
// skip, we'll do a sanity for the currentUri value later anyway
}
}

//fmt.Println(string(body))
if strings.Count(currentUri, "-") < 2 {
log.Fatal("queryForgeAPI(): Error: Something went wrong while trying to figure out what version is current for Forge module ", name, " ", currentUri, " should contain three '-' characters")
} else {
// modified the split because I found a module with version 4.0.0-beta1 mayflower-php
version := strings.Split(currentUri, name+"-")[1]
Debugf("queryForgeAPI(): found version " + version + " for " + name + "-latest")
mutex.Lock()
latestForgeModules[name] = version
mutex.Unlock()
return ForgeResult{true, version}
}

if err != nil {
panic(err)
}
Expand All @@ -174,11 +199,15 @@ func queryForgeAPI(name string, file string) ForgeResult {
}
}

func downloadForgeModule(name string, version string) {
func downloadForgeModule(name string, version string, fm ForgeModule) {
//url := "https://forgeapi.puppetlabs.com/v3/files/puppetlabs-apt-2.1.1.tar.gz"
fileName := name + "-" + version + ".tar.gz"
if _, err := os.Stat(config.ForgeCacheDir + name + "-" + version); os.IsNotExist(err) {
url := "https://forgeapi.puppetlabs.com/v3/files/" + fileName
baseUrl := config.Forge.Baseurl
if len(fm.baseUrl) > 0 {
baseUrl = fm.baseUrl
}
url := baseUrl + "/v3/files/" + fileName
req, err := http.NewRequest("GET", url, nil)
req.Header.Set("User-Agent", "https://github.com/xorpaul/g10k/")
req.Header.Set("Connection", "close")
Expand Down Expand Up @@ -307,15 +336,15 @@ func readModuleMetadata(file string) ForgeModule {
return ForgeModule{name: strings.Split(m["name"].(string), "-")[1], version: m["version"].(string), author: strings.ToLower(m["author"].(string))}
}

func resolveForgeModules(modules map[string]struct{}) {
func resolveForgeModules(modules map[string]ForgeModule) {
var wgForge sync.WaitGroup
for m := range modules {
for m, fm := range modules {
wgForge.Add(1)
go func(m string) {
go func(m string, fm ForgeModule) {
defer wgForge.Done()
Debugf("Trying to get forge module " + m)
doModuleInstallOrNothing(m)
}(m)
Debugf("Trying to get forge module " + m + " with Forge base url " + fm.baseUrl)
doModuleInstallOrNothing(m, fm)
}(m, fm)
}
wgForge.Wait()
}
Expand Down Expand Up @@ -371,8 +400,7 @@ func syncForgeToModuleDir(name string, m ForgeModule, moduleDir string) {
}
workDir := config.ForgeCacheDir + moduleName + "-" + m.version + "/"
if _, err := os.Stat(workDir); os.IsNotExist(err) {
log.Print("syncForgeToModuleDir(): Forge module not found in dir: ", workDir)
os.Exit(1)
Fatalf("syncForgeToModuleDir(): Forge module not found in dir: " + workDir)
} else {
Infof("Need to sync " + targetDir)
cmd := "cp --link --archive " + workDir + "* " + targetDir
Expand Down
23 changes: 16 additions & 7 deletions g10k.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ var (
cpGitTime float64
cpForgeTime float64
buildtime string
uniqueForgeModules map[string]struct{}
uniqueForgeModules map[string]ForgeModule
latestForgeModules map[string]string
)

Expand All @@ -46,13 +46,15 @@ type ConfigSettings struct {
privateKey string `yaml:"private_key"`
username string
}
Forge struct {
Baseurl string `yaml:"baseurl"`
}
Forge Forge
Sources map[string]Source
Timeout int `yaml:"timeout"`
}

type Forge struct {
Baseurl string `yaml:"baseurl"`
}

// Source contains basic information about a Puppet environment repository
type Source struct {
Remote string
Expand All @@ -64,17 +66,19 @@ type Source struct {
// Puppetfile contains the key value pairs from the Puppetfile
type Puppetfile struct {
moduleDir string
forgeBaseURL string
forgeModules map[string]ForgeModule
gitModules map[string]GitModule
privateKey string
source string
}

// ForgeModule contains information (Version, Name, Author) about a Puppetlabs Forge module
// ForgeModule contains information (Version, Name, Author, Forge BaseURL if custom) about a Puppetlabs Forge module
type ForgeModule struct {
version string
name string
author string
baseUrl string
}

// GitModule contains information about a Git Puppet module
Expand Down Expand Up @@ -146,7 +150,10 @@ func main() {
before := time.Now()
if len(*configFile) > 0 {
if usemove {
log.Fatalln("Error: -usemove parameter is only allowed in -puppetfile mode!")
Fatalf("Error: -usemove parameter is only allowed in -puppetfile mode!")
}
if pfMode {
Fatalf("Error: -puppetfile parameter is not allowed with -config parameter!")
}
Debugf("Using as config file: " + *configFile)
config = readConfigfile(*configFile)
Expand All @@ -170,7 +177,9 @@ func main() {
} else {
cachedir = checkDirAndCreate(cachedir, "cachedir default value")
}
config = ConfigSettings{CacheDir: cachedir, ForgeCacheDir: cachedir, ModulesCacheDir: cachedir, EnvCacheDir: cachedir, Sources: sm}
//config = ConfigSettings{CacheDir: cachedir, ForgeCacheDir: cachedir, ModulesCacheDir: cachedir, EnvCacheDir: cachedir, Forge:{Baseurl: "https://forgeapi.puppetlabs.com"}, Sources: sm}
forgeDefaultSettings := Forge{Baseurl: "https://forgeapi.puppetlabs.com"}
config = ConfigSettings{CacheDir: cachedir, ForgeCacheDir: cachedir, ModulesCacheDir: cachedir, EnvCacheDir: cachedir, Sources: sm, Forge: forgeDefaultSettings}
target = "./Puppetfile"
puppetfile := readPuppetfile("./Puppetfile", "", "cmdlineparam")
pfm := make(map[string]Puppetfile)
Expand Down
17 changes: 11 additions & 6 deletions helper.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,14 +20,14 @@ func Debugf(s string) {
}
}

// Verbosef is a helper function for debug logging if global variable verbose is set to true
// Verbosef is a helper function for verbose logging if global variable verbose is set to true
func Verbosef(s string) {
if debug != false || verbose != false {
log.Print(fmt.Sprint(s))
}
}

// Infof is a helper function for debug logging if global variable info is set to true
// Infof is a helper function for info logging if global variable info is set to true
func Infof(s string) {
if debug != false || verbose != false || info != false {
color.Set(color.FgGreen)
Expand All @@ -36,6 +36,13 @@ func Infof(s string) {
}
}

// Fatalf is a helper function for fatal logging
func Fatalf(s string) {
color.Set(color.FgRed)
log.Fatal(s)
color.Unset()
}

// fileExists checks if the given file exists and return a bool
func fileExists(file string) bool {
if _, err := os.Stat(file); os.IsNotExist(err) {
Expand All @@ -51,14 +58,12 @@ func checkDirAndCreate(dir string, name string) string {
if _, err := os.Stat(dir); os.IsNotExist(err) {
//log.Printf("checkDirAndCreate(): trying to create dir '%s' as %s", dir, name)
if err := os.MkdirAll(dir, 0777); err != nil {
log.Print("checkDirAndCreate(): Error: failed to create directory: ", dir)
os.Exit(1)
Fatalf("checkDirAndCreate(): Error: failed to create directory: " + dir)
}
}
} else {
// TODO make dir optional
log.Print("dir setting '" + name + "' missing! Exiting!")
os.Exit(1)
Fatalf("checkDirAndCreate(): Error: dir setting '" + name + "' missing! Exiting!")
}
}
if !strings.HasSuffix(dir, "/") {
Expand Down
9 changes: 7 additions & 2 deletions puppetfile.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ func resolvePuppetEnvironment(envBranch string) {
func resolvePuppetfile(allPuppetfiles map[string]Puppetfile) {
var wg sync.WaitGroup
uniqueGitModules := make(map[string]GitModule)
uniqueForgeModules = make(map[string]struct{})
uniqueForgeModules = make(map[string]ForgeModule)
latestForgeModules = make(map[string]string)
exisitingModuleDirs := make(map[string]struct{})
for env, pf := range allPuppetfiles {
Expand All @@ -112,10 +112,15 @@ func resolvePuppetfile(allPuppetfiles map[string]Puppetfile) {
}
for forgeModuleName, fm := range pf.forgeModules {
//fmt.Println("Found Forge module ", forgeModuleName, " with version", fm.version)
if len(pf.forgeBaseURL) > 0 {
fm.baseUrl = pf.forgeBaseURL
} else {
fm.baseUrl = ""
}
mutex.Lock()
forgeModuleName = strings.Replace(forgeModuleName, "/", "-", -1)
if _, ok := uniqueForgeModules[forgeModuleName+"-"+fm.version]; !ok {
uniqueForgeModules[forgeModuleName+"-"+fm.version] = empty
uniqueForgeModules[forgeModuleName+"-"+fm.version] = fm
}
mutex.Unlock()
}
Expand Down

0 comments on commit 7f67550

Please sign in to comment.