MaxThreads configurable; defaults to number of cpus

builder.go

@@ -44,6 +44,22 @@ func copyImages(source string, dest string) {
 	}
 }
 
+func prepOutput(dir string) {
+	log.Printf("Prepping %s\n", dir)
+	if dir == "" {
+		log.Fatal("dir empty, unexpected")
+	}
+	os.MkdirAll(dir, 0755)   // Make sure it exists, so that RemoveAll won't fail
+	err := os.RemoveAll(dir) // Remove all - including old files, subdirs, etc.
+	if err != nil {
+		log.Fatal(err)
+	}
+	err = os.MkdirAll(dir, 0755) // Make sure the directory now exists, for real.
+	if err != nil {
+		log.Fatal(err)
+	}
+}
+
 func main() {
 	flag.Parse()
 	log.SetFlags(log.LstdFlags | log.Lshortfile)
@@ -57,6 +73,8 @@ func main() {
 		log.Fatal(err)
 	}
 
+	prepOutput(conf.Directories.OutputDir)
+
 	var postTable = []job.PostInfoType{
 		{
 			Directory:   "css",
@@ -109,7 +127,7 @@ func main() {
 	}
 
 	// Start the job queue for templates
-	jobTracker := job.StartQueue()
+	jobTracker := job.StartQueue(conf.Options.MaxThreads)
 
 	// Load all langauges, calculate all percentages of completion.
 	languages, err := po.LoadAll(conf.Directories.PoDir+"/falling-sky.pot", conf.Directories.PoDir+"/dl")

config/config.go

@@ -22,7 +22,10 @@ type Record struct {
 		PHP    []string
 		Apache []string
 	}
-	Map map[string]string
+	Map     map[string]string
+	Options struct {
+		MaxThreads int
+	}
 }
 
 // Defaults will update a config record with safe defaults for any missing values
@@ -75,7 +78,7 @@ func (r *Record) Defaults() {
 		}
 	}
 	if len(r.Processors.Apache) == 0 {
-		r.Processors.PHP = []string{
+		r.Processors.Apache = []string{
 		//			`mv [NAME].orig [NAME]`,
 		}
 	}

job/job.go

@@ -320,7 +320,7 @@ func ProcessContent(qi *QueueItem, content string) {
 
 		// Compress in memory
 		b := &bytes.Buffer{}
-		w, err := gzip.NewWriterLevel(b, 9)
+		w, err := gzip.NewWriterLevel(b, gzip.BestCompression)
 		if err != nil {
 			log.Fatal(err)
 		}
@@ -399,13 +399,15 @@ func (qt *QueueTracker) Wait() {
 
 // StartQueue will start a goroutine for jobs, and return
 // a handle to be used for adding and waiting on jobs.
-func StartQueue() *QueueTracker {
+func StartQueue(maxjobs int) *QueueTracker {
 	qt := &QueueTracker{}
 	qt.Channel = make(chan *QueueItem, 10000)
 	qt.WG = &sync.WaitGroup{}
 
-	maxjobs := runtime.NumCPU()
-	maxjobs = 1
+	if maxjobs == 0 {
+		maxjobs = runtime.NumCPU()
+	}
+	//maxjobs = 1
 	for i := 0; i < maxjobs; i++ {
 		go qt.RunQueue()
 	}

signature/signature.go

@@ -25,7 +25,7 @@ func ScanDir(directory string, otherstuff ...string) string {
 		}
 
 		fn := directory + "/" + file
-		log.Printf("scanning %s\n", fn)
+		//	log.Printf("scanning %s\n", fn)
 
 		// This will cache, saving a trip for other jobs
 		content, err := fileutil.ReadFile(fn)