reworked cacher interface and http cache implementation. #96

cache/cache.go

@@ -1,22 +1,14 @@
 package cache
 
-import (
-	"fmt"
-	"io"
-)
+import "fmt"
 
 //	Cacher defines a cache back end
 type Cacher interface {
-	Get(key string) (Item, error)
-	Set(key string, value io.Reader) error
+	Get(key string) ([]byte, error)
+	Set(key string, value []byte) error
 	Purge(key string) error
 }
 
-type Item interface {
-	Read(p []byte) (int, error)
-	Write(p []byte) (int, error)
-}
-
 // InitFunc initilize a cache given a config map.
 // The InitFunc should validate the config map, and report any errors.
 // This is called by the For function.

cache/filecache/filecache.go

@@ -2,8 +2,10 @@ package filecache
 
 import (
 	"errors"
+	"io/ioutil"
 	"os"
 	"path/filepath"
+	"sync"
 
 	"github.com/terranodo/tegola/cache"
 	"github.com/terranodo/tegola/util/dict"
@@ -49,43 +51,60 @@ func New(config map[string]interface{}) (cache.Cacher, error) {
 
 	return &Filecache{
 		Basepath: basepath,
+		Locker:   map[string]sync.RWMutex{},
 	}, nil
 }
 
 type Filecache struct {
 	Basepath string
 
-	//	locker tracks which cache keys are being operated on.
-	//	when the cache is being written to a Lock() is used.
+	//	Locker tracks which cache keys are being operated on.
+	//	when the cache is being written to a Lock() is set.
 	//	when being read from an RLock() is used so we don't
-	//	block concurrent reads
+	//	block concurrent reads.
 	//
 	//	TODO: currently the map keys are not cleaned up after they're
 	//	created. this will cause more memory to be used.
-	locker map[string]Item
+	Locker map[string]sync.RWMutex
 }
 
-func (fc *Filecache) Get(key string) (*os.File, error) {
+func (fc *Filecache) Get(key string) ([]byte, error) {
 	path := filepath.Join(fc.Basepath, key)
-	/*
-		mutex, ok := fc.locker[key]
-		if !ok {
-			fc.locker[key] = sync.RWMutex{}
-			mutex = fc.locker[key]
-		}
-		mutex.RLock()
-		defer mutex.RUnlock()
-	*/
-	return os.Open(path)
+
+	//	lookup our mutex
+	mutex, ok := fc.Locker[key]
+	if !ok {
+		fc.Locker[key] = sync.RWMutex{}
+		mutex = fc.Locker[key]
+	}
+	//	read lock
+	mutex.RLock()
+	defer mutex.RUnlock()
+
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+
+	return ioutil.ReadAll(f)
 }
 
-/*
-func (fc *Filecache) Set(key string) (*os.File, error) {
+func (fc *Filecache) Set(key string, val []byte) error {
 	var err error
 
 	//	build our filepath
 	path := filepath.Join(fc.Basepath, key)
 
+	//	lookup our mutex
+	mutex, ok := fc.Locker[key]
+	if !ok {
+		fc.Locker[key] = sync.RWMutex{}
+		mutex = fc.Locker[key]
+	}
+	//	write lock
+	mutex.Lock()
+	defer mutex.Unlock()
+
 	//	the key can have a directory syntax so we need to makeAll
 	if err = os.MkdirAll(filepath.Dir(path), os.ModePerm); err != nil {
 		return err
@@ -99,29 +118,13 @@ func (fc *Filecache) Set(key string) (*os.File, error) {
 	defer f.Close()
 
 	//	copy the contents
-	_, err = io.Copy(f, value)
+	_, err = f.Write(val)
 	if err != nil {
 		return err
 	}
 
 	return nil
 }
-*/
-
-func (fc *Filecache) Set(key string) (*os.File, error) {
-	var err error
-
-	//	build our filepath
-	path := filepath.Join(fc.Basepath, key)
-
-	//	the key can have a directory syntax so we need to makeAll
-	if err = os.MkdirAll(filepath.Dir(path), os.ModePerm); err != nil {
-		return nil, err
-	}
-
-	//	create the file
-	return os.Create(path)
-}
 
 func (fc *Filecache) Purge(key string) error {
 	path := filepath.Join(fc.Basepath, key)

cache/filecache/filecache_test.go

@@ -1,8 +1,8 @@
 package filecache_test
 
 import (
-	"io/ioutil"
 	"reflect"
+	"sync"
 	"testing"
 
 	"github.com/terranodo/tegola/cache/filecache"
@@ -19,6 +19,7 @@ func TestNew(t *testing.T) {
 			},
 			expected: &filecache.Filecache{
 				Basepath: "testfiles/tegola-cache",
+				Locker:   map[string]sync.RWMutex{},
 			},
 		},
 	}
@@ -37,7 +38,7 @@ func TestNew(t *testing.T) {
 	}
 }
 
-func TestWriteReadPurge(t *testing.T) {
+func TestSetGetPurge(t *testing.T) {
 	testcases := []struct {
 		config   map[string]interface{}
 		key      string
@@ -53,40 +54,24 @@ func TestWriteReadPurge(t *testing.T) {
 	}
 
 	for i, tc := range testcases {
-		var err error
-
 		fc, err := filecache.New(tc.config)
 		if err != nil {
 			t.Errorf("testcase (%v) failed. err: %v", i, err)
 			continue
 		}
 
 		//	test write
-		w, err := fc.Set(tc.key)
-		if err != nil {
+		if err = fc.Set(tc.key, tc.expected); err != nil {
 			t.Errorf("testcase (%v) write failed. err: %v", i, err)
 			continue
 		}
 
-		_, err = w.Write(input)
-		if err != nil {
-			t.Errorf("testcase (%v) write failed. err: %v", i, err)
-			continue
-		}
-
-		r, err := fc.Get(tc.key)
+		output, err := fc.Get(tc.key)
 		if err != nil {
 			t.Errorf("testcase (%v) read failed. err: %v", i, err)
 			continue
 		}
 
-		//	test read
-		output, err := ioutil.ReadAll(r)
-		if err != nil {
-			t.Errorf("testcase (%v) readAll failed. err: %v", i, err)
-			continue
-		}
-
 		if !reflect.DeepEqual(output, tc.expected) {
 			t.Errorf("testcase (%v) failed. output (%v) does not match expected (%v)", i, output, tc.expected)
 			continue

server/middleware_cache.go

@@ -1,10 +1,6 @@
 package server
 
-import (
-	"io"
-	"log"
-	"net/http"
-)
+import "net/http"
 
 func CacheHandler(next http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -19,14 +15,13 @@ func CacheHandler(next http.Handler) http.Handler {
 		cachedTile, err := Cache.Get(r.URL.Path)
 		if err != nil {
 			//	TODO: this should be a debug warning
-			log.Printf("cache err: %v", err)
-			cWriter, err := Cache.GetWriter(r.URL.Path)
-			if err != nil {
-				log.Printf("cache newWriter err: %v", err)
-			}
+			//	log.Printf("cache err: %v", err)
 
-			//	ovewrite our current response writer with the cache writer
-			w = newCacheResponseWriter(w, cWriter)
+			//	ovewrite our current responseWriter with a cacheResponseWriter
+			w = &cacheResponseWriter{
+				cacheKey: r.URL.Path,
+				resp:     w,
+			}
 
 		} else {
 			//	TODO: how configurable do we want the CORS policy to be?
@@ -39,39 +34,33 @@ func CacheHandler(next http.Handler) http.Handler {
 			//	communicate the cache is being used
 			w.Header().Add("Tegola-Cache", "HIT")
 
-			io.Copy(w, cachedTile)
+			w.Write(cachedTile)
 			return
 		}
 
 		next.ServeHTTP(w, r)
 	})
 }
 
-func newCacheResponseWriter(resp http.ResponseWriter, writers ...io.Writer) http.ResponseWriter {
-	//	communicate the cache is being used
-	resp.Header().Add("Tegola-Cache", "MISS")
-
-	writers = append(writers, resp)
-
-	return &cacheResponseWriter{
-		resp:  resp,
-		multi: io.MultiWriter(writers...),
-	}
-}
-
+//	cacheResponsWriter wraps http.ResponseWriter (https://golang.org/pkg/net/http/#ResponseWriter)
+//	to also write the response to a cache when there is a cache MISS
 type cacheResponseWriter struct {
-	resp  http.ResponseWriter
-	multi io.Writer
+	cacheKey string
+	resp     http.ResponseWriter
 }
 
-// implement http.ResponseWriter
-// https://golang.org/pkg/net/http/#ResponseWriter
 func (w *cacheResponseWriter) Header() http.Header {
+	//	communicate the tegola cache is being used
+	w.resp.Header().Add("Tegola-Cache", "MISS")
+
 	return w.resp.Header()
 }
 
 func (w *cacheResponseWriter) Write(b []byte) (int, error) {
-	return w.multi.Write(b)
+	//	after we write the response, persist the data to the cache
+	defer Cache.Set(w.cacheKey, b)
+
+	return w.resp.Write(b)
 }
 
 func (w *cacheResponseWriter) WriteHeader(i int) {