Merge branch 'master' into cli-multi-files

CHANGELOG.md

@@ -25,6 +25,7 @@ Unreleased changes are available as `avenga/couper:edge` container.
     * added `request.context.beta_required_permission` and `request.context.beta_granted_permissions` [request variables](./docs/REFERENCE.md#request)
   * Clarified the type of various [attributes/variables](./docs/REFERENCE.md) ([#485](https://github.com/avenga/couper/pull/485))
   * [`spa` block](./docs/REFERENCE.md#spa-block) can be defined multiple times now ([#510](https://github.com/avenga/couper/pull/510))
+  * [`files` block](./docs/REFERENCE.md#files-block) can be defined multiple times now ([#513](https://github.com/avenga/couper/pull/513))
 
 * **Fixed**
   * Keys in object type attribute values are only handled case-insensitively if reasonable (e.g. they represent HTTP methods or header field values) ([#461](https://github.com/avenga/couper/pull/461))

cache/memory.go

@@ -2,6 +2,7 @@ package cache
 
 import (
 	"runtime/debug"
+	"strings"
 	"sync"
 	"time"
 
@@ -60,6 +61,23 @@ func (ms *MemoryStore) Get(k string) interface{} {
 	return nil
 }
 
+func (ms *MemoryStore) GetAllWithPrefix(prefix string) []interface{} {
+	var list []interface{}
+
+	ms.mu.RLock()
+	defer ms.mu.RUnlock()
+
+	for k, v := range ms.db {
+		if time.Now().Unix() >= v.expAt || !strings.HasPrefix(k, prefix) {
+			continue
+		}
+
+		list = append(list, v.value)
+	}
+
+	return list
+}
+
 // Set stores a key/value pair for <ttl> second(s) into the <MemoryStore>.
 func (ms *MemoryStore) Set(k string, v interface{}, ttl int64) {
 	if ttl < 0 {

command/run.go

@@ -20,6 +20,7 @@ import (
 	"github.com/avenga/couper/config/env"
 	"github.com/avenga/couper/config/runtime"
 	"github.com/avenga/couper/errors"
+	"github.com/avenga/couper/eval"
 	"github.com/avenga/couper/server"
 	"github.com/avenga/couper/server/writer"
 	"github.com/avenga/couper/telemetry"
@@ -100,6 +101,9 @@ func (r *Run) Execute(args Args, config *config.Couper, logEntry *logrus.Entry)
 	*r.settings = *config.Settings
 	r.settingsMu.Unlock()
 
+	// apply command context
+	config.Context = config.Context.(*eval.Context).WithContext(r.context)
+
 	if f := r.flagSet.Lookup("accept-forwarded-url"); f != nil {
 		if afv, ok := f.Value.(*AcceptForwardedValue); ok {
 			afv.settings = r.settings
@@ -136,6 +140,14 @@ func (r *Run) Execute(args Args, config *config.Couper, logEntry *logrus.Entry)
 		logEntry.Infof("configured with ca-certificate: %s", config.Settings.CAFile)
 	}
 
+	memStore := cache.New(logEntry, r.context.Done())
+	// logEntry has still the 'daemon' type which can be used for config related load errors.
+	srvConf, err := runtime.NewServerConfiguration(config, logEntry, memStore)
+	if err != nil {
+		return err
+	}
+	errors.SetLogger(logEntry)
+
 	telemetry.InitExporter(r.context, &telemetry.Options{
 		MetricsCollectPeriod: time.Second * 2,
 		Metrics:              r.settings.TelemetryMetrics,
@@ -145,14 +157,7 @@ func (r *Run) Execute(args Args, config *config.Couper, logEntry *logrus.Entry)
 		ServiceName:          r.settings.TelemetryServiceName,
 		Traces:               r.settings.TelemetryTraces,
 		TracesEndpoint:       r.settings.TelemetryTracesEndpoint,
-	}, logEntry)
-
-	// logEntry has still the 'daemon' type which can be used for config related load errors.
-	srvConf, err := runtime.NewServerConfiguration(config, logEntry, cache.New(logEntry, r.context.Done()))
-	if err != nil {
-		return err
-	}
-	errors.SetLogger(logEntry)
+	}, memStore, logEntry)
 
 	if limitFn != nil {
 		limitFn(logEntry)

command/run_test.go

@@ -11,6 +11,8 @@ import (
 	"reflect"
 	"runtime"
 	"strconv"
+	"strings"
+	"sync/atomic"
 	"testing"
 	"time"
 
@@ -253,7 +255,6 @@ func TestArgs_CAFile(t *testing.T) {
 	}
 
 	expiresIn := time.Minute
-	//expires := time.After(expiresIn)
 	selfSigned, err := server.NewCertificate(expiresIn, nil, nil)
 	helper.Must(err)
 
@@ -264,7 +265,12 @@ func TestArgs_CAFile(t *testing.T) {
 	helper.Must(tmpFile.Close())
 	defer os.Remove(tmpFile.Name())
 
+	var healthCheckSeen uint32
+
 	srv := httptest.NewTLSServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
+		if strings.HasSuffix(request.Header.Get("User-Agent"), "health-check") {
+			atomic.StoreUint32(&healthCheckSeen, 1)
+		}
 		writer.WriteHeader(http.StatusNoContent)
 
 		// force close to trigger a new handshake
@@ -286,7 +292,16 @@ func TestArgs_CAFile(t *testing.T) {
 	couperHCL := `server {
 	endpoint "/" {
 		request {
-			url = "` + srv.URL + `"
+			backend = "tls"
+		}
+	}
+}
+
+definitions {
+	backend "tls" {
+		origin = "` + srv.URL + `"
+		beta_health {
+			failure_threshold = 0
 		}
 	}
 }`
@@ -318,16 +333,9 @@ func TestArgs_CAFile(t *testing.T) {
 		t.Error("unexpected status code")
 	}
 
-	// ca after
-	//<-expires
-
-	// handshake error
-	//res, err = client.Do(req)
-	//helper.Must(err)
-	//
-	//if res.StatusCode != http.StatusBadGateway {
-	//	t.Error("unexpected status code")
-	//}
+	if atomic.LoadUint32(&healthCheckSeen) != 1 {
+		t.Error("expected a successful tls health check")
+	}
 }
 
 func TestCAFile_Run(t *testing.T) {

config/backend.go

@@ -53,6 +53,7 @@ func (b Backend) Inline() interface{} {
 		ResponseStatus *uint8                    `hcl:"set_response_status,optional"`
 		TTFBTimeout    string                    `hcl:"ttfb_timeout,optional"`
 		Timeout        string                    `hcl:"timeout,optional"`
+		UseUnhealthy   bool                      `hcl:"use_when_unhealthy,optional"`
 
 		// set by backend preparation
 		BackendURL string `hcl:"backend_url,optional"`

config/configload/load.go

@@ -31,6 +31,7 @@ const (
 	definitions  = "definitions"
 	endpoint     = "endpoint"
 	errorHandler = "error_handler"
+	files        = "files"
 	nameLabel    = "name"
 	oauth2       = "oauth2"
 	proxy        = "proxy"
@@ -400,9 +401,23 @@ func LoadConfig(body hcl.Body, src []byte, filename string) (*config.Couper, err
 			if len(spaBlock.Labels) > 0 {
 				spaConfig.Name = spaBlock.Labels[0]
 			}
+
 			serverConfig.SPAs = append(serverConfig.SPAs, spaConfig)
 		}
 
+		for _, filesBlock := range bodyToContent(serverConfig.Remain).Blocks.OfType(files) {
+			filesConfig := &config.Files{}
+			if diags := gohcl.DecodeBody(filesBlock.Body, helper.context, filesConfig); diags.HasErrors() {
+				return nil, diags
+			}
+
+			if len(filesBlock.Labels) > 0 {
+				filesConfig.Name = filesBlock.Labels[0]
+			}
+
+			serverConfig.Files = append(serverConfig.Files, filesConfig)
+		}
+
 		// standalone endpoints
 		err = refineEndpoints(helper, serverConfig.Endpoints, true)
 		if err != nil {

config/configload/merge.go

@@ -41,8 +41,18 @@ func mergeServers(bodies []*hclsyntax.Body) (hclsyntax.Blocks, error) {
 			attributes      hclsyntax.Attributes
 			blocks          namedBlocks
 		}
+		filesDefinition struct {
+			labels          []string
+			typeRange       hcl.Range
+			labelRanges     []hcl.Range
+			openBraceRange  hcl.Range
+			closeBraceRange hcl.Range
+			attributes      hclsyntax.Attributes
+			blocks          namedBlocks
+		}
 		namedAPIs        map[string]*apiDefinition
 		namedSPAs        map[string]*spaDefinition
+		namedFiles       map[string]*filesDefinition
 		serverDefinition struct {
 			labels          []string
 			typeRange       hcl.Range
@@ -54,6 +64,7 @@ func mergeServers(bodies []*hclsyntax.Body) (hclsyntax.Blocks, error) {
 			endpoints       namedBlocks
 			apis            namedAPIs
 			spas            namedSPAs
+			files           namedFiles
 		}
 
 		servers map[string]*serverDefinition
@@ -70,6 +81,14 @@ func mergeServers(bodies []*hclsyntax.Body) (hclsyntax.Blocks, error) {
 				endpoints[<key>]     = hclsyntax.Block
 				error_handler[<key>] = hclsyntax.Block
 			}
+			spas[<key>]      = {
+				attributes           = hclsyntax.Attributes
+				blocks[<name>]       = hclsyntax.Block (cors)
+			}
+			files[<key>]      = {
+				attributes           = hclsyntax.Attributes
+				blocks[<name>]       = hclsyntax.Block (cors)
+			}
 		}
 	*/
 
@@ -112,6 +131,7 @@ func mergeServers(bodies []*hclsyntax.Body) (hclsyntax.Blocks, error) {
 					endpoints:       make(namedBlocks),
 					apis:            make(namedAPIs),
 					spas:            make(namedSPAs),
+					files:           make(namedFiles),
 				}
 			}
 
@@ -120,7 +140,7 @@ func mergeServers(bodies []*hclsyntax.Body) (hclsyntax.Blocks, error) {
 			}
 
 			for _, block := range outerBlock.Body.Blocks {
-				uniqueAPILabels, uniqueSPALabels := make(map[string]struct{}), make(map[string]struct{})
+				uniqueAPILabels, uniqueSPALabels, uniqueFilesLabels := make(map[string]struct{}), make(map[string]struct{}), make(map[string]struct{})
 
 				if block.Type == endpoint {
 					if err := absInBackends(block); err != nil { // Backend block inside a free endpoint block
@@ -228,7 +248,43 @@ func mergeServers(bodies []*hclsyntax.Body) (hclsyntax.Blocks, error) {
 					for _, subBlock := range block.Body.Blocks {
 						results[serverKey].spas[spaKey].blocks[subBlock.Type] = subBlock
 					}
+				} else if block.Type == files {
+					var filesKey string
+
+					if len(block.Labels) > 0 {
+						filesKey = block.Labels[0]
+					}
+
+					if len(bodies) > 1 {
+						if _, ok := uniqueFilesLabels[filesKey]; ok {
+							return nil, newMergeError(errUniqueLabels, block)
+						}
+
+						uniqueFilesLabels[filesKey] = struct{}{}
+					} else {
+						// Create unique key for multiple files blocks inside a single config file.
+						filesKey += fmt.Sprintf("|%p", &filesKey)
+					}
 
+					if results[serverKey].files[filesKey] == nil {
+						results[serverKey].files[filesKey] = &filesDefinition{
+							labels:          block.Labels,
+							typeRange:       block.TypeRange,
+							labelRanges:     block.LabelRanges,
+							openBraceRange:  block.OpenBraceRange,
+							closeBraceRange: block.CloseBraceRange,
+							attributes:      make(hclsyntax.Attributes),
+							blocks:          make(namedBlocks),
+						}
+					}
+
+					for name, attr := range block.Body.Attributes {
+						results[serverKey].files[filesKey].attributes[name] = attr
+					}
+
+					for _, subBlock := range block.Body.Blocks {
+						results[serverKey].files[filesKey].blocks[subBlock.Type] = subBlock
+					}
 				} else {
 					results[serverKey].blocks[block.Type] = block
 				}
@@ -303,6 +359,29 @@ func mergeServers(bodies []*hclsyntax.Body) (hclsyntax.Blocks, error) {
 			serverBlocks = append(serverBlocks, mergedSPA)
 		}
 
+		for _, filesBlock := range serverBlock.files {
+			var filesBlocks hclsyntax.Blocks
+
+			for _, b := range filesBlock.blocks {
+				filesBlocks = append(filesBlocks, b)
+			}
+
+			mergedFiles := &hclsyntax.Block{
+				Type:   files,
+				Labels: filesBlock.labels,
+				Body: &hclsyntax.Body{
+					Attributes: filesBlock.attributes,
+					Blocks:     filesBlocks,
+				},
+				TypeRange:       filesBlock.typeRange,
+				LabelRanges:     filesBlock.labelRanges,
+				OpenBraceRange:  filesBlock.openBraceRange,
+				CloseBraceRange: filesBlock.closeBraceRange,
+			}
+
+			serverBlocks = append(serverBlocks, mergedFiles)
+		}
+
 		mergedServer := &hclsyntax.Block{
 			Type:   server,
 			Labels: serverBlock.labels,

config/configload/schema.go

@@ -229,7 +229,7 @@ func completeSchemaComponents(body hcl.Body, schema *hcl.BodySchema, attrs hcl.A
 			added := false
 			for _, block := range bodyContent.Blocks {
 				switch block.Type {
-				case api, backend, errorHandler, proxy, request, server, spa:
+				case api, backend, errorHandler, proxy, request, server, spa, files:
 					blocks = append(blocks, block)
 
 					added = true

config/files.go

@@ -7,6 +7,8 @@ import (
 
 var _ Inline = &Files{}
 
+type FilesBlocks []*Files
+
 // Files represents the <Files> object.
 type Files struct {
 	AccessControl        []string `hcl:"access_control,optional"`
@@ -15,6 +17,7 @@ type Files struct {
 	DisableAccessControl []string `hcl:"disable_access_control,optional"`
 	DocumentRoot         string   `hcl:"document_root"`
 	ErrorFile            string   `hcl:"error_file,optional"`
+	Name                 string   `hcl:"name,label"`
 	Remain               hcl.Body `hcl:",remain"`
 }