Setup test fixtures for resolvers

.github/workflows/go-test.yml

@@ -25,3 +25,5 @@ jobs:
       run: test -z $(gofmt -l .)
     - name: Go test
       run: go test -short -v ./...
+      env:
+        RILL_RUNTIME_DRUID_TEST_DSN: ${{ secrets.RILL_RUNTIME_DRUID_TEST_DSN }}

go.mod

@@ -261,6 +261,7 @@ require (
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-runewidth v0.0.15 // indirect
 	github.com/mattn/go-sqlite3 v2.0.3+incompatible // indirect
+	github.com/maxatome/go-testdeep v1.14.0 // indirect
 	github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect
 	github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
 	github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect

go.sum

@@ -1718,6 +1718,8 @@ github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJK
 github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
+github.com/maxatome/go-testdeep v1.14.0 h1:rRlLv1+kI8eOI3OaBXZwb3O7xY3exRzdW5QyX48g9wI=
+github.com/maxatome/go-testdeep v1.14.0/go.mod h1:lPZc/HAcJMP92l7yI6TRz1aZN5URwUBUAfUNvrclaNM=
 github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88Jz2VyhSmden33/aXg4oVIY=
 github.com/mazznoer/csscolorparser v0.1.3 h1:vug4zh6loQxAUxfU1DZEu70gTPufDPspamZlHAkKcxE=
 github.com/mazznoer/csscolorparser v0.1.3/go.mod h1:Aj22+L/rYN/Y6bj3bYqO3N6g1dtdHtGfQ32xZ5PJQic=

runtime/drivers/clickhouse/olap.go

@@ -567,6 +567,8 @@ func databaseTypeToPB(dbt string, nullable bool) (*runtimev1.Type, error) {
 		t.Code = runtimev1.Type_CODE_STRING
 	case "OTHER":
 		t.Code = runtimev1.Type_CODE_JSON
+	case "NOTHING":
+		t.Code = runtimev1.Type_CODE_STRING
 	default:
 		match = false
 	}

runtime/drivers/druid/druidsqldriver/druid_api_sql_driver.go

@@ -11,6 +11,7 @@ import (
 	"net/http"
 	"reflect"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/google/uuid"
@@ -65,12 +66,18 @@ func (c *sqlConnection) QueryContext(ctx context.Context, query string, args []d
 	})
 	req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.dsn, bodyReader)
 	if err != nil {
+		if strings.Contains(err.Error(), c.dsn) { // avoid returning the actual DSN with the password which will be logged
+			return nil, fmt.Errorf("invalid dsn")
+		}
 		return nil, err
 	}
 
 	req.Header.Add("Content-Type", "application/json")
 	resp, err := c.client.Do(req)
 	if err != nil {
+		if strings.Contains(err.Error(), c.dsn) { // avoid returning the actual DSN with the password which will be logged
+			return nil, fmt.Errorf("invalid dsn")
+		}
 		return nil, err
 	}
 

runtime/resolver.go

@@ -8,7 +8,9 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"strconv"
 
+	"github.com/mitchellh/hashstructure/v2"
 	runtimev1 "github.com/rilldata/rill/proto/gen/rill/runtime/v1"
 	"github.com/rilldata/rill/runtime/drivers"
 	"github.com/rilldata/rill/runtime/pkg/jsonval"
@@ -135,6 +137,15 @@ func (r *Runtime) Resolve(ctx context.Context, opts *ResolveOptions) (ResolverRe
 	if _, err := hash.Write([]byte(resolver.Key())); err != nil {
 		return nil, err
 	}
+	if opts.Claims.UserAttributes != nil {
+		h, err := hashstructure.Hash(opts.Claims.UserAttributes, hashstructure.FormatV2, nil)
+		if err != nil {
+			return nil, err
+		}
+		if _, err = hash.Write([]byte(strconv.FormatUint(h, 16))); err != nil {
+			return nil, err
+		}
+	}
 	for _, ref := range resolver.Refs() {
 		res, err := ctrl.Get(ctx, ref, false)
 		if err != nil {

runtime/resolvers/clickhouse_resolvers_test.yaml

@@ -0,0 +1,91 @@
+project:
+  models:
+    ad_bids_mini.yaml: 
+      type: model
+      sql: SELECT * FROM url('https://raw.githubusercontent.com/rilldata/rill/main/runtime/testruntime/testdata/ad_bids/data/AdBids_mini.csv', CSV)
+      output:
+        columns: (id UInt32,timestamp DateTime64,publisher varchar,domain varchar,bid_price Float32,volume UInt8,impressions UInt8,"ad words" varchar,clicks Float32,device varchar)
+        materialize: true
+        incremental_strategy: append
+  dashboards:
+    ad_bids_mini_metrics_with_policy.yaml:
+      model: ad_bids_mini
+      display_name: Ad bids
+      description:
+
+      timeseries: timestamp
+      smallest_time_grain: ""
+
+      dimensions:
+      - label: Publisher
+        name: publisher
+        expression: publisher
+        description: ""
+      - label: Domain
+        property: domain
+        description: ""
+
+      measures:
+      - label: "Number of bids"
+        name: bid's number
+        expression: count(*)
+      - label: "Total volume"
+        name: total volume
+        expression: sum(volume)
+      - label: "Total impressions"
+        name: total impressions
+        expression: sum(impressions)
+      - label: "Total clicks"
+        name: total click"s
+        expression: sum(clicks)
+
+      security:
+        access: true
+        row_filter: "domain = '{{ .user.domain }}'"
+        exclude:
+          - if: "'{{ .user.domain }}' != 'msn.com'"
+            names: 
+              - total volume
+  apis:
+    mv_sql_policy_api.yaml:
+      kind : api
+      metrics_sql: | 
+        select 
+            publisher,
+            domain, 
+            "total impressions"
+        FROM 
+          ad_bids_mini_metrics_with_policy 
+connectors:
+   clickhouse: ~
+tests:
+  simple:
+    resolver: mv_sql_policy_api 
+    options:
+      claims:
+        user_attributes:
+          domain: yahoo.com 
+          email: user@yahoo.com
+    result:
+      - domain: yahoo.com
+        publisher: Yahoo
+        "total impressions": 3.0
+  msn:
+    resolver: mv_sql_policy_api 
+    options:
+      claims:
+        user_attributes:
+          domain: msn.com 
+          email: user@msn.com
+    result:
+      - domain: msn.com
+        publisher: ""
+        "total impressions": 3.0
+  empty:
+    resolver: mv_sql_policy_api 
+    options:
+      claims:
+        user_attributes:
+          domain: google.com 
+          email: user@google.com
+    result: ~

runtime/resolvers/druid_resolvers_test.yaml

@@ -0,0 +1,57 @@
+project:
+  sources: {}
+  models: {}
+  dashboards:
+    ad_bids_mini_metrics_with_policy.yaml:
+      model: AdBids
+      display_name: Ad bids
+      description:
+      timeseries: __time
+      smallest_time_grain: ""
+      dimensions:
+        - label: Publisher
+          name: publisher
+          expression: publisher
+          description: ""
+        - label: Domain
+          property: domain
+          description: ""
+      measures:
+        - label: "Number of bids"
+          name: bid's number
+          expression: count(*)
+        - label: "Max bid price"
+          name: max bid price
+          expression: max(bid_price)
+        - label: "min bid price"
+          name: min bid price
+          expression: min(bid_price)
+      security:
+        access: true
+        # row_filter: "domain = '{{ .user.domain }}'" is not supported in Druid
+        # exclude: is not supported in Druid
+  apis:
+    mv_sql_policy_api.yaml:
+      kind: api
+      metrics_sql: "select \n    publisher,\n    domain, \n    \"min bid price\",\n    \"max bid price\"\nFROM \n  ad_bids_mini_metrics_with_policy \nWHERE\n  publisher is not null AND domain = 'news.yahoo.com'\nORDER BY \n  publisher,\n  domain\nLIMIT 1\n"
+connectors:
+  druid: null
+tests:
+  simple:
+    resolver: mv_sql_policy_api
+    options:
+      instanceid: ""
+      resolver: ""
+      resolverproperties: {}
+      args: {}
+      claims:
+        user_attributes:
+          domain: yahoo.com
+          email: user@yahoo.com
+        additionalrules: []
+        skipchecks: false
+    result:
+      - domain: news.yahoo.com
+        max bid price: 6.00
+        min bid price: 1.00
+        publisher: Yahoo

runtime/resolvers/duckdb_resolvers_test.yaml

@@ -0,0 +1,99 @@
+project:
+  sources:
+    ad_bids_mini_source.yaml:
+      connector: https
+      path: https://raw.githubusercontent.com/rilldata/rill/main/runtime/testruntime/testdata/ad_bids/data/AdBids_mini.csv
+  models:
+    "ad_bids_mini.yaml": 
+      sql: |
+        select
+          id,
+          timestamp,
+          publisher,
+          domain,
+          volume,
+          impressions,
+          clicks
+        from ad_bids_mini_source
+  dashboards:
+    ad_bids_mini_metrics_with_policy.yaml:
+      model: ad_bids_mini
+      display_name: Ad bids
+      description:
+
+      timeseries: timestamp
+      smallest_time_grain: ""
+
+      dimensions:
+      - label: Publisher
+        name: publisher
+        expression: upper(publisher)
+        description: ""
+      - label: Domain
+        property: domain
+        description: ""
+
+      measures:
+      - label: "Number of bids"
+        name: bid's number
+        expression: count(*)
+      - label: "Total volume"
+        name: total volume
+        expression: sum(volume)
+      - label: "Total impressions"
+        name: total impressions
+        expression: sum(impressions)
+      - label: "Total clicks"
+        name: total click"s
+        expression: sum(clicks)
+
+      security:
+        access: true
+        row_filter: "domain = '{{ .user.domain }}'"
+        exclude:
+          - if: "'{{ .user.domain }}' != 'msn.com'"
+            names: 
+              - total volume
+  apis:
+    mv_sql_policy_api.yaml:
+      kind : api
+      metrics_sql: | 
+        select 
+            publisher,
+            domain, 
+            "total impressions" 
+        FROM 
+          ad_bids_mini_metrics_with_policy 
+connectors:
+   duckdb: ~
+tests:
+  simple:
+    resolver: mv_sql_policy_api 
+    options:
+      claims:
+        user_attributes:
+          domain: yahoo.com 
+          email: user@yahoo.com
+    result:
+      - domain: yahoo.com
+        publisher: YAHOO
+        "total impressions": 3.0
+  msn:
+    resolver: mv_sql_policy_api 
+    options:
+      claims:
+        user_attributes:
+          domain: msn.com 
+          email: user@msn.com
+    result:
+      - domain: msn.com
+        publisher: ~
+        "total impressions": 3.0
+  empty:
+    resolver: mv_sql_policy_api 
+    options:
+      claims:
+        user_attributes:
+          domain: google.com 
+          email: user@google.com
+    result: ~