Add http api for fetching accounts

config/config_test.go

@@ -652,8 +652,7 @@ func TestValidateAccountsConfigRestrictions(t *testing.T) {
 	cfg.Accounts.Postgres.ConnectionInfo.Database = "accounts"
 
 	errs := cfg.validate()
-	assert.Len(t, errs, 2)
-	assert.Contains(t, errs, errors.New("accounts.http: retrieving accounts via http not available, use accounts.files"))
+	assert.Len(t, errs, 1)
 	assert.Contains(t, errs, errors.New("accounts.postgres: retrieving accounts via postgres not available, use accounts.files"))
 }
 

config/stored_requests.go

@@ -137,9 +137,6 @@ func resolvedStoredRequestsConfig(cfg *Configuration) {
 }
 
 func (cfg *StoredRequests) validate(errs configErrors) configErrors {
-	if cfg.DataType() == AccountDataType && cfg.HTTP.Endpoint != "" {
-		errs = append(errs, fmt.Errorf("%s.http: retrieving accounts via http not available, use accounts.files", cfg.Section()))
-	}
 	if cfg.DataType() == AccountDataType && cfg.Postgres.ConnectionInfo.Database != "" {
 		errs = append(errs, fmt.Errorf("%s.postgres: retrieving accounts via postgres not available, use accounts.files", cfg.Section()))
 	} else {

stored_requests/backends/http_fetcher/fetcher.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
+	"net/url"
 	"strings"
 
 	"github.com/prebid/prebid-server/stored_requests"
@@ -19,9 +20,13 @@ import (
 //
 // This file expects the endpoint to satisfy the following API:
 //
+// Stored requests
 // GET {endpoint}?request-ids=["req1","req2"]&imp-ids=["imp1","imp2","imp3"]
 //
-// This endpoint should return a payload like:
+// Accounts
+// GET {endpoint}?account-ids=["acc1","acc2"]
+//
+// The above endpoints should return a payload like:
 //
 // {
 //   "requests": {
@@ -34,20 +39,32 @@ import (
 //     "imp3": null // If imp3 is not found
 //   }
 // }
+// or
+// {
+//   "accounts": {
+//     "acc1": { ... config data for acc1 ... },
+//     "acc2": { ... config data for acc2 ... },
+//   },
+// }
 //
 //
 func NewFetcher(client *http.Client, endpoint string) *HttpFetcher {
 	// Do some work up-front to figure out if the (configurable) endpoint has a query string or not.
 	// When we build requests, we'll either want to add `?request-ids=...&imp-ids=...` _or_
-	// `&request-ids=...&imp-ids=...`, depending.
+	// `&request-ids=...&imp-ids=...`.
+
 	urlPrefix := endpoint
 	if strings.Contains(endpoint, "?") {
 		urlPrefix = urlPrefix + "&"
 	} else {
 		urlPrefix = urlPrefix + "?"
 	}
 
-	glog.Info("Making http_fetcher which calls GET " + urlPrefix + "request-ids=%REQUEST_ID_LIST%&imp-ids=%IMP_ID_LIST%")
+	if url, err := url.Parse(endpoint); err != nil {
+		glog.Warningf(`Invalid endpoint "%s": %v`, endpoint, err)
+	} else {
+		glog.Infof("Making http_fetcher for endpoint %v", url)
+	}
 
 	return &HttpFetcher{
 		client:   client,
@@ -81,8 +98,68 @@ func (fetcher *HttpFetcher) FetchRequests(ctx context.Context, requestIDs []stri
 	return
 }
 
-func (fetcher *HttpFetcher) FetchAccount(ctx context.Context, accountID string) (json.RawMessage, []error) {
-	return nil, []error{stored_requests.NotFoundError{accountID, "Account"}}
+// FetchAccounts retrieves account configurations
+//
+// Request format is similar to the one for requests:
+// GET {endpoint}?account-ids=["account1","account2",...]
+//
+// The endpoint is expected to respond with a JSON map with accountID -> json.RawMessage
+// {
+//   "account1": { ... account json ... }
+// }
+// The JSON contents of account config is returned as-is (NOT validated)
+func (fetcher *HttpFetcher) FetchAccounts(ctx context.Context, accountIDs []string) (accountData map[string]json.RawMessage, errs []error) {
+	if len(accountIDs) == 0 {
+		return nil, nil
+	}
+	httpReq, err := http.NewRequestWithContext(ctx, "GET", fetcher.Endpoint+"account-ids=[\""+strings.Join(accountIDs, "\",\"")+"\"]", nil)
+	if err != nil {
+		return nil, []error{
+			fmt.Errorf(`Error fetching accounts "%v" via http: %v`, accountIDs, err),
+		}
+	}
+	httpResp, err := ctxhttp.Do(ctx, fetcher.client, httpReq)
+	if err != nil {
+		return nil, []error{
+			fmt.Errorf(`Error fetching accounts %v via http: %v`, accountIDs, err),
+		}
+	}
+	defer httpResp.Body.Close()
+	respBytes, err := ioutil.ReadAll(httpResp.Body)
+	if err != nil {
+		return nil, []error{
+			fmt.Errorf(`Error fetching accounts %v via http: could not read response`, accountIDs),
+		}
+	}
+	if httpResp.StatusCode != http.StatusOK {
+		return nil, []error{
+			fmt.Errorf(`Error fetching accounts %v via http: unexpected response status %d`, accountIDs, httpResp.StatusCode),
+		}
+	}
+	var responseData accountsResponseContract
+	if err = json.Unmarshal(respBytes, &responseData); err != nil {
+		return nil, []error{
+			fmt.Errorf(`Error fetching accounts %v via http: malformed response`, accountIDs),
+		}
+	}
+	errs = convertNullsToErrs(responseData.Accounts, "Account", errs)
+	return responseData.Accounts, errs
+}
+
+// FetchAccount fetchers a single accountID and returns its corresponding json
+func (fetcher *HttpFetcher) FetchAccount(ctx context.Context, accountID string) (accountJSON json.RawMessage, errs []error) {
+	accountData, errs := fetcher.FetchAccounts(ctx, []string{accountID})
+	if len(errs) > 0 {
+		return nil, errs
+	}
+	accountJSON, ok := accountData[accountID]
+	if !ok {
+		return nil, []error{stored_requests.NotFoundError{
+			ID:       accountID,
+			DataType: "Account",
+		}}
+	}
+	return accountJSON, []error{}
 }
 
 func (fetcher *HttpFetcher) FetchCategories(ctx context.Context, primaryAdServer, publisherId, iabCategory string) (string, error) {
@@ -190,3 +267,7 @@ type responseContract struct {
 	Requests map[string]json.RawMessage `json:"requests"`
 	Imps     map[string]json.RawMessage `json:"imps"`
 }
+
+type accountsResponseContract struct {
+	Accounts map[string]json.RawMessage `json:"accounts"`
+}

stored_requests/backends/http_fetcher/fetcher_test.go

@@ -9,6 +9,8 @@ import (
 	"net/http/httptest"
 	"strings"
 	"testing"
+
+	"github.com/stretchr/testify/assert"
 )
 
 func TestSingleReq(t *testing.T) {
@@ -61,14 +63,39 @@ func TestReqsAndImps(t *testing.T) {
 	assertErrLength(t, errs, 0)
 }
 
-func TestMissingValues(t *testing.T) {
-	fetcher, close := newEmptyFetcher(t, []string{"req-1", "req-2"}, []string{"imp-1"})
+func TestFetchAccounts(t *testing.T) {
+	fetcher, close := newTestAccountFetcher(t, []string{"acc-1", "acc-2"})
 	defer close()
 
-	reqData, impData, errs := fetcher.FetchRequests(context.Background(), []string{"req-1", "req-2"}, []string{"imp-1"})
-	assertMapKeys(t, reqData)
-	assertMapKeys(t, impData)
-	assertErrLength(t, errs, 3)
+	accData, errs := fetcher.FetchAccounts(context.Background(), []string{"acc-1", "acc-2"})
+	assertMapKeys(t, accData, "acc-1", "acc-2")
+	assertErrLength(t, errs, 0)
+}
+
+func TestFetchAccountsNoData(t *testing.T) {
+	fetcher, close := newFetcherBrokenBackend()
+	defer close()
+	accData, errs := fetcher.FetchAccounts(context.Background(), []string{"req-1"})
+	assertMapKeys(t, accData)
+	assertErrLength(t, errs, 1)
+}
+
+func TestFetchAccount(t *testing.T) {
+	fetcher, close := newTestAccountFetcher(t, []string{"acc-1"})
+	defer close()
+
+	account, errs := fetcher.FetchAccount(context.Background(), "acc-1")
+	assert.Empty(t, errs, "Unexpected error fetching existing account")
+	assert.JSONEq(t, `"acc-1"`, string(account))
+}
+
+func TestFetchAccountNoData(t *testing.T) {
+	fetcher, close := newFetcherBrokenBackend()
+	defer close()
+
+	unknownAccount, errs := fetcher.FetchAccount(context.Background(), "unknown-acc")
+	assert.NotEmpty(t, errs, "Retrieving unknown account should have returned an error")
+	assert.Nil(t, unknownAccount)
 }
 
 func TestErrResponse(t *testing.T) {
@@ -139,12 +166,12 @@ func newTestFetcher(t *testing.T, expectReqIDs []string, expectImpIDs []string)
 func newHandler(t *testing.T, expectReqIDs []string, expectImpIDs []string, jsonifier func(string) json.RawMessage) func(w http.ResponseWriter, r *http.Request) {
 	return func(w http.ResponseWriter, r *http.Request) {
 		query := r.URL.Query()
-		assertMatches(t, query.Get("request-ids"), expectReqIDs)
-		assertMatches(t, query.Get("imp-ids"), expectImpIDs)
-
 		gotReqIDs := richSplit(query.Get("request-ids"))
 		gotImpIDs := richSplit(query.Get("imp-ids"))
 
+		assertMatches(t, gotReqIDs, expectReqIDs)
+		assertMatches(t, gotImpIDs, expectImpIDs)
+
 		reqIDResponse := make(map[string]json.RawMessage, len(gotReqIDs))
 		impIDResponse := make(map[string]json.RawMessage, len(gotImpIDs))
 
@@ -174,10 +201,43 @@ func newHandler(t *testing.T, expectReqIDs []string, expectImpIDs []string, json
 	}
 }
 
-func assertMatches(t *testing.T, query string, expected []string) {
+func newTestAccountFetcher(t *testing.T, expectAccIDs []string) (fetcher *HttpFetcher, closer func()) {
+	handler := newAccountHandler(t, expectAccIDs, jsonifyID)
+	server := httptest.NewServer(http.HandlerFunc(handler))
+	return NewFetcher(server.Client(), server.URL), server.Close
+}
+
+func newAccountHandler(t *testing.T, expectAccIDs []string, jsonifier func(string) json.RawMessage) func(w http.ResponseWriter, r *http.Request) {
+	return func(w http.ResponseWriter, r *http.Request) {
+		query := r.URL.Query()
+		gotAccIDs := richSplit(query.Get("account-ids"))
+
+		assertMatches(t, gotAccIDs, expectAccIDs)
+
+		accIDResponse := make(map[string]json.RawMessage, len(gotAccIDs))
+
+		for _, accID := range gotAccIDs {
+			if accID != "" {
+				accIDResponse[accID] = jsonifier(accID)
+			}
+		}
+
+		respObj := accountsResponseContract{
+			Accounts: accIDResponse,
+		}
+
+		if respBytes, err := json.Marshal(respObj); err != nil {
+			t.Errorf("failed to marshal responseContract in test:  %v", err)
+			w.WriteHeader(http.StatusInternalServerError)
+		} else {
+			w.Write(respBytes)
+		}
+	}
+}
+
+func assertMatches(t *testing.T, queryVals []string, expected []string) {
 	t.Helper()
 
-	queryVals := richSplit(query)
 	if len(queryVals) == 1 && queryVals[0] == "" {
 		if len(expected) != 0 {
 			t.Errorf("Expected no query vals, but got %v", queryVals)