Add /api/manifest endpoint

util/populate_dev_data.go

@@ -41,7 +41,7 @@ func main() {
 		log.Fatal(err)
 	}
 
-	emptySecretToken := []interface{}{&base.Token{}}
+	emptySecretToken := &base.Token{}
 	staticDataTime, _ := time.Parse(time.RFC3339, "2017-10-18T00:00:00Z")
 
 	// Follow pattern established in run/*.py data collection code.
@@ -134,15 +134,17 @@ func main() {
 		},
 	}
 
-	tokenKindName := "Token"
 	testRunKindName := "TestRun"
 	passRateMetadataKindName := metrics.GetDatastoreKindName(
 		metrics.PassRateMetadata{})
 	failuresMetadataKindName := metrics.GetDatastoreKindName(
 		metrics.FailuresMetadata{})
 
+	log.Print("Adding local (empty) secrets...")
+	addSecretToken(ctx, "upload-token", emptySecretToken)
+	addSecretToken(ctx, "github-api-token", emptySecretToken)
+
 	log.Print("Adding local mock data (static/)...")
-	addData(ctx, tokenKindName, emptySecretToken)
 	addData(ctx, testRunKindName, staticTestRunMetadata)
 	addData(ctx, passRateMetadataKindName, staticPassRateMetadata)
 	addData(ctx, failuresMetadataKindName, staticFailuresMetadata)
@@ -164,6 +166,14 @@ func main() {
 	addData(ctx, testRunKindName, latestProductionTestRunMetadata)
 }
 
+func addSecretToken(ctx context.Context, id string, data interface{}) {
+	key := datastore.NewKey(ctx, "Token", id, 0, nil)
+	if _, err := datastore.Put(ctx, key, data); err != nil {
+		log.Fatalf("Failed to add %s secret: %s", id, err.Error())
+	}
+	log.Printf("Added %s secret", id)
+}
+
 func addData(ctx context.Context, kindName string, data []interface{}) {
 	keys := make([]*datastore.Key, len(data))
 	for i := range data {

webapp/api_handlers.go

@@ -5,12 +5,15 @@
 package webapp
 
 import (
+	"bytes"
+	"compress/gzip"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"net/http"
 	"net/url"
 	"strconv"
+	"strings"
 	"time"
 
 	models "github.com/web-platform-tests/wpt.fyi/shared"
@@ -368,3 +371,113 @@ func handleAPIDiffPost(w http.ResponseWriter, r *http.Request) {
 	}
 	w.Write(bytes)
 }
+
+func apiManifestHandler(w http.ResponseWriter, r *http.Request) {
+	sha, err := ParseSHAParamFull(r)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+	ctx := appengine.NewContext(r)
+	if manifest, err := getManifestForSHA(ctx, sha); err != nil {
+		http.Error(w, err.Error(), http.StatusNotFound)
+	} else {
+		w.Header().Add("content-type", "application/json")
+		w.Write(manifest)
+	}
+}
+
+func gitHubSHASearchURL(sha string) string {
+	return fmt.Sprintf(`https://api.github.com/search/issues?q=SHA:%s+user:w3c+repo:web-platform-tests`, sha)
+}
+
+func gitHubReleaseURL(tag string) string {
+	return fmt.Sprintf(`https://api.github.com/repos/w3c/web-platform-tests/releases/tags/%s`, tag)
+}
+
+type gitHubClient interface {
+	fetch(url string) ([]byte, error)
+}
+
+func getManifestForSHA(ctx context.Context, sha string) (manifest []byte, err error) {
+	// Fetch models.Token entity for GitHub API Token.
+	tokenKey := datastore.NewKey(ctx, "Token", "github-api-token", 0, nil)
+	var token models.Token
+	datastore.Get(ctx, tokenKey, &token)
+
+	client := gitHubClientImpl{
+		Token:   &token,
+		Context: ctx,
+	}
+	return getGitHubReleaseAssetForSHA(&client, sha)
+}
+
+func getGitHubReleaseAssetForSHA(client gitHubClient, sha string) (manifest []byte, err error) {
+	// Search for the PR associated with the SHA.
+	url := gitHubSHASearchURL(sha)
+	var body []byte
+	if body, err = client.fetch(url); err != nil {
+		return nil, err
+	}
+
+	var queryResults map[string]*json.RawMessage
+	if err = json.Unmarshal(body, &queryResults); err != nil {
+		return nil, err
+	}
+	var issues []map[string]*json.RawMessage
+	if err = json.Unmarshal(*queryResults["items"], &issues); err != nil {
+		return nil, err
+	}
+	if len(issues) < 1 {
+		return nil, fmt.Errorf("No search results found for SHA %s", sha)
+	}
+
+	// Load the release by the presumed tag name merge_pr_*
+	var prNumber int
+	if err = json.Unmarshal(*issues[0]["number"], &prNumber); err != nil {
+		return nil, err
+	}
+
+	releaseTag := fmt.Sprintf("merge_pr_%d", prNumber)
+	url = gitHubReleaseURL(releaseTag)
+	if body, err = client.fetch(url); err != nil {
+		return nil, err
+	}
+
+	var release map[string]*json.RawMessage
+	if err = json.Unmarshal(body, &release); err != nil {
+		return nil, err
+	}
+	var assets []map[string]*json.RawMessage
+	if err = json.Unmarshal(*release["assets"], &assets); err != nil {
+		return nil, err
+	}
+	if len(assets) < 1 {
+		return nil, fmt.Errorf("No assets found for release %s", releaseTag)
+	}
+	// Get (and unzip) the asset with name "MANIFEST-{sha}.json.gz"
+	for _, asset := range assets {
+		var name string
+		if err = json.Unmarshal(*asset["name"], &name); err != nil {
+			return nil, err
+		}
+		if strings.Contains(name, sha) {
+			if err = json.Unmarshal(*asset["browser_download_url"], &url); err != nil {
+				return nil, err
+			}
+
+			if body, err = client.fetch(url); err != nil {
+				return nil, err
+			}
+			gzReader, err := gzip.NewReader(bytes.NewReader(body))
+			if err != nil {
+				return nil, err
+			}
+			if body, err = ioutil.ReadAll(gzReader); err != nil {
+				return nil, err
+			}
+			return body, nil
+		}
+	}
+	return nil, fmt.Errorf("No manifest asset found for release %s", releaseTag)
+}

webapp/api_handlers_test.go

@@ -0,0 +1,96 @@
+// Copyright 2017 The WPT Dashboard Project. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package webapp
+
+import (
+	"bytes"
+	"compress/gzip"
+	"encoding/json"
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+type mockGitHubClient struct {
+	Responses map[string][]byte
+}
+
+func (m *mockGitHubClient) fetch(url string) ([]byte, error) {
+	if _, ok := m.Responses[url]; !ok {
+		return nil, fmt.Errorf("fore! oh; for: %s", url)
+	}
+	return m.Responses[url], nil
+}
+
+func TestGetGitHubReleaseAssetForSHA_SHANotFound(t *testing.T) {
+	client := mockGitHubClient{}
+	const sha = "abcdef1234"
+	manifest, err := getGitHubReleaseAssetForSHA(&client, sha)
+	assert.Nil(t, manifest)
+	assert.NotNil(t, err)
+}
+
+func TestGetGitHubReleaseAssetForSHA(t *testing.T) {
+	const sha = "abcdef1234"
+	searchResults, _ := json.Marshal(
+		object{
+			"items": []object{
+				object{
+					"number": 123,
+				},
+			},
+		},
+	)
+	downloadURL := "http://github.com/magic_url"
+
+	releasesJSON := object{
+		"assets": []object{
+			object{
+				"name":                 "MANIFEST-abcdef1234.json.gz",
+				"browser_download_url": downloadURL,
+			},
+		},
+	}
+
+	var buf bytes.Buffer
+	zw := gzip.NewWriter(&buf)
+	zw.Write([]byte("magic data"))
+	zw.Close()
+	data := buf.Bytes()
+
+	client := mockGitHubClient{
+		Responses: map[string][]byte{
+			gitHubSHASearchURL(sha):          searchResults,
+			gitHubReleaseURL("merge_pr_123"): unsafeMarshal(releasesJSON),
+			downloadURL:                      data,
+		},
+	}
+
+	// 1) Data is unzipped.
+	manifest, err := getGitHubReleaseAssetForSHA(&client, sha)
+	assert.Nil(t, err)
+	assert.Equal(t, []byte("magic data"), manifest)
+
+	// 2) Correct asset picked when first asset is some other asset.
+	releasesJSON["assets"] = []object{
+		object{
+			"name":                 "Some other asset.txt",
+			"browser_download_url": "http://huh.com?",
+		},
+		releasesJSON["assets"].([]object)[0],
+	}
+	client.Responses[gitHubReleaseURL("merge_pr_123")] = unsafeMarshal(releasesJSON)
+	manifest, err = getGitHubReleaseAssetForSHA(&client, sha)
+	assert.Nil(t, err)
+	assert.Equal(t, []byte("magic data"), manifest)
+
+	// 3) Error when no matching asset found.
+	releasesJSON["assets"] = releasesJSON["assets"].([]object)[0:1] // Just the other asset
+	client.Responses[gitHubReleaseURL("merge_pr_123")] = unsafeMarshal(releasesJSON)
+	manifest, err = getGitHubReleaseAssetForSHA(&client, sha)
+	assert.NotNil(t, err)
+	assert.Nil(t, manifest)
+}

webapp/github_client.go

@@ -0,0 +1,41 @@
+package webapp
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net/http"
+
+	models "github.com/web-platform-tests/wpt.fyi/shared"
+	"golang.org/x/net/context"
+	"google.golang.org/appengine/urlfetch"
+)
+
+type gitHubClientImpl struct {
+	Token   *models.Token
+	Context context.Context
+}
+
+func (g *gitHubClientImpl) fetch(url string) ([]byte, error) {
+	client := urlfetch.Client(g.Context)
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return nil, err
+	}
+	if g.Token != nil {
+		req.Header.Add("Authorization", fmt.Sprintf("token %s", g.Token.Secret))
+	}
+	var resp *http.Response
+	if resp, err = client.Do(req); err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	var body []byte
+	if body, err = ioutil.ReadAll(resp.Body); err != nil {
+		return nil, err
+	}
+	if resp.StatusCode != 200 {
+		return nil, fmt.Errorf("%s returned HTTP status %d:\n%s", url, resp.StatusCode, string(body))
+	}
+	return body, nil
+}

webapp/main.go

@@ -33,6 +33,9 @@ func init() {
 	// API endpoint for diff of two test run summary JSON blobs.
 	http.HandleFunc("/api/diff", apiDiffHandler)
 
+	// API endpoint for fetching a manifest for a commit SHA.
+	http.HandleFunc("/api/manifest", apiManifestHandler)
+
 	// API endpoint for listing all test runs for a given SHA.
 	http.HandleFunc("/api/runs", apiTestRunsHandler)
 

webapp/params.go

@@ -28,9 +28,19 @@ const MaxCountMinValue = 1
 // SHARegex is a regex for SHA[0:10] slice of a git hash.
 var SHARegex = regexp.MustCompile("[0-9a-fA-F]{10,40}")
 
-// ParseSHAParam parses and validates the 'sha' param for the request.
-// It returns "latest" by default (and in error cases).
+// ParseSHAParam parses and validates the 'sha' param for the request,
+// cropping it to 10 chars. It returns "latest" by default. (and in error cases).
 func ParseSHAParam(r *http.Request) (runSHA string, err error) {
+	sha, err := ParseSHAParamFull(r)
+	if err != nil || !SHARegex.MatchString(sha) {
+		return sha, err
+	}
+	return sha[:10], nil
+}
+
+// ParseSHAParamFull parses and validates the 'sha' param for the request.
+// It returns "latest" by default (and in error cases).
+func ParseSHAParamFull(r *http.Request) (runSHA string, err error) {
 	// Get the SHA for the run being loaded (the first part of the path.)
 	runSHA = "latest"
 	params, err := url.ParseQuery(r.URL.RawQuery)
@@ -40,10 +50,10 @@ func ParseSHAParam(r *http.Request) (runSHA string, err error) {
 
 	runParam := params.Get("sha")
 	if runParam != "" && runParam != "latest" {
+		runSHA = runParam
 		if !SHARegex.MatchString(runParam) {
-			return "", fmt.Errorf("Invalid sha param value: %s", runParam)
+			return "latest", fmt.Errorf("Invalid sha param value: %s", runParam)
 		}
-		runSHA = runParam[:10]
 	}
 	return runSHA, err
 }

webapp/util_test.go

@@ -5,12 +5,20 @@
 package webapp
 
 import (
+	"encoding/json"
 	"sort"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 )
 
+type object map[string]interface{}
+
+func unsafeMarshal(i interface{}) []byte {
+	result, _ := json.Marshal(i)
+	return result
+}
+
 func TestGetBrowserNames(t *testing.T) {
 	names, _ := GetBrowserNames()
 	assert.True(t, sort.StringsAreSorted(names))