From 8bcf7b44a6a752bec36a9b33b8895957c911ad2a Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Fri, 2 Dec 2022 03:39:12 +0100 Subject: [PATCH 001/123] feat: fetch sourcemaps from Elasticsearch Add a new caching metadata fetcher that fetch sourcemap metadata from ES and forward the request to the backend fetcher if it exists. The backend fetcher is an ES fetcher wrapped in a LRU cache to cache the sourcemap body. Sourcemap metadata are periodically synced with ES by the caching metadata fetcher. Keep fleet and kibana fetcher until the UI team has updated the sourcemap uploading flow. Update ES sourcemap response to include sourcemap metadata. --- internal/beater/beater.go | 37 +++-- internal/sourcemap/elasticsearch.go | 7 +- internal/sourcemap/metadata.go | 206 ++++++++++++++++++++++++++++ systemtest/sourcemap_test.go | 76 ++++++++++ 4 files changed, 315 insertions(+), 11 deletions(-) create mode 100644 internal/sourcemap/metadata.go diff --git a/internal/beater/beater.go b/internal/beater/beater.go index 7242f382485..169b6e01c59 100644 --- a/internal/beater/beater.go +++ b/internal/beater/beater.go @@ -320,13 +320,7 @@ func (s *Runner) Run(ctx context.Context) error { if err != nil { return err } - cachingFetcher, err := sourcemap.NewCachingFetcher( - fetcher, s.config.RumConfig.SourceMapping.Cache.Expiration, - ) - if err != nil { - return err - } - sourcemapFetcher = cachingFetcher + sourcemapFetcher = fetcher } // Create the runServer function. We start with newBaseRunServer, and then @@ -827,18 +821,33 @@ func newSourcemapFetcher( } } - return sourcemap.NewFleetFetcher( + ff, err := sourcemap.NewFleetFetcher( &client, fleetCfg.AccessAPIKey, fleetServerURLs, artifactRefs, ) + if err != nil { + return nil, err + } + + cachingFetcher, err := sourcemap.NewCachingFetcher(ff, cfg.Cache.Expiration) + if err != nil { + return nil, err + } + + return cachingFetcher, nil } // For standalone, we query both Kibana and Elasticsearch for backwards compatibility. var chained sourcemap.ChainedFetcher if kibanaClient != nil { - chained = append(chained, sourcemap.NewKibanaFetcher(kibanaClient)) + cachingFetcher, err := sourcemap.NewCachingFetcher(sourcemap.NewKibanaFetcher(kibanaClient), cfg.Cache.Expiration) + if err != nil { + return nil, err + } + + chained = append(chained, cachingFetcher) } esClient, err := newElasticsearchClient(cfg.ESConfig) if err != nil { @@ -846,7 +855,15 @@ func newSourcemapFetcher( } index := strings.ReplaceAll(cfg.IndexPattern, "%{[observer.version]}", version.Version) esFetcher := sourcemap.NewElasticsearchFetcher(esClient, index) - chained = append(chained, esFetcher) + cachingFetcher, err := sourcemap.NewCachingFetcher(esFetcher, cfg.Cache.Expiration) + if err != nil { + return nil, err + } + metadataCachingFetcher := sourcemap.NewMetadataCachingFetcher(esClient, cachingFetcher, index) + metadataCachingFetcher.StartBackgroundSync() + + chained = append(chained, metadataCachingFetcher) + return chained, nil } diff --git a/internal/sourcemap/elasticsearch.go b/internal/sourcemap/elasticsearch.go index bd530e16e39..9ebada693b4 100644 --- a/internal/sourcemap/elasticsearch.go +++ b/internal/sourcemap/elasticsearch.go @@ -59,7 +59,12 @@ type esSourcemapResponse struct { Hits []struct { Source struct { Sourcemap struct { - Sourcemap string + Service struct { + Name string + Version string + } + BundleFilepath string `json:"bundle_filepath"` + Sourcemap string } } `json:"_source"` } diff --git a/internal/sourcemap/metadata.go b/internal/sourcemap/metadata.go new file mode 100644 index 00000000000..12ab1866cbe --- /dev/null +++ b/internal/sourcemap/metadata.go @@ -0,0 +1,206 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package sourcemap + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "sync" + "time" + + "github.com/go-sourcemap/sourcemap" + "github.com/pkg/errors" + + "github.com/elastic/apm-server/internal/elasticsearch" + "github.com/elastic/apm-server/internal/logs" + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/go-elasticsearch/v8/esapi" +) + +type Identifier struct { + name string + version string + path string +} + +type Metadata struct { + id Identifier + contentHash string +} + +type MetadataCachingFetcher struct { + esClient elasticsearch.Client + set map[Identifier]bool + mu sync.RWMutex + backend Fetcher + logger *logp.Logger + once sync.Once + index string +} + +func NewMetadataCachingFetcher( + c elasticsearch.Client, + backend Fetcher, + index string, +) *MetadataCachingFetcher { + return &MetadataCachingFetcher{ + esClient: c, + index: index, + set: make(map[Identifier]bool), + backend: backend, + logger: logp.NewLogger(logs.Sourcemap), + } +} + +func (s *MetadataCachingFetcher) Fetch(ctx context.Context, name, version, path string) (*sourcemap.Consumer, error) { + if len(s.set) == 0 { + // First run, populate cache + s.once.Do(func() { + s.sync(ctx) + }) + } + + key := Identifier{ + name: name, + version: version, + path: path, + } + + s.mu.RLock() + defer s.mu.RUnlock() + + if _, found := s.set[key]; found { + // Only fetch from ES if the sourcemap id exists + return s.backend.Fetch(ctx, name, version, path) + } + + return nil, nil +} + +func (s *MetadataCachingFetcher) Update(ids []Identifier) { + s.mu.Lock() + defer s.mu.Unlock() + + for k := range s.set { + delete(s.set, k) + } + + for _, k := range ids { + s.set[k] = true + } +} + +func (s *MetadataCachingFetcher) StartBackgroundSync() { + go func() { + // TODO make this a config option ? + t := time.NewTicker(30 * time.Second) + defer t.Stop() + + for { + select { + case <-t.C: + ctx, cleanup := context.WithTimeout(context.Background(), 10*time.Second) + s.sync(ctx) + cleanup() + } + } + }() +} + +func (s *MetadataCachingFetcher) sync(ctx context.Context) error { + resp, err := s.runSearchQuery(ctx) + if err != nil { + return errors.Wrap(err, errMsgESFailure) + } + defer resp.Body.Close() + + // handle error response + if resp.StatusCode >= http.StatusMultipleChoices { + if resp.StatusCode == http.StatusNotFound { + return nil + } + b, err := io.ReadAll(resp.Body) + if err != nil { + return errors.Wrap(err, errMsgParseSourcemap) + } + return errors.New(fmt.Sprintf("%s %s", errMsgParseSourcemap, b)) + } + + // parse response + body, err := parseResponse(resp.Body, s.logger) + if err != nil { + return err + } + + var ids []Identifier + for _, v := range body.Hits.Hits { + id := Identifier{ + name: v.Source.Sourcemap.Service.Name, + version: v.Source.Sourcemap.Service.Version, + path: v.Source.Sourcemap.BundleFilepath, + } + + ids = append(ids, id) + } + + // Update cache + s.Update(ids) + return nil +} + +func (s *MetadataCachingFetcher) runSearchQuery(ctx context.Context) (*esapi.Response, error) { + var buf bytes.Buffer + if err := json.NewEncoder(&buf).Encode(queryMetadata()); err != nil { + return nil, err + } + + req := esapi.SearchRequest{ + Index: []string{s.index}, + Body: &buf, + TrackTotalHits: true, + } + return req.Do(ctx, s.esClient) +} + +func queryMetadata() map[string]interface{} { + return map[string]interface{}{ + "_source": []string{"sourcemap.service.*", "sourcemap.bundle_filepath"}, + } +} + +func parseResponse(body io.ReadCloser, logger *logp.Logger) (esSourcemapResponse, error) { + b, err := io.ReadAll(body) + if err != nil { + return esSourcemapResponse{}, err + } + + var esSourcemapResponse esSourcemapResponse + if err := json.Unmarshal(b, &esSourcemapResponse); err != nil { + return esSourcemapResponse, err + } + hits := esSourcemapResponse.Hits.Total.Value + if hits == 0 || len(esSourcemapResponse.Hits.Hits) == 0 { + return esSourcemapResponse, nil + } + + return esSourcemapResponse, nil +} diff --git a/systemtest/sourcemap_test.go b/systemtest/sourcemap_test.go index 4a58c4b9f3f..47104ae5c72 100644 --- a/systemtest/sourcemap_test.go +++ b/systemtest/sourcemap_test.go @@ -18,6 +18,10 @@ package systemtest_test import ( + "bytes" + "context" + "encoding/json" + "io" "os" "testing" @@ -27,6 +31,7 @@ import ( "github.com/elastic/apm-server/systemtest" "github.com/elastic/apm-server/systemtest/apmservertest" "github.com/elastic/apm-server/systemtest/estest" + "github.com/elastic/go-elasticsearch/v8/esapi" ) func TestRUMErrorSourcemapping(t *testing.T) { @@ -131,6 +136,77 @@ func TestNoMatchingSourcemap(t *testing.T) { ) } +func TestSourcemapElasticsearch(t *testing.T) { + t.Skip("DISABLED FOR NOW") + + systemtest.CleanupElasticsearch(t) + srv := apmservertest.NewUnstartedServerTB(t) + srv.Config.RUM = &apmservertest.RUMConfig{Enabled: true} + err := srv.Start() + require.NoError(t, err) + + sourcemap, err := os.ReadFile("../testdata/sourcemap/bundle.js.map") + require.NoError(t, err) + + type Source struct { + Timestamp string `json:"@timestamp"` + Processor struct { + Name string `json:"name"` + } `json:"processor"` + Sourcemap struct { + Service struct { + Name string `json:"name"` + Version string `json:"version"` + } `json:"service"` + BundleFilepath string `json:"bundle_filepath"` + Sourcemap string `json:"sourcemap"` + } `json:"sourcemap"` + } + + s := Source{ + Timestamp: "1970-01-01T00:00:01.000Z", + Processor: struct { + Name string `json:"name"` + }{ + Name: "sourcemap", + }, + Sourcemap: struct { + Service struct { + Name string `json:"name"` + Version string `json:"version"` + } `json:"service"` + BundleFilepath string `json:"bundle_filepath"` + Sourcemap string `json:"sourcemap"` + }{ + Service: struct { + Name string `json:"name"` + Version string `json:"version"` + }{ + Name: "apm-agent-js", + Version: "1.0.1", + }, + BundleFilepath: "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", + Sourcemap: string(sourcemap), + }, + } + + b, err := json.Marshal(s) + require.NoError(t, err) + + _, err = systemtest.Elasticsearch.Do(context.Background(), &esapi.IndexRequest{ + Index: "apm-test-sourcemap", + Body: bytes.NewReader(b), + }, nil) + require.NoError(t, err) + + systemtest.Elasticsearch.ExpectMinDocs(t, 1, "apm-*-sourcemap", nil) + + // Index an error, applying source mapping and caching the source map in the process. + systemtest.SendRUMEventsPayload(t, srv.URL, "../testdata/intake-v2/errors_rum.ndjson") + result := systemtest.Elasticsearch.ExpectDocs(t, "logs-apm.error-*", nil) + assertSourcemapUpdated(t, result, true) +} + func TestSourcemapCaching(t *testing.T) { systemtest.CleanupElasticsearch(t) srv := apmservertest.NewUnstartedServerTB(t) From 3d61fc116cb76d73fedd67aac5e3226e2461f293 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Fri, 2 Dec 2022 03:55:14 +0100 Subject: [PATCH 002/123] lint: fix linter issues --- systemtest/sourcemap_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/systemtest/sourcemap_test.go b/systemtest/sourcemap_test.go index 47104ae5c72..56a0a2e2ec2 100644 --- a/systemtest/sourcemap_test.go +++ b/systemtest/sourcemap_test.go @@ -21,7 +21,6 @@ import ( "bytes" "context" "encoding/json" - "io" "os" "testing" From d18afa0dfab6d88a06e8529e3e5bc2861316dfa1 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Sun, 4 Dec 2022 14:01:36 +0100 Subject: [PATCH 003/123] lint: fix staticcheck issue --- internal/sourcemap/metadata.go | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/internal/sourcemap/metadata.go b/internal/sourcemap/metadata.go index 12ab1866cbe..22ee82f2f58 100644 --- a/internal/sourcemap/metadata.go +++ b/internal/sourcemap/metadata.go @@ -115,13 +115,10 @@ func (s *MetadataCachingFetcher) StartBackgroundSync() { t := time.NewTicker(30 * time.Second) defer t.Stop() - for { - select { - case <-t.C: - ctx, cleanup := context.WithTimeout(context.Background(), 10*time.Second) - s.sync(ctx) - cleanup() - } + for range t.C { + ctx, cleanup := context.WithTimeout(context.Background(), 10*time.Second) + s.sync(ctx) + cleanup() } }() } From 6b13bb31928ef3d5d7a1e316c44bb44f51e17bfe Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Tue, 13 Dec 2022 00:02:33 +0100 Subject: [PATCH 004/123] feat: update code to support latest kibana PR sourcemaps are compressed with zlib and encoded as base64, keep that in mind when retrieving the content from ES. The document structure has been updated to reflect the current latest commit in the kibana PR. Use the _id to fetch the actual sourcemap since the service fields are doc-value-only and cannot be used for filtering. Log an error whenever sync fails. --- internal/beater/beater.go | 9 +++-- internal/sourcemap/elasticsearch.go | 60 +++++++++++++++++------------ internal/sourcemap/metadata.go | 39 ++++++++++++++----- 3 files changed, 70 insertions(+), 38 deletions(-) diff --git a/internal/beater/beater.go b/internal/beater/beater.go index 169b6e01c59..c22426570ed 100644 --- a/internal/beater/beater.go +++ b/internal/beater/beater.go @@ -26,7 +26,6 @@ import ( "net/url" "os" "runtime" - "strings" "time" "github.com/dustin/go-humanize" @@ -770,6 +769,8 @@ func (s *Runner) newLibbeatFinalBatchProcessor( return publisher, stop, nil } +const apmSourcemapIndex = ".apm-source-map" + func newSourcemapFetcher( cfg config.SourceMapping, fleetCfg *config.Fleet, @@ -853,13 +854,13 @@ func newSourcemapFetcher( if err != nil { return nil, err } - index := strings.ReplaceAll(cfg.IndexPattern, "%{[observer.version]}", version.Version) - esFetcher := sourcemap.NewElasticsearchFetcher(esClient, index) + + esFetcher := sourcemap.NewElasticsearchFetcher(esClient, apmSourcemapIndex) cachingFetcher, err := sourcemap.NewCachingFetcher(esFetcher, cfg.Cache.Expiration) if err != nil { return nil, err } - metadataCachingFetcher := sourcemap.NewMetadataCachingFetcher(esClient, cachingFetcher, index) + metadataCachingFetcher := sourcemap.NewMetadataCachingFetcher(esClient, cachingFetcher, apmSourcemapIndex) metadataCachingFetcher.StartBackgroundSync() chained = append(chained, metadataCachingFetcher) diff --git a/internal/sourcemap/elasticsearch.go b/internal/sourcemap/elasticsearch.go index 9ebada693b4..2044d2079b6 100644 --- a/internal/sourcemap/elasticsearch.go +++ b/internal/sourcemap/elasticsearch.go @@ -19,7 +19,9 @@ package sourcemap import ( "bytes" + "compress/zlib" "context" + "encoding/base64" "encoding/json" "fmt" "io" @@ -58,14 +60,11 @@ type esSourcemapResponse struct { } Hits []struct { Source struct { - Sourcemap struct { - Service struct { - Name string - Version string - } - BundleFilepath string `json:"bundle_filepath"` - Sourcemap string - } + ServiceName string `json:"service.name"` + ServiceVersion string `json:"service.version"` + BundleFilepath string `json:"file.path"` + Sourcemap string `json:"content"` + ContentHash string `json:"content_sha256"` } `json:"_source"` } } `json:"hits"` @@ -102,7 +101,24 @@ func (s *esFetcher) Fetch(ctx context.Context, name, version, path string) (*sou if err != nil { return nil, err } - return parseSourceMap(body) + + d, err := base64.StdEncoding.DecodeString(body) + if err != nil { + return nil, fmt.Errorf("failed to base64 decode string: %w", err) + } + + r, err := zlib.NewReader(bytes.NewReader(d)) + if err != nil { + return nil, fmt.Errorf("failed to create zlib reader: %w", err) + } + defer r.Close() + + bbb, err := io.ReadAll(r) + if err != nil { + return nil, fmt.Errorf("failed to read sourcemap content: %w", err) + } + + return parseSourceMap(string(bbb)) } func (s *esFetcher) runSearchQuery(ctx context.Context, name, version, path string) (*esapi.Response, error) { @@ -119,10 +135,16 @@ func (s *esFetcher) runSearchQuery(ctx context.Context, name, version, path stri } func parse(body io.ReadCloser, name, version, path string, logger *logp.Logger) (string, error) { + b, err := io.ReadAll(body) + if err != nil { + return "", err + } + var esSourcemapResponse esSourcemapResponse - if err := json.NewDecoder(body).Decode(&esSourcemapResponse); err != nil { + if err := json.Unmarshal(b, &esSourcemapResponse); err != nil { return "", err } + hits := esSourcemapResponse.Hits.Total.Value if hits == 0 || len(esSourcemapResponse.Hits.Hits) == 0 { return "", nil @@ -133,7 +155,7 @@ func parse(body io.ReadCloser, name, version, path string, logger *logp.Logger) logger.Warnf("%d sourcemaps found for service %s version %s and file %s, using the most recent one", hits, name, version, path) } - esSourcemap = esSourcemapResponse.Hits.Hits[0].Source.Sourcemap.Sourcemap + esSourcemap = esSourcemapResponse.Hits.Hits[0].Source.Sourcemap // until https://github.com/golang/go/issues/19858 is resolved if esSourcemap == "" { return "", errSourcemapWrongFormat @@ -142,25 +164,15 @@ func parse(body io.ReadCloser, name, version, path string, logger *logp.Logger) } func query(name, version, path string) map[string]interface{} { + id := name + "-" + version + "-" + path return searchFirst( boolean( must( - term("processor.name", "sourcemap"), - term("sourcemap.service.name", name), - term("sourcemap.service.version", version), - term("processor.name", "sourcemap"), - boolean( - should( - // prefer full URL match - boostedTerm("sourcemap.bundle_filepath", path, 2.0), - term("sourcemap.bundle_filepath", maybeParseURLPath(path)), - ), - ), + term("_id", id), ), ), - "sourcemap.sourcemap", + "content", desc("_score"), - desc("@timestamp"), ) } diff --git a/internal/sourcemap/metadata.go b/internal/sourcemap/metadata.go index 22ee82f2f58..2823457d749 100644 --- a/internal/sourcemap/metadata.go +++ b/internal/sourcemap/metadata.go @@ -49,7 +49,7 @@ type Metadata struct { type MetadataCachingFetcher struct { esClient elasticsearch.Client - set map[Identifier]bool + set map[Identifier]string mu sync.RWMutex backend Fetcher logger *logp.Logger @@ -65,7 +65,7 @@ func NewMetadataCachingFetcher( return &MetadataCachingFetcher{ esClient: c, index: index, - set: make(map[Identifier]bool), + set: make(map[Identifier]string), backend: backend, logger: logp.NewLogger(logs.Sourcemap), } @@ -73,9 +73,12 @@ func NewMetadataCachingFetcher( func (s *MetadataCachingFetcher) Fetch(ctx context.Context, name, version, path string) (*sourcemap.Consumer, error) { if len(s.set) == 0 { - // First run, populate cache + // If cache is empty and we're trying to fetch a sourcemap + // make sure the initial cache population has ended. s.once.Do(func() { - s.sync(ctx) + if err := s.sync(ctx); err != nil { + s.logger.Error("failed to fetch sourcemaps metadata: %v", err) + } }) } @@ -105,11 +108,23 @@ func (s *MetadataCachingFetcher) Update(ids []Identifier) { } for _, k := range ids { - s.set[k] = true + s.set[k] = "" } } func (s *MetadataCachingFetcher) StartBackgroundSync() { + go func() { + // First run, populate cache + s.once.Do(func() { + ctx, cleanup := context.WithTimeout(context.Background(), 10*time.Second) + defer cleanup() + + if err := s.sync(ctx); err != nil { + s.logger.Error("failed to fetch sourcemaps metadata: %v", err) + } + }) + }() + go func() { // TODO make this a config option ? t := time.NewTicker(30 * time.Second) @@ -117,7 +132,11 @@ func (s *MetadataCachingFetcher) StartBackgroundSync() { for range t.C { ctx, cleanup := context.WithTimeout(context.Background(), 10*time.Second) - s.sync(ctx) + + if err := s.sync(ctx); err != nil { + s.logger.Error("failed to sync sourcemaps metadata: %v", err) + } + cleanup() } }() @@ -151,9 +170,9 @@ func (s *MetadataCachingFetcher) sync(ctx context.Context) error { var ids []Identifier for _, v := range body.Hits.Hits { id := Identifier{ - name: v.Source.Sourcemap.Service.Name, - version: v.Source.Sourcemap.Service.Version, - path: v.Source.Sourcemap.BundleFilepath, + name: v.Source.ServiceName, + version: v.Source.ServiceVersion, + path: v.Source.BundleFilepath, } ids = append(ids, id) @@ -180,7 +199,7 @@ func (s *MetadataCachingFetcher) runSearchQuery(ctx context.Context) (*esapi.Res func queryMetadata() map[string]interface{} { return map[string]interface{}{ - "_source": []string{"sourcemap.service.*", "sourcemap.bundle_filepath"}, + "_source": []string{"service.*", "file.path", "content_sha256"}, } } From 27883dfdba0623674182b37fe3cfea3797eaf878 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Tue, 13 Dec 2022 00:39:28 +0100 Subject: [PATCH 005/123] test: udpdate sourcemap test to use the kibana sourcemap API the new kibana PR is upload sourcemaps to ES and migrating old one to it. We can restore the sourcemap test to use the standard method instead of upload the sourcemap manually to ES. --- systemtest/sourcemap_test.go | 63 +++--------------------------------- 1 file changed, 5 insertions(+), 58 deletions(-) diff --git a/systemtest/sourcemap_test.go b/systemtest/sourcemap_test.go index 56a0a2e2ec2..f67e2cfd6bd 100644 --- a/systemtest/sourcemap_test.go +++ b/systemtest/sourcemap_test.go @@ -18,9 +18,6 @@ package systemtest_test import ( - "bytes" - "context" - "encoding/json" "os" "testing" @@ -30,7 +27,6 @@ import ( "github.com/elastic/apm-server/systemtest" "github.com/elastic/apm-server/systemtest/apmservertest" "github.com/elastic/apm-server/systemtest/estest" - "github.com/elastic/go-elasticsearch/v8/esapi" ) func TestRUMErrorSourcemapping(t *testing.T) { @@ -136,8 +132,6 @@ func TestNoMatchingSourcemap(t *testing.T) { } func TestSourcemapElasticsearch(t *testing.T) { - t.Skip("DISABLED FOR NOW") - systemtest.CleanupElasticsearch(t) srv := apmservertest.NewUnstartedServerTB(t) srv.Config.RUM = &apmservertest.RUMConfig{Enabled: true} @@ -146,59 +140,12 @@ func TestSourcemapElasticsearch(t *testing.T) { sourcemap, err := os.ReadFile("../testdata/sourcemap/bundle.js.map") require.NoError(t, err) + sourcemapID := systemtest.CreateSourceMap(t, string(sourcemap), "apm-agent-js", "1.0.1", + "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", + ) - type Source struct { - Timestamp string `json:"@timestamp"` - Processor struct { - Name string `json:"name"` - } `json:"processor"` - Sourcemap struct { - Service struct { - Name string `json:"name"` - Version string `json:"version"` - } `json:"service"` - BundleFilepath string `json:"bundle_filepath"` - Sourcemap string `json:"sourcemap"` - } `json:"sourcemap"` - } - - s := Source{ - Timestamp: "1970-01-01T00:00:01.000Z", - Processor: struct { - Name string `json:"name"` - }{ - Name: "sourcemap", - }, - Sourcemap: struct { - Service struct { - Name string `json:"name"` - Version string `json:"version"` - } `json:"service"` - BundleFilepath string `json:"bundle_filepath"` - Sourcemap string `json:"sourcemap"` - }{ - Service: struct { - Name string `json:"name"` - Version string `json:"version"` - }{ - Name: "apm-agent-js", - Version: "1.0.1", - }, - BundleFilepath: "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", - Sourcemap: string(sourcemap), - }, - } - - b, err := json.Marshal(s) - require.NoError(t, err) - - _, err = systemtest.Elasticsearch.Do(context.Background(), &esapi.IndexRequest{ - Index: "apm-test-sourcemap", - Body: bytes.NewReader(b), - }, nil) - require.NoError(t, err) - - systemtest.Elasticsearch.ExpectMinDocs(t, 1, "apm-*-sourcemap", nil) + systemtest.Elasticsearch.ExpectMinDocs(t, 1, ".apm-source-map", nil) + require.NotEmpty(t, sourcemapID) // Index an error, applying source mapping and caching the source map in the process. systemtest.SendRUMEventsPayload(t, srv.URL, "../testdata/intake-v2/errors_rum.ndjson") From 4d8c011aeb111193788d56176a26f31cee811e6c Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Tue, 13 Dec 2022 00:51:39 +0100 Subject: [PATCH 006/123] feat: implement cache invalidation in sourcemap cache The sourcemap cache is now refreshed whenever the content hashcode changes or a sourcemap is removed from ES. Update the sourcemap cache to use a LRU cache. --- internal/beater/beater.go | 15 ++++---- internal/sourcemap/caching.go | 68 ++++++++++------------------------ internal/sourcemap/metadata.go | 68 +++++++++++++++++++++++----------- 3 files changed, 74 insertions(+), 77 deletions(-) diff --git a/internal/beater/beater.go b/internal/beater/beater.go index c22426570ed..82d8d9c7e7d 100644 --- a/internal/beater/beater.go +++ b/internal/beater/beater.go @@ -832,7 +832,7 @@ func newSourcemapFetcher( return nil, err } - cachingFetcher, err := sourcemap.NewCachingFetcher(ff, cfg.Cache.Expiration) + cachingFetcher, err := sourcemap.NewCachingFetcher(ff, make(<-chan sourcemap.Identifier), 128) if err != nil { return nil, err } @@ -843,7 +843,7 @@ func newSourcemapFetcher( // For standalone, we query both Kibana and Elasticsearch for backwards compatibility. var chained sourcemap.ChainedFetcher if kibanaClient != nil { - cachingFetcher, err := sourcemap.NewCachingFetcher(sourcemap.NewKibanaFetcher(kibanaClient), cfg.Cache.Expiration) + cachingFetcher, err := sourcemap.NewCachingFetcher(sourcemap.NewKibanaFetcher(kibanaClient), make(<-chan sourcemap.Identifier), 128) if err != nil { return nil, err } @@ -855,17 +855,18 @@ func newSourcemapFetcher( return nil, err } + size := 128 + invalidationChan := make(chan sourcemap.Identifier, size) + esFetcher := sourcemap.NewElasticsearchFetcher(esClient, apmSourcemapIndex) - cachingFetcher, err := sourcemap.NewCachingFetcher(esFetcher, cfg.Cache.Expiration) + cachingFetcher, err := sourcemap.NewCachingFetcher(esFetcher, invalidationChan, size) if err != nil { return nil, err } - metadataCachingFetcher := sourcemap.NewMetadataCachingFetcher(esClient, cachingFetcher, apmSourcemapIndex) + metadataCachingFetcher := sourcemap.NewMetadataCachingFetcher(esClient, cachingFetcher, apmSourcemapIndex, invalidationChan) metadataCachingFetcher.StartBackgroundSync() - chained = append(chained, metadataCachingFetcher) - - return chained, nil + return metadataCachingFetcher, nil } // TODO: This is copying behavior from libbeat: diff --git a/internal/sourcemap/caching.go b/internal/sourcemap/caching.go index 3e6f9722d60..693015448e9 100644 --- a/internal/sourcemap/caching.go +++ b/internal/sourcemap/caching.go @@ -21,11 +21,10 @@ import ( "context" "math" "strings" - "sync" "time" "github.com/go-sourcemap/sourcemap" - gocache "github.com/patrickmn/go-cache" + "github.com/hashicorp/golang-lru" "github.com/pkg/errors" "github.com/elastic/apm-server/internal/logs" @@ -52,27 +51,33 @@ type Fetcher interface { // CachingFetcher wraps a Fetcher, caching source maps in memory and fetching from the wrapped Fetcher on cache misses. type CachingFetcher struct { - cache *gocache.Cache + cache *lru.Cache backend Fetcher logger *logp.Logger - - mu sync.Mutex - inflight map[string]chan struct{} } // NewCachingFetcher returns a CachingFetcher that wraps backend, caching results for the configured cacheExpiration. func NewCachingFetcher( backend Fetcher, - cacheExpiration time.Duration, + invalidationChan <-chan Identifier, + cacheSize int, ) (*CachingFetcher, error) { - if cacheExpiration < 0 { - return nil, errInit + c, err := lru.New(cacheSize) + if err != nil { + return nil, err } + + go func() { + for i := range invalidationChan { + key := cacheKey([]string{i.name, i.version, i.path}) + c.Remove(key) + } + }() + return &CachingFetcher{ - cache: gocache.New(cacheExpiration, cleanupInterval(cacheExpiration)), - backend: backend, - logger: logp.NewLogger(logs.Sourcemap), - inflight: make(map[string]chan struct{}), + cache: c, + backend: backend, + logger: logp.NewLogger(logs.Sourcemap), }, nil } @@ -86,39 +91,6 @@ func (s *CachingFetcher) Fetch(ctx context.Context, name, version, path string) return consumer, nil } - // if the value hasn't been found, check to see if there's an inflight - // request to update the value. - s.mu.Lock() - wait, ok := s.inflight[key] - if ok { - // found an inflight request, wait for it to complete. - s.mu.Unlock() - - select { - case <-wait: - case <-ctx.Done(): - return nil, ctx.Err() - } - // Try to read the value again - return s.Fetch(ctx, name, version, path) - } - - // no inflight request found, add a channel to the map and then - // make the fetch request. - wait = make(chan struct{}) - s.inflight[key] = wait - - s.mu.Unlock() - - // Once the fetch request is complete, close and remove the channel - // from the syncronization map. - defer func() { - s.mu.Lock() - delete(s.inflight, key) - close(wait) - s.mu.Unlock() - }() - // fetch from the store and ensure caching for all non-temporary results consumer, err := s.backend.Fetch(ctx, name, version, path) if err != nil { @@ -132,11 +104,11 @@ func (s *CachingFetcher) Fetch(ctx context.Context, name, version, path string) } func (s *CachingFetcher) add(key string, consumer *sourcemap.Consumer) { - s.cache.SetDefault(key, consumer) + s.cache.Add(key, consumer) if !s.logger.IsDebug() { return } - s.logger.Debugf("Added id %v. Cache now has %v entries.", key, s.cache.ItemCount()) + s.logger.Debugf("Added id %v. Cache now has %v entries.", key, s.cache.Len()) } func cacheKey(s []string) string { diff --git a/internal/sourcemap/metadata.go b/internal/sourcemap/metadata.go index 2823457d749..581cab0f125 100644 --- a/internal/sourcemap/metadata.go +++ b/internal/sourcemap/metadata.go @@ -48,26 +48,29 @@ type Metadata struct { } type MetadataCachingFetcher struct { - esClient elasticsearch.Client - set map[Identifier]string - mu sync.RWMutex - backend Fetcher - logger *logp.Logger - once sync.Once - index string + esClient elasticsearch.Client + set map[Identifier]string + mu sync.RWMutex + backend Fetcher + logger *logp.Logger + once sync.Once + index string + invalidationChan chan<- Identifier } func NewMetadataCachingFetcher( c elasticsearch.Client, backend Fetcher, index string, + invalidationChan chan<- Identifier, ) *MetadataCachingFetcher { return &MetadataCachingFetcher{ - esClient: c, - index: index, - set: make(map[Identifier]string), - backend: backend, - logger: logp.NewLogger(logs.Sourcemap), + esClient: c, + index: index, + set: make(map[Identifier]string), + backend: backend, + logger: logp.NewLogger(logs.Sourcemap), + invalidationChan: invalidationChan, } } @@ -99,16 +102,34 @@ func (s *MetadataCachingFetcher) Fetch(ctx context.Context, name, version, path return nil, nil } -func (s *MetadataCachingFetcher) Update(ids []Identifier) { +func (s *MetadataCachingFetcher) Update(ms []Metadata) { s.mu.Lock() defer s.mu.Unlock() + for _, m := range ms { + if contentHash, ok := s.set[m.id]; ok { + // seen + delete(s.set, m.id) + + // content hash changed, invalidate the sourcemap cache + if contentHash != m.contentHash { + s.invalidationChan <- m.id + } + } + } + + // Loop for any unseed sourcemap for k := range s.set { + // the sourcemap no longer exists in ES. + // invalidate the sourcemap cache. + s.invalidationChan <- k + + // remove from metadata cache delete(s.set, k) } - for _, k := range ids { - s.set[k] = "" + for _, m := range ms { + s.set[m.id] = m.contentHash } } @@ -167,19 +188,22 @@ func (s *MetadataCachingFetcher) sync(ctx context.Context) error { return err } - var ids []Identifier + var ms []Metadata for _, v := range body.Hits.Hits { - id := Identifier{ - name: v.Source.ServiceName, - version: v.Source.ServiceVersion, - path: v.Source.BundleFilepath, + m := Metadata{ + id: Identifier{ + name: v.Source.ServiceName, + version: v.Source.ServiceVersion, + path: v.Source.BundleFilepath, + }, + contentHash: v.Source.ContentHash, } - ids = append(ids, id) + ms = append(ms, m) } // Update cache - s.Update(ids) + s.Update(ms) return nil } From f04473edb7d02089c725d1d3ef4f8e4ade06a520 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Tue, 13 Dec 2022 00:57:08 +0100 Subject: [PATCH 007/123] feat: remove fleet and kibana sourcemap fetchers --- internal/beater/beater.go | 78 ----------- internal/sourcemap/chained.go | 47 ------- internal/sourcemap/fleet.go | 186 ------------------------- internal/sourcemap/fleet_test.go | 217 ------------------------------ internal/sourcemap/kibana.go | 86 ------------ internal/sourcemap/kibana_test.go | 148 -------------------- 6 files changed, 762 deletions(-) delete mode 100644 internal/sourcemap/chained.go delete mode 100644 internal/sourcemap/fleet.go delete mode 100644 internal/sourcemap/fleet_test.go delete mode 100644 internal/sourcemap/kibana.go delete mode 100644 internal/sourcemap/kibana_test.go diff --git a/internal/beater/beater.go b/internal/beater/beater.go index 82d8d9c7e7d..9728d9ede62 100644 --- a/internal/beater/beater.go +++ b/internal/beater/beater.go @@ -23,7 +23,6 @@ import ( "fmt" "net" "net/http" - "net/url" "os" "runtime" "time" @@ -32,13 +31,11 @@ import ( "github.com/hashicorp/go-multierror" "github.com/pkg/errors" "go.elastic.co/apm/module/apmgrpc/v2" - "go.elastic.co/apm/module/apmhttp/v2" "go.elastic.co/apm/v2" "golang.org/x/sync/errgroup" "google.golang.org/grpc" "github.com/elastic/beats/v7/libbeat/beat" - "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/esleg/eslegclient" "github.com/elastic/beats/v7/libbeat/instrumentation" "github.com/elastic/beats/v7/libbeat/licenser" @@ -49,8 +46,6 @@ import ( agentconfig "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/monitoring" - "github.com/elastic/elastic-agent-libs/transport" - "github.com/elastic/elastic-agent-libs/transport/tlscommon" "github.com/elastic/go-ucfg" "github.com/elastic/apm-server/internal/agentcfg" @@ -777,79 +772,6 @@ func newSourcemapFetcher( kibanaClient *kibana.Client, newElasticsearchClient func(*elasticsearch.Config) (elasticsearch.Client, error), ) (sourcemap.Fetcher, error) { - // When running under Fleet we only fetch via Fleet Server. - if fleetCfg != nil { - var tlsConfig *tlscommon.TLSConfig - var err error - if fleetCfg.TLS.IsEnabled() { - if tlsConfig, err = tlscommon.LoadTLSConfig(fleetCfg.TLS); err != nil { - return nil, err - } - } - - timeout := 30 * time.Second - dialer := transport.NetDialer(timeout) - tlsDialer := transport.TLSDialer(dialer, tlsConfig, timeout) - - client := *http.DefaultClient - client.Transport = apmhttp.WrapRoundTripper(&http.Transport{ - Proxy: http.ProxyFromEnvironment, - Dial: dialer.Dial, - DialTLS: tlsDialer.Dial, - TLSClientConfig: tlsConfig.ToConfig(), - }) - - fleetServerURLs := make([]*url.URL, len(fleetCfg.Hosts)) - for i, host := range fleetCfg.Hosts { - urlString, err := common.MakeURL(fleetCfg.Protocol, "", host, 8220) - if err != nil { - return nil, err - } - u, err := url.Parse(urlString) - if err != nil { - return nil, err - } - fleetServerURLs[i] = u - } - - artifactRefs := make([]sourcemap.FleetArtifactReference, len(cfg.Metadata)) - for i, meta := range cfg.Metadata { - artifactRefs[i] = sourcemap.FleetArtifactReference{ - ServiceName: meta.ServiceName, - ServiceVersion: meta.ServiceVersion, - BundleFilepath: meta.BundleFilepath, - FleetServerURLPath: meta.SourceMapURL, - } - } - - ff, err := sourcemap.NewFleetFetcher( - &client, - fleetCfg.AccessAPIKey, - fleetServerURLs, - artifactRefs, - ) - if err != nil { - return nil, err - } - - cachingFetcher, err := sourcemap.NewCachingFetcher(ff, make(<-chan sourcemap.Identifier), 128) - if err != nil { - return nil, err - } - - return cachingFetcher, nil - } - - // For standalone, we query both Kibana and Elasticsearch for backwards compatibility. - var chained sourcemap.ChainedFetcher - if kibanaClient != nil { - cachingFetcher, err := sourcemap.NewCachingFetcher(sourcemap.NewKibanaFetcher(kibanaClient), make(<-chan sourcemap.Identifier), 128) - if err != nil { - return nil, err - } - - chained = append(chained, cachingFetcher) - } esClient, err := newElasticsearchClient(cfg.ESConfig) if err != nil { return nil, err diff --git a/internal/sourcemap/chained.go b/internal/sourcemap/chained.go deleted file mode 100644 index daeb11f4104..00000000000 --- a/internal/sourcemap/chained.go +++ /dev/null @@ -1,47 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package sourcemap - -import ( - "context" - - "github.com/go-sourcemap/sourcemap" -) - -// ChainedFetcher is a Fetcher that attempts fetching from each Fetcher in sequence. -type ChainedFetcher []Fetcher - -// Fetch fetches a source map from Kibana. -// -// Fetch calls Fetch on each Fetcher in the chain, in sequence, until one returns -// a non-nil Consumer and nil error. If no Fetch call succeeds, then the last error -// will be returned. -func (c ChainedFetcher) Fetch(ctx context.Context, name, version, path string) (*sourcemap.Consumer, error) { - var lastErr error - for _, f := range c { - consumer, err := f.Fetch(ctx, name, version, path) - if err != nil { - lastErr = err - continue - } - if consumer != nil { - return consumer, nil - } - } - return nil, lastErr -} diff --git a/internal/sourcemap/fleet.go b/internal/sourcemap/fleet.go deleted file mode 100644 index 1fe830f2628..00000000000 --- a/internal/sourcemap/fleet.go +++ /dev/null @@ -1,186 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package sourcemap - -import ( - "compress/zlib" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "net/url" - "path" - "sync" - - "github.com/go-sourcemap/sourcemap" - "github.com/pkg/errors" -) - -var errMsgFleetFailure = errMsgFailure + " fleet" - -type fleetFetcher struct { - authorization string - httpClient *http.Client - fleetServerURLs []*url.URL - sourceMapURLPaths map[key]string -} - -type key struct { - ServiceName string - ServiceVersion string - BundleFilepath string -} - -// FleetArtifactReference holds information mapping a source map to a -// Fleet Artifact URL path. The server uses this to fetch the source -// map content via Fleet Server. -type FleetArtifactReference struct { - // ServiceName holds the service name to which the source map relates. - ServiceName string - - // ServiceVersion holds the service version to which the source map relates. - ServiceVersion string - - // BundleFilepath holds the bundle file path to which the source map relates. - // - // If BundleFilepath is an absolute URL, only the URL path is matched on. - BundleFilepath string - - // FleetServerURLPath holds the URL path for fetching the source map, - // by appending it to each of the Fleet Server URLs. - FleetServerURLPath string -} - -// NewFleetFetcher returns a Fetcher which fetches source maps via Fleet Server. -func NewFleetFetcher( - httpClient *http.Client, apiKey string, - fleetServerURLs []*url.URL, - refs []FleetArtifactReference, -) (Fetcher, error) { - - if len(fleetServerURLs) == 0 { - return nil, errors.New("no fleet-server hosts present for fleet store") - } - - sourceMapURLPaths := make(map[key]string) - for _, ref := range refs { - k := key{ref.ServiceName, ref.ServiceVersion, maybeParseURLPath(ref.BundleFilepath)} - sourceMapURLPaths[k] = ref.FleetServerURLPath - } - - return fleetFetcher{ - authorization: "ApiKey " + apiKey, - httpClient: httpClient, - fleetServerURLs: fleetServerURLs, - sourceMapURLPaths: sourceMapURLPaths, - }, nil -} - -// Fetch fetches a source map from Fleet Server. -func (f fleetFetcher) Fetch(ctx context.Context, name, version, bundleFilepath string) (*sourcemap.Consumer, error) { - sourceMapURLPath, ok := f.sourceMapURLPaths[key{name, version, maybeParseURLPath(bundleFilepath)}] - if !ok { - return nil, fmt.Errorf("unable to find sourcemap.url for service.name=%s service.version=%s bundle.path=%s", - name, version, bundleFilepath, - ) - } - - ctx, cancel := context.WithCancel(ctx) - defer cancel() - - type result struct { - sourcemap string - err error - } - - results := make(chan result) - var wg sync.WaitGroup - for _, baseURL := range f.fleetServerURLs { - // TODO(axw) use URL.JoinPath when we upgrade to Go 1.19. - u := *baseURL - u.Path = path.Join(u.Path, sourceMapURLPath) - artifactURL := u.String() - - wg.Add(1) - go func() { - defer wg.Done() - sourcemap, err := sendRequest(ctx, f, artifactURL) - select { - case <-ctx.Done(): - case results <- result{sourcemap, err}: - } - }() - } - - go func() { - wg.Wait() - close(results) - }() - - var err error - for result := range results { - err = result.err - if err == nil { - return parseSourceMap(result.sourcemap) - } - } - - if err != nil { - return nil, err - } - // No results were received: context was cancelled. - return nil, ctx.Err() -} - -func sendRequest(ctx context.Context, f fleetFetcher, fleetURL string) (string, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, fleetURL, nil) - if err != nil { - return "", err - } - req.Header.Add("Authorization", f.authorization) - - resp, err := f.httpClient.Do(req) - if err != nil { - return "", err - } - defer resp.Body.Close() - - // Verify that we should only get 200 back from fleet-server - if resp.StatusCode != http.StatusOK { - body, err := io.ReadAll(resp.Body) - if err != nil { - return "", fmt.Errorf(errMsgFleetFailure, ": statuscode=%d response=(failed to read body)", resp.StatusCode) - } - return "", fmt.Errorf(errMsgFleetFailure, ": statuscode=%d response=%s", resp.StatusCode, body) - } - - // Looking at the index in elasticsearch, currently - // - no encryption - // - zlib compression - r, err := zlib.NewReader(resp.Body) - if err != nil { - return "", err - } - - var m map[string]json.RawMessage - if err := json.NewDecoder(r).Decode(&m); err != nil { - return "", err - } - return string(m["sourceMap"]), nil -} diff --git a/internal/sourcemap/fleet_test.go b/internal/sourcemap/fleet_test.go deleted file mode 100644 index 7ee30e0a3ac..00000000000 --- a/internal/sourcemap/fleet_test.go +++ /dev/null @@ -1,217 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package sourcemap - -import ( - "compress/zlib" - "context" - "net/http" - "net/http/httptest" - "net/url" - "sync" - "sync/atomic" - "testing" - - "github.com/go-sourcemap/sourcemap" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestFleetFetch(t *testing.T) { - var ( - apikey = "supersecret" - name = "webapp" - version = "1.0.0" - path1 = "/my/path/to/bundle1.js.map" - path2 = "/my/path/to/bundle2.js.map" - c = http.DefaultClient - sourceMapPath = "/api/fleet/artifact" - ) - - h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - assert.Equal(t, sourceMapPath, r.URL.Path) - if auth := r.Header.Get("Authorization"); auth != "ApiKey "+apikey { - w.WriteHeader(http.StatusUnauthorized) - return - } - // zlib compress - wr := zlib.NewWriter(w) - defer wr.Close() - wr.Write([]byte(resp)) - }) - - ts0 := httptest.NewServer(h) - defer ts0.Close() - ts0URL, _ := url.Parse(ts0.URL) - - ts1 := httptest.NewServer(h) - defer ts1.Close() - ts1URL, _ := url.Parse(ts1.URL) - - fleetServerURLs := []*url.URL{ts0URL, ts1URL} - f, err := NewFleetFetcher(c, apikey, fleetServerURLs, []FleetArtifactReference{{ - ServiceName: name, - ServiceVersion: version, - BundleFilepath: path1, - FleetServerURLPath: sourceMapPath, - }, { - ServiceName: name, - ServiceVersion: version, - BundleFilepath: "http://not_testing.invalid" + path2, - FleetServerURLPath: sourceMapPath, - }}) - assert.NoError(t, err) - - for _, path := range []string{path1, path2} { - consumer, err := f.Fetch(context.Background(), name, version, path) - assert.NoError(t, err) - assert.NotNil(t, consumer) - - // Check that only the URL *path* is used, and matches entries in the - // fetcher stored with either absolute or relative URLs. - consumer, err = f.Fetch(context.Background(), name, version, "http://testing.invalid"+path) - assert.NoError(t, err) - assert.NotNil(t, consumer) - } -} - -func TestFailedAndSuccessfulFleetHostsFetch(t *testing.T) { - type response struct { - consumer *sourcemap.Consumer - err error - } - var ( - apikey = "supersecret" - name = "webapp" - version = "1.0.0" - path = "/my/path/to/bundle.js.map" - c = http.DefaultClient - sourceMapPath = "/api/fleet/artifact" - successc = make(chan struct{}) - errc = make(chan struct{}) - lastc = make(chan struct{}) - waitc = make(chan struct{}) - resc = make(chan response) - wg sync.WaitGroup - ) - wg.Add(3) - defer func() { - close(successc) - close(errc) - close(lastc) - close(resc) - }() - - hError := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - wg.Done() - errc <- struct{}{} - http.Error(w, "err", http.StatusInternalServerError) - }) - ts0 := httptest.NewServer(hError) - ts0URL, _ := url.Parse(ts0.URL) - - h1 := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - wg.Done() - successc <- struct{}{} - wr := zlib.NewWriter(w) - defer wr.Close() - wr.Write([]byte(resp)) - }) - ts1 := httptest.NewServer(h1) - ts1URL, _ := url.Parse(ts1.URL) - - h2 := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - wg.Done() - lastc <- struct{}{} - close(waitc) - }) - ts2 := httptest.NewServer(h2) - ts2URL, _ := url.Parse(ts2.URL) - - fleetServerURLs := []*url.URL{ts0URL, ts1URL, ts2URL} - f, err := NewFleetFetcher(c, apikey, fleetServerURLs, []FleetArtifactReference{{ - ServiceName: name, - ServiceVersion: version, - BundleFilepath: path, - FleetServerURLPath: sourceMapPath, - }}) - assert.NoError(t, err) - - go func() { - consumer, err := f.Fetch(context.Background(), name, version, path) - resc <- response{consumer, err} - }() - // Make sure every server has received a request - wg.Wait() - <-errc - <-successc - res := <-resc - assert.NoError(t, res.err) - assert.NotNil(t, res.consumer) - - // Wait for h2 - <-lastc - <-waitc -} - -func TestAllFailedFleetHostsFetch(t *testing.T) { - var ( - requestCount int32 - apikey = "supersecret" - name = "webapp" - version = "1.0.0" - path = "/my/path/to/bundle.js.map" - c = http.DefaultClient - sourceMapPath = "/api/fleet/artifact" - ) - - h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - http.Error(w, "err", http.StatusInternalServerError) - atomic.AddInt32(&requestCount, 1) - }) - - ts0 := httptest.NewServer(h) - defer ts0.Close() - ts0URL, _ := url.Parse(ts0.URL) - - ts1 := httptest.NewServer(h) - defer ts1.Close() - ts1URL, _ := url.Parse(ts1.URL) - - ts2 := httptest.NewServer(h) - defer ts2.Close() - ts2URL, _ := url.Parse(ts2.URL) - - fleetServerURLs := []*url.URL{ts0URL, ts1URL, ts2URL} - f, err := NewFleetFetcher(c, apikey, fleetServerURLs, []FleetArtifactReference{{ - ServiceName: name, - ServiceVersion: version, - BundleFilepath: path, - FleetServerURLPath: sourceMapPath, - }}) - assert.NoError(t, err) - - consumer, err := f.Fetch(context.Background(), name, version, path) - assert.EqualValues(t, len(fleetServerURLs), requestCount) - require.Error(t, err) - assert.Contains(t, err.Error(), errMsgFleetFailure) - assert.Nil(t, consumer) -} - -var resp = "{\"serviceName\":\"web-app\",\"serviceVersion\":\"1.0.0\",\"bundleFilepath\":\"/test/e2e/general-usecase/bundle.js.map\",\"sourceMap\":{\"version\":3,\"sources\":[\"webpack:///bundle.js\",\"\",\"webpack:///./scripts/index.js\",\"webpack:///./index.html\",\"webpack:///./scripts/app.js\"],\"names\":[\"modules\",\"__webpack_require__\",\"moduleId\",\"installedModules\",\"exports\",\"module\",\"id\",\"loaded\",\"call\",\"m\",\"c\",\"p\",\"foo\",\"console\",\"log\",\"foobar\"],\"mappings\":\"CAAS,SAAUA,GCInB,QAAAC,GAAAC,GAGA,GAAAC,EAAAD,GACA,MAAAC,GAAAD,GAAAE,OAGA,IAAAC,GAAAF,EAAAD,IACAE,WACAE,GAAAJ,EACAK,QAAA,EAUA,OANAP,GAAAE,GAAAM,KAAAH,EAAAD,QAAAC,IAAAD,QAAAH,GAGAI,EAAAE,QAAA,EAGAF,EAAAD,QAvBA,GAAAD,KAqCA,OATAF,GAAAQ,EAAAT,EAGAC,EAAAS,EAAAP,EAGAF,EAAAU,EAAA,GAGAV,EAAA,KDMM,SAASI,EAAQD,EAASH,GE3ChCA,EAAA,GAEAA,EAAA,GAEAW,OFmDM,SAASP,EAAQD,EAASH,GGxDhCI,EAAAD,QAAAH,EAAAU,EAAA,cH8DM,SAASN,EAAQD,GI9DvB,QAAAQ,KACAC,QAAAC,IAAAC,QAGAH\",\"file\":\"bundle.js\",\"sourcesContent\":[\"/******/ (function(modules) { // webpackBootstrap\\n/******/ \\t// The module cache\\n/******/ \\tvar installedModules = {};\\n/******/\\n/******/ \\t// The require function\\n/******/ \\tfunction __webpack_require__(moduleId) {\\n/******/\\n/******/ \\t\\t// Check if module is in cache\\n/******/ \\t\\tif(installedModules[moduleId])\\n/******/ \\t\\t\\treturn installedModules[moduleId].exports;\\n/******/\\n/******/ \\t\\t// Create a new module (and put it into the cache)\\n/******/ \\t\\tvar module = installedModules[moduleId] = {\\n/******/ \\t\\t\\texports: {},\\n/******/ \\t\\t\\tid: moduleId,\\n/******/ \\t\\t\\tloaded: false\\n/******/ \\t\\t};\\n/******/\\n/******/ \\t\\t// Execute the module function\\n/******/ \\t\\tmodules[moduleId].call(module.exports, module, module.exports, __webpack_require__);\\n/******/\\n/******/ \\t\\t// Flag the module as loaded\\n/******/ \\t\\tmodule.loaded = true;\\n/******/\\n/******/ \\t\\t// Return the exports of the module\\n/******/ \\t\\treturn module.exports;\\n/******/ \\t}\\n/******/\\n/******/\\n/******/ \\t// expose the modules object (__webpack_modules__)\\n/******/ \\t__webpack_require__.m = modules;\\n/******/\\n/******/ \\t// expose the module cache\\n/******/ \\t__webpack_require__.c = installedModules;\\n/******/\\n/******/ \\t// __webpack_public_path__\\n/******/ \\t__webpack_require__.p = \\\"\\\";\\n/******/\\n/******/ \\t// Load entry module and return exports\\n/******/ \\treturn __webpack_require__(0);\\n/******/ })\\n/************************************************************************/\\n/******/ ([\\n/* 0 */\\n/***/ function(module, exports, __webpack_require__) {\\n\\n\\t// Webpack\\n\\t__webpack_require__(1)\\n\\t\\n\\t__webpack_require__(2)\\n\\t\\n\\tfoo()\\n\\n\\n/***/ },\\n/* 1 */\\n/***/ function(module, exports, __webpack_require__) {\\n\\n\\tmodule.exports = __webpack_require__.p + \\\"index.html\\\"\\n\\n/***/ },\\n/* 2 */\\n/***/ function(module, exports) {\\n\\n\\tfunction foo() {\\n\\t console.log(foobar)\\n\\t}\\n\\t\\n\\tfoo()\\n\\n\\n/***/ }\\n/******/ ]);\\n\\n\\n/** WEBPACK FOOTER **\\n ** bundle.js\\n **/\",\" \\t// The module cache\\n \\tvar installedModules = {};\\n\\n \\t// The require function\\n \\tfunction __webpack_require__(moduleId) {\\n\\n \\t\\t// Check if module is in cache\\n \\t\\tif(installedModules[moduleId])\\n \\t\\t\\treturn installedModules[moduleId].exports;\\n\\n \\t\\t// Create a new module (and put it into the cache)\\n \\t\\tvar module = installedModules[moduleId] = {\\n \\t\\t\\texports: {},\\n \\t\\t\\tid: moduleId,\\n \\t\\t\\tloaded: false\\n \\t\\t};\\n\\n \\t\\t// Execute the module function\\n \\t\\tmodules[moduleId].call(module.exports, module, module.exports, __webpack_require__);\\n\\n \\t\\t// Flag the module as loaded\\n \\t\\tmodule.loaded = true;\\n\\n \\t\\t// Return the exports of the module\\n \\t\\treturn module.exports;\\n \\t}\\n\\n\\n \\t// expose the modules object (__webpack_modules__)\\n \\t__webpack_require__.m = modules;\\n\\n \\t// expose the module cache\\n \\t__webpack_require__.c = installedModules;\\n\\n \\t// __webpack_public_path__\\n \\t__webpack_require__.p = \\\"\\\";\\n\\n \\t// Load entry module and return exports\\n \\treturn __webpack_require__(0);\\n\\n\\n\\n/** WEBPACK FOOTER **\\n ** webpack/bootstrap 6002740481c9666b0d38\\n **/\",\"// Webpack\\nrequire('../index.html')\\n\\nrequire('./app')\\n\\nfoo()\\n\\n\\n\\n/*****************\\n ** WEBPACK FOOTER\\n ** ./scripts/index.js\\n ** module id = 0\\n ** module chunks = 0\\n **/\",\"module.exports = __webpack_public_path__ + \\\"index.html\\\"\\n\\n\\n/*****************\\n ** WEBPACK FOOTER\\n ** ./index.html\\n ** module id = 1\\n ** module chunks = 0\\n **/\",\"function foo() {\\n console.log(foobar)\\n}\\n\\nfoo()\\n\\n\\n\\n/*****************\\n ** WEBPACK FOOTER\\n ** ./scripts/app.js\\n ** module id = 2\\n ** module chunks = 0\\n **/\"],\"sourceRoot\":\"\"}}" diff --git a/internal/sourcemap/kibana.go b/internal/sourcemap/kibana.go deleted file mode 100644 index 61608504505..00000000000 --- a/internal/sourcemap/kibana.go +++ /dev/null @@ -1,86 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package sourcemap - -import ( - "context" - "encoding/json" - "fmt" - "io" - "net/http" - - "github.com/go-sourcemap/sourcemap" - - "github.com/elastic/elastic-agent-libs/logp" - - "github.com/elastic/apm-server/internal/kibana" - "github.com/elastic/apm-server/internal/logs" -) - -const sourcemapArtifactType = "sourcemap" - -type kibanaFetcher struct { - client *kibana.Client - logger *logp.Logger -} - -type kibanaSourceMapArtifact struct { - Type string `json:"type"` - Body struct { - ServiceName string `json:"serviceName"` - ServiceVersion string `json:"serviceVersion"` - BundleFilepath string `json:"bundleFilepath"` - SourceMap json.RawMessage `json:"sourceMap"` - } `json:"body"` -} - -// NewKibanaFetcher returns a Fetcher that fetches source maps stored by Kibana. -func NewKibanaFetcher(c *kibana.Client) Fetcher { - logger := logp.NewLogger(logs.Sourcemap) - return &kibanaFetcher{c, logger} -} - -// Fetch fetches a source map from Kibana. -func (s *kibanaFetcher) Fetch(ctx context.Context, name, version, path string) (*sourcemap.Consumer, error) { - resp, err := s.client.Send(ctx, "GET", "/api/apm/sourcemaps", nil, nil, nil) - if err != nil { - return nil, err - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - body, _ := io.ReadAll(resp.Body) - return nil, fmt.Errorf("failed to query source maps (%s): %s", resp.Status, body) - } - - var result struct { - Artifacts []kibanaSourceMapArtifact `json:"artifacts"` - } - if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { - return nil, err - } - path = maybeParseURLPath(path) - for _, a := range result.Artifacts { - if a.Type != sourcemapArtifactType { - continue - } - if a.Body.ServiceName == name && a.Body.ServiceVersion == version && maybeParseURLPath(a.Body.BundleFilepath) == path { - return parseSourceMap(string(a.Body.SourceMap)) - } - } - return nil, nil -} diff --git a/internal/sourcemap/kibana_test.go b/internal/sourcemap/kibana_test.go deleted file mode 100644 index 2565d255a0d..00000000000 --- a/internal/sourcemap/kibana_test.go +++ /dev/null @@ -1,148 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package sourcemap - -import ( - "context" - "encoding/json" - "fmt" - "net/http" - "net/http/httptest" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - libkibana "github.com/elastic/elastic-agent-libs/kibana" - - "github.com/elastic/apm-server/internal/kibana" -) - -func TestKibanaFetcher(t *testing.T) { - fetcher := newTestKibanaFetcher(t, func(w http.ResponseWriter, r *http.Request) { - result := map[string]interface{}{ - "artifacts": []interface{}{ - map[string]interface{}{ - "type": "not_a_sourcemap", - "body": map[string]interface{}{ - "serviceName": "service_name", - "serviceVersion": "service_version", - "bundleFilepath": "http://another_host:456/path", - "sourceMap": "invalid_sourcemap", - }, - }, - map[string]interface{}{ - "type": "sourcemap", - "body": map[string]interface{}{ - "serviceName": "non_matching_service_name", - "serviceVersion": "service_version", - "bundleFilepath": "http://another_host:456/path", - "sourceMap": "invalid_sourcemap", - }, - }, - map[string]interface{}{ - "type": "sourcemap", - "body": map[string]interface{}{ - "serviceName": "service_name", - "serviceVersion": "non_matching_service_version", - "bundleFilepath": "http://another_host:456/path", - "sourceMap": "invalid_sourcemap", - }, - }, - map[string]interface{}{ - "type": "sourcemap", - "body": map[string]interface{}{ - "serviceName": "service_name", - "serviceVersion": "service_version", - "bundleFilepath": "http://another_host:456/non_matching_path", - "sourceMap": "invalid_sourcemap", - }, - }, - map[string]interface{}{ - "type": "sourcemap", - "body": map[string]interface{}{ - "serviceName": "service_name", - "serviceVersion": "service_version", - "bundleFilepath": "http://another_host:456/path", - "sourceMap": json.RawMessage(validSourcemap), - }, - }, - }, - } - json.NewEncoder(w).Encode(result) - }) - consumer, err := fetcher.Fetch(context.Background(), "service_name", "service_version", "http://host:123/path") - require.NoError(t, err) - assert.NotNil(t, consumer) -} - -func TestKibanaFetcherInvalidSourcemap(t *testing.T) { - fetcher := newTestKibanaFetcher(t, func(w http.ResponseWriter, r *http.Request) { - result := map[string]interface{}{ - "artifacts": []interface{}{ - map[string]interface{}{ - "type": "sourcemap", - "body": map[string]interface{}{ - "serviceName": "service_name", - "serviceVersion": "service_version", - "bundleFilepath": "http://another_host:456/path", - "sourceMap": "invalid_sourcemap", - }, - }, - }, - } - json.NewEncoder(w).Encode(result) - }) - consumer, err := fetcher.Fetch(context.Background(), "service_name", "service_version", "http://host:123/path") - require.Error(t, err) - assert.EqualError(t, err, "Could not parse Sourcemap: json: cannot unmarshal string into Go value of type sourcemap.v3") - assert.Nil(t, consumer) -} - -func TestKibanaFetcherNotFound(t *testing.T) { - fetcher := newTestKibanaFetcher(t, func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintf(w, `{"artifacts":[]}`) - }) - consumer, err := fetcher.Fetch(context.Background(), "service_name", "service_version", "http://host:123/path") - require.NoError(t, err) - assert.Nil(t, consumer) -} - -func TestKibanaFetcherServerError(t *testing.T) { - fetcher := newTestKibanaFetcher(t, func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte("terrible things")) - }) - consumer, err := fetcher.Fetch(context.Background(), "service_name", "service_version", "http://host:123/path") - require.Error(t, err) - assert.EqualError(t, err, "failed to query source maps (500 Internal Server Error): terrible things") - assert.Nil(t, consumer) -} - -func newTestKibanaFetcher(t testing.TB, h http.HandlerFunc) Fetcher { - mux := http.NewServeMux() - mux.HandleFunc("/api/apm/sourcemaps", h) - srv := httptest.NewServer(mux) - t.Cleanup(srv.Close) - - kibanaClient, err := kibana.NewClient(libkibana.ClientConfig{ - Host: srv.Listener.Addr().String(), - }) - require.NoError(t, err) - return NewKibanaFetcher(kibanaClient) -} From 7a64c06c6ed5cc6e11a6b15e05834ec8ba1e4d73 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Tue, 13 Dec 2022 02:18:25 +0100 Subject: [PATCH 008/123] refactor: code cleanup --- .../model/modelprocessor/internal_metrics.go | 17 ---- internal/sourcemap/caching.go | 52 +++++------ internal/sourcemap/elasticsearch.go | 61 +++---------- internal/sourcemap/fetcher.go | 44 ++++++++++ internal/sourcemap/metadata.go | 21 ++--- internal/sourcemap/search.go | 88 +++++++++++++++++++ 6 files changed, 168 insertions(+), 115 deletions(-) create mode 100644 internal/sourcemap/fetcher.go create mode 100644 internal/sourcemap/search.go diff --git a/internal/model/modelprocessor/internal_metrics.go b/internal/model/modelprocessor/internal_metrics.go index adde5da0f05..b31ddb44052 100644 --- a/internal/model/modelprocessor/internal_metrics.go +++ b/internal/model/modelprocessor/internal_metrics.go @@ -1,20 +1,3 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - package modelprocessor func isInternalMetricName(name string) bool { diff --git a/internal/sourcemap/caching.go b/internal/sourcemap/caching.go index 693015448e9..d1a49bcf76c 100644 --- a/internal/sourcemap/caching.go +++ b/internal/sourcemap/caching.go @@ -19,36 +19,21 @@ package sourcemap import ( "context" - "math" + "fmt" "strings" - "time" "github.com/go-sourcemap/sourcemap" - "github.com/hashicorp/golang-lru" + lru "github.com/hashicorp/golang-lru" "github.com/pkg/errors" "github.com/elastic/apm-server/internal/logs" "github.com/elastic/elastic-agent-libs/logp" ) -const ( - minCleanupIntervalSeconds float64 = 60 -) - var ( errMsgFailure = "failure querying" - errInit = errors.New("Cache cannot be initialized. Expiration and CleanupInterval need to be >= 0") ) -// Fetcher is an interface for fetching a source map with a given service name, service version, -// and bundle filepath. -type Fetcher interface { - // Fetch fetches a source map with a given service name, service version, and bundle filepath. - // - // If there is no such source map available, Fetch returns a nil Consumer. - Fetch(ctx context.Context, serviceName, serviceVersion, bundleFilepath string) (*sourcemap.Consumer, error) -} - // CachingFetcher wraps a Fetcher, caching source maps in memory and fetching from the wrapped Fetcher on cache misses. type CachingFetcher struct { cache *lru.Cache @@ -62,28 +47,39 @@ func NewCachingFetcher( invalidationChan <-chan Identifier, cacheSize int, ) (*CachingFetcher, error) { - c, err := lru.New(cacheSize) + logger := logp.NewLogger(logs.Sourcemap) + + c, err := lru.NewWithEvict(cacheSize, func(key, value interface{}) { + if !logger.IsDebug() { + return + } + logger.Debugf("Removed id %v", key) + }) + if err != nil { - return nil, err + return nil, fmt.Errorf("failed to create lru cache for caching fetcher: %w", err) } go func() { for i := range invalidationChan { - key := cacheKey([]string{i.name, i.version, i.path}) - c.Remove(key) + c.Remove(i) } }() return &CachingFetcher{ cache: c, backend: backend, - logger: logp.NewLogger(logs.Sourcemap), + logger: logger, }, nil } // Fetch fetches a source map from the cache or wrapped backend. func (s *CachingFetcher) Fetch(ctx context.Context, name, version, path string) (*sourcemap.Consumer, error) { - key := cacheKey([]string{name, version, path}) + key := Identifier{ + name: name, + version: version, + path: path, + } // fetch from cache if val, found := s.cache.Get(key); found { @@ -103,7 +99,7 @@ func (s *CachingFetcher) Fetch(ctx context.Context, name, version, path string) return consumer, nil } -func (s *CachingFetcher) add(key string, consumer *sourcemap.Consumer) { +func (s *CachingFetcher) add(key Identifier, consumer *sourcemap.Consumer) { s.cache.Add(key, consumer) if !s.logger.IsDebug() { return @@ -111,14 +107,6 @@ func (s *CachingFetcher) add(key string, consumer *sourcemap.Consumer) { s.logger.Debugf("Added id %v. Cache now has %v entries.", key, s.cache.Len()) } -func cacheKey(s []string) string { - return strings.Join(s, "_") -} - -func cleanupInterval(ttl time.Duration) time.Duration { - return time.Duration(math.Max(ttl.Seconds(), minCleanupIntervalSeconds)) * time.Second -} - func parseSourceMap(data string) (*sourcemap.Consumer, error) { if data == "" { return nil, nil diff --git a/internal/sourcemap/elasticsearch.go b/internal/sourcemap/elasticsearch.go index 2044d2079b6..dde79996af4 100644 --- a/internal/sourcemap/elasticsearch.go +++ b/internal/sourcemap/elasticsearch.go @@ -123,7 +123,7 @@ func (s *esFetcher) Fetch(ctx context.Context, name, version, path string) (*sou func (s *esFetcher) runSearchQuery(ctx context.Context, name, version, path string) (*esapi.Response, error) { var buf bytes.Buffer - if err := json.NewEncoder(&buf).Encode(query(name, version, path)); err != nil { + if err := json.NewEncoder(&buf).Encode(requestBody(name, version, path)); err != nil { return nil, err } req := esapi.SearchRequest{ @@ -163,61 +163,22 @@ func parse(body io.ReadCloser, name, version, path string, logger *logp.Logger) return esSourcemap, nil } -func query(name, version, path string) map[string]interface{} { +func requestBody(name, version, path string) map[string]interface{} { id := name + "-" + version + "-" + path - return searchFirst( - boolean( - must( - term("_id", id), + return search( + size(1), + sort(desc("_score")), + source("content"), + query( + boolean( + must( + term("_id", id), + ), ), ), - "content", - desc("_score"), ) } -func wrap(k string, v map[string]interface{}) map[string]interface{} { - return map[string]interface{}{k: v} -} - -func boolean(clause map[string]interface{}) map[string]interface{} { - return wrap("bool", clause) -} - -func should(clauses ...map[string]interface{}) map[string]interface{} { - return map[string]interface{}{"should": clauses} -} - -func must(clauses ...map[string]interface{}) map[string]interface{} { - return map[string]interface{}{"must": clauses} -} - -func term(k, v string) map[string]interface{} { - return map[string]interface{}{"term": map[string]interface{}{k: v}} -} - -func boostedTerm(k, v string, boost float32) map[string]interface{} { - return wrap("term", - wrap(k, map[string]interface{}{ - "value": v, - "boost": boost, - }), - ) -} - -func desc(by string) map[string]interface{} { - return wrap(by, map[string]interface{}{"order": "desc"}) -} - -func searchFirst(query map[string]interface{}, source string, sort ...map[string]interface{}) map[string]interface{} { - return map[string]interface{}{ - "query": query, - "size": 1, - "sort": sort, - "_source": source, - } -} - // maybeParseURLPath attempts to parse s as a URL, returning its path component // if successful. If s cannot be parsed as a URL, s is returned. func maybeParseURLPath(s string) string { diff --git a/internal/sourcemap/fetcher.go b/internal/sourcemap/fetcher.go new file mode 100644 index 00000000000..ebbdbef4c11 --- /dev/null +++ b/internal/sourcemap/fetcher.go @@ -0,0 +1,44 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package sourcemap + +import ( + "context" + + "github.com/go-sourcemap/sourcemap" +) + +// Fetcher is an interface for fetching a source map with a given service name, service version, +// and bundle filepath. +type Fetcher interface { + // Fetch fetches a source map with a given service name, service version, and bundle filepath. + // + // If there is no such source map available, Fetch returns a nil Consumer. + Fetch(ctx context.Context, name string, version string, bundleFilepath string) (*sourcemap.Consumer, error) +} + +type Identifier struct { + name string + version string + path string +} + +type Metadata struct { + id Identifier + contentHash string +} diff --git a/internal/sourcemap/metadata.go b/internal/sourcemap/metadata.go index 581cab0f125..7c9b9dab740 100644 --- a/internal/sourcemap/metadata.go +++ b/internal/sourcemap/metadata.go @@ -36,17 +36,6 @@ import ( "github.com/elastic/go-elasticsearch/v8/esapi" ) -type Identifier struct { - name string - version string - path string -} - -type Metadata struct { - id Identifier - contentHash string -} - type MetadataCachingFetcher struct { esClient elasticsearch.Client set map[Identifier]string @@ -102,7 +91,7 @@ func (s *MetadataCachingFetcher) Fetch(ctx context.Context, name, version, path return nil, nil } -func (s *MetadataCachingFetcher) Update(ms []Metadata) { +func (s *MetadataCachingFetcher) update(ms []Metadata) { s.mu.Lock() defer s.mu.Unlock() @@ -203,7 +192,7 @@ func (s *MetadataCachingFetcher) sync(ctx context.Context) error { } // Update cache - s.Update(ms) + s.update(ms) return nil } @@ -222,9 +211,9 @@ func (s *MetadataCachingFetcher) runSearchQuery(ctx context.Context) (*esapi.Res } func queryMetadata() map[string]interface{} { - return map[string]interface{}{ - "_source": []string{"service.*", "file.path", "content_sha256"}, - } + return search( + sources([]string{"service.*", "file.path", "content_sha256"}), + ) } func parseResponse(body io.ReadCloser, logger *logp.Logger) (esSourcemapResponse, error) { diff --git a/internal/sourcemap/search.go b/internal/sourcemap/search.go new file mode 100644 index 00000000000..109c12ae930 --- /dev/null +++ b/internal/sourcemap/search.go @@ -0,0 +1,88 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package sourcemap + +type searchOption func(map[string]interface{}) + +func search(opts ...searchOption) map[string]interface{} { + m := make(map[string]interface{}, len(opts)) + + for _, opt := range opts { + opt(m) + } + + return m +} + +func source(s string) searchOption { + return func(m map[string]interface{}) { + m["_source"] = s + } +} + +func sources(s []string) searchOption { + return func(m map[string]interface{}) { + m["_source"] = s + } +} + +func size(i int) searchOption { + return func(m map[string]interface{}) { + m["size"] = 1 + } +} + +func sort(opts ...searchOption) searchOption { + return func(m map[string]interface{}) { + s := make(map[string]interface{}, len(opts)) + + for _, opt := range opts { + opt(s) + } + + m["sort"] = s + } +} + +func desc(k string) searchOption { + return func(m map[string]interface{}) { + m[k] = map[string]interface{}{"order": "desc"} + } +} + +func query(q map[string]interface{}) searchOption { + return func(m map[string]interface{}) { + m["query"] = q + } +} + +func wrap(k string, v map[string]interface{}) map[string]interface{} { + return map[string]interface{}{k: v} +} + +func boolean(clause map[string]interface{}) map[string]interface{} { + return wrap("bool", clause) +} + +func must(clauses ...map[string]interface{}) map[string]interface{} { + return map[string]interface{}{"must": clauses} +} + +func term(k, v string) map[string]interface{} { + return map[string]interface{}{"term": map[string]interface{}{k: v}} +} From c96587df006bfb001b5b7876e20091d381c8c2cc Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Tue, 13 Dec 2022 02:29:49 +0100 Subject: [PATCH 009/123] fix: do not try to decode the body if the sourcemap was not found --- internal/sourcemap/elasticsearch.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/sourcemap/elasticsearch.go b/internal/sourcemap/elasticsearch.go index dde79996af4..15d401dd4b5 100644 --- a/internal/sourcemap/elasticsearch.go +++ b/internal/sourcemap/elasticsearch.go @@ -102,6 +102,10 @@ func (s *esFetcher) Fetch(ctx context.Context, name, version, path string) (*sou return nil, err } + if body == "" { + return nil, nil + } + d, err := base64.StdEncoding.DecodeString(body) if err != nil { return nil, fmt.Errorf("failed to base64 decode string: %w", err) From 855cb02f9e0639e803350ccefda685677015a052 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Tue, 13 Dec 2022 02:30:43 +0100 Subject: [PATCH 010/123] test: update tests for sourcemap changes --- internal/beater/beater_test.go | 75 --------- internal/sourcemap/caching_test.go | 188 +---------------------- internal/sourcemap/elasticsearch_test.go | 15 +- internal/sourcemap/processor_test.go | 2 +- 4 files changed, 21 insertions(+), 259 deletions(-) diff --git a/internal/beater/beater_test.go b/internal/beater/beater_test.go index 09ed3e799b8..9e0b657f238 100644 --- a/internal/beater/beater_test.go +++ b/internal/beater/beater_test.go @@ -19,15 +19,12 @@ package beater import ( "bytes" - "compress/zlib" "context" "encoding/json" - "fmt" "io" "net/http" "net/http/httptest" "os" - "strings" "testing" "github.com/stretchr/testify/assert" @@ -35,46 +32,9 @@ import ( "github.com/elastic/apm-server/internal/beater/config" "github.com/elastic/apm-server/internal/elasticsearch" - "github.com/elastic/apm-server/internal/version" "github.com/elastic/elastic-agent-libs/monitoring" ) -func TestSourcemapIndexPattern(t *testing.T) { - test := func(t *testing.T, indexPattern, expected string) { - var requestPaths []string - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - requestPaths = append(requestPaths, r.URL.Path) - })) - defer srv.Close() - - cfg := config.DefaultConfig() - cfg.RumConfig.Enabled = true - cfg.RumConfig.SourceMapping.ESConfig.Hosts = []string{srv.URL} - if indexPattern != "" { - cfg.RumConfig.SourceMapping.IndexPattern = indexPattern - } - - fetcher, err := newSourcemapFetcher( - cfg.RumConfig.SourceMapping, nil, - nil, elasticsearch.NewClient, - ) - require.NoError(t, err) - fetcher.Fetch(context.Background(), "name", "version", "path") - require.Len(t, requestPaths, 1) - - path := requestPaths[0] - path = strings.TrimPrefix(path, "/") - path = strings.TrimSuffix(path, "/_search") - assert.Equal(t, expected, path) - } - t.Run("default-pattern", func(t *testing.T) { - test(t, "", "apm-*-sourcemap*") - }) - t.Run("with-observer-version", func(t *testing.T) { - test(t, "blah-%{[observer.version]}-blah", fmt.Sprintf("blah-%s-blah", version.Version)) - }) -} - var validSourcemap, _ = os.ReadFile("../../testdata/sourcemap/bundle.js.map") func TestStoreUsesRUMElasticsearchConfig(t *testing.T) { @@ -105,41 +65,6 @@ func TestStoreUsesRUMElasticsearchConfig(t *testing.T) { assert.True(t, called) } -func TestFleetStoreUsed(t *testing.T) { - var called bool - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - called = true - wr := zlib.NewWriter(w) - defer wr.Close() - wr.Write([]byte(fmt.Sprintf(`{"sourceMap":%s}`, validSourcemap))) - })) - defer ts.Close() - - cfg := config.DefaultConfig() - cfg.RumConfig.Enabled = true - cfg.RumConfig.SourceMapping.Enabled = true - cfg.RumConfig.SourceMapping.Metadata = []config.SourceMapMetadata{{ - ServiceName: "app", - ServiceVersion: "1.0", - BundleFilepath: "/bundle/path", - SourceMapURL: "/my/path", - }} - - fleetCfg := &config.Fleet{ - Hosts: []string{ts.URL[7:]}, - Protocol: "http", - AccessAPIKey: "my-key", - TLS: nil, - } - - fetcher, err := newSourcemapFetcher(cfg.RumConfig.SourceMapping, fleetCfg, nil, nil) - require.NoError(t, err) - _, err = fetcher.Fetch(context.Background(), "app", "1.0", "/bundle/path") - require.NoError(t, err) - - assert.True(t, called) -} - func TestQueryClusterUUIDRegistriesExist(t *testing.T) { stateRegistry := monitoring.GetNamespace("state").GetRegistry() stateRegistry.Clear() diff --git a/internal/sourcemap/caching_test.go b/internal/sourcemap/caching_test.go index ee6bb390fa3..26ab0159fac 100644 --- a/internal/sourcemap/caching_test.go +++ b/internal/sourcemap/caching_test.go @@ -18,20 +18,11 @@ package sourcemap import ( - "compress/zlib" "context" - "errors" - "fmt" "net/http" - "net/http/httptest" - "net/url" - "sync" - "sync/atomic" "testing" - "time" "github.com/go-sourcemap/sourcemap" - gocache "github.com/patrickmn/go-cache" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -49,17 +40,21 @@ var unsupportedVersionSourcemap = `{ }` func Test_NewCachingFetcher(t *testing.T) { - _, err := NewCachingFetcher(nil, -1) + _, err := NewCachingFetcher(nil, nil, -1) require.Error(t, err) - f, err := NewCachingFetcher(nil, 100) + f, err := NewCachingFetcher(nil, nil, 100) require.NoError(t, err) assert.NotNil(t, f.cache) } func TestStore_Fetch(t *testing.T) { serviceName, serviceVersion, path := "foo", "1.0.1", "/tmp" - key := "foo_1.0.1_/tmp" + key := Identifier{ + name: "foo", + version: "1.0.1", + path: "/tmp", + } t.Run("cache", func(t *testing.T) { t.Run("nil", func(t *testing.T) { @@ -164,176 +159,9 @@ func TestStore_Fetch(t *testing.T) { }) } -func TestFetchContext(t *testing.T) { - var ( - apikey = "supersecret" - name = "webapp" - version = "1.0.0" - path = "/my/path/to/bundle.js.map" - c = http.DefaultClient - ) - - requestReceived := make(chan struct{}) - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - select { - case requestReceived <- struct{}{}: - case <-r.Context().Done(): - return - } - // block until the client cancels the request - <-r.Context().Done() - })) - defer ts.Close() - tsURL, _ := url.Parse(ts.URL) - - fleetServerURLs := []*url.URL{tsURL} - fleetFetcher, err := NewFleetFetcher(c, apikey, fleetServerURLs, []FleetArtifactReference{{ - ServiceName: name, - ServiceVersion: version, - BundleFilepath: path, - FleetServerURLPath: "", - }}) - assert.NoError(t, err) - - store, err := NewCachingFetcher(fleetFetcher, time.Minute) - require.NoError(t, err) - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - fetchReturned := make(chan error, 1) - go func() { - defer close(fetchReturned) - _, err := store.Fetch(ctx, name, version, path) - fetchReturned <- err - }() - select { - case <-requestReceived: - case <-time.After(10 * time.Second): - t.Fatal("timed out waiting for server to receive request") - } - - // Check that cancelling the context unblocks the request. - cancel() - select { - case err := <-fetchReturned: - assert.True(t, errors.Is(err, context.Canceled)) - case <-time.After(10 * time.Second): - t.Fatal("timed out waiting for Fetch to return") - } -} - -func TestConcurrentFetch(t *testing.T) { - for _, tc := range []struct { - calledWant, errWant, succsWant int64 - }{ - {calledWant: 1, errWant: 0, succsWant: 10}, - {calledWant: 2, errWant: 1, succsWant: 9}, - {calledWant: 4, errWant: 3, succsWant: 7}, - } { - var ( - called, errs, succs int64 - - apikey = "supersecret" - name = "webapp" - version = "1.0.0" - path = "/my/path/to/bundle.js.map" - c = http.DefaultClient - res = fmt.Sprintf(`{"sourceMap":%s}`, validSourcemap) - - errsLeft = tc.errWant - ) - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - atomic.AddInt64(&called, 1) - // Simulate the wait for a network request. - time.Sleep(50 * time.Millisecond) - if errsLeft > 0 { - errsLeft-- - http.Error(w, "err", http.StatusInternalServerError) - return - } - wr := zlib.NewWriter(w) - defer wr.Close() - wr.Write([]byte(res)) - })) - defer ts.Close() - tsURL, _ := url.Parse(ts.URL) - - fleetServerURLs := []*url.URL{tsURL} - fleetFetcher, err := NewFleetFetcher(c, apikey, fleetServerURLs, []FleetArtifactReference{{ - ServiceName: name, - ServiceVersion: version, - BundleFilepath: path, - FleetServerURLPath: "", - }}) - assert.NoError(t, err) - - fetcher, err := NewCachingFetcher(fleetFetcher, time.Minute) - assert.NoError(t, err) - - var wg sync.WaitGroup - for i := 0; i < int(tc.succsWant+tc.errWant); i++ { - wg.Add(1) - go func() { - consumer, err := fetcher.Fetch(context.Background(), name, version, path) - if err != nil { - atomic.AddInt64(&errs, 1) - } else { - assert.NotNil(t, consumer) - atomic.AddInt64(&succs, 1) - } - - wg.Done() - }() - } - - wg.Wait() - assert.Equal(t, tc.errWant, errs) - assert.Equal(t, tc.calledWant, called) - assert.Equal(t, tc.succsWant, succs) - } -} - -func TestExpiration(t *testing.T) { - store := testCachingFetcher(t, newUnavailableElasticsearchClient(t)) //if ES was queried it would return an error - store.cache = gocache.New(25*time.Millisecond, 100) - store.add("foo_1.0.1_/tmp", &sourcemap.Consumer{}) - name, version, path := "foo", "1.0.1", "/tmp" - - // sourcemap is cached - mapper, err := store.Fetch(context.Background(), name, version, path) - require.NoError(t, err) - assert.Equal(t, &sourcemap.Consumer{}, mapper) - - time.Sleep(25 * time.Millisecond) - // cache is cleared, sourcemap is fetched from ES leading to an error - mapper, err = store.Fetch(context.Background(), name, version, path) - require.Error(t, err) - assert.Nil(t, mapper) -} - -func TestCleanupInterval(t *testing.T) { - tests := []struct { - ttl time.Duration - expected float64 - }{ - {expected: 1}, - {ttl: 30 * time.Second, expected: 1}, - {ttl: 30 * time.Second, expected: 1}, - {ttl: 60 * time.Second, expected: 1}, - {ttl: 61 * time.Second, expected: 61.0 / 60}, - {ttl: 5 * time.Minute, expected: 5}, - } - for idx, test := range tests { - out := cleanupInterval(test.ttl) - assert.Equal(t, test.expected, out.Minutes(), - fmt.Sprintf("(%v) expected %v minutes, received %v minutes", idx, test.expected, out.Minutes())) - } -} - func testCachingFetcher(t *testing.T, client elasticsearch.Client) *CachingFetcher { esFetcher := NewElasticsearchFetcher(client, "apm-*sourcemap*") - cachingFetcher, err := NewCachingFetcher(esFetcher, time.Minute) + cachingFetcher, err := NewCachingFetcher(esFetcher, nil, 100) require.NoError(t, err) return cachingFetcher } diff --git a/internal/sourcemap/elasticsearch_test.go b/internal/sourcemap/elasticsearch_test.go index 8ed51adfa21..693ed8b714f 100644 --- a/internal/sourcemap/elasticsearch_test.go +++ b/internal/sourcemap/elasticsearch_test.go @@ -19,7 +19,9 @@ package sourcemap import ( "bytes" + "compress/zlib" "context" + "encoding/base64" "encoding/json" "io" "net/http" @@ -138,11 +140,18 @@ func sourcemapSearchResponseBody(hitsTotal int, hits []map[string]interface{}) i } func sourcemapHit(sourcemap string) map[string]interface{} { + b := &bytes.Buffer{} + + z := zlib.NewWriter(b) + z.Write([]byte(sourcemap)) + z.Close() + + s := base64.StdEncoding.EncodeToString(b.Bytes()) + + return map[string]interface{}{ "_source": map[string]interface{}{ - "sourcemap": map[string]interface{}{ - "sourcemap": sourcemap, - }, + "content": s, }, } } diff --git a/internal/sourcemap/processor_test.go b/internal/sourcemap/processor_test.go index ea9760836b3..9b56d820e2e 100644 --- a/internal/sourcemap/processor_test.go +++ b/internal/sourcemap/processor_test.go @@ -37,7 +37,7 @@ func TestBatchProcessor(t *testing.T) { sourcemapSearchResponseBody(1, []map[string]interface{}{sourcemapHit(string(validSourcemap))}), ) esFetcher := NewElasticsearchFetcher(client, "index") - fetcher, err := NewCachingFetcher(esFetcher, time.Minute) + fetcher, err := NewCachingFetcher(esFetcher, nil, 100) require.NoError(t, err) originalLinenoWithFilename := 1 From cd01a102e385a36014d6dc4cd951545e0e768d04 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Tue, 13 Dec 2022 02:34:32 +0100 Subject: [PATCH 011/123] lint: revert update task changes --- .../model/modelprocessor/internal_metrics.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/internal/model/modelprocessor/internal_metrics.go b/internal/model/modelprocessor/internal_metrics.go index b31ddb44052..adde5da0f05 100644 --- a/internal/model/modelprocessor/internal_metrics.go +++ b/internal/model/modelprocessor/internal_metrics.go @@ -1,3 +1,20 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + package modelprocessor func isInternalMetricName(name string) bool { From 9518384ed01397ce5c5561b3cb66eb9ef626dcbc Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Mon, 19 Dec 2022 06:42:55 +0100 Subject: [PATCH 012/123] build: remove unused imports --- internal/beater/beater.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/internal/beater/beater.go b/internal/beater/beater.go index 788288ad680..2e38a002404 100644 --- a/internal/beater/beater.go +++ b/internal/beater/beater.go @@ -47,8 +47,6 @@ import ( agentconfig "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/monitoring" - "github.com/elastic/elastic-agent-libs/transport" - "github.com/elastic/elastic-agent-libs/transport/tlscommon" "github.com/elastic/go-docappender" "github.com/elastic/go-ucfg" From ac529cde112ec26ec5f82124771e21c5d016d380 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Mon, 19 Dec 2022 06:50:25 +0100 Subject: [PATCH 013/123] lint: fix linter issues --- internal/sourcemap/elasticsearch_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/sourcemap/elasticsearch_test.go b/internal/sourcemap/elasticsearch_test.go index 4f94e519fa9..efc2c1e371f 100644 --- a/internal/sourcemap/elasticsearch_test.go +++ b/internal/sourcemap/elasticsearch_test.go @@ -148,7 +148,6 @@ func sourcemapHit(sourcemap string) map[string]interface{} { s := base64.StdEncoding.EncodeToString(b.Bytes()) - return map[string]interface{}{ "_source": map[string]interface{}{ "content": s, From a9957ea066e2e60ab38ea7fd66e5a3b544695b7a Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Mon, 19 Dec 2022 12:35:31 +0100 Subject: [PATCH 014/123] feat: add support for relative path --- internal/sourcemap/elasticsearch.go | 11 ----------- internal/sourcemap/metadata.go | 19 +++++++++++++++++++ 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/internal/sourcemap/elasticsearch.go b/internal/sourcemap/elasticsearch.go index 286eb48d16c..7c9c3a7df90 100644 --- a/internal/sourcemap/elasticsearch.go +++ b/internal/sourcemap/elasticsearch.go @@ -26,7 +26,6 @@ import ( "fmt" "io" "net/http" - "net/url" "github.com/go-sourcemap/sourcemap" "github.com/pkg/errors" @@ -182,13 +181,3 @@ func requestBody(name, version, path string) map[string]interface{} { ), ) } - -// maybeParseURLPath attempts to parse s as a URL, returning its path component -// if successful. If s cannot be parsed as a URL, s is returned. -func maybeParseURLPath(s string) string { - url, err := url.Parse(s) - if err != nil { - return s - } - return url.Path -} diff --git a/internal/sourcemap/metadata.go b/internal/sourcemap/metadata.go index d23eafedc16..802fe678352 100644 --- a/internal/sourcemap/metadata.go +++ b/internal/sourcemap/metadata.go @@ -24,6 +24,7 @@ import ( "fmt" "io" "net/http" + "net/url" "sync" "time" @@ -88,9 +89,27 @@ func (s *MetadataCachingFetcher) Fetch(ctx context.Context, name, version, path return s.backend.Fetch(ctx, name, version, path) } + // Try again + key.path = maybeParseURLPath(path) + + if _, found := s.set[key]; found { + // Only fetch from ES if the sourcemap id exists + return s.backend.Fetch(ctx, name, version, path) + } + return nil, nil } +// maybeParseURLPath attempts to parse s as a URL, returning its path component +// if successful. If s cannot be parsed as a URL, s is returned. +func maybeParseURLPath(s string) string { + url, err := url.Parse(s) + if err != nil { + return s + } + return url.Path +} + func (s *MetadataCachingFetcher) update(ms []Metadata) { s.mu.Lock() defer s.mu.Unlock() From 995e25453303b7130b1be1efbd4e2990d19de8c0 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Fri, 23 Dec 2022 01:06:46 +0100 Subject: [PATCH 015/123] feat: add support for injected api key --- apm-server.docker.yml | 10 +++++++--- apm-server.yml | 10 +++++++--- internal/beater/beater.go | 7 ++++++- internal/beater/beater_test.go | 1 + internal/beater/config/agentconfig.go | 7 ++++++- 5 files changed, 27 insertions(+), 8 deletions(-) diff --git a/apm-server.docker.yml b/apm-server.docker.yml index f87f81bb754..ecbf52677c0 100644 --- a/apm-server.docker.yml +++ b/apm-server.docker.yml @@ -198,9 +198,13 @@ apm-server: #---------------------------- APM Server - Agent Configuration ---------------------------- - # When using APM agent configuration, information fetched from Kibana will be cached in memory for some time. - # Specify cache key expiration via this setting. Default is 30 seconds. - #agent.config.cache.expiration: 30s + #agent: + # When using APM agent configuration, information fetched from Kibana will be cached in memory for some time. + # Specify cache key expiration via this setting. Default is 30 seconds. + #config.cache.expiration: 30s + + #elasticsearch: + #api_key: "id:api_key" #kibana: # Enabled must be true to enable APM Agent configuration, and for fetching source maps uploaded through Kibana. diff --git a/apm-server.yml b/apm-server.yml index a3547cf8810..c8b448e6364 100644 --- a/apm-server.yml +++ b/apm-server.yml @@ -198,9 +198,13 @@ apm-server: #---------------------------- APM Server - Agent Configuration ---------------------------- - # When using APM agent configuration, information fetched from Kibana will be cached in memory for some time. - # Specify cache key expiration via this setting. Default is 30 seconds. - #agent.config.cache.expiration: 30s + #agent: + # When using APM agent configuration, information fetched from Kibana will be cached in memory for some time. + # Specify cache key expiration via this setting. Default is 30 seconds. + #config.cache.expiration: 30s + + #elasticsearch: + #api_key: "id:api_key" #kibana: # Enabled must be true to enable APM Agent configuration, and for fetching source maps uploaded through Kibana. diff --git a/internal/beater/beater.go b/internal/beater/beater.go index 2e38a002404..08aee410637 100644 --- a/internal/beater/beater.go +++ b/internal/beater/beater.go @@ -303,9 +303,13 @@ func (s *Runner) Run(ctx context.Context) error { var sourcemapFetcher sourcemap.Fetcher if s.config.RumConfig.Enabled && s.config.RumConfig.SourceMapping.Enabled { + esConfig := *s.config.RumConfig.SourceMapping.ESConfig + esConfig.APIKey= s.config.KibanaAgentConfig.Elasticsearch.ApiKey + fetcher, err := newSourcemapFetcher( s.config.RumConfig.SourceMapping, s.fleetConfig, kibanaClient, newElasticsearchClient, + &esConfig, ) if err != nil { return err @@ -768,8 +772,9 @@ func newSourcemapFetcher( fleetCfg *config.Fleet, kibanaClient *kibana.Client, newElasticsearchClient func(*elasticsearch.Config) (*elasticsearch.Client, error), + esConfig *elasticsearch.Config, ) (sourcemap.Fetcher, error) { - esClient, err := newElasticsearchClient(cfg.ESConfig) + esClient, err := newElasticsearchClient(esConfig) if err != nil { return nil, err } diff --git a/internal/beater/beater_test.go b/internal/beater/beater_test.go index 5d2137b07c4..2d61559b090 100644 --- a/internal/beater/beater_test.go +++ b/internal/beater/beater_test.go @@ -53,6 +53,7 @@ func TestStoreUsesRUMElasticsearchConfig(t *testing.T) { fetcher, err := newSourcemapFetcher( cfg.RumConfig.SourceMapping, nil, nil, elasticsearch.NewClient, + cfg.RumConfig.SourceMapping.ESConfig, ) require.NoError(t, err) // Check that the provided rum elasticsearch config was used and diff --git a/internal/beater/config/agentconfig.go b/internal/beater/config/agentconfig.go index 87d52258714..5ecc2ee42c8 100644 --- a/internal/beater/config/agentconfig.go +++ b/internal/beater/config/agentconfig.go @@ -27,7 +27,8 @@ import ( // KibanaAgentConfig holds remote agent config information type KibanaAgentConfig struct { - Cache Cache `config:"cache"` + Cache Cache `config:"cache"` + Elasticsearch Elasticsearch `config:"elasticsearch"` } // Cache holds config information about cache expiration @@ -35,6 +36,10 @@ type Cache struct { Expiration time.Duration `config:"expiration"` } +type Elasticsearch struct { + ApiKey string `config:"api_key"` +} + // defaultKibanaAgentConfig holds the default KibanaAgentConfig func defaultKibanaAgentConfig() KibanaAgentConfig { return KibanaAgentConfig{ From cf86bb35014e265d040fbd9ae1b4a9810a6217b2 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Fri, 23 Dec 2022 01:07:20 +0100 Subject: [PATCH 016/123] lint: fix formatting issue --- internal/beater/beater.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/beater/beater.go b/internal/beater/beater.go index 08aee410637..ff9192476d9 100644 --- a/internal/beater/beater.go +++ b/internal/beater/beater.go @@ -304,7 +304,7 @@ func (s *Runner) Run(ctx context.Context) error { var sourcemapFetcher sourcemap.Fetcher if s.config.RumConfig.Enabled && s.config.RumConfig.SourceMapping.Enabled { esConfig := *s.config.RumConfig.SourceMapping.ESConfig - esConfig.APIKey= s.config.KibanaAgentConfig.Elasticsearch.ApiKey + esConfig.APIKey = s.config.KibanaAgentConfig.Elasticsearch.ApiKey fetcher, err := newSourcemapFetcher( s.config.RumConfig.SourceMapping, s.fleetConfig, From 6f0fb18994c8116572e14c230d317b73acb1c817 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Thu, 29 Dec 2022 03:55:09 +0100 Subject: [PATCH 017/123] refactor: use meaningful variable names --- internal/sourcemap/caching.go | 8 ++++---- internal/sourcemap/elasticsearch.go | 12 ++++++------ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/internal/sourcemap/caching.go b/internal/sourcemap/caching.go index d1a49bcf76c..89d06af6ab9 100644 --- a/internal/sourcemap/caching.go +++ b/internal/sourcemap/caching.go @@ -49,7 +49,7 @@ func NewCachingFetcher( ) (*CachingFetcher, error) { logger := logp.NewLogger(logs.Sourcemap) - c, err := lru.NewWithEvict(cacheSize, func(key, value interface{}) { + lruCache, err := lru.NewWithEvict(cacheSize, func(key, value interface{}) { if !logger.IsDebug() { return } @@ -61,13 +61,13 @@ func NewCachingFetcher( } go func() { - for i := range invalidationChan { - c.Remove(i) + for identifier := range invalidationChan { + lruCache.Remove(identifier) } }() return &CachingFetcher{ - cache: c, + cache: lruCache, backend: backend, logger: logger, }, nil diff --git a/internal/sourcemap/elasticsearch.go b/internal/sourcemap/elasticsearch.go index 7c9c3a7df90..bbf7a6dfa35 100644 --- a/internal/sourcemap/elasticsearch.go +++ b/internal/sourcemap/elasticsearch.go @@ -88,11 +88,11 @@ func (s *esFetcher) Fetch(ctx context.Context, name, version, path string) (*sou if resp.StatusCode == http.StatusNotFound { return nil, nil } - b, err := io.ReadAll(resp.Body) + body, err := io.ReadAll(resp.Body) if err != nil { return nil, errors.Wrap(err, errMsgParseSourcemap) } - return nil, errors.New(fmt.Sprintf("%s %s", errMsgParseSourcemap, b)) + return nil, errors.New(fmt.Sprintf("%s %s", errMsgParseSourcemap, body)) } // parse response @@ -105,23 +105,23 @@ func (s *esFetcher) Fetch(ctx context.Context, name, version, path string) (*sou return nil, nil } - d, err := base64.StdEncoding.DecodeString(body) + decodedBody, err := base64.StdEncoding.DecodeString(body) if err != nil { return nil, fmt.Errorf("failed to base64 decode string: %w", err) } - r, err := zlib.NewReader(bytes.NewReader(d)) + r, err := zlib.NewReader(bytes.NewReader(decodedBody)) if err != nil { return nil, fmt.Errorf("failed to create zlib reader: %w", err) } defer r.Close() - bbb, err := io.ReadAll(r) + uncompressedBody, err := io.ReadAll(r) if err != nil { return nil, fmt.Errorf("failed to read sourcemap content: %w", err) } - return parseSourceMap(string(bbb)) + return parseSourceMap(string(uncompressedBody)) } func (s *esFetcher) runSearchQuery(ctx context.Context, name, version, path string) (*esapi.Response, error) { From 094dbfcfe26afbaa53df7da9a0d326d71a6f019e Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Thu, 29 Dec 2022 03:56:27 +0100 Subject: [PATCH 018/123] lint: fix linter issues --- internal/beater/beater.go | 2 +- internal/beater/config/agentconfig.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/beater/beater.go b/internal/beater/beater.go index 21c9caabbbb..2c698b4420b 100644 --- a/internal/beater/beater.go +++ b/internal/beater/beater.go @@ -325,7 +325,7 @@ func (s *Runner) Run(ctx context.Context) error { var sourcemapFetcher sourcemap.Fetcher if s.config.RumConfig.Enabled && s.config.RumConfig.SourceMapping.Enabled { esConfig := *s.config.RumConfig.SourceMapping.ESConfig - esConfig.APIKey = s.config.KibanaAgentConfig.Elasticsearch.ApiKey + esConfig.APIKey = s.config.KibanaAgentConfig.Elasticsearch.APIKey fetcher, err := newSourcemapFetcher( s.config.RumConfig.SourceMapping, s.fleetConfig, diff --git a/internal/beater/config/agentconfig.go b/internal/beater/config/agentconfig.go index 5ecc2ee42c8..4ad8c574612 100644 --- a/internal/beater/config/agentconfig.go +++ b/internal/beater/config/agentconfig.go @@ -37,7 +37,7 @@ type Cache struct { } type Elasticsearch struct { - ApiKey string `config:"api_key"` + APIKey string `config:"api_key"` } // defaultKibanaAgentConfig holds the default KibanaAgentConfig From 3f0c2beb26d8d4921a0178d0038be43c1c1a31a3 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Thu, 29 Dec 2022 04:07:54 +0100 Subject: [PATCH 019/123] refactor: use map data structure to reduce iteration count and improve performance --- internal/sourcemap/metadata.go | 38 +++++++++++++--------------------- 1 file changed, 14 insertions(+), 24 deletions(-) diff --git a/internal/sourcemap/metadata.go b/internal/sourcemap/metadata.go index 802fe678352..d492aff7b1c 100644 --- a/internal/sourcemap/metadata.go +++ b/internal/sourcemap/metadata.go @@ -110,34 +110,24 @@ func maybeParseURLPath(s string) string { return url.Path } -func (s *MetadataCachingFetcher) update(ms []Metadata) { +func (s *MetadataCachingFetcher) update(ms map[Identifier]Metadata) { s.mu.Lock() defer s.mu.Unlock() - for _, m := range ms { - if contentHash, ok := s.set[m.id]; ok { - // seen - delete(s.set, m.id) - + for id, contentHash := range s.set { + if metadata, ok := ms[id]; ok { // content hash changed, invalidate the sourcemap cache - if contentHash != m.contentHash { - s.invalidationChan <- m.id + if contentHash != metadata.contentHash { + s.invalidationChan <- id } - } - } - - // Loop for any unseed sourcemap - for k := range s.set { - // the sourcemap no longer exists in ES. - // invalidate the sourcemap cache. - s.invalidationChan <- k + } else { + // the sourcemap no longer exists in ES. + // invalidate the sourcemap cache. + s.invalidationChan <- id - // remove from metadata cache - delete(s.set, k) - } - - for _, m := range ms { - s.set[m.id] = m.contentHash + // remove from metadata cache + delete(s.set, id) + } } } @@ -196,7 +186,7 @@ func (s *MetadataCachingFetcher) sync(ctx context.Context) error { return err } - var ms []Metadata + ms := make(map[Identifier]Metadata, len(body.Hits.Hits)) for _, v := range body.Hits.Hits { m := Metadata{ id: Identifier{ @@ -207,7 +197,7 @@ func (s *MetadataCachingFetcher) sync(ctx context.Context) error { contentHash: v.Source.ContentHash, } - ms = append(ms, m) + ms[m.id] = m } // Update cache From cc35612daf541bc2bd57d1d2c8d08fdbb65db9c9 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Thu, 29 Dec 2022 04:11:58 +0100 Subject: [PATCH 020/123] refactor: do not block when retrieving a sourcemap before the cache is initialized --- internal/sourcemap/metadata.go | 23 +++++------------------ 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/internal/sourcemap/metadata.go b/internal/sourcemap/metadata.go index d492aff7b1c..0c8c9ad7169 100644 --- a/internal/sourcemap/metadata.go +++ b/internal/sourcemap/metadata.go @@ -43,7 +43,6 @@ type MetadataCachingFetcher struct { mu sync.RWMutex backend Fetcher logger *logp.Logger - once sync.Once index string invalidationChan chan<- Identifier } @@ -65,16 +64,6 @@ func NewMetadataCachingFetcher( } func (s *MetadataCachingFetcher) Fetch(ctx context.Context, name, version, path string) (*sourcemap.Consumer, error) { - if len(s.set) == 0 { - // If cache is empty and we're trying to fetch a sourcemap - // make sure the initial cache population has ended. - s.once.Do(func() { - if err := s.sync(ctx); err != nil { - s.logger.Error("failed to fetch sourcemaps metadata: %v", err) - } - }) - } - key := Identifier{ name: name, version: version, @@ -134,14 +123,12 @@ func (s *MetadataCachingFetcher) update(ms map[Identifier]Metadata) { func (s *MetadataCachingFetcher) StartBackgroundSync() { go func() { // First run, populate cache - s.once.Do(func() { - ctx, cleanup := context.WithTimeout(context.Background(), 10*time.Second) - defer cleanup() + ctx, cleanup := context.WithTimeout(context.Background(), 10*time.Second) + defer cleanup() - if err := s.sync(ctx); err != nil { - s.logger.Error("failed to fetch sourcemaps metadata: %v", err) - } - }) + if err := s.sync(ctx); err != nil { + s.logger.Error("failed to fetch sourcemaps metadata: %v", err) + } }() go func() { From a3e65cb9d21b9ba4a5aa0fd8351b82f0eee2e43d Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Thu, 29 Dec 2022 04:17:38 +0100 Subject: [PATCH 021/123] refactor: reduce diff noise --- internal/sourcemap/elasticsearch.go | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/internal/sourcemap/elasticsearch.go b/internal/sourcemap/elasticsearch.go index bbf7a6dfa35..44af14e64cb 100644 --- a/internal/sourcemap/elasticsearch.go +++ b/internal/sourcemap/elasticsearch.go @@ -138,16 +138,10 @@ func (s *esFetcher) runSearchQuery(ctx context.Context, name, version, path stri } func parse(body io.ReadCloser, name, version, path string, logger *logp.Logger) (string, error) { - b, err := io.ReadAll(body) - if err != nil { - return "", err - } - var esSourcemapResponse esSourcemapResponse - if err := json.Unmarshal(b, &esSourcemapResponse); err != nil { + if err := json.NewDecoder(body).Decode(&esSourcemapResponse); err != nil { return "", err } - hits := esSourcemapResponse.Hits.Total.Value if hits == 0 || len(esSourcemapResponse.Hits.Hits) == 0 { return "", nil From b1a2bf2ae2052034e5a3ef33606bf1f99f33d75e Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Thu, 29 Dec 2022 15:36:26 +0100 Subject: [PATCH 022/123] fix: actually add new sourcemaps to the metadata cache --- internal/sourcemap/metadata.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/internal/sourcemap/metadata.go b/internal/sourcemap/metadata.go index 0c8c9ad7169..d63081a8359 100644 --- a/internal/sourcemap/metadata.go +++ b/internal/sourcemap/metadata.go @@ -105,6 +105,9 @@ func (s *MetadataCachingFetcher) update(ms map[Identifier]Metadata) { for id, contentHash := range s.set { if metadata, ok := ms[id]; ok { + // already in the cache, remove from the updates. + delete(ms, id) + // content hash changed, invalidate the sourcemap cache if contentHash != metadata.contentHash { s.invalidationChan <- id @@ -118,6 +121,10 @@ func (s *MetadataCachingFetcher) update(ms map[Identifier]Metadata) { delete(s.set, id) } } + // add new sourcemaps to the metadata cache. + for id, m := range ms { + s.set[id] = m.contentHash + } } func (s *MetadataCachingFetcher) StartBackgroundSync() { From 0f4b3fa75c93a8b5383bd952c75e61395d7f66e1 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Thu, 29 Dec 2022 15:41:13 +0100 Subject: [PATCH 023/123] refactor: don't send the whole metadata as part of the update we only need a map[id]hash, not the whole metadata. --- internal/sourcemap/metadata.go | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/internal/sourcemap/metadata.go b/internal/sourcemap/metadata.go index d63081a8359..0b980a1ab57 100644 --- a/internal/sourcemap/metadata.go +++ b/internal/sourcemap/metadata.go @@ -99,17 +99,17 @@ func maybeParseURLPath(s string) string { return url.Path } -func (s *MetadataCachingFetcher) update(ms map[Identifier]Metadata) { +func (s *MetadataCachingFetcher) update(updates map[Identifier]string) { s.mu.Lock() defer s.mu.Unlock() for id, contentHash := range s.set { - if metadata, ok := ms[id]; ok { + if updatedHash, ok := updates[id]; ok { // already in the cache, remove from the updates. - delete(ms, id) + delete(updates, id) // content hash changed, invalidate the sourcemap cache - if contentHash != metadata.contentHash { + if contentHash != updatedHash { s.invalidationChan <- id } } else { @@ -122,8 +122,8 @@ func (s *MetadataCachingFetcher) update(ms map[Identifier]Metadata) { } } // add new sourcemaps to the metadata cache. - for id, m := range ms { - s.set[id] = m.contentHash + for id, contentHash := range updates { + s.set[id] = contentHash } } @@ -180,22 +180,19 @@ func (s *MetadataCachingFetcher) sync(ctx context.Context) error { return err } - ms := make(map[Identifier]Metadata, len(body.Hits.Hits)) + updates := make(map[Identifier]string, len(body.Hits.Hits)) for _, v := range body.Hits.Hits { - m := Metadata{ - id: Identifier{ - name: v.Source.ServiceName, - version: v.Source.ServiceVersion, - path: v.Source.BundleFilepath, - }, - contentHash: v.Source.ContentHash, + id := Identifier{ + name: v.Source.ServiceName, + version: v.Source.ServiceVersion, + path: v.Source.BundleFilepath, } - ms[m.id] = m + updates[id] = v.Source.ContentHash } // Update cache - s.update(ms) + s.update(updates) return nil } From b0b8646a54d9160f1d77ee488343e7a768885a2a Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Mon, 16 Jan 2023 20:58:42 +0100 Subject: [PATCH 024/123] feat: use sourcemapping rum es api_key --- internal/beater/beater.go | 7 +------ internal/beater/beater_test.go | 1 - 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/internal/beater/beater.go b/internal/beater/beater.go index c2e4ef7d45b..f5cef4570ba 100644 --- a/internal/beater/beater.go +++ b/internal/beater/beater.go @@ -324,13 +324,9 @@ func (s *Runner) Run(ctx context.Context) error { var sourcemapFetcher sourcemap.Fetcher if s.config.RumConfig.Enabled && s.config.RumConfig.SourceMapping.Enabled { - esConfig := *s.config.RumConfig.SourceMapping.ESConfig - esConfig.APIKey = s.config.KibanaAgentConfig.Elasticsearch.APIKey - fetcher, err := newSourcemapFetcher( s.config.RumConfig.SourceMapping, s.fleetConfig, kibanaClient, newElasticsearchClient, - &esConfig, ) if err != nil { return err @@ -817,9 +813,8 @@ func newSourcemapFetcher( fleetCfg *config.Fleet, kibanaClient *kibana.Client, newElasticsearchClient func(*elasticsearch.Config) (*elasticsearch.Client, error), - esConfig *elasticsearch.Config, ) (sourcemap.Fetcher, error) { - esClient, err := newElasticsearchClient(esConfig) + esClient, err := newElasticsearchClient(cfg.ESConfig) if err != nil { return nil, err } diff --git a/internal/beater/beater_test.go b/internal/beater/beater_test.go index 2d61559b090..5d2137b07c4 100644 --- a/internal/beater/beater_test.go +++ b/internal/beater/beater_test.go @@ -53,7 +53,6 @@ func TestStoreUsesRUMElasticsearchConfig(t *testing.T) { fetcher, err := newSourcemapFetcher( cfg.RumConfig.SourceMapping, nil, nil, elasticsearch.NewClient, - cfg.RumConfig.SourceMapping.ESConfig, ) require.NoError(t, err) // Check that the provided rum elasticsearch config was used and From 5e80d7768cc6d84611b42e03b2e9d0496a24d0c5 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Wed, 18 Jan 2023 06:30:50 +0100 Subject: [PATCH 025/123] fix: forward the request to the backend fetcher if the metadata cache is not populated --- internal/sourcemap/metadata.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/internal/sourcemap/metadata.go b/internal/sourcemap/metadata.go index 0b980a1ab57..831771901c3 100644 --- a/internal/sourcemap/metadata.go +++ b/internal/sourcemap/metadata.go @@ -44,6 +44,7 @@ type MetadataCachingFetcher struct { backend Fetcher logger *logp.Logger index string + init chan struct{} invalidationChan chan<- Identifier } @@ -59,6 +60,7 @@ func NewMetadataCachingFetcher( set: make(map[Identifier]string), backend: backend, logger: logp.NewLogger(logs.Sourcemap), + init: make(chan struct{}), invalidationChan: invalidationChan, } } @@ -73,6 +75,14 @@ func (s *MetadataCachingFetcher) Fetch(ctx context.Context, name, version, path s.mu.RLock() defer s.mu.RUnlock() + select { + case <-s.init: + default: + // metadata cache is not populated yet + // forward the request to the backend fetcher + return s.backend.Fetch(ctx, name, version, path) + } + if _, found := s.set[key]; found { // Only fetch from ES if the sourcemap id exists return s.backend.Fetch(ctx, name, version, path) @@ -136,6 +146,8 @@ func (s *MetadataCachingFetcher) StartBackgroundSync() { if err := s.sync(ctx); err != nil { s.logger.Error("failed to fetch sourcemaps metadata: %v", err) } + + close(s.init) }() go func() { From 61fed2d8f8c9b8ce157d0493a5e2df0c464f363a Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Wed, 18 Jan 2023 12:00:48 +0100 Subject: [PATCH 026/123] fix: update ES sourcemap result format --- internal/sourcemap/elasticsearch.go | 16 ++++++++++------ internal/sourcemap/metadata.go | 6 +++--- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/internal/sourcemap/elasticsearch.go b/internal/sourcemap/elasticsearch.go index 44af14e64cb..a03db8392f7 100644 --- a/internal/sourcemap/elasticsearch.go +++ b/internal/sourcemap/elasticsearch.go @@ -59,13 +59,17 @@ type esSourcemapResponse struct { } Hits []struct { Source struct { - ServiceName string `json:"service.name"` - ServiceVersion string `json:"service.version"` - BundleFilepath string `json:"file.path"` - Sourcemap string `json:"content"` - ContentHash string `json:"content_sha256"` + Service struct { + Name string `json:"name"` + Version string `json:"version"` + } `json:"service"` + File struct { + BundleFilepath string `json:"path"` + } `json:"file"` + Sourcemap string `json:"content"` + ContentHash string `json:"content_sha256"` } `json:"_source"` - } + } `json:"hits"` } `json:"hits"` } diff --git a/internal/sourcemap/metadata.go b/internal/sourcemap/metadata.go index 831771901c3..2bdab9719a1 100644 --- a/internal/sourcemap/metadata.go +++ b/internal/sourcemap/metadata.go @@ -195,9 +195,9 @@ func (s *MetadataCachingFetcher) sync(ctx context.Context) error { updates := make(map[Identifier]string, len(body.Hits.Hits)) for _, v := range body.Hits.Hits { id := Identifier{ - name: v.Source.ServiceName, - version: v.Source.ServiceVersion, - path: v.Source.BundleFilepath, + name: v.Source.Service.Name, + version: v.Source.Service.Version, + path: v.Source.File.BundleFilepath, } updates[id] = v.Source.ContentHash From 2131a333a53a8463ef34fccd9db334c83800b9a7 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Wed, 18 Jan 2023 12:01:27 +0100 Subject: [PATCH 027/123] refactor: do not sort by score if we limited the size to 1 --- internal/sourcemap/elasticsearch.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/sourcemap/elasticsearch.go b/internal/sourcemap/elasticsearch.go index a03db8392f7..afd086e71b8 100644 --- a/internal/sourcemap/elasticsearch.go +++ b/internal/sourcemap/elasticsearch.go @@ -168,7 +168,6 @@ func requestBody(name, version, path string) map[string]interface{} { id := name + "-" + version + "-" + path return search( size(1), - sort(desc("_score")), source("content"), query( boolean( From 4d6633427c02f21f4bc3842b919e55f45bd590ad Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Wed, 18 Jan 2023 12:03:06 +0100 Subject: [PATCH 028/123] test: always wait for document to be indexed when creating sourcemaps --- systemtest/kibana.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/systemtest/kibana.go b/systemtest/kibana.go index 5f80c069652..f2e8839e9ef 100644 --- a/systemtest/kibana.go +++ b/systemtest/kibana.go @@ -39,6 +39,7 @@ import ( "gopkg.in/yaml.v3" "github.com/elastic/apm-server/systemtest/apmservertest" + "github.com/elastic/apm-server/systemtest/estest" "github.com/elastic/apm-server/systemtest/fleettest" ) @@ -296,6 +297,13 @@ func CreateSourceMap(t testing.TB, sourcemap, serviceName, serviceVersion, bundl } err = json.Unmarshal(respBody, &result) require.NoError(t, err) + + id := serviceName + "-" + serviceVersion + "-" + bundleFilepath + Elasticsearch.ExpectMinDocs(t, 1, ".apm-source-map", estest.TermQuery{ + Field: "_id", + Value: id, + }) + t.Cleanup(func() { DeleteSourceMap(t, result.ID) }) From 16839e86954734780a712b7fd6700787d814e44c Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Wed, 18 Jan 2023 12:51:57 +0100 Subject: [PATCH 029/123] test: create sourcemap in ES before starting APM server --- systemtest/sourcemap_test.go | 40 +++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/systemtest/sourcemap_test.go b/systemtest/sourcemap_test.go index f67e2cfd6bd..c662c763550 100644 --- a/systemtest/sourcemap_test.go +++ b/systemtest/sourcemap_test.go @@ -77,16 +77,18 @@ func TestRUMErrorSourcemapping(t *testing.T) { func TestRUMSpanSourcemapping(t *testing.T) { systemtest.CleanupElasticsearch(t) - srv := apmservertest.NewUnstartedServerTB(t) - srv.Config.RUM = &apmservertest.RUMConfig{Enabled: true} - err := srv.Start() - require.NoError(t, err) sourcemap, err := os.ReadFile("../testdata/sourcemap/bundle.js.map") require.NoError(t, err) systemtest.CreateSourceMap(t, string(sourcemap), "apm-agent-js", "1.0.0", "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", ) + + srv := apmservertest.NewUnstartedServerTB(t) + srv.Config.RUM = &apmservertest.RUMConfig{Enabled: true} + err = srv.Start() + require.NoError(t, err) + systemtest.SendRUMEventsPayload(t, srv.URL, "../testdata/intake-v2/transactions_spans_rum_2.ndjson") result := systemtest.Elasticsearch.ExpectDocs(t, "traces-apm*", estest.TermQuery{ Field: "processor.event", @@ -104,10 +106,6 @@ func TestRUMSpanSourcemapping(t *testing.T) { func TestNoMatchingSourcemap(t *testing.T) { systemtest.CleanupElasticsearch(t) - srv := apmservertest.NewUnstartedServerTB(t) - srv.Config.RUM = &apmservertest.RUMConfig{Enabled: true} - err := srv.Start() - require.NoError(t, err) // upload sourcemap with a wrong service version sourcemap, err := os.ReadFile("../testdata/sourcemap/bundle.js.map") @@ -116,6 +114,11 @@ func TestNoMatchingSourcemap(t *testing.T) { "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", ) + srv := apmservertest.NewUnstartedServerTB(t) + srv.Config.RUM = &apmservertest.RUMConfig{Enabled: true} + err = srv.Start() + require.NoError(t, err) + systemtest.SendRUMEventsPayload(t, srv.URL, "../testdata/intake-v2/transactions_spans_rum_2.ndjson") result := systemtest.Elasticsearch.ExpectDocs(t, "traces-apm*", estest.TermQuery{ Field: "processor.event", @@ -133,19 +136,17 @@ func TestNoMatchingSourcemap(t *testing.T) { func TestSourcemapElasticsearch(t *testing.T) { systemtest.CleanupElasticsearch(t) - srv := apmservertest.NewUnstartedServerTB(t) - srv.Config.RUM = &apmservertest.RUMConfig{Enabled: true} - err := srv.Start() - require.NoError(t, err) sourcemap, err := os.ReadFile("../testdata/sourcemap/bundle.js.map") require.NoError(t, err) - sourcemapID := systemtest.CreateSourceMap(t, string(sourcemap), "apm-agent-js", "1.0.1", + systemtest.CreateSourceMap(t, string(sourcemap), "apm-agent-js", "1.0.1", "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", ) - systemtest.Elasticsearch.ExpectMinDocs(t, 1, ".apm-source-map", nil) - require.NotEmpty(t, sourcemapID) + srv := apmservertest.NewUnstartedServerTB(t) + srv.Config.RUM = &apmservertest.RUMConfig{Enabled: true} + err = srv.Start() + require.NoError(t, err) // Index an error, applying source mapping and caching the source map in the process. systemtest.SendRUMEventsPayload(t, srv.URL, "../testdata/intake-v2/errors_rum.ndjson") @@ -155,10 +156,6 @@ func TestSourcemapElasticsearch(t *testing.T) { func TestSourcemapCaching(t *testing.T) { systemtest.CleanupElasticsearch(t) - srv := apmservertest.NewUnstartedServerTB(t) - srv.Config.RUM = &apmservertest.RUMConfig{Enabled: true} - err := srv.Start() - require.NoError(t, err) sourcemap, err := os.ReadFile("../testdata/sourcemap/bundle.js.map") require.NoError(t, err) @@ -166,6 +163,11 @@ func TestSourcemapCaching(t *testing.T) { "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", ) + srv := apmservertest.NewUnstartedServerTB(t) + srv.Config.RUM = &apmservertest.RUMConfig{Enabled: true} + err = srv.Start() + require.NoError(t, err) + // Index an error, applying source mapping and caching the source map in the process. systemtest.SendRUMEventsPayload(t, srv.URL, "../testdata/intake-v2/errors_rum.ndjson") result := systemtest.Elasticsearch.ExpectDocs(t, "logs-apm.error-*", nil) From 1031ac2684cf79740bc847e53771520512a75004 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Wed, 18 Jan 2023 13:29:14 +0100 Subject: [PATCH 030/123] test: add read privilege for .apm-source-map index to apm_server role --- testing/docker/elasticsearch/roles.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/testing/docker/elasticsearch/roles.yml b/testing/docker/elasticsearch/roles.yml index 72f8fbbbe69..40484d004b5 100644 --- a/testing/docker/elasticsearch/roles.yml +++ b/testing/docker/elasticsearch/roles.yml @@ -3,6 +3,8 @@ apm_server: indices: - names: ['apm-*', 'traces-apm*', 'logs-apm*', 'metrics-apm*'] privileges: ['write','create_index','manage','manage_ilm'] + - names: ['.apm-source-map'] + privileges: ['read'] applications: - application: 'apm' privileges: ['sourcemap:write','event:write','config_agent:read'] From 4532dfb4d20f8b240c26d7ca3ed08d2034056828 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Thu, 19 Jan 2023 04:09:58 +0100 Subject: [PATCH 031/123] fix: ignore query and fragment on bundlefilepath and avoid double fetching --- internal/sourcemap/metadata.go | 42 +++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/internal/sourcemap/metadata.go b/internal/sourcemap/metadata.go index 2bdab9719a1..b99b97ad353 100644 --- a/internal/sourcemap/metadata.go +++ b/internal/sourcemap/metadata.go @@ -66,10 +66,25 @@ func NewMetadataCachingFetcher( } func (s *MetadataCachingFetcher) Fetch(ctx context.Context, name, version, path string) (*sourcemap.Consumer, error) { + var cleanPath string + var urlPath string + + u, err := url.Parse(path) + if err != nil { + // path is not a valid url + // assume path + cleanPath = path + } else { + u.RawQuery = "" + u.Fragment = "" + cleanPath = u.String() + urlPath = u.Path + } + key := Identifier{ name: name, version: version, - path: path, + path: cleanPath, } s.mu.RLock() @@ -80,35 +95,30 @@ func (s *MetadataCachingFetcher) Fetch(ctx context.Context, name, version, path default: // metadata cache is not populated yet // forward the request to the backend fetcher - return s.backend.Fetch(ctx, name, version, path) + return s.backend.Fetch(ctx, key.name, key.version, key.path) } if _, found := s.set[key]; found { // Only fetch from ES if the sourcemap id exists - return s.backend.Fetch(ctx, name, version, path) + return s.backend.Fetch(ctx, key.name, key.version, key.path) + } + + if urlPath == "" { + // return early if path is not a valid url + return nil, nil } - // Try again - key.path = maybeParseURLPath(path) + // Try again using url.Path + key.path = urlPath if _, found := s.set[key]; found { // Only fetch from ES if the sourcemap id exists - return s.backend.Fetch(ctx, name, version, path) + return s.backend.Fetch(ctx, key.name, key.version, key.path) } return nil, nil } -// maybeParseURLPath attempts to parse s as a URL, returning its path component -// if successful. If s cannot be parsed as a URL, s is returned. -func maybeParseURLPath(s string) string { - url, err := url.Parse(s) - if err != nil { - return s - } - return url.Path -} - func (s *MetadataCachingFetcher) update(updates map[Identifier]string) { s.mu.Lock() defer s.mu.Unlock() From cff8dd0c4d5065b5bf67bd9abf46752ee4c02c96 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Thu, 19 Jan 2023 04:11:02 +0100 Subject: [PATCH 032/123] test: empty sourcemap index as part of the cleanup process --- systemtest/elasticsearch.go | 1 + 1 file changed, 1 insertion(+) diff --git a/systemtest/elasticsearch.go b/systemtest/elasticsearch.go index 84cd8cbe3d8..9cf1996556b 100644 --- a/systemtest/elasticsearch.go +++ b/systemtest/elasticsearch.go @@ -97,6 +97,7 @@ func cleanupElasticsearch() error { "traces-apm*", "metrics-apm*", "logs-apm*", + ".apm-source-map", }}, nil) return err } From 7946c09ff8ed1baffe8f5fb150a53a1524fc66ad Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Thu, 19 Jan 2023 04:25:43 +0100 Subject: [PATCH 033/123] feat: add debug log statement when forwarding requesting before init --- internal/sourcemap/metadata.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/sourcemap/metadata.go b/internal/sourcemap/metadata.go index b99b97ad353..b781093f783 100644 --- a/internal/sourcemap/metadata.go +++ b/internal/sourcemap/metadata.go @@ -93,8 +93,7 @@ func (s *MetadataCachingFetcher) Fetch(ctx context.Context, name, version, path select { case <-s.init: default: - // metadata cache is not populated yet - // forward the request to the backend fetcher + s.logger.Debugf("Metadata cache not populated. Falling back to backend fetcher for id: %v", key) return s.backend.Fetch(ctx, key.name, key.version, key.path) } From 39510c4dce1ac9f3e039d1ac49638f3e7853bb55 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Thu, 19 Jan 2023 04:38:18 +0100 Subject: [PATCH 034/123] fix: try with both names when forwarding requests --- internal/sourcemap/metadata.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/internal/sourcemap/metadata.go b/internal/sourcemap/metadata.go index b781093f783..24c569eb25c 100644 --- a/internal/sourcemap/metadata.go +++ b/internal/sourcemap/metadata.go @@ -90,14 +90,16 @@ func (s *MetadataCachingFetcher) Fetch(ctx context.Context, name, version, path s.mu.RLock() defer s.mu.RUnlock() + var forwardRequest bool + select { case <-s.init: default: s.logger.Debugf("Metadata cache not populated. Falling back to backend fetcher for id: %v", key) - return s.backend.Fetch(ctx, key.name, key.version, key.path) + forwardRequest = true } - if _, found := s.set[key]; found { + if _, found := s.set[key]; found || forwardRequest { // Only fetch from ES if the sourcemap id exists return s.backend.Fetch(ctx, key.name, key.version, key.path) } @@ -110,7 +112,7 @@ func (s *MetadataCachingFetcher) Fetch(ctx context.Context, name, version, path // Try again using url.Path key.path = urlPath - if _, found := s.set[key]; found { + if _, found := s.set[key]; found || forwardRequest { // Only fetch from ES if the sourcemap id exists return s.backend.Fetch(ctx, key.name, key.version, key.path) } From 020f869a7146d51b87b6468b839fa8d4a75f271e Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Thu, 19 Jan 2023 04:40:26 +0100 Subject: [PATCH 035/123] test: update RUM test to use standalone APM server --- ...on.approved.json => TestRUMRouting.approved.json} | 3 --- systemtest/rum_test.go | 12 +++++++++--- 2 files changed, 9 insertions(+), 6 deletions(-) rename systemtest/approvals/{TestRUMRoutingIntegration.approved.json => TestRUMRouting.approved.json} (98%) diff --git a/systemtest/approvals/TestRUMRoutingIntegration.approved.json b/systemtest/approvals/TestRUMRouting.approved.json similarity index 98% rename from systemtest/approvals/TestRUMRoutingIntegration.approved.json rename to systemtest/approvals/TestRUMRouting.approved.json index de5259186e5..1ec96e24e9d 100644 --- a/systemtest/approvals/TestRUMRoutingIntegration.approved.json +++ b/systemtest/approvals/TestRUMRouting.approved.json @@ -447,9 +447,6 @@ "line": { "column": 3, "number": 7666 - }, - "sourcemap": { - "error": "unable to find sourcemap.url for service.name=apm-a-rum-test-e2e-general-usecase service.version=0.0.1 bundle.path=http://localhost:8000/test/e2e/general-usecase/app.e2e-bundle.min.js?token=secret" } } ], diff --git a/systemtest/rum_test.go b/systemtest/rum_test.go index b53876aebda..e55d2b63bff 100644 --- a/systemtest/rum_test.go +++ b/systemtest/rum_test.go @@ -193,17 +193,23 @@ func TestRUMCORS(t *testing.T) { assert.Equal(t, "stick, door, Content-Type, Content-Encoding, Accept", resp.Header.Get("Access-Control-Allow-Headers")) } -func TestRUMRoutingIntegration(t *testing.T) { +func TestRUMRouting(t *testing.T) { // This test asserts that the events that are coming from the RUM JS agent // are sent to the appropriate datastream. systemtest.CleanupElasticsearch(t) - apmIntegration := newAPMIntegration(t, nil) + + srv := apmservertest.NewUnstartedServerTB(t) + srv.Config.RUM = &apmservertest.RUMConfig{ + Enabled: true, + } + err := srv.Start() + require.NoError(t, err) body, err := os.Open(filepath.Join("..", "testdata", "intake-v3", "rum_events.ndjson")) require.NoError(t, err) defer body.Close() - req, err := http.NewRequest("POST", apmIntegration.URL+"/intake/v3/rum/events", body) + req, err := http.NewRequest("POST", srv.URL+"/intake/v3/rum/events", body) require.NoError(t, err) req.Header.Add("Content-Type", "application/x-ndjson") From 870b63531b80739d41b498a4787f14d73cdb8f19 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Thu, 19 Jan 2023 09:57:10 +0100 Subject: [PATCH 036/123] fix: properly support aliases --- internal/sourcemap/elasticsearch.go | 17 ++++++-- internal/sourcemap/fetcher.go | 35 +++++++++++++++++ internal/sourcemap/metadata.go | 61 ++++++++++++----------------- internal/sourcemap/search.go | 15 +++++++ 4 files changed, 89 insertions(+), 39 deletions(-) diff --git a/internal/sourcemap/elasticsearch.go b/internal/sourcemap/elasticsearch.go index afd086e71b8..a479312bff1 100644 --- a/internal/sourcemap/elasticsearch.go +++ b/internal/sourcemap/elasticsearch.go @@ -165,16 +165,27 @@ func parse(body io.ReadCloser, name, version, path string, logger *logp.Logger) } func requestBody(name, version, path string) map[string]interface{} { - id := name + "-" + version + "-" + path + identifiers := GetIdentifiers(name, version, path) + + m := make([]map[string]interface{}, 0, len(identifiers)) + + m = append(m, boostedTerm("_id", identifiers[0].name+"-"+identifiers[0].version+"-"+identifiers[0].path, 2.0)) + + for _, k := range identifiers[1:] { + id := k.name + "-" + k.version + "-" + k.path + m = append(m, term("_id", id)) + } + return search( size(1), source("content"), query( boolean( - must( - term("_id", id), + should( + m..., ), ), ), + sort(desc("_score")), ) } diff --git a/internal/sourcemap/fetcher.go b/internal/sourcemap/fetcher.go index ebbdbef4c11..c82d4050d47 100644 --- a/internal/sourcemap/fetcher.go +++ b/internal/sourcemap/fetcher.go @@ -19,6 +19,7 @@ package sourcemap import ( "context" + "net/url" "github.com/go-sourcemap/sourcemap" ) @@ -42,3 +43,37 @@ type Metadata struct { id Identifier contentHash string } + +func GetIdentifiers(name string, version string, bundleFilepath string) []Identifier { + urlPath, err := url.Parse(bundleFilepath) + if err != nil { + // bundleFilepath is not an url + // use full match + return []Identifier{{ + name: name, + version: version, + path: bundleFilepath, + }} + } + + identifiers := make([]Identifier, 0, 2) + + urlPath.RawQuery = "" + urlPath.Fragment = "" + + // first try to match the full url + identifiers = append(identifiers, Identifier{ + name: name, + version: version, + path: urlPath.String(), + }) + + // then try to match the url path + identifiers = append(identifiers, Identifier{ + name: name, + version: version, + path: urlPath.Path, + }) + + return identifiers +} diff --git a/internal/sourcemap/metadata.go b/internal/sourcemap/metadata.go index 24c569eb25c..4bea9460d4f 100644 --- a/internal/sourcemap/metadata.go +++ b/internal/sourcemap/metadata.go @@ -24,7 +24,6 @@ import ( "fmt" "io" "net/http" - "net/url" "sync" "time" @@ -40,6 +39,7 @@ import ( type MetadataCachingFetcher struct { esClient *elasticsearch.Client set map[Identifier]string + alias map[Identifier]struct{} mu sync.RWMutex backend Fetcher logger *logp.Logger @@ -58,6 +58,7 @@ func NewMetadataCachingFetcher( esClient: c, index: index, set: make(map[Identifier]string), + alias: make(map[Identifier]struct{}), backend: backend, logger: logp.NewLogger(logs.Sourcemap), init: make(chan struct{}), @@ -66,27 +67,6 @@ func NewMetadataCachingFetcher( } func (s *MetadataCachingFetcher) Fetch(ctx context.Context, name, version, path string) (*sourcemap.Consumer, error) { - var cleanPath string - var urlPath string - - u, err := url.Parse(path) - if err != nil { - // path is not a valid url - // assume path - cleanPath = path - } else { - u.RawQuery = "" - u.Fragment = "" - cleanPath = u.String() - urlPath = u.Path - } - - key := Identifier{ - name: name, - version: version, - path: cleanPath, - } - s.mu.RLock() defer s.mu.RUnlock() @@ -95,26 +75,28 @@ func (s *MetadataCachingFetcher) Fetch(ctx context.Context, name, version, path select { case <-s.init: default: - s.logger.Debugf("Metadata cache not populated. Falling back to backend fetcher for id: %v", key) + s.logger.Debugf("Metadata cache not populated. Falling back to backend fetcher for id: %s, %s, %s", name, version, path) forwardRequest = true } - if _, found := s.set[key]; found || forwardRequest { - // Only fetch from ES if the sourcemap id exists - return s.backend.Fetch(ctx, key.name, key.version, key.path) - } + keys := GetIdentifiers(name, version, path) - if urlPath == "" { - // return early if path is not a valid url - return nil, nil + if _, found := s.set[keys[0]]; found || forwardRequest { + // Only fetch from ES if the sourcemap id exists + c, err := s.backend.Fetch(ctx, keys[0].name, keys[0].version, keys[0].path) + if c != nil || err != nil { + return c, err + } } - // Try again using url.Path - key.path = urlPath - - if _, found := s.set[key]; found || forwardRequest { - // Only fetch from ES if the sourcemap id exists - return s.backend.Fetch(ctx, key.name, key.version, key.path) + for _, key := range keys[1:] { + if _, found := s.alias[key]; found || forwardRequest { + // Only fetch from ES if the sourcemap id exists + c, err := s.backend.Fetch(ctx, key.name, key.version, key.path) + if c != nil || err != nil { + return c, err + } + } } return nil, nil @@ -140,11 +122,18 @@ func (s *MetadataCachingFetcher) update(updates map[Identifier]string) { // remove from metadata cache delete(s.set, id) + // remove alias + for _, k := range GetIdentifiers(id.name, id.version, id.path)[1:] { + delete(s.alias, k) + } } } // add new sourcemaps to the metadata cache. for id, contentHash := range updates { s.set[id] = contentHash + for _, k := range GetIdentifiers(id.name, id.version, id.path)[1:] { + s.alias[k] = struct{}{} + } } } diff --git a/internal/sourcemap/search.go b/internal/sourcemap/search.go index 109c12ae930..ef521be4ae7 100644 --- a/internal/sourcemap/search.go +++ b/internal/sourcemap/search.go @@ -83,6 +83,21 @@ func must(clauses ...map[string]interface{}) map[string]interface{} { return map[string]interface{}{"must": clauses} } +func should(clauses ...map[string]interface{}) map[string]interface{} { + return map[string]interface{}{"should": clauses} +} + func term(k, v string) map[string]interface{} { return map[string]interface{}{"term": map[string]interface{}{k: v}} } + +func boostedTerm(k, v string, boost float32) map[string]interface{} { + return map[string]interface{}{ + "term": map[string]map[string]interface{}{ + k: { + "value": v, + "boost": boost, + }, + }, + } +} From 06a59b1670e25e2d0090a7ce34b325e790072234 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Thu, 19 Jan 2023 11:05:59 +0100 Subject: [PATCH 037/123] test: clean path before waiting for sourcemap to be indexed --- systemtest/kibana.go | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/systemtest/kibana.go b/systemtest/kibana.go index f2e8839e9ef..da34dea2ed9 100644 --- a/systemtest/kibana.go +++ b/systemtest/kibana.go @@ -29,6 +29,7 @@ import ( "net/url" "os" "os/exec" + "path" "path/filepath" "runtime" "strings" @@ -278,9 +279,9 @@ func CreateSourceMap(t testing.TB, sourcemap, serviceName, serviceVersion, bundl sourcemapFileWriter.Write([]byte(sourcemap)) require.NoError(t, mw.Close()) - url := *KibanaURL - url.Path += "/api/apm/sourcemaps" - req, _ := http.NewRequest("POST", url.String(), &data) + apiURL := *KibanaURL + apiURL.Path += "/api/apm/sourcemaps" + req, _ := http.NewRequest("POST", apiURL.String(), &data) req.Header.Add("Content-Type", mw.FormDataContentType()) req.Header.Set("kbn-xsrf", "1") @@ -298,7 +299,16 @@ func CreateSourceMap(t testing.TB, sourcemap, serviceName, serviceVersion, bundl err = json.Unmarshal(respBody, &result) require.NoError(t, err) - id := serviceName + "-" + serviceVersion + "-" + bundleFilepath + cleanPath := bundleFilepath + u, err := url.Parse(bundleFilepath) + if err == nil { + u.Fragment = "" + u.RawQuery = "" + u.Path = path.Clean(u.Path) + cleanPath = u.String() + } + + id := serviceName + "-" + serviceVersion + "-" + cleanPath Elasticsearch.ExpectMinDocs(t, 1, ".apm-source-map", estest.TermQuery{ Field: "_id", Value: id, From b881da387a50b6695ef079c4d81825245f01ba42 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Thu, 19 Jan 2023 11:07:36 +0100 Subject: [PATCH 038/123] test: only run absolute and relative bundle filepath test on standalone --- ...=> absolute_bundle_filepath.approved.json} | 0 ...=> relative_bundle_filepath.approved.json} | 0 .../integration.approved.json | 302 ------------------ .../standalone.approved.json | 302 ------------------ systemtest/sourcemap_test.go | 43 +-- 5 files changed, 15 insertions(+), 632 deletions(-) rename systemtest/approvals/TestRUMErrorSourcemapping/{absolute_bundle_filepath/integration.approved.json => absolute_bundle_filepath.approved.json} (100%) rename systemtest/approvals/TestRUMErrorSourcemapping/{absolute_bundle_filepath/standalone.approved.json => relative_bundle_filepath.approved.json} (100%) delete mode 100644 systemtest/approvals/TestRUMErrorSourcemapping/relative_bundle_filepath/integration.approved.json delete mode 100644 systemtest/approvals/TestRUMErrorSourcemapping/relative_bundle_filepath/standalone.approved.json diff --git a/systemtest/approvals/TestRUMErrorSourcemapping/absolute_bundle_filepath/integration.approved.json b/systemtest/approvals/TestRUMErrorSourcemapping/absolute_bundle_filepath.approved.json similarity index 100% rename from systemtest/approvals/TestRUMErrorSourcemapping/absolute_bundle_filepath/integration.approved.json rename to systemtest/approvals/TestRUMErrorSourcemapping/absolute_bundle_filepath.approved.json diff --git a/systemtest/approvals/TestRUMErrorSourcemapping/absolute_bundle_filepath/standalone.approved.json b/systemtest/approvals/TestRUMErrorSourcemapping/relative_bundle_filepath.approved.json similarity index 100% rename from systemtest/approvals/TestRUMErrorSourcemapping/absolute_bundle_filepath/standalone.approved.json rename to systemtest/approvals/TestRUMErrorSourcemapping/relative_bundle_filepath.approved.json diff --git a/systemtest/approvals/TestRUMErrorSourcemapping/relative_bundle_filepath/integration.approved.json b/systemtest/approvals/TestRUMErrorSourcemapping/relative_bundle_filepath/integration.approved.json deleted file mode 100644 index 99a821f4b47..00000000000 --- a/systemtest/approvals/TestRUMErrorSourcemapping/relative_bundle_filepath/integration.approved.json +++ /dev/null @@ -1,302 +0,0 @@ -{ - "events": [ - { - "@timestamp": "dynamic", - "agent": { - "name": "rum-js", - "version": "0.0.0" - }, - "client": { - "ip": "dynamic" - }, - "data_stream.dataset": "apm.error", - "data_stream.namespace": "default", - "data_stream.type": "logs", - "error": { - "culprit": "webpack:///webpack/bootstrap 6002740481c9666b0d38 in \u003canonymous\u003e", - "exception": [ - { - "message": "Uncaught Error: timeout test error", - "stacktrace": [ - { - "abs_path": "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", - "context": { - "post": [ - "", - " \t\t// Check if module is in cache", - " \t\tif(installedModules[moduleId])", - " \t\t\treturn installedModules[moduleId].exports;", - "" - ], - "pre": [ - " \t// The module cache", - " \tvar installedModules = {};", - "", - " \t// The require function" - ] - }, - "exclude_from_grouping": false, - "filename": "webpack:///webpack/bootstrap 6002740481c9666b0d38", - "function": "__webpack_require__", - "line": { - "column": 0, - "context": " \tfunction __webpack_require__(moduleId) {", - "number": 5 - }, - "original": { - "abs_path": "http://localhost:8000/test/../test/e2e/general-usecase/bundle.js.map", - "colno": 18, - "filename": "test/e2e/general-usecase/bundle.js.map", - "function": "\u003canonymous\u003e", - "library_frame": true, - "lineno": 1 - }, - "sourcemap": { - "updated": true - } - }, - { - "abs_path": "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", - "context": { - "post": [ - "", - " \t// __webpack_public_path__", - " \t__webpack_require__.p = \"\";", - "", - " \t// Load entry module and return exports" - ], - "pre": [ - "", - " \t// expose the modules object (__webpack_modules__)", - " \t__webpack_require__.m = modules;", - "", - " \t// expose the module cache" - ] - }, - "exclude_from_grouping": false, - "filename": "webpack:///webpack/bootstrap 6002740481c9666b0d38", - "function": "\u003cunknown\u003e", - "line": { - "column": 0, - "context": " \t__webpack_require__.c = installedModules;", - "number": 33 - }, - "original": { - "abs_path": "http://localhost:8000/test/./e2e/general-usecase/bundle.js.map", - "colno": 181, - "filename": "~/test/e2e/general-usecase/bundle.js.map", - "function": "invokeTask", - "lineno": 1 - }, - "sourcemap": { - "updated": true - } - }, - { - "abs_path": "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", - "context": { - "post": [ - "", - " \t\t// Check if module is in cache", - " \t\tif(installedModules[moduleId])", - " \t\t\treturn installedModules[moduleId].exports;", - "" - ], - "pre": [ - " \t// The module cache", - " \tvar installedModules = {};", - "", - " \t// The require function" - ] - }, - "exclude_from_grouping": false, - "filename": "webpack:///webpack/bootstrap 6002740481c9666b0d38", - "function": "\u003cunknown\u003e", - "line": { - "column": 0, - "context": " \tfunction __webpack_require__(moduleId) {", - "number": 5 - }, - "original": { - "abs_path": "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", - "colno": 15, - "filename": "~/test/e2e/general-usecase/bundle.js.map", - "function": "runTask", - "lineno": 1 - }, - "sourcemap": { - "updated": true - } - }, - { - "abs_path": "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", - "context": { - "post": [ - "", - "", - "", - "/** WEBPACK FOOTER **", - " ** webpack/bootstrap 6002740481c9666b0d38" - ], - "pre": [ - "", - " \t// __webpack_public_path__", - " \t__webpack_require__.p = \"\";", - "", - " \t// Load entry module and return exports" - ] - }, - "exclude_from_grouping": false, - "filename": "webpack:///webpack/bootstrap 6002740481c9666b0d38", - "function": "moduleId", - "line": { - "column": 0, - "context": " \treturn __webpack_require__(0);", - "number": 39 - }, - "original": { - "abs_path": "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", - "colno": 199, - "filename": "~/test/e2e/general-usecase/bundle.js.map", - "function": "invoke", - "lineno": 1 - }, - "sourcemap": { - "updated": true - } - }, - { - "abs_path": "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", - "context": { - "post": [ - " \t\t\treturn installedModules[moduleId].exports;", - "", - " \t\t// Create a new module (and put it into the cache)", - " \t\tvar module = installedModules[moduleId] = {", - " \t\t\texports: {}," - ], - "pre": [ - "", - " \t// The require function", - " \tfunction __webpack_require__(moduleId) {", - "", - " \t\t// Check if module is in cache" - ] - }, - "exclude_from_grouping": false, - "filename": "webpack:///webpack/bootstrap 6002740481c9666b0d38", - "function": "\u003canonymous\u003e", - "line": { - "column": 0, - "context": " \t\tif(installedModules[moduleId])", - "number": 8 - }, - "original": { - "abs_path": "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", - "colno": 33, - "filename": "~/test/e2e/general-usecase/bundle.js.map", - "function": "timer", - "lineno": 1 - }, - "sourcemap": { - "updated": true - } - } - ], - "type": "Error" - } - ], - "grouping_key": "89e23da755c2dd759d2d529e37c92b8f", - "grouping_name": "Uncaught Error: log timeout test error", - "id": "aba2688e033848ce9c4e4005f1caa534", - "log": { - "message": "Uncaught Error: log timeout test error", - "stacktrace": [ - { - "abs_path": "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", - "context": { - "post": [ - "", - " \t\t// Check if module is in cache", - " \t\tif(installedModules[moduleId])", - " \t\t\treturn installedModules[moduleId].exports;", - "" - ], - "pre": [ - " \t// The module cache", - " \tvar installedModules = {};", - "", - " \t// The require function" - ] - }, - "exclude_from_grouping": false, - "filename": "webpack:///webpack/bootstrap 6002740481c9666b0d38", - "function": "\u003canonymous\u003e", - "line": { - "column": 0, - "context": " \tfunction __webpack_require__(moduleId) {", - "number": 5 - }, - "original": { - "abs_path": "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", - "colno": 18, - "filename": "~/test/e2e/general-usecase/bundle.js.map", - "function": "\u003canonymous\u003e", - "lineno": 1 - }, - "sourcemap": { - "updated": true - } - } - ] - } - }, - "event": { - "agent_id_status": "missing", - "ingested": "dynamic" - }, - "http": { - "request": { - "referrer": "http://localhost:8000/test/e2e/" - } - }, - "message": "Uncaught Error: log timeout test error", - "observer": { - "hostname": "dynamic", - "type": "apm-server", - "version": "dynamic" - }, - "processor": { - "event": "error", - "name": "error" - }, - "service": { - "name": "apm-agent-js", - "version": "1.0.1" - }, - "source": { - "ip": "dynamic", - "port": "dynamic" - }, - "timestamp": { - "us": "dynamic" - }, - "url": { - "domain": "localhost", - "full": "http://localhost:8000/test/e2e/general-usecase/", - "original": "http://localhost:8000/test/e2e/general-usecase/", - "path": "/test/e2e/general-usecase/", - "port": 8000, - "scheme": "http" - }, - "user_agent": { - "device": { - "name": "Other" - }, - "name": "Go-http-client", - "original": "Go-http-client/1.1", - "version": "1.1" - } - } - ] -} diff --git a/systemtest/approvals/TestRUMErrorSourcemapping/relative_bundle_filepath/standalone.approved.json b/systemtest/approvals/TestRUMErrorSourcemapping/relative_bundle_filepath/standalone.approved.json deleted file mode 100644 index 99a821f4b47..00000000000 --- a/systemtest/approvals/TestRUMErrorSourcemapping/relative_bundle_filepath/standalone.approved.json +++ /dev/null @@ -1,302 +0,0 @@ -{ - "events": [ - { - "@timestamp": "dynamic", - "agent": { - "name": "rum-js", - "version": "0.0.0" - }, - "client": { - "ip": "dynamic" - }, - "data_stream.dataset": "apm.error", - "data_stream.namespace": "default", - "data_stream.type": "logs", - "error": { - "culprit": "webpack:///webpack/bootstrap 6002740481c9666b0d38 in \u003canonymous\u003e", - "exception": [ - { - "message": "Uncaught Error: timeout test error", - "stacktrace": [ - { - "abs_path": "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", - "context": { - "post": [ - "", - " \t\t// Check if module is in cache", - " \t\tif(installedModules[moduleId])", - " \t\t\treturn installedModules[moduleId].exports;", - "" - ], - "pre": [ - " \t// The module cache", - " \tvar installedModules = {};", - "", - " \t// The require function" - ] - }, - "exclude_from_grouping": false, - "filename": "webpack:///webpack/bootstrap 6002740481c9666b0d38", - "function": "__webpack_require__", - "line": { - "column": 0, - "context": " \tfunction __webpack_require__(moduleId) {", - "number": 5 - }, - "original": { - "abs_path": "http://localhost:8000/test/../test/e2e/general-usecase/bundle.js.map", - "colno": 18, - "filename": "test/e2e/general-usecase/bundle.js.map", - "function": "\u003canonymous\u003e", - "library_frame": true, - "lineno": 1 - }, - "sourcemap": { - "updated": true - } - }, - { - "abs_path": "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", - "context": { - "post": [ - "", - " \t// __webpack_public_path__", - " \t__webpack_require__.p = \"\";", - "", - " \t// Load entry module and return exports" - ], - "pre": [ - "", - " \t// expose the modules object (__webpack_modules__)", - " \t__webpack_require__.m = modules;", - "", - " \t// expose the module cache" - ] - }, - "exclude_from_grouping": false, - "filename": "webpack:///webpack/bootstrap 6002740481c9666b0d38", - "function": "\u003cunknown\u003e", - "line": { - "column": 0, - "context": " \t__webpack_require__.c = installedModules;", - "number": 33 - }, - "original": { - "abs_path": "http://localhost:8000/test/./e2e/general-usecase/bundle.js.map", - "colno": 181, - "filename": "~/test/e2e/general-usecase/bundle.js.map", - "function": "invokeTask", - "lineno": 1 - }, - "sourcemap": { - "updated": true - } - }, - { - "abs_path": "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", - "context": { - "post": [ - "", - " \t\t// Check if module is in cache", - " \t\tif(installedModules[moduleId])", - " \t\t\treturn installedModules[moduleId].exports;", - "" - ], - "pre": [ - " \t// The module cache", - " \tvar installedModules = {};", - "", - " \t// The require function" - ] - }, - "exclude_from_grouping": false, - "filename": "webpack:///webpack/bootstrap 6002740481c9666b0d38", - "function": "\u003cunknown\u003e", - "line": { - "column": 0, - "context": " \tfunction __webpack_require__(moduleId) {", - "number": 5 - }, - "original": { - "abs_path": "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", - "colno": 15, - "filename": "~/test/e2e/general-usecase/bundle.js.map", - "function": "runTask", - "lineno": 1 - }, - "sourcemap": { - "updated": true - } - }, - { - "abs_path": "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", - "context": { - "post": [ - "", - "", - "", - "/** WEBPACK FOOTER **", - " ** webpack/bootstrap 6002740481c9666b0d38" - ], - "pre": [ - "", - " \t// __webpack_public_path__", - " \t__webpack_require__.p = \"\";", - "", - " \t// Load entry module and return exports" - ] - }, - "exclude_from_grouping": false, - "filename": "webpack:///webpack/bootstrap 6002740481c9666b0d38", - "function": "moduleId", - "line": { - "column": 0, - "context": " \treturn __webpack_require__(0);", - "number": 39 - }, - "original": { - "abs_path": "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", - "colno": 199, - "filename": "~/test/e2e/general-usecase/bundle.js.map", - "function": "invoke", - "lineno": 1 - }, - "sourcemap": { - "updated": true - } - }, - { - "abs_path": "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", - "context": { - "post": [ - " \t\t\treturn installedModules[moduleId].exports;", - "", - " \t\t// Create a new module (and put it into the cache)", - " \t\tvar module = installedModules[moduleId] = {", - " \t\t\texports: {}," - ], - "pre": [ - "", - " \t// The require function", - " \tfunction __webpack_require__(moduleId) {", - "", - " \t\t// Check if module is in cache" - ] - }, - "exclude_from_grouping": false, - "filename": "webpack:///webpack/bootstrap 6002740481c9666b0d38", - "function": "\u003canonymous\u003e", - "line": { - "column": 0, - "context": " \t\tif(installedModules[moduleId])", - "number": 8 - }, - "original": { - "abs_path": "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", - "colno": 33, - "filename": "~/test/e2e/general-usecase/bundle.js.map", - "function": "timer", - "lineno": 1 - }, - "sourcemap": { - "updated": true - } - } - ], - "type": "Error" - } - ], - "grouping_key": "89e23da755c2dd759d2d529e37c92b8f", - "grouping_name": "Uncaught Error: log timeout test error", - "id": "aba2688e033848ce9c4e4005f1caa534", - "log": { - "message": "Uncaught Error: log timeout test error", - "stacktrace": [ - { - "abs_path": "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", - "context": { - "post": [ - "", - " \t\t// Check if module is in cache", - " \t\tif(installedModules[moduleId])", - " \t\t\treturn installedModules[moduleId].exports;", - "" - ], - "pre": [ - " \t// The module cache", - " \tvar installedModules = {};", - "", - " \t// The require function" - ] - }, - "exclude_from_grouping": false, - "filename": "webpack:///webpack/bootstrap 6002740481c9666b0d38", - "function": "\u003canonymous\u003e", - "line": { - "column": 0, - "context": " \tfunction __webpack_require__(moduleId) {", - "number": 5 - }, - "original": { - "abs_path": "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", - "colno": 18, - "filename": "~/test/e2e/general-usecase/bundle.js.map", - "function": "\u003canonymous\u003e", - "lineno": 1 - }, - "sourcemap": { - "updated": true - } - } - ] - } - }, - "event": { - "agent_id_status": "missing", - "ingested": "dynamic" - }, - "http": { - "request": { - "referrer": "http://localhost:8000/test/e2e/" - } - }, - "message": "Uncaught Error: log timeout test error", - "observer": { - "hostname": "dynamic", - "type": "apm-server", - "version": "dynamic" - }, - "processor": { - "event": "error", - "name": "error" - }, - "service": { - "name": "apm-agent-js", - "version": "1.0.1" - }, - "source": { - "ip": "dynamic", - "port": "dynamic" - }, - "timestamp": { - "us": "dynamic" - }, - "url": { - "domain": "localhost", - "full": "http://localhost:8000/test/e2e/general-usecase/", - "original": "http://localhost:8000/test/e2e/general-usecase/", - "path": "/test/e2e/general-usecase/", - "port": 8000, - "scheme": "http" - }, - "user_agent": { - "device": { - "name": "Other" - }, - "name": "Go-http-client", - "original": "Go-http-client/1.1", - "version": "1.1" - } - } - ] -} diff --git a/systemtest/sourcemap_test.go b/systemtest/sourcemap_test.go index c662c763550..f4a49b47d5d 100644 --- a/systemtest/sourcemap_test.go +++ b/systemtest/sourcemap_test.go @@ -31,39 +31,26 @@ import ( func TestRUMErrorSourcemapping(t *testing.T) { test := func(t *testing.T, bundleFilepath string) { + systemtest.CleanupElasticsearch(t) + sourcemap, err := os.ReadFile("../testdata/sourcemap/bundle.js.map") require.NoError(t, err) systemtest.CreateSourceMap(t, string(sourcemap), "apm-agent-js", "1.0.1", bundleFilepath) - // Create the integration after uploading the sourcemap, so that it is added - // to the integration policy from the start. Otherwise we would have to wait - // for the policy to be reloaded. - apmIntegration := newAPMIntegration(t, map[string]interface{}{"enable_rum": true}) - - test := func(t *testing.T, serverURL string) { - systemtest.CleanupElasticsearch(t) - systemtest.SendRUMEventsPayload(t, serverURL, "../testdata/intake-v2/errors_rum.ndjson") - result := systemtest.Elasticsearch.ExpectDocs(t, "logs-apm.error-*", nil) - systemtest.ApproveEvents( - t, t.Name(), result.Hits.Hits, - // RUM timestamps are set by the server based on the time the payload is received. - "@timestamp", "timestamp.us", - // RUM events have the source IP and port recorded, which are dynamic in the tests - "client.ip", "source.ip", "source.port", - ) - } + srv := apmservertest.NewUnstartedServerTB(t) + srv.Config.RUM = &apmservertest.RUMConfig{Enabled: true} + err = srv.Start() + require.NoError(t, err) - t.Run("standalone", func(t *testing.T) { - srv := apmservertest.NewUnstartedServerTB(t) - srv.Config.RUM = &apmservertest.RUMConfig{Enabled: true} - err := srv.Start() - require.NoError(t, err) - test(t, srv.URL) - }) - - t.Run("integration", func(t *testing.T) { - test(t, apmIntegration.URL) - }) + systemtest.SendRUMEventsPayload(t, srv.URL, "../testdata/intake-v2/errors_rum.ndjson") + result := systemtest.Elasticsearch.ExpectDocs(t, "logs-apm.error-*", nil) + systemtest.ApproveEvents( + t, t.Name(), result.Hits.Hits, + // RUM timestamps are set by the server based on the time the payload is received. + "@timestamp", "timestamp.us", + // RUM events have the source IP and port recorded, which are dynamic in the tests + "client.ip", "source.ip", "source.port", + ) } t.Run("absolute_bundle_filepath", func(t *testing.T) { From df99000acbc6099d4da8ac7876461cda9830eddf Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Thu, 19 Jan 2023 14:24:40 +0100 Subject: [PATCH 039/123] refactor: add more debug messages for empty search results --- internal/sourcemap/metadata.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/internal/sourcemap/metadata.go b/internal/sourcemap/metadata.go index 4bea9460d4f..e43a29d5740 100644 --- a/internal/sourcemap/metadata.go +++ b/internal/sourcemap/metadata.go @@ -84,8 +84,13 @@ func (s *MetadataCachingFetcher) Fetch(ctx context.Context, name, version, path if _, found := s.set[keys[0]]; found || forwardRequest { // Only fetch from ES if the sourcemap id exists c, err := s.backend.Fetch(ctx, keys[0].name, keys[0].version, keys[0].path) - if c != nil || err != nil { + if err != nil { + return nil, err + } + if c != nil { return c, err + } else if found { + s.logger.Debugf("Backed fetcher failed to retrieve sourcemap: %v", keys[0]) } } @@ -93,8 +98,13 @@ func (s *MetadataCachingFetcher) Fetch(ctx context.Context, name, version, path if _, found := s.alias[key]; found || forwardRequest { // Only fetch from ES if the sourcemap id exists c, err := s.backend.Fetch(ctx, key.name, key.version, key.path) - if c != nil || err != nil { + if err != nil { + return nil, err + } + if c != nil { return c, err + } else if found { + s.logger.Debugf("Backed fetcher failed to retrieve sourcemap from alias %v", key) } } } From ffa89415b8a2108356e13c962237a8ec69740bff Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Thu, 19 Jan 2023 14:25:09 +0100 Subject: [PATCH 040/123] fix: do not include url path twice in identifiers --- internal/sourcemap/fetcher.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/internal/sourcemap/fetcher.go b/internal/sourcemap/fetcher.go index c82d4050d47..1b0ccabc810 100644 --- a/internal/sourcemap/fetcher.go +++ b/internal/sourcemap/fetcher.go @@ -68,12 +68,16 @@ func GetIdentifiers(name string, version string, bundleFilepath string) []Identi path: urlPath.String(), }) - // then try to match the url path - identifiers = append(identifiers, Identifier{ - name: name, - version: version, - path: urlPath.Path, - }) + // "/foo.bundle.js.map" is a valid url + // make sure it is included twice + if urlPath.String() != urlPath.Path { + // then try to match the url path + identifiers = append(identifiers, Identifier{ + name: name, + version: version, + path: urlPath.Path, + }) + } return identifiers } From c5508d9cebe566b8807fdbf7074df52855329ef9 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Thu, 19 Jan 2023 15:36:13 +0100 Subject: [PATCH 041/123] fix: refactor aliases handling and avoid init deadlock --- internal/sourcemap/elasticsearch.go | 8 ++-- internal/sourcemap/fetcher.go | 40 ++++++++-------- internal/sourcemap/metadata.go | 74 ++++++++++++++++++++--------- 3 files changed, 75 insertions(+), 47 deletions(-) diff --git a/internal/sourcemap/elasticsearch.go b/internal/sourcemap/elasticsearch.go index a479312bff1..294862ea9bd 100644 --- a/internal/sourcemap/elasticsearch.go +++ b/internal/sourcemap/elasticsearch.go @@ -165,13 +165,13 @@ func parse(body io.ReadCloser, name, version, path string, logger *logp.Logger) } func requestBody(name, version, path string) map[string]interface{} { - identifiers := GetIdentifiers(name, version, path) + aliases := GetAliases(name, version, path) - m := make([]map[string]interface{}, 0, len(identifiers)) + m := make([]map[string]interface{}, 0, 1+len(aliases)) - m = append(m, boostedTerm("_id", identifiers[0].name+"-"+identifiers[0].version+"-"+identifiers[0].path, 2.0)) + m = append(m, boostedTerm("_id", name+"-"+version+"-"+path, 2.0)) - for _, k := range identifiers[1:] { + for _, k := range aliases { id := k.name + "-" + k.version + "-" + k.path m = append(m, term("_id", id)) } diff --git a/internal/sourcemap/fetcher.go b/internal/sourcemap/fetcher.go index 1b0ccabc810..8add7089075 100644 --- a/internal/sourcemap/fetcher.go +++ b/internal/sourcemap/fetcher.go @@ -44,40 +44,38 @@ type Metadata struct { contentHash string } -func GetIdentifiers(name string, version string, bundleFilepath string) []Identifier { +func GetAliases(name string, version string, bundleFilepath string) []Identifier { urlPath, err := url.Parse(bundleFilepath) if err != nil { - // bundleFilepath is not an url + // bundleFilepath is not an url so it + // has no alias. // use full match - return []Identifier{{ - name: name, - version: version, - path: bundleFilepath, - }} + return nil } - identifiers := make([]Identifier, 0, 2) + if urlPath.String() == urlPath.Path { + // "/foo.bundle.js.map" is a valid url + // bundleFilepath is an url path + // no alias + return nil + } urlPath.RawQuery = "" urlPath.Fragment = "" - // first try to match the full url - identifiers = append(identifiers, Identifier{ - name: name, - version: version, - path: urlPath.String(), - }) + return []Identifier{ + // first try to match the full url + { + name: name, + version: version, + path: urlPath.String(), + }, - // "/foo.bundle.js.map" is a valid url - // make sure it is included twice - if urlPath.String() != urlPath.Path { // then try to match the url path - identifiers = append(identifiers, Identifier{ + { name: name, version: version, path: urlPath.Path, - }) + }, } - - return identifiers } diff --git a/internal/sourcemap/metadata.go b/internal/sourcemap/metadata.go index e43a29d5740..70f2f8c5f89 100644 --- a/internal/sourcemap/metadata.go +++ b/internal/sourcemap/metadata.go @@ -39,7 +39,7 @@ import ( type MetadataCachingFetcher struct { esClient *elasticsearch.Client set map[Identifier]string - alias map[Identifier]struct{} + alias map[Identifier]*Identifier mu sync.RWMutex backend Fetcher logger *logp.Logger @@ -58,7 +58,7 @@ func NewMetadataCachingFetcher( esClient: c, index: index, set: make(map[Identifier]string), - alias: make(map[Identifier]struct{}), + alias: make(map[Identifier]*Identifier), backend: backend, logger: logp.NewLogger(logs.Sourcemap), init: make(chan struct{}), @@ -67,44 +67,58 @@ func NewMetadataCachingFetcher( } func (s *MetadataCachingFetcher) Fetch(ctx context.Context, name, version, path string) (*sourcemap.Consumer, error) { - s.mu.RLock() - defer s.mu.RUnlock() - - var forwardRequest bool + var initPending bool select { case <-s.init: default: s.logger.Debugf("Metadata cache not populated. Falling back to backend fetcher for id: %s, %s, %s", name, version, path) - forwardRequest = true + initPending = true } - keys := GetIdentifiers(name, version, path) + original := Identifier{name: name, version: version, path: path} - if _, found := s.set[keys[0]]; found || forwardRequest { + // try to minimize lock contention so that init can finish faster + // avoid defer since later down we are waiting for the init routine to + // finish and that would create a deadlock + s.mu.RLock() + _, found := s.set[original] + s.mu.RUnlock() + + if found || initPending { // Only fetch from ES if the sourcemap id exists - c, err := s.backend.Fetch(ctx, keys[0].name, keys[0].version, keys[0].path) + c, err := s.backend.Fetch(ctx, original.name, original.version, original.path) if err != nil { return nil, err } if c != nil { return c, err } else if found { - s.logger.Debugf("Backed fetcher failed to retrieve sourcemap: %v", keys[0]) + s.logger.Debugf("Backend fetcher failed to retrieve sourcemap: %v", original) } } - for _, key := range keys[1:] { - if _, found := s.alias[key]; found || forwardRequest { + keys := GetAliases(name, version, path) + if len(keys) != 0 && initPending { + s.logger.Debug("Found aliases. Blocking until init is completed") + // aliases only work after init + <-s.init + } + + s.mu.RLock() + defer s.mu.RUnlock() + + for _, key := range keys { + if id, found := s.alias[key]; found { // Only fetch from ES if the sourcemap id exists - c, err := s.backend.Fetch(ctx, key.name, key.version, key.path) + c, err := s.backend.Fetch(ctx, id.name, id.version, id.path) if err != nil { return nil, err } if c != nil { return c, err } else if found { - s.logger.Debugf("Backed fetcher failed to retrieve sourcemap from alias %v", key) + s.logger.Debugf("Backend fetcher failed to retrieve sourcemap from alias %v", key) } } } @@ -112,7 +126,7 @@ func (s *MetadataCachingFetcher) Fetch(ctx context.Context, name, version, path return nil, nil } -func (s *MetadataCachingFetcher) update(updates map[Identifier]string) { +func (s *MetadataCachingFetcher) update(ctx context.Context, updates map[Identifier]string) { s.mu.Lock() defer s.mu.Unlock() @@ -123,17 +137,28 @@ func (s *MetadataCachingFetcher) update(updates map[Identifier]string) { // content hash changed, invalidate the sourcemap cache if contentHash != updatedHash { - s.invalidationChan <- id + select { + case s.invalidationChan <- id: + case <-ctx.Done(): + s.logger.Errorf("ctx finished while invaliding id: %v", ctx.Err()) + return + } + } } else { // the sourcemap no longer exists in ES. // invalidate the sourcemap cache. - s.invalidationChan <- id + select { + case s.invalidationChan <- id: + case <-ctx.Done(): + s.logger.Errorf("ctx finished while invaliding id: %v", ctx.Err()) + return + } // remove from metadata cache delete(s.set, id) // remove alias - for _, k := range GetIdentifiers(id.name, id.version, id.path)[1:] { + for _, k := range GetAliases(id.name, id.version, id.path)[1:] { delete(s.alias, k) } } @@ -141,8 +166,12 @@ func (s *MetadataCachingFetcher) update(updates map[Identifier]string) { // add new sourcemaps to the metadata cache. for id, contentHash := range updates { s.set[id] = contentHash - for _, k := range GetIdentifiers(id.name, id.version, id.path)[1:] { - s.alias[k] = struct{}{} + // store aliases with a pointer to the original id. + // The id is then passed over to the backend fetcher + // to minimize the size of the lru cache and + // and increase cache hits. + for _, k := range GetAliases(id.name, id.version, id.path)[1:] { + s.alias[k] = &id } } } @@ -157,6 +186,7 @@ func (s *MetadataCachingFetcher) StartBackgroundSync() { s.logger.Error("failed to fetch sourcemaps metadata: %v", err) } + s.logger.Info("init routine completed") close(s.init) }() @@ -214,7 +244,7 @@ func (s *MetadataCachingFetcher) sync(ctx context.Context) error { } // Update cache - s.update(updates) + s.update(ctx, updates) return nil } From b342665dfeccef0c2ad4e9a8d2ea93d2aac90ae3 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Thu, 19 Jan 2023 17:09:56 +0100 Subject: [PATCH 042/123] fix: do not return duplicate alias if bundlefilepath is a full url --- internal/sourcemap/fetcher.go | 13 +++++++++++++ internal/sourcemap/metadata.go | 4 ++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/internal/sourcemap/fetcher.go b/internal/sourcemap/fetcher.go index 8add7089075..7036f0d9c17 100644 --- a/internal/sourcemap/fetcher.go +++ b/internal/sourcemap/fetcher.go @@ -63,6 +63,19 @@ func GetAliases(name string, version string, bundleFilepath string) []Identifier urlPath.RawQuery = "" urlPath.Fragment = "" + if urlPath.String() == bundleFilepath { + // bundleFilepath is a valid url and it is + // already clean. + // Only return the url path as an alias + return []Identifier{ + { + name: name, + version: version, + path: urlPath.Path, + }, + } + } + return []Identifier{ // first try to match the full url { diff --git a/internal/sourcemap/metadata.go b/internal/sourcemap/metadata.go index 70f2f8c5f89..a78e5a1599f 100644 --- a/internal/sourcemap/metadata.go +++ b/internal/sourcemap/metadata.go @@ -158,7 +158,7 @@ func (s *MetadataCachingFetcher) update(ctx context.Context, updates map[Identif // remove from metadata cache delete(s.set, id) // remove alias - for _, k := range GetAliases(id.name, id.version, id.path)[1:] { + for _, k := range GetAliases(id.name, id.version, id.path) { delete(s.alias, k) } } @@ -170,7 +170,7 @@ func (s *MetadataCachingFetcher) update(ctx context.Context, updates map[Identif // The id is then passed over to the backend fetcher // to minimize the size of the lru cache and // and increase cache hits. - for _, k := range GetAliases(id.name, id.version, id.path)[1:] { + for _, k := range GetAliases(id.name, id.version, id.path) { s.alias[k] = &id } } From 695c495ff92b2724b71edcb9516d0b1e790855b3 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Thu, 19 Jan 2023 17:15:09 +0100 Subject: [PATCH 043/123] refactor: add more verbose debug messages to make metadata fetcher consistent with caching fetcher --- internal/sourcemap/metadata.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/internal/sourcemap/metadata.go b/internal/sourcemap/metadata.go index a78e5a1599f..d41fc278608 100644 --- a/internal/sourcemap/metadata.go +++ b/internal/sourcemap/metadata.go @@ -137,6 +137,7 @@ func (s *MetadataCachingFetcher) update(ctx context.Context, updates map[Identif // content hash changed, invalidate the sourcemap cache if contentHash != updatedHash { + s.logger.Debugf("Hash changed: %s -> %s: invaliding %v", contentHash, updatedHash, id) select { case s.invalidationChan <- id: case <-ctx.Done(): @@ -166,14 +167,18 @@ func (s *MetadataCachingFetcher) update(ctx context.Context, updates map[Identif // add new sourcemaps to the metadata cache. for id, contentHash := range updates { s.set[id] = contentHash + s.logger.Debugf("Added metadata id %v", id) // store aliases with a pointer to the original id. // The id is then passed over to the backend fetcher // to minimize the size of the lru cache and // and increase cache hits. for _, k := range GetAliases(id.name, id.version, id.path) { + s.logger.Debugf("Added metadata alias %v -> %v", k, id) s.alias[k] = &id } } + + s.logger.Debugf("Cache now has %d entries.", len(s.set)) } func (s *MetadataCachingFetcher) StartBackgroundSync() { From b3c5792b22cee07ba1a4f04c45fb8fb98565d662 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Thu, 19 Jan 2023 17:17:30 +0100 Subject: [PATCH 044/123] refactor: clarify cache size log message --- internal/sourcemap/metadata.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/sourcemap/metadata.go b/internal/sourcemap/metadata.go index d41fc278608..7c7a312809c 100644 --- a/internal/sourcemap/metadata.go +++ b/internal/sourcemap/metadata.go @@ -178,7 +178,7 @@ func (s *MetadataCachingFetcher) update(ctx context.Context, updates map[Identif } } - s.logger.Debugf("Cache now has %d entries.", len(s.set)) + s.logger.Debugf("Metadata cache now has %d entries.", len(s.set)) } func (s *MetadataCachingFetcher) StartBackgroundSync() { From b48a88247f3ccb161d00a07a378cb7045b98ae47 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Thu, 19 Jan 2023 17:28:58 +0100 Subject: [PATCH 045/123] fix: try to lookup the aliases in the metadata cache to account for edge cases --- internal/sourcemap/metadata.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/internal/sourcemap/metadata.go b/internal/sourcemap/metadata.go index 7c7a312809c..86279504c2d 100644 --- a/internal/sourcemap/metadata.go +++ b/internal/sourcemap/metadata.go @@ -108,6 +108,25 @@ func (s *MetadataCachingFetcher) Fetch(ctx context.Context, name, version, path s.mu.RLock() defer s.mu.RUnlock() + // Try again from the original cache + // This is because you might be storing the sourcemap in ES with an alias + // or original request id might not be using a clear url. + for _, key := range keys { + if _, found := s.set[key]; found { + // Only fetch from ES if the sourcemap id exists + c, err := s.backend.Fetch(ctx, key.name, key.version, key.path) + if err != nil { + return nil, err + } + if c != nil { + return c, err + } else if found { + s.logger.Debugf("Backend fetcher failed to retrieve alias sourcemap %v", key) + } + } + } + + // Try to retrieve the sourcemap from alias for _, key := range keys { if id, found := s.alias[key]; found { // Only fetch from ES if the sourcemap id exists From 7aded273263d8b357895f25e673c06fec1660b0e Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Thu, 19 Jan 2023 18:11:08 +0100 Subject: [PATCH 046/123] refactor: cleanup and reduce duplicate code --- internal/sourcemap/metadata.go | 81 ++++++++++++++-------------------- 1 file changed, 34 insertions(+), 47 deletions(-) diff --git a/internal/sourcemap/metadata.go b/internal/sourcemap/metadata.go index 86279504c2d..7bdd89858b8 100644 --- a/internal/sourcemap/metadata.go +++ b/internal/sourcemap/metadata.go @@ -68,34 +68,28 @@ func NewMetadataCachingFetcher( func (s *MetadataCachingFetcher) Fetch(ctx context.Context, name, version, path string) (*sourcemap.Consumer, error) { var initPending bool + original := Identifier{name: name, version: version, path: path} select { case <-s.init: + // try to minimize lock contention so that update can finish faster + // avoid defer since later down we are waiting for the update routine to + // finish and that would create a deadlock + s.mu.RLock() + if _, found := s.set[original]; found { + s.mu.RUnlock() + // Only fetch from ES if the sourcemap id exists + return s.fetch(ctx, &original) + } + s.mu.RUnlock() default: s.logger.Debugf("Metadata cache not populated. Falling back to backend fetcher for id: %s, %s, %s", name, version, path) - initPending = true - } - - original := Identifier{name: name, version: version, path: path} - - // try to minimize lock contention so that init can finish faster - // avoid defer since later down we are waiting for the init routine to - // finish and that would create a deadlock - s.mu.RLock() - _, found := s.set[original] - s.mu.RUnlock() - - if found || initPending { - // Only fetch from ES if the sourcemap id exists - c, err := s.backend.Fetch(ctx, original.name, original.version, original.path) - if err != nil { - return nil, err - } - if c != nil { + // init is in progress, ignore the metadata cache and fetch the sourcemap directly + // return if we get a result or an error + if c, err := s.backend.Fetch(ctx, original.name, original.version, original.path); c != nil || err != nil { return c, err - } else if found { - s.logger.Debugf("Backend fetcher failed to retrieve sourcemap: %v", original) } + initPending = true } keys := GetAliases(name, version, path) @@ -108,43 +102,36 @@ func (s *MetadataCachingFetcher) Fetch(ctx context.Context, name, version, path s.mu.RLock() defer s.mu.RUnlock() - // Try again from the original cache - // This is because you might be storing the sourcemap in ES with an alias - // or original request id might not be using a clear url. for _, key := range keys { + // Try again from the original cache + // This is because you might be storing the sourcemap in ES with an alias + // or original request id might not be using a clear url. if _, found := s.set[key]; found { - // Only fetch from ES if the sourcemap id exists - c, err := s.backend.Fetch(ctx, key.name, key.version, key.path) - if err != nil { - return nil, err - } - if c != nil { - return c, err - } else if found { - s.logger.Debugf("Backend fetcher failed to retrieve alias sourcemap %v", key) - } + return s.fetch(ctx, &key) } - } - // Try to retrieve the sourcemap from alias - for _, key := range keys { + // Try to retrieve the sourcemap from alias + // Only fetch from ES if the sourcemap alias exists if id, found := s.alias[key]; found { - // Only fetch from ES if the sourcemap id exists - c, err := s.backend.Fetch(ctx, id.name, id.version, id.path) - if err != nil { - return nil, err - } - if c != nil { - return c, err - } else if found { - s.logger.Debugf("Backend fetcher failed to retrieve sourcemap from alias %v", key) - } + return s.fetch(ctx, id) } } return nil, nil } +func (s *MetadataCachingFetcher) fetch(ctx context.Context, key *Identifier) (*sourcemap.Consumer, error) { + c, err := s.backend.Fetch(ctx, key.name, key.version, key.path) + + // log a message if the sourcemap is present in the cache but the backend fetcher did not + // find it. + if err == nil && c == nil { + s.logger.Debugf("Backend fetcher failed to retrieve sourcemap: %v", key) + } + + return c, err +} + func (s *MetadataCachingFetcher) update(ctx context.Context, updates map[Identifier]string) { s.mu.Lock() defer s.mu.Unlock() From ba117b462da821b6a7a910cbd0d0d45c28326cd1 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Thu, 19 Jan 2023 18:12:07 +0100 Subject: [PATCH 047/123] lint: fix linter issues --- systemtest/rum_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/systemtest/rum_test.go b/systemtest/rum_test.go index e55d2b63bff..eeff759fa7f 100644 --- a/systemtest/rum_test.go +++ b/systemtest/rum_test.go @@ -200,7 +200,7 @@ func TestRUMRouting(t *testing.T) { srv := apmservertest.NewUnstartedServerTB(t) srv.Config.RUM = &apmservertest.RUMConfig{ - Enabled: true, + Enabled: true, } err := srv.Start() require.NoError(t, err) From d4fccb27ed6784f8aba03210927e241e26c90f50 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Thu, 19 Jan 2023 18:14:40 +0100 Subject: [PATCH 048/123] fix: use correct log format method --- internal/sourcemap/metadata.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/sourcemap/metadata.go b/internal/sourcemap/metadata.go index 7bdd89858b8..6198c675bc9 100644 --- a/internal/sourcemap/metadata.go +++ b/internal/sourcemap/metadata.go @@ -194,7 +194,7 @@ func (s *MetadataCachingFetcher) StartBackgroundSync() { defer cleanup() if err := s.sync(ctx); err != nil { - s.logger.Error("failed to fetch sourcemaps metadata: %v", err) + s.logger.Errorf("failed to fetch sourcemaps metadata: %v", err) } s.logger.Info("init routine completed") @@ -210,7 +210,7 @@ func (s *MetadataCachingFetcher) StartBackgroundSync() { ctx, cleanup := context.WithTimeout(context.Background(), 10*time.Second) if err := s.sync(ctx); err != nil { - s.logger.Error("failed to sync sourcemaps metadata: %v", err) + s.logger.Errorf("failed to sync sourcemaps metadata: %v", err) } cleanup() From 54ab5b9c4adc40e1ff7fc7b387f3a3aeca20ccb0 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Thu, 19 Jan 2023 18:36:41 +0100 Subject: [PATCH 049/123] fix: readd support and test for indexpattern --- internal/beater/beater.go | 9 ++++--- internal/beater/beater_test.go | 45 ++++++++++++++++++++++++++++++++++ internal/beater/config/rum.go | 2 +- 3 files changed, 51 insertions(+), 5 deletions(-) diff --git a/internal/beater/beater.go b/internal/beater/beater.go index f5cef4570ba..01194d88743 100644 --- a/internal/beater/beater.go +++ b/internal/beater/beater.go @@ -25,6 +25,7 @@ import ( "net/http" "os" "runtime" + "strings" "time" "github.com/dustin/go-humanize" @@ -806,8 +807,6 @@ func (s *Runner) newLibbeatFinalBatchProcessor( return publisher, stop, nil } -const apmSourcemapIndex = ".apm-source-map" - func newSourcemapFetcher( cfg config.SourceMapping, fleetCfg *config.Fleet, @@ -822,12 +821,14 @@ func newSourcemapFetcher( size := 128 invalidationChan := make(chan sourcemap.Identifier, size) - esFetcher := sourcemap.NewElasticsearchFetcher(esClient, apmSourcemapIndex) + index := strings.ReplaceAll(cfg.IndexPattern, "%{[observer.version]}", version.Version) + + esFetcher := sourcemap.NewElasticsearchFetcher(esClient, index) cachingFetcher, err := sourcemap.NewCachingFetcher(esFetcher, invalidationChan, size) if err != nil { return nil, err } - metadataCachingFetcher := sourcemap.NewMetadataCachingFetcher(esClient, cachingFetcher, apmSourcemapIndex, invalidationChan) + metadataCachingFetcher := sourcemap.NewMetadataCachingFetcher(esClient, cachingFetcher, index, invalidationChan) metadataCachingFetcher.StartBackgroundSync() return metadataCachingFetcher, nil diff --git a/internal/beater/beater_test.go b/internal/beater/beater_test.go index 5d2137b07c4..a739a1d9789 100644 --- a/internal/beater/beater_test.go +++ b/internal/beater/beater_test.go @@ -23,6 +23,7 @@ import ( "net/http" "net/http/httptest" "os" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -30,11 +31,55 @@ import ( "github.com/elastic/apm-server/internal/beater/config" "github.com/elastic/apm-server/internal/elasticsearch" + "github.com/elastic/apm-server/internal/version" "github.com/elastic/elastic-agent-libs/monitoring" ) var validSourcemap, _ = os.ReadFile("../../testdata/sourcemap/bundle.js.map") +func TestSourcemapIndexPattern(t *testing.T) { + test := func(t *testing.T, indexPattern, expected string) { + var requestPaths []string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Ignore duplicates: we might have two requests since the first one + // will be from the init goroutine + for _, v := range requestPaths { + if v == r.URL.Path { + return + } + } + requestPaths = append(requestPaths, r.URL.Path) + })) + defer srv.Close() + + cfg := config.DefaultConfig() + cfg.RumConfig.Enabled = true + cfg.RumConfig.SourceMapping.ESConfig.Hosts = []string{srv.URL} + if indexPattern != "" { + cfg.RumConfig.SourceMapping.IndexPattern = indexPattern + } + + fetcher, err := newSourcemapFetcher( + cfg.RumConfig.SourceMapping, nil, + nil, elasticsearch.NewClient, + ) + require.NoError(t, err) + fetcher.Fetch(context.Background(), "name", "version", "path") + require.Len(t, requestPaths, 1) + + path := requestPaths[0] + path = strings.TrimPrefix(path, "/") + path = strings.TrimSuffix(path, "/_search") + assert.Equal(t, expected, path) + } + t.Run("default-pattern", func(t *testing.T) { + test(t, "", ".apm-source-map") + }) + t.Run("with-observer-version", func(t *testing.T) { + test(t, "blah-%{[observer.version]}-blah", fmt.Sprintf("blah-%s-blah", version.Version)) + }) +} + func TestStoreUsesRUMElasticsearchConfig(t *testing.T) { var called bool ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/internal/beater/config/rum.go b/internal/beater/config/rum.go index 2e6ca3f6236..5d8d75671b6 100644 --- a/internal/beater/config/rum.go +++ b/internal/beater/config/rum.go @@ -34,7 +34,7 @@ const ( defaultExcludeFromGrouping = "^/webpack" defaultLibraryPattern = "node_modules|bower_components|~" defaultSourcemapCacheExpiration = 5 * time.Minute - defaultSourcemapIndexPattern = "apm-*-sourcemap*" + defaultSourcemapIndexPattern = ".apm-source-map" defaultSourcemapTimeout = 5 * time.Second ) From 529b12978ce5fc49406b40827db13a9357c05b5d Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Sat, 21 Jan 2023 17:44:20 +0100 Subject: [PATCH 050/123] test: add back system tests for apm integration --- .../integration.approved.json} | 0 .../standalone.approved.json} | 0 .../integration.approved.json | 302 +++++++ .../standalone.approved.json | 302 +++++++ .../TestRUMRoutingIntegration.approved.json | 774 ++++++++++++++++++ systemtest/rum_test.go | 24 + systemtest/sourcemap_test.go | 59 +- 7 files changed, 1437 insertions(+), 24 deletions(-) rename systemtest/approvals/TestRUMErrorSourcemapping/{absolute_bundle_filepath.approved.json => absolute_bundle_filepath/integration.approved.json} (100%) rename systemtest/approvals/TestRUMErrorSourcemapping/{relative_bundle_filepath.approved.json => absolute_bundle_filepath/standalone.approved.json} (100%) create mode 100644 systemtest/approvals/TestRUMErrorSourcemapping/relative_bundle_filepath/integration.approved.json create mode 100644 systemtest/approvals/TestRUMErrorSourcemapping/relative_bundle_filepath/standalone.approved.json create mode 100644 systemtest/approvals/TestRUMRoutingIntegration.approved.json diff --git a/systemtest/approvals/TestRUMErrorSourcemapping/absolute_bundle_filepath.approved.json b/systemtest/approvals/TestRUMErrorSourcemapping/absolute_bundle_filepath/integration.approved.json similarity index 100% rename from systemtest/approvals/TestRUMErrorSourcemapping/absolute_bundle_filepath.approved.json rename to systemtest/approvals/TestRUMErrorSourcemapping/absolute_bundle_filepath/integration.approved.json diff --git a/systemtest/approvals/TestRUMErrorSourcemapping/relative_bundle_filepath.approved.json b/systemtest/approvals/TestRUMErrorSourcemapping/absolute_bundle_filepath/standalone.approved.json similarity index 100% rename from systemtest/approvals/TestRUMErrorSourcemapping/relative_bundle_filepath.approved.json rename to systemtest/approvals/TestRUMErrorSourcemapping/absolute_bundle_filepath/standalone.approved.json diff --git a/systemtest/approvals/TestRUMErrorSourcemapping/relative_bundle_filepath/integration.approved.json b/systemtest/approvals/TestRUMErrorSourcemapping/relative_bundle_filepath/integration.approved.json new file mode 100644 index 00000000000..99a821f4b47 --- /dev/null +++ b/systemtest/approvals/TestRUMErrorSourcemapping/relative_bundle_filepath/integration.approved.json @@ -0,0 +1,302 @@ +{ + "events": [ + { + "@timestamp": "dynamic", + "agent": { + "name": "rum-js", + "version": "0.0.0" + }, + "client": { + "ip": "dynamic" + }, + "data_stream.dataset": "apm.error", + "data_stream.namespace": "default", + "data_stream.type": "logs", + "error": { + "culprit": "webpack:///webpack/bootstrap 6002740481c9666b0d38 in \u003canonymous\u003e", + "exception": [ + { + "message": "Uncaught Error: timeout test error", + "stacktrace": [ + { + "abs_path": "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", + "context": { + "post": [ + "", + " \t\t// Check if module is in cache", + " \t\tif(installedModules[moduleId])", + " \t\t\treturn installedModules[moduleId].exports;", + "" + ], + "pre": [ + " \t// The module cache", + " \tvar installedModules = {};", + "", + " \t// The require function" + ] + }, + "exclude_from_grouping": false, + "filename": "webpack:///webpack/bootstrap 6002740481c9666b0d38", + "function": "__webpack_require__", + "line": { + "column": 0, + "context": " \tfunction __webpack_require__(moduleId) {", + "number": 5 + }, + "original": { + "abs_path": "http://localhost:8000/test/../test/e2e/general-usecase/bundle.js.map", + "colno": 18, + "filename": "test/e2e/general-usecase/bundle.js.map", + "function": "\u003canonymous\u003e", + "library_frame": true, + "lineno": 1 + }, + "sourcemap": { + "updated": true + } + }, + { + "abs_path": "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", + "context": { + "post": [ + "", + " \t// __webpack_public_path__", + " \t__webpack_require__.p = \"\";", + "", + " \t// Load entry module and return exports" + ], + "pre": [ + "", + " \t// expose the modules object (__webpack_modules__)", + " \t__webpack_require__.m = modules;", + "", + " \t// expose the module cache" + ] + }, + "exclude_from_grouping": false, + "filename": "webpack:///webpack/bootstrap 6002740481c9666b0d38", + "function": "\u003cunknown\u003e", + "line": { + "column": 0, + "context": " \t__webpack_require__.c = installedModules;", + "number": 33 + }, + "original": { + "abs_path": "http://localhost:8000/test/./e2e/general-usecase/bundle.js.map", + "colno": 181, + "filename": "~/test/e2e/general-usecase/bundle.js.map", + "function": "invokeTask", + "lineno": 1 + }, + "sourcemap": { + "updated": true + } + }, + { + "abs_path": "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", + "context": { + "post": [ + "", + " \t\t// Check if module is in cache", + " \t\tif(installedModules[moduleId])", + " \t\t\treturn installedModules[moduleId].exports;", + "" + ], + "pre": [ + " \t// The module cache", + " \tvar installedModules = {};", + "", + " \t// The require function" + ] + }, + "exclude_from_grouping": false, + "filename": "webpack:///webpack/bootstrap 6002740481c9666b0d38", + "function": "\u003cunknown\u003e", + "line": { + "column": 0, + "context": " \tfunction __webpack_require__(moduleId) {", + "number": 5 + }, + "original": { + "abs_path": "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", + "colno": 15, + "filename": "~/test/e2e/general-usecase/bundle.js.map", + "function": "runTask", + "lineno": 1 + }, + "sourcemap": { + "updated": true + } + }, + { + "abs_path": "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", + "context": { + "post": [ + "", + "", + "", + "/** WEBPACK FOOTER **", + " ** webpack/bootstrap 6002740481c9666b0d38" + ], + "pre": [ + "", + " \t// __webpack_public_path__", + " \t__webpack_require__.p = \"\";", + "", + " \t// Load entry module and return exports" + ] + }, + "exclude_from_grouping": false, + "filename": "webpack:///webpack/bootstrap 6002740481c9666b0d38", + "function": "moduleId", + "line": { + "column": 0, + "context": " \treturn __webpack_require__(0);", + "number": 39 + }, + "original": { + "abs_path": "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", + "colno": 199, + "filename": "~/test/e2e/general-usecase/bundle.js.map", + "function": "invoke", + "lineno": 1 + }, + "sourcemap": { + "updated": true + } + }, + { + "abs_path": "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", + "context": { + "post": [ + " \t\t\treturn installedModules[moduleId].exports;", + "", + " \t\t// Create a new module (and put it into the cache)", + " \t\tvar module = installedModules[moduleId] = {", + " \t\t\texports: {}," + ], + "pre": [ + "", + " \t// The require function", + " \tfunction __webpack_require__(moduleId) {", + "", + " \t\t// Check if module is in cache" + ] + }, + "exclude_from_grouping": false, + "filename": "webpack:///webpack/bootstrap 6002740481c9666b0d38", + "function": "\u003canonymous\u003e", + "line": { + "column": 0, + "context": " \t\tif(installedModules[moduleId])", + "number": 8 + }, + "original": { + "abs_path": "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", + "colno": 33, + "filename": "~/test/e2e/general-usecase/bundle.js.map", + "function": "timer", + "lineno": 1 + }, + "sourcemap": { + "updated": true + } + } + ], + "type": "Error" + } + ], + "grouping_key": "89e23da755c2dd759d2d529e37c92b8f", + "grouping_name": "Uncaught Error: log timeout test error", + "id": "aba2688e033848ce9c4e4005f1caa534", + "log": { + "message": "Uncaught Error: log timeout test error", + "stacktrace": [ + { + "abs_path": "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", + "context": { + "post": [ + "", + " \t\t// Check if module is in cache", + " \t\tif(installedModules[moduleId])", + " \t\t\treturn installedModules[moduleId].exports;", + "" + ], + "pre": [ + " \t// The module cache", + " \tvar installedModules = {};", + "", + " \t// The require function" + ] + }, + "exclude_from_grouping": false, + "filename": "webpack:///webpack/bootstrap 6002740481c9666b0d38", + "function": "\u003canonymous\u003e", + "line": { + "column": 0, + "context": " \tfunction __webpack_require__(moduleId) {", + "number": 5 + }, + "original": { + "abs_path": "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", + "colno": 18, + "filename": "~/test/e2e/general-usecase/bundle.js.map", + "function": "\u003canonymous\u003e", + "lineno": 1 + }, + "sourcemap": { + "updated": true + } + } + ] + } + }, + "event": { + "agent_id_status": "missing", + "ingested": "dynamic" + }, + "http": { + "request": { + "referrer": "http://localhost:8000/test/e2e/" + } + }, + "message": "Uncaught Error: log timeout test error", + "observer": { + "hostname": "dynamic", + "type": "apm-server", + "version": "dynamic" + }, + "processor": { + "event": "error", + "name": "error" + }, + "service": { + "name": "apm-agent-js", + "version": "1.0.1" + }, + "source": { + "ip": "dynamic", + "port": "dynamic" + }, + "timestamp": { + "us": "dynamic" + }, + "url": { + "domain": "localhost", + "full": "http://localhost:8000/test/e2e/general-usecase/", + "original": "http://localhost:8000/test/e2e/general-usecase/", + "path": "/test/e2e/general-usecase/", + "port": 8000, + "scheme": "http" + }, + "user_agent": { + "device": { + "name": "Other" + }, + "name": "Go-http-client", + "original": "Go-http-client/1.1", + "version": "1.1" + } + } + ] +} diff --git a/systemtest/approvals/TestRUMErrorSourcemapping/relative_bundle_filepath/standalone.approved.json b/systemtest/approvals/TestRUMErrorSourcemapping/relative_bundle_filepath/standalone.approved.json new file mode 100644 index 00000000000..99a821f4b47 --- /dev/null +++ b/systemtest/approvals/TestRUMErrorSourcemapping/relative_bundle_filepath/standalone.approved.json @@ -0,0 +1,302 @@ +{ + "events": [ + { + "@timestamp": "dynamic", + "agent": { + "name": "rum-js", + "version": "0.0.0" + }, + "client": { + "ip": "dynamic" + }, + "data_stream.dataset": "apm.error", + "data_stream.namespace": "default", + "data_stream.type": "logs", + "error": { + "culprit": "webpack:///webpack/bootstrap 6002740481c9666b0d38 in \u003canonymous\u003e", + "exception": [ + { + "message": "Uncaught Error: timeout test error", + "stacktrace": [ + { + "abs_path": "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", + "context": { + "post": [ + "", + " \t\t// Check if module is in cache", + " \t\tif(installedModules[moduleId])", + " \t\t\treturn installedModules[moduleId].exports;", + "" + ], + "pre": [ + " \t// The module cache", + " \tvar installedModules = {};", + "", + " \t// The require function" + ] + }, + "exclude_from_grouping": false, + "filename": "webpack:///webpack/bootstrap 6002740481c9666b0d38", + "function": "__webpack_require__", + "line": { + "column": 0, + "context": " \tfunction __webpack_require__(moduleId) {", + "number": 5 + }, + "original": { + "abs_path": "http://localhost:8000/test/../test/e2e/general-usecase/bundle.js.map", + "colno": 18, + "filename": "test/e2e/general-usecase/bundle.js.map", + "function": "\u003canonymous\u003e", + "library_frame": true, + "lineno": 1 + }, + "sourcemap": { + "updated": true + } + }, + { + "abs_path": "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", + "context": { + "post": [ + "", + " \t// __webpack_public_path__", + " \t__webpack_require__.p = \"\";", + "", + " \t// Load entry module and return exports" + ], + "pre": [ + "", + " \t// expose the modules object (__webpack_modules__)", + " \t__webpack_require__.m = modules;", + "", + " \t// expose the module cache" + ] + }, + "exclude_from_grouping": false, + "filename": "webpack:///webpack/bootstrap 6002740481c9666b0d38", + "function": "\u003cunknown\u003e", + "line": { + "column": 0, + "context": " \t__webpack_require__.c = installedModules;", + "number": 33 + }, + "original": { + "abs_path": "http://localhost:8000/test/./e2e/general-usecase/bundle.js.map", + "colno": 181, + "filename": "~/test/e2e/general-usecase/bundle.js.map", + "function": "invokeTask", + "lineno": 1 + }, + "sourcemap": { + "updated": true + } + }, + { + "abs_path": "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", + "context": { + "post": [ + "", + " \t\t// Check if module is in cache", + " \t\tif(installedModules[moduleId])", + " \t\t\treturn installedModules[moduleId].exports;", + "" + ], + "pre": [ + " \t// The module cache", + " \tvar installedModules = {};", + "", + " \t// The require function" + ] + }, + "exclude_from_grouping": false, + "filename": "webpack:///webpack/bootstrap 6002740481c9666b0d38", + "function": "\u003cunknown\u003e", + "line": { + "column": 0, + "context": " \tfunction __webpack_require__(moduleId) {", + "number": 5 + }, + "original": { + "abs_path": "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", + "colno": 15, + "filename": "~/test/e2e/general-usecase/bundle.js.map", + "function": "runTask", + "lineno": 1 + }, + "sourcemap": { + "updated": true + } + }, + { + "abs_path": "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", + "context": { + "post": [ + "", + "", + "", + "/** WEBPACK FOOTER **", + " ** webpack/bootstrap 6002740481c9666b0d38" + ], + "pre": [ + "", + " \t// __webpack_public_path__", + " \t__webpack_require__.p = \"\";", + "", + " \t// Load entry module and return exports" + ] + }, + "exclude_from_grouping": false, + "filename": "webpack:///webpack/bootstrap 6002740481c9666b0d38", + "function": "moduleId", + "line": { + "column": 0, + "context": " \treturn __webpack_require__(0);", + "number": 39 + }, + "original": { + "abs_path": "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", + "colno": 199, + "filename": "~/test/e2e/general-usecase/bundle.js.map", + "function": "invoke", + "lineno": 1 + }, + "sourcemap": { + "updated": true + } + }, + { + "abs_path": "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", + "context": { + "post": [ + " \t\t\treturn installedModules[moduleId].exports;", + "", + " \t\t// Create a new module (and put it into the cache)", + " \t\tvar module = installedModules[moduleId] = {", + " \t\t\texports: {}," + ], + "pre": [ + "", + " \t// The require function", + " \tfunction __webpack_require__(moduleId) {", + "", + " \t\t// Check if module is in cache" + ] + }, + "exclude_from_grouping": false, + "filename": "webpack:///webpack/bootstrap 6002740481c9666b0d38", + "function": "\u003canonymous\u003e", + "line": { + "column": 0, + "context": " \t\tif(installedModules[moduleId])", + "number": 8 + }, + "original": { + "abs_path": "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", + "colno": 33, + "filename": "~/test/e2e/general-usecase/bundle.js.map", + "function": "timer", + "lineno": 1 + }, + "sourcemap": { + "updated": true + } + } + ], + "type": "Error" + } + ], + "grouping_key": "89e23da755c2dd759d2d529e37c92b8f", + "grouping_name": "Uncaught Error: log timeout test error", + "id": "aba2688e033848ce9c4e4005f1caa534", + "log": { + "message": "Uncaught Error: log timeout test error", + "stacktrace": [ + { + "abs_path": "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", + "context": { + "post": [ + "", + " \t\t// Check if module is in cache", + " \t\tif(installedModules[moduleId])", + " \t\t\treturn installedModules[moduleId].exports;", + "" + ], + "pre": [ + " \t// The module cache", + " \tvar installedModules = {};", + "", + " \t// The require function" + ] + }, + "exclude_from_grouping": false, + "filename": "webpack:///webpack/bootstrap 6002740481c9666b0d38", + "function": "\u003canonymous\u003e", + "line": { + "column": 0, + "context": " \tfunction __webpack_require__(moduleId) {", + "number": 5 + }, + "original": { + "abs_path": "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", + "colno": 18, + "filename": "~/test/e2e/general-usecase/bundle.js.map", + "function": "\u003canonymous\u003e", + "lineno": 1 + }, + "sourcemap": { + "updated": true + } + } + ] + } + }, + "event": { + "agent_id_status": "missing", + "ingested": "dynamic" + }, + "http": { + "request": { + "referrer": "http://localhost:8000/test/e2e/" + } + }, + "message": "Uncaught Error: log timeout test error", + "observer": { + "hostname": "dynamic", + "type": "apm-server", + "version": "dynamic" + }, + "processor": { + "event": "error", + "name": "error" + }, + "service": { + "name": "apm-agent-js", + "version": "1.0.1" + }, + "source": { + "ip": "dynamic", + "port": "dynamic" + }, + "timestamp": { + "us": "dynamic" + }, + "url": { + "domain": "localhost", + "full": "http://localhost:8000/test/e2e/general-usecase/", + "original": "http://localhost:8000/test/e2e/general-usecase/", + "path": "/test/e2e/general-usecase/", + "port": 8000, + "scheme": "http" + }, + "user_agent": { + "device": { + "name": "Other" + }, + "name": "Go-http-client", + "original": "Go-http-client/1.1", + "version": "1.1" + } + } + ] +} diff --git a/systemtest/approvals/TestRUMRoutingIntegration.approved.json b/systemtest/approvals/TestRUMRoutingIntegration.approved.json new file mode 100644 index 00000000000..1ec96e24e9d --- /dev/null +++ b/systemtest/approvals/TestRUMRoutingIntegration.approved.json @@ -0,0 +1,774 @@ +{ + "events": [ + { + "@timestamp": "dynamic", + "agent": { + "name": "js-base", + "version": "4.8.1" + }, + "client": "dynamic", + "data_stream.dataset": "apm.rum", + "data_stream.namespace": "default", + "data_stream.type": "traces", + "destination": { + "address": "localhost", + "port": 8003 + }, + "event": { + "agent_id_status": "missing", + "ingested": "dynamic", + "outcome": "success" + }, + "http": { + "request": { + "method": "POST" + }, + "response": { + "status_code": 200 + } + }, + "labels": { + "testTagKey": "testTagValue" + }, + "network": { + "connection": { + "type": "5G" + } + }, + "observer": { + "hostname": "dynamic", + "type": "apm-server", + "version": "dynamic" + }, + "parent": { + "id": "ec2e280be8345240" + }, + "processor": { + "event": "span", + "name": "transaction" + }, + "service": { + "environment": "prod", + "name": "apm-a-rum-test-e2e-general-usecase" + }, + "source": { + "ip": "dynamic", + "port": "dynamic" + }, + "span": { + "destination": { + "service": { + "name": "http://localhost:8003", + "resource": "localhost:8003", + "type": "external" + } + }, + "duration": { + "us": 11584 + }, + "id": "27f45fd274f976d4", + "name": "POST http://localhost:8003/data", + "subtype": "h", + "sync": true, + "type": "external" + }, + "timestamp": { + "us": "dynamic" + }, + "trace": { + "id": "286ac3ad697892c406528f13c82e0ce1" + }, + "transaction": { + "id": "ec2e280be8345240" + }, + "url": { + "original": "http://localhost:8003/data" + } + }, + { + "@timestamp": "dynamic", + "agent": { + "name": "js-base", + "version": "4.8.1" + }, + "client": "dynamic", + "data_stream.dataset": "apm.rum", + "data_stream.namespace": "default", + "data_stream.type": "traces", + "destination": { + "address": "localhost", + "port": 8000 + }, + "event": { + "agent_id_status": "missing", + "ingested": "dynamic", + "outcome": "success" + }, + "http": { + "request": { + "method": "GET" + }, + "response": { + "status_code": 200 + } + }, + "labels": { + "testTagKey": "testTagValue" + }, + "network": { + "connection": { + "type": "5G" + } + }, + "observer": { + "hostname": "dynamic", + "type": "apm-server", + "version": "dynamic" + }, + "parent": { + "id": "ec2e280be8345240" + }, + "processor": { + "event": "span", + "name": "transaction" + }, + "service": { + "environment": "prod", + "name": "apm-a-rum-test-e2e-general-usecase" + }, + "source": { + "ip": "dynamic", + "port": "dynamic" + }, + "span": { + "destination": { + "service": { + "name": "http://localhost:8000", + "resource": "localhost:8000", + "type": "external" + } + }, + "duration": { + "us": 6724 + }, + "id": "5ecb8ee030749715", + "name": "GET /test/e2e/common/data.json", + "subtype": "h", + "sync": true, + "type": "external" + }, + "timestamp": { + "us": "dynamic" + }, + "trace": { + "id": "286ac3ad697892c406528f13c82e0ce1" + }, + "transaction": { + "id": "ec2e280be8345240" + }, + "url": { + "original": "http://localhost:8000/test/e2e/common/data.json?test=hamid" + } + }, + { + "@timestamp": "dynamic", + "agent": { + "name": "js-base", + "version": "4.8.1" + }, + "client": "dynamic", + "data_stream.dataset": "apm.rum", + "data_stream.namespace": "default", + "data_stream.type": "traces", + "event": { + "agent_id_status": "missing", + "ingested": "dynamic", + "outcome": "unknown" + }, + "labels": { + "testTagKey": "testTagValue" + }, + "network": { + "connection": { + "type": "5G" + } + }, + "observer": { + "hostname": "dynamic", + "type": "apm-server", + "version": "dynamic" + }, + "parent": { + "id": "ec2e280be8345240" + }, + "processor": { + "event": "span", + "name": "transaction" + }, + "service": { + "environment": "prod", + "name": "apm-a-rum-test-e2e-general-usecase" + }, + "source": { + "ip": "dynamic", + "port": "dynamic" + }, + "span": { + "duration": { + "us": 198070 + }, + "id": "9b80535c4403c9fb", + "name": "OpenTracing y", + "type": "cu" + }, + "timestamp": { + "us": "dynamic" + }, + "trace": { + "id": "286ac3ad697892c406528f13c82e0ce1" + }, + "transaction": { + "id": "ec2e280be8345240" + } + }, + { + "@timestamp": "dynamic", + "agent": { + "name": "js-base", + "version": "4.8.1" + }, + "client": "dynamic", + "data_stream.dataset": "apm.rum", + "data_stream.namespace": "default", + "data_stream.type": "traces", + "destination": { + "address": "localhost", + "port": 8003 + }, + "event": { + "agent_id_status": "missing", + "ingested": "dynamic", + "outcome": "success" + }, + "http": { + "request": { + "method": "POST" + }, + "response": { + "status_code": 200 + } + }, + "labels": { + "testTagKey": "testTagValue" + }, + "network": { + "connection": { + "type": "5G" + } + }, + "observer": { + "hostname": "dynamic", + "type": "apm-server", + "version": "dynamic" + }, + "parent": { + "id": "bbd8bcc3be14d814" + }, + "processor": { + "event": "span", + "name": "transaction" + }, + "service": { + "environment": "prod", + "name": "apm-a-rum-test-e2e-general-usecase" + }, + "source": { + "ip": "dynamic", + "port": "dynamic" + }, + "span": { + "action": "action", + "destination": { + "service": { + "name": "http://localhost:8003", + "resource": "localhost:8003", + "type": "external" + } + }, + "duration": { + "us": 15949 + }, + "id": "a3c043330bc2015e", + "name": "POST http://localhost:8003/fetch", + "subtype": "h", + "sync": false, + "type": "external" + }, + "timestamp": { + "us": "dynamic" + }, + "trace": { + "id": "286ac3ad697892c406528f13c82e0ce1" + }, + "transaction": { + "id": "ec2e280be8345240" + }, + "url": { + "original": "http://localhost:8003/fetch" + } + }, + { + "@timestamp": "dynamic", + "agent": { + "name": "js-base", + "version": "4.8.1" + }, + "client": "dynamic", + "data_stream.dataset": "apm.rum", + "data_stream.namespace": "default", + "data_stream.type": "traces", + "event": { + "agent_id_status": "missing", + "ingested": "dynamic", + "outcome": "unknown" + }, + "labels": { + "testTagKey": "testTagValue" + }, + "network": { + "connection": { + "type": "5G" + } + }, + "observer": { + "hostname": "dynamic", + "type": "apm-server", + "version": "dynamic" + }, + "parent": { + "id": "ec2e280be8345240" + }, + "processor": { + "event": "span", + "name": "transaction" + }, + "service": { + "environment": "prod", + "name": "apm-a-rum-test-e2e-general-usecase" + }, + "source": { + "ip": "dynamic", + "port": "dynamic" + }, + "span": { + "duration": { + "us": 2000 + }, + "id": "bbd8bcc3be14d814", + "name": "Requesting and receiving the document", + "subtype": "browser-timing", + "type": "hard-navigation" + }, + "timestamp": { + "us": "dynamic" + }, + "trace": { + "id": "286ac3ad697892c406528f13c82e0ce1" + }, + "transaction": { + "id": "ec2e280be8345240" + } + }, + { + "@timestamp": "dynamic", + "agent": { + "name": "js-base", + "version": "4.8.1" + }, + "client": "dynamic", + "data_stream.dataset": "apm.rum", + "data_stream.namespace": "default", + "data_stream.type": "traces", + "event": { + "agent_id_status": "missing", + "ingested": "dynamic", + "outcome": "success" + }, + "labels": { + "testTagKey": "testTagValue" + }, + "network": { + "connection": { + "type": "5G" + } + }, + "observer": { + "hostname": "dynamic", + "type": "apm-server", + "version": "dynamic" + }, + "parent": { + "id": "ec2e280be8345240" + }, + "processor": { + "event": "span", + "name": "transaction" + }, + "service": { + "environment": "prod", + "name": "apm-a-rum-test-e2e-general-usecase" + }, + "source": { + "ip": "dynamic", + "port": "dynamic" + }, + "span": { + "duration": { + "us": 2000 + }, + "id": "bc7665dc25629379", + "name": "Fire \"DOMContentLoaded\" event", + "stacktrace": [ + { + "abs_path": "http://localhost:8000/test/e2e/general-usecase/app.e2e-bundle.min.js?token=secret", + "exclude_from_grouping": false, + "filename": "test/e2e/general-usecase/app.e2e-bundle.min.js?token=secret", + "function": "generateError", + "line": { + "column": 9, + "number": 7662 + } + }, + { + "abs_path": "http://localhost:8000/test/e2e/general-usecase/app.e2e-bundle.min.js?token=secret", + "exclude_from_grouping": false, + "filename": "test/e2e/general-usecase/app.e2e-bundle.min.js?token=secret", + "function": "\u003canonymous\u003e", + "line": { + "column": 3, + "number": 7666 + } + } + ], + "subtype": "browser-timing", + "type": "hard-navigation" + }, + "timestamp": { + "us": "dynamic" + }, + "trace": { + "id": "286ac3ad697892c406528f13c82e0ce1" + }, + "transaction": { + "id": "ec2e280be8345240" + } + }, + { + "@timestamp": "dynamic", + "agent": { + "name": "js-base", + "version": "4.8.1" + }, + "client": "dynamic", + "data_stream.dataset": "apm.rum", + "data_stream.namespace": "default", + "data_stream.type": "traces", + "destination": { + "address": "localhost", + "port": 8000 + }, + "event": { + "agent_id_status": "missing", + "ingested": "dynamic", + "outcome": "unknown" + }, + "http": { + "response": { + "decoded_body_size": 676864, + "encoded_body_size": 676864, + "transfer_size": 677175 + } + }, + "labels": { + "testTagKey": "testTagValue" + }, + "network": { + "connection": { + "type": "5G" + } + }, + "observer": { + "hostname": "dynamic", + "type": "apm-server", + "version": "dynamic" + }, + "parent": { + "id": "ec2e280be8345240" + }, + "processor": { + "event": "span", + "name": "transaction" + }, + "service": { + "environment": "prod", + "name": "apm-a-rum-test-e2e-general-usecase" + }, + "source": { + "ip": "dynamic", + "port": "dynamic" + }, + "span": { + "destination": { + "service": { + "name": "http://localhost:8000", + "resource": "localhost:8000", + "type": "rc" + } + }, + "duration": { + "us": 35060 + }, + "id": "fb8f717930697299", + "name": "http://localhost:8000/test/e2e/general-usecase/app.e2e-bundle.min.js", + "subtype": "script", + "type": "rc" + }, + "timestamp": { + "us": "dynamic" + }, + "trace": { + "id": "286ac3ad697892c406528f13c82e0ce1" + }, + "transaction": { + "id": "ec2e280be8345240" + }, + "url": { + "original": "http://localhost:8000/test/e2e/general-usecase/app.e2e-bundle.min.js?token=REDACTED" + } + }, + { + "@timestamp": "dynamic", + "agent": { + "name": "js-base", + "version": "4.8.1" + }, + "client": "dynamic", + "data_stream.dataset": "apm.rum", + "data_stream.namespace": "default", + "data_stream.type": "traces", + "event": { + "agent_id_status": "missing", + "ingested": "dynamic", + "outcome": "unknown" + }, + "labels": { + "testTagKey": "testTagValue" + }, + "network": { + "connection": { + "type": "5G" + } + }, + "observer": { + "hostname": "dynamic", + "type": "apm-server", + "version": "dynamic" + }, + "parent": { + "id": "ec2e280be8345240" + }, + "processor": { + "event": "span", + "name": "transaction" + }, + "service": { + "environment": "prod", + "name": "apm-a-rum-test-e2e-general-usecase" + }, + "source": { + "ip": "dynamic", + "port": "dynamic" + }, + "span": { + "duration": { + "us": 106000 + }, + "id": "fc546e87a90a774f", + "name": "Parsing the document, executing sy. scripts", + "subtype": "browser-timing", + "type": "hard-navigation" + }, + "timestamp": { + "us": "dynamic" + }, + "trace": { + "id": "286ac3ad697892c406528f13c82e0ce1" + }, + "transaction": { + "id": "ec2e280be8345240" + } + }, + { + "@timestamp": "dynamic", + "agent": { + "name": "js-base", + "version": "4.8.1" + }, + "client": "dynamic", + "data_stream.dataset": "apm.rum", + "data_stream.namespace": "default", + "data_stream.type": "traces", + "event": { + "agent_id_status": "missing", + "ingested": "dynamic", + "outcome": "success" + }, + "http": { + "request": { + "headers": { + "Accept": [ + "application/json" + ] + }, + "method": "GET", + "referrer": "http://localhost:8000/test/e2e/" + }, + "response": { + "decoded_body_size": 690, + "encoded_body_size": 690, + "headers": { + "Content-Type": [ + "application/json" + ] + }, + "status_code": 200, + "transfer_size": 983 + }, + "version": "1.1" + }, + "labels": { + "testTagKey": "testTagValue" + }, + "network": { + "connection": { + "type": "5G" + } + }, + "observer": { + "hostname": "dynamic", + "type": "apm-server", + "version": "dynamic" + }, + "parent": { + "id": "1ef08ac234fca23b455d9e27c660f1ab" + }, + "processor": { + "event": "transaction", + "name": "transaction" + }, + "service": { + "environment": "prod", + "framework": { + "name": "angular", + "version": "2" + }, + "language": { + "name": "javascript", + "version": "6" + }, + "name": "apm-a-rum-test-e2e-general-usecase", + "runtime": { + "name": "v8", + "version": "8.0" + }, + "version": "0.0.1" + }, + "source": { + "ip": "dynamic", + "port": "dynamic" + }, + "timestamp": { + "us": "dynamic" + }, + "trace": { + "id": "286ac3ad697892c406528f13c82e0ce1" + }, + "transaction": { + "custom": { + "testContext": "testContext" + }, + "duration": { + "us": 295000 + }, + "experience": { + "cls": 1, + "fid": 2, + "longtask": { + "count": 3, + "max": 1, + "sum": 2.5 + }, + "tbt": 3.4 + }, + "id": "ec2e280be8345240", + "marks": { + "agent": { + "domComplete": 138, + "domContentLoadedEventEnd": 110, + "domContentLoadedEventStart": 100, + "domInteractive": 120, + "firstContentfulPaint": 70.82500003930181, + "largestContentfulPaint": 131.03000004775822, + "timeToFirstByte": 5 + }, + "navigationTiming": { + "connectEnd": 0, + "connectStart": 0, + "domComplete": 138, + "domContentLoadedEventEnd": 122, + "domContentLoadedEventStart": 120, + "domInteractive": 120, + "domLoading": 14, + "domainLookupEnd": 0, + "domainLookupStart": 0, + "fetchStart": 0, + "loadEventEnd": 138, + "loadEventStart": 138, + "requestStart": 4, + "responseEnd": 6, + "responseStart": 5 + } + }, + "name": "general-usecase-initial-p-load", + "representative_count": 1, + "sampled": true, + "span_count": { + "dropped": 1, + "started": 8 + }, + "type": "p-load" + }, + "url": { + "domain": "localhost", + "full": "http://localhost:8000/test/e2e/general-usecase/", + "original": "http://localhost:8000/test/e2e/general-usecase/", + "path": "/test/e2e/general-usecase/", + "port": 8000, + "scheme": "http" + }, + "user": { + "email": "em", + "id": "uId", + "name": "un" + }, + "user_agent": { + "device": { + "name": "Other" + }, + "name": "Go-http-client", + "original": "Go-http-client/1.1", + "version": "1.1" + } + } + ] +} diff --git a/systemtest/rum_test.go b/systemtest/rum_test.go index eeff759fa7f..923b1eaa6f1 100644 --- a/systemtest/rum_test.go +++ b/systemtest/rum_test.go @@ -223,3 +223,27 @@ func TestRUMRouting(t *testing.T) { "source.port", "source.ip", "client", ) } + +func TestRUMRoutingIntegration(t *testing.T) { + // This test asserts that the events that are coming from the RUM JS agent + // are sent to the appropriate datastream. + systemtest.CleanupElasticsearch(t) + apmIntegration := newAPMIntegration(t, map[string]interface{}{"enable_rum": true}) + + body, err := os.Open(filepath.Join("..", "testdata", "intake-v3", "rum_events.ndjson")) + require.NoError(t, err) + defer body.Close() + + req, err := http.NewRequest("POST", apmIntegration.URL+"/intake/v3/rum/events", body) + require.NoError(t, err) + req.Header.Add("Content-Type", "application/x-ndjson") + + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + defer resp.Body.Close() + result := systemtest.Elasticsearch.ExpectDocs(t, "traces-apm.rum*", nil) + systemtest.ApproveEvents( + t, t.Name(), result.Hits.Hits, "@timestamp", "timestamp.us", + "source.port", "source.ip", "client", + ) +} diff --git a/systemtest/sourcemap_test.go b/systemtest/sourcemap_test.go index f4a49b47d5d..6361fd8fe59 100644 --- a/systemtest/sourcemap_test.go +++ b/systemtest/sourcemap_test.go @@ -37,20 +37,31 @@ func TestRUMErrorSourcemapping(t *testing.T) { require.NoError(t, err) systemtest.CreateSourceMap(t, string(sourcemap), "apm-agent-js", "1.0.1", bundleFilepath) - srv := apmservertest.NewUnstartedServerTB(t) - srv.Config.RUM = &apmservertest.RUMConfig{Enabled: true} - err = srv.Start() - require.NoError(t, err) + apmIntegration := newAPMIntegration(t, map[string]interface{}{"enable_rum": true}) + + test := func(t *testing.T, serverURL string) { + systemtest.SendRUMEventsPayload(t, serverURL, "../testdata/intake-v2/errors_rum.ndjson") + result := systemtest.Elasticsearch.ExpectDocs(t, "logs-apm.error-*", nil) + systemtest.ApproveEvents( + t, t.Name(), result.Hits.Hits, + // RUM timestamps are set by the server based on the time the payload is received. + "@timestamp", "timestamp.us", + // RUM events have the source IP and port recorded, which are dynamic in the tests + "client.ip", "source.ip", "source.port", + ) + } - systemtest.SendRUMEventsPayload(t, srv.URL, "../testdata/intake-v2/errors_rum.ndjson") - result := systemtest.Elasticsearch.ExpectDocs(t, "logs-apm.error-*", nil) - systemtest.ApproveEvents( - t, t.Name(), result.Hits.Hits, - // RUM timestamps are set by the server based on the time the payload is received. - "@timestamp", "timestamp.us", - // RUM events have the source IP and port recorded, which are dynamic in the tests - "client.ip", "source.ip", "source.port", - ) + t.Run("standalone", func(t *testing.T) { + srv := apmservertest.NewUnstartedServerTB(t) + srv.Config.RUM = &apmservertest.RUMConfig{Enabled: true} + err := srv.Start() + require.NoError(t, err) + test(t, srv.URL) + }) + + t.Run("integration", func(t *testing.T) { + test(t, apmIntegration.URL) + }) } t.Run("absolute_bundle_filepath", func(t *testing.T) { @@ -121,12 +132,12 @@ func TestNoMatchingSourcemap(t *testing.T) { ) } -func TestSourcemapElasticsearch(t *testing.T) { +func TestSourcemapCaching(t *testing.T) { systemtest.CleanupElasticsearch(t) sourcemap, err := os.ReadFile("../testdata/sourcemap/bundle.js.map") require.NoError(t, err) - systemtest.CreateSourceMap(t, string(sourcemap), "apm-agent-js", "1.0.1", + sourcemapID := systemtest.CreateSourceMap(t, string(sourcemap), "apm-agent-js", "1.0.1", "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", ) @@ -139,14 +150,21 @@ func TestSourcemapElasticsearch(t *testing.T) { systemtest.SendRUMEventsPayload(t, srv.URL, "../testdata/intake-v2/errors_rum.ndjson") result := systemtest.Elasticsearch.ExpectDocs(t, "logs-apm.error-*", nil) assertSourcemapUpdated(t, result, true) + + // Delete the source map and error, and try again. + systemtest.DeleteSourceMap(t, sourcemapID) + systemtest.CleanupElasticsearch(t) + systemtest.SendRUMEventsPayload(t, srv.URL, "../testdata/intake-v2/errors_rum.ndjson") + result = systemtest.Elasticsearch.ExpectMinDocs(t, 1, "logs-apm.error-*", nil) + assertSourcemapUpdated(t, result, true) } -func TestSourcemapCaching(t *testing.T) { +func TestSourcemapElasticsearch(t *testing.T) { systemtest.CleanupElasticsearch(t) sourcemap, err := os.ReadFile("../testdata/sourcemap/bundle.js.map") require.NoError(t, err) - sourcemapID := systemtest.CreateSourceMap(t, string(sourcemap), "apm-agent-js", "1.0.1", + systemtest.CreateSourceMap(t, string(sourcemap), "apm-agent-js", "1.0.1", "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", ) @@ -159,13 +177,6 @@ func TestSourcemapCaching(t *testing.T) { systemtest.SendRUMEventsPayload(t, srv.URL, "../testdata/intake-v2/errors_rum.ndjson") result := systemtest.Elasticsearch.ExpectDocs(t, "logs-apm.error-*", nil) assertSourcemapUpdated(t, result, true) - - // Delete the source map and error, and try again. - systemtest.DeleteSourceMap(t, sourcemapID) - systemtest.CleanupElasticsearch(t) - systemtest.SendRUMEventsPayload(t, srv.URL, "../testdata/intake-v2/errors_rum.ndjson") - result = systemtest.Elasticsearch.ExpectMinDocs(t, 1, "logs-apm.error-*", nil) - assertSourcemapUpdated(t, result, true) } func deleteIndex(t *testing.T, name string) { From 778ec6df079e785d8a8b6807e52d187d3de9a7bb Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Sat, 21 Jan 2023 17:45:41 +0100 Subject: [PATCH 051/123] feat: bump sourcemap error log message level to error --- internal/sourcemap/processor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/sourcemap/processor.go b/internal/sourcemap/processor.go index 4863f4fe9fb..e2831908e27 100644 --- a/internal/sourcemap/processor.go +++ b/internal/sourcemap/processor.go @@ -121,7 +121,7 @@ func (p BatchProcessor) processStacktraceFrame( mapper, err := p.Fetcher.Fetch(ctx, service.Name, service.Version, path) if err != nil { frame.SourcemapError = err.Error() - getProcessorLogger().Debugf("failed to fetch source map: %s", frame.SourcemapError) + getProcessorLogger().Errorf("failed to fetch source map: %s", frame.SourcemapError) return false, "" } if mapper == nil { From 69e28597f563c658ea214d072f5076a676860ccf Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Sat, 21 Jan 2023 17:49:39 +0100 Subject: [PATCH 052/123] fix: prevent sourcemapping specific ES config from being overwritten --- internal/beater/config/rum.go | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/internal/beater/config/rum.go b/internal/beater/config/rum.go index 5d8d75671b6..6cce6808940 100644 --- a/internal/beater/config/rum.go +++ b/internal/beater/config/rum.go @@ -58,6 +58,7 @@ type SourceMapping struct { Metadata []SourceMapMetadata `config:"metadata"` Timeout time.Duration `config:"timeout" validate:"positive"` esConfigured bool + es *config.C } func (c *RumConfig) setup(log *logp.Logger, outputESCfg *config.C) error { @@ -72,20 +73,35 @@ func (c *RumConfig) setup(log *logp.Logger, outputESCfg *config.C) error { return errors.Wrapf(err, "Invalid regex for `exclude_from_grouping`: ") } - // No need to unpack the ESConfig if SourceMapMetadata exist if len(c.SourceMapping.Metadata) > 0 { - return nil + // We don't have the fleet fetcher anymore. + // Ignore metadata and setup the elasticsearch config. + log.Warn("Ignoring sourcemap metadata") } - // fall back to elasticsearch output configuration for sourcemap storage if possible if outputESCfg == nil { log.Info("Unable to determine sourcemap storage, sourcemaps will not be applied") return nil } - log.Info("Falling back to elasticsearch output for sourcemap storage") + + // Unpack the output elasticsearch config first if err := outputESCfg.Unpack(c.SourceMapping.ESConfig); err != nil { - return errors.Wrap(err, "unpacking Elasticsearch config into Sourcemap config") + return errors.Wrap(err, "unpacking Elasticsearch output config into Sourcemap config") + } + + // SourceMapping ES config not configured, use the main one and return early + if c.SourceMapping.es == nil { + log.Info("Using default sourcemap Elasticsearch config") + return nil } + + // Unpack the SourceMapping ES config on top of the output elasticsearch config + if err := c.SourceMapping.es.Unpack(c.SourceMapping.ESConfig); err != nil { + return errors.Wrap(err, "unpacking Elasticsearch sourcemap config into Sourcemap config") + } + + c.SourceMapping.es = nil + return nil } @@ -95,6 +111,10 @@ func (s *SourceMapping) Unpack(inp *config.C) error { return errors.Wrap(err, "error unpacking sourcemapping config") } s.esConfigured = inp.HasField("elasticsearch") + var err error + if s.es, err = inp.Child("elasticsearch", -1); err != nil { + return errors.Wrap(err, "error storing sourcemap elasticsearch config") + } return nil } From faece2a59cee43a579e93889f8a433543fabb615 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Sat, 21 Jan 2023 17:50:40 +0100 Subject: [PATCH 053/123] fix: defer closing the init channel to avoid potential deadlock --- internal/sourcemap/metadata.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/sourcemap/metadata.go b/internal/sourcemap/metadata.go index 6198c675bc9..a579cf43547 100644 --- a/internal/sourcemap/metadata.go +++ b/internal/sourcemap/metadata.go @@ -193,12 +193,13 @@ func (s *MetadataCachingFetcher) StartBackgroundSync() { ctx, cleanup := context.WithTimeout(context.Background(), 10*time.Second) defer cleanup() + defer close(s.init) + if err := s.sync(ctx); err != nil { s.logger.Errorf("failed to fetch sourcemaps metadata: %v", err) } s.logger.Info("init routine completed") - close(s.init) }() go func() { From 878b0258de2f6f98d75a8eaf1d0f62554b3a8d29 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Sat, 21 Jan 2023 19:41:07 +0100 Subject: [PATCH 054/123] refactor: reduce diff noise --- internal/beater/beater_test.go | 4 ++-- internal/sourcemap/elasticsearch.go | 4 ++-- systemtest/rum_test.go | 34 ++--------------------------- 3 files changed, 6 insertions(+), 36 deletions(-) diff --git a/internal/beater/beater_test.go b/internal/beater/beater_test.go index a739a1d9789..6c88fd6d156 100644 --- a/internal/beater/beater_test.go +++ b/internal/beater/beater_test.go @@ -35,8 +35,6 @@ import ( "github.com/elastic/elastic-agent-libs/monitoring" ) -var validSourcemap, _ = os.ReadFile("../../testdata/sourcemap/bundle.js.map") - func TestSourcemapIndexPattern(t *testing.T) { test := func(t *testing.T, indexPattern, expected string) { var requestPaths []string @@ -80,6 +78,8 @@ func TestSourcemapIndexPattern(t *testing.T) { }) } +var validSourcemap, _ = os.ReadFile("../../testdata/sourcemap/bundle.js.map") + func TestStoreUsesRUMElasticsearchConfig(t *testing.T) { var called bool ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/internal/sourcemap/elasticsearch.go b/internal/sourcemap/elasticsearch.go index 294862ea9bd..ec2f05bf96d 100644 --- a/internal/sourcemap/elasticsearch.go +++ b/internal/sourcemap/elasticsearch.go @@ -92,11 +92,11 @@ func (s *esFetcher) Fetch(ctx context.Context, name, version, path string) (*sou if resp.StatusCode == http.StatusNotFound { return nil, nil } - body, err := io.ReadAll(resp.Body) + b, err := io.ReadAll(resp.Body) if err != nil { return nil, errors.Wrap(err, errMsgParseSourcemap) } - return nil, errors.New(fmt.Sprintf("%s %s", errMsgParseSourcemap, body)) + return nil, errors.New(fmt.Sprintf("%s %s", errMsgParseSourcemap, b)) } // parse response diff --git a/systemtest/rum_test.go b/systemtest/rum_test.go index 923b1eaa6f1..b53876aebda 100644 --- a/systemtest/rum_test.go +++ b/systemtest/rum_test.go @@ -193,42 +193,11 @@ func TestRUMCORS(t *testing.T) { assert.Equal(t, "stick, door, Content-Type, Content-Encoding, Accept", resp.Header.Get("Access-Control-Allow-Headers")) } -func TestRUMRouting(t *testing.T) { - // This test asserts that the events that are coming from the RUM JS agent - // are sent to the appropriate datastream. - systemtest.CleanupElasticsearch(t) - - srv := apmservertest.NewUnstartedServerTB(t) - srv.Config.RUM = &apmservertest.RUMConfig{ - Enabled: true, - } - err := srv.Start() - require.NoError(t, err) - - body, err := os.Open(filepath.Join("..", "testdata", "intake-v3", "rum_events.ndjson")) - require.NoError(t, err) - defer body.Close() - - req, err := http.NewRequest("POST", srv.URL+"/intake/v3/rum/events", body) - require.NoError(t, err) - req.Header.Add("Content-Type", "application/x-ndjson") - - resp, err := http.DefaultClient.Do(req) - require.NoError(t, err) - defer resp.Body.Close() - - result := systemtest.Elasticsearch.ExpectDocs(t, "traces-apm.rum*", nil) - systemtest.ApproveEvents( - t, t.Name(), result.Hits.Hits, "@timestamp", "timestamp.us", - "source.port", "source.ip", "client", - ) -} - func TestRUMRoutingIntegration(t *testing.T) { // This test asserts that the events that are coming from the RUM JS agent // are sent to the appropriate datastream. systemtest.CleanupElasticsearch(t) - apmIntegration := newAPMIntegration(t, map[string]interface{}{"enable_rum": true}) + apmIntegration := newAPMIntegration(t, nil) body, err := os.Open(filepath.Join("..", "testdata", "intake-v3", "rum_events.ndjson")) require.NoError(t, err) @@ -241,6 +210,7 @@ func TestRUMRoutingIntegration(t *testing.T) { resp, err := http.DefaultClient.Do(req) require.NoError(t, err) defer resp.Body.Close() + result := systemtest.Elasticsearch.ExpectDocs(t, "traces-apm.rum*", nil) systemtest.ApproveEvents( t, t.Name(), result.Hits.Hits, "@timestamp", "timestamp.us", From 558a473f8088bf80899985ef51deb8dca4370331 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Sat, 21 Jan 2023 19:41:58 +0100 Subject: [PATCH 055/123] refactor: remove unused approval document --- .../approvals/TestRUMRouting.approved.json | 774 ------------------ 1 file changed, 774 deletions(-) delete mode 100644 systemtest/approvals/TestRUMRouting.approved.json diff --git a/systemtest/approvals/TestRUMRouting.approved.json b/systemtest/approvals/TestRUMRouting.approved.json deleted file mode 100644 index 1ec96e24e9d..00000000000 --- a/systemtest/approvals/TestRUMRouting.approved.json +++ /dev/null @@ -1,774 +0,0 @@ -{ - "events": [ - { - "@timestamp": "dynamic", - "agent": { - "name": "js-base", - "version": "4.8.1" - }, - "client": "dynamic", - "data_stream.dataset": "apm.rum", - "data_stream.namespace": "default", - "data_stream.type": "traces", - "destination": { - "address": "localhost", - "port": 8003 - }, - "event": { - "agent_id_status": "missing", - "ingested": "dynamic", - "outcome": "success" - }, - "http": { - "request": { - "method": "POST" - }, - "response": { - "status_code": 200 - } - }, - "labels": { - "testTagKey": "testTagValue" - }, - "network": { - "connection": { - "type": "5G" - } - }, - "observer": { - "hostname": "dynamic", - "type": "apm-server", - "version": "dynamic" - }, - "parent": { - "id": "ec2e280be8345240" - }, - "processor": { - "event": "span", - "name": "transaction" - }, - "service": { - "environment": "prod", - "name": "apm-a-rum-test-e2e-general-usecase" - }, - "source": { - "ip": "dynamic", - "port": "dynamic" - }, - "span": { - "destination": { - "service": { - "name": "http://localhost:8003", - "resource": "localhost:8003", - "type": "external" - } - }, - "duration": { - "us": 11584 - }, - "id": "27f45fd274f976d4", - "name": "POST http://localhost:8003/data", - "subtype": "h", - "sync": true, - "type": "external" - }, - "timestamp": { - "us": "dynamic" - }, - "trace": { - "id": "286ac3ad697892c406528f13c82e0ce1" - }, - "transaction": { - "id": "ec2e280be8345240" - }, - "url": { - "original": "http://localhost:8003/data" - } - }, - { - "@timestamp": "dynamic", - "agent": { - "name": "js-base", - "version": "4.8.1" - }, - "client": "dynamic", - "data_stream.dataset": "apm.rum", - "data_stream.namespace": "default", - "data_stream.type": "traces", - "destination": { - "address": "localhost", - "port": 8000 - }, - "event": { - "agent_id_status": "missing", - "ingested": "dynamic", - "outcome": "success" - }, - "http": { - "request": { - "method": "GET" - }, - "response": { - "status_code": 200 - } - }, - "labels": { - "testTagKey": "testTagValue" - }, - "network": { - "connection": { - "type": "5G" - } - }, - "observer": { - "hostname": "dynamic", - "type": "apm-server", - "version": "dynamic" - }, - "parent": { - "id": "ec2e280be8345240" - }, - "processor": { - "event": "span", - "name": "transaction" - }, - "service": { - "environment": "prod", - "name": "apm-a-rum-test-e2e-general-usecase" - }, - "source": { - "ip": "dynamic", - "port": "dynamic" - }, - "span": { - "destination": { - "service": { - "name": "http://localhost:8000", - "resource": "localhost:8000", - "type": "external" - } - }, - "duration": { - "us": 6724 - }, - "id": "5ecb8ee030749715", - "name": "GET /test/e2e/common/data.json", - "subtype": "h", - "sync": true, - "type": "external" - }, - "timestamp": { - "us": "dynamic" - }, - "trace": { - "id": "286ac3ad697892c406528f13c82e0ce1" - }, - "transaction": { - "id": "ec2e280be8345240" - }, - "url": { - "original": "http://localhost:8000/test/e2e/common/data.json?test=hamid" - } - }, - { - "@timestamp": "dynamic", - "agent": { - "name": "js-base", - "version": "4.8.1" - }, - "client": "dynamic", - "data_stream.dataset": "apm.rum", - "data_stream.namespace": "default", - "data_stream.type": "traces", - "event": { - "agent_id_status": "missing", - "ingested": "dynamic", - "outcome": "unknown" - }, - "labels": { - "testTagKey": "testTagValue" - }, - "network": { - "connection": { - "type": "5G" - } - }, - "observer": { - "hostname": "dynamic", - "type": "apm-server", - "version": "dynamic" - }, - "parent": { - "id": "ec2e280be8345240" - }, - "processor": { - "event": "span", - "name": "transaction" - }, - "service": { - "environment": "prod", - "name": "apm-a-rum-test-e2e-general-usecase" - }, - "source": { - "ip": "dynamic", - "port": "dynamic" - }, - "span": { - "duration": { - "us": 198070 - }, - "id": "9b80535c4403c9fb", - "name": "OpenTracing y", - "type": "cu" - }, - "timestamp": { - "us": "dynamic" - }, - "trace": { - "id": "286ac3ad697892c406528f13c82e0ce1" - }, - "transaction": { - "id": "ec2e280be8345240" - } - }, - { - "@timestamp": "dynamic", - "agent": { - "name": "js-base", - "version": "4.8.1" - }, - "client": "dynamic", - "data_stream.dataset": "apm.rum", - "data_stream.namespace": "default", - "data_stream.type": "traces", - "destination": { - "address": "localhost", - "port": 8003 - }, - "event": { - "agent_id_status": "missing", - "ingested": "dynamic", - "outcome": "success" - }, - "http": { - "request": { - "method": "POST" - }, - "response": { - "status_code": 200 - } - }, - "labels": { - "testTagKey": "testTagValue" - }, - "network": { - "connection": { - "type": "5G" - } - }, - "observer": { - "hostname": "dynamic", - "type": "apm-server", - "version": "dynamic" - }, - "parent": { - "id": "bbd8bcc3be14d814" - }, - "processor": { - "event": "span", - "name": "transaction" - }, - "service": { - "environment": "prod", - "name": "apm-a-rum-test-e2e-general-usecase" - }, - "source": { - "ip": "dynamic", - "port": "dynamic" - }, - "span": { - "action": "action", - "destination": { - "service": { - "name": "http://localhost:8003", - "resource": "localhost:8003", - "type": "external" - } - }, - "duration": { - "us": 15949 - }, - "id": "a3c043330bc2015e", - "name": "POST http://localhost:8003/fetch", - "subtype": "h", - "sync": false, - "type": "external" - }, - "timestamp": { - "us": "dynamic" - }, - "trace": { - "id": "286ac3ad697892c406528f13c82e0ce1" - }, - "transaction": { - "id": "ec2e280be8345240" - }, - "url": { - "original": "http://localhost:8003/fetch" - } - }, - { - "@timestamp": "dynamic", - "agent": { - "name": "js-base", - "version": "4.8.1" - }, - "client": "dynamic", - "data_stream.dataset": "apm.rum", - "data_stream.namespace": "default", - "data_stream.type": "traces", - "event": { - "agent_id_status": "missing", - "ingested": "dynamic", - "outcome": "unknown" - }, - "labels": { - "testTagKey": "testTagValue" - }, - "network": { - "connection": { - "type": "5G" - } - }, - "observer": { - "hostname": "dynamic", - "type": "apm-server", - "version": "dynamic" - }, - "parent": { - "id": "ec2e280be8345240" - }, - "processor": { - "event": "span", - "name": "transaction" - }, - "service": { - "environment": "prod", - "name": "apm-a-rum-test-e2e-general-usecase" - }, - "source": { - "ip": "dynamic", - "port": "dynamic" - }, - "span": { - "duration": { - "us": 2000 - }, - "id": "bbd8bcc3be14d814", - "name": "Requesting and receiving the document", - "subtype": "browser-timing", - "type": "hard-navigation" - }, - "timestamp": { - "us": "dynamic" - }, - "trace": { - "id": "286ac3ad697892c406528f13c82e0ce1" - }, - "transaction": { - "id": "ec2e280be8345240" - } - }, - { - "@timestamp": "dynamic", - "agent": { - "name": "js-base", - "version": "4.8.1" - }, - "client": "dynamic", - "data_stream.dataset": "apm.rum", - "data_stream.namespace": "default", - "data_stream.type": "traces", - "event": { - "agent_id_status": "missing", - "ingested": "dynamic", - "outcome": "success" - }, - "labels": { - "testTagKey": "testTagValue" - }, - "network": { - "connection": { - "type": "5G" - } - }, - "observer": { - "hostname": "dynamic", - "type": "apm-server", - "version": "dynamic" - }, - "parent": { - "id": "ec2e280be8345240" - }, - "processor": { - "event": "span", - "name": "transaction" - }, - "service": { - "environment": "prod", - "name": "apm-a-rum-test-e2e-general-usecase" - }, - "source": { - "ip": "dynamic", - "port": "dynamic" - }, - "span": { - "duration": { - "us": 2000 - }, - "id": "bc7665dc25629379", - "name": "Fire \"DOMContentLoaded\" event", - "stacktrace": [ - { - "abs_path": "http://localhost:8000/test/e2e/general-usecase/app.e2e-bundle.min.js?token=secret", - "exclude_from_grouping": false, - "filename": "test/e2e/general-usecase/app.e2e-bundle.min.js?token=secret", - "function": "generateError", - "line": { - "column": 9, - "number": 7662 - } - }, - { - "abs_path": "http://localhost:8000/test/e2e/general-usecase/app.e2e-bundle.min.js?token=secret", - "exclude_from_grouping": false, - "filename": "test/e2e/general-usecase/app.e2e-bundle.min.js?token=secret", - "function": "\u003canonymous\u003e", - "line": { - "column": 3, - "number": 7666 - } - } - ], - "subtype": "browser-timing", - "type": "hard-navigation" - }, - "timestamp": { - "us": "dynamic" - }, - "trace": { - "id": "286ac3ad697892c406528f13c82e0ce1" - }, - "transaction": { - "id": "ec2e280be8345240" - } - }, - { - "@timestamp": "dynamic", - "agent": { - "name": "js-base", - "version": "4.8.1" - }, - "client": "dynamic", - "data_stream.dataset": "apm.rum", - "data_stream.namespace": "default", - "data_stream.type": "traces", - "destination": { - "address": "localhost", - "port": 8000 - }, - "event": { - "agent_id_status": "missing", - "ingested": "dynamic", - "outcome": "unknown" - }, - "http": { - "response": { - "decoded_body_size": 676864, - "encoded_body_size": 676864, - "transfer_size": 677175 - } - }, - "labels": { - "testTagKey": "testTagValue" - }, - "network": { - "connection": { - "type": "5G" - } - }, - "observer": { - "hostname": "dynamic", - "type": "apm-server", - "version": "dynamic" - }, - "parent": { - "id": "ec2e280be8345240" - }, - "processor": { - "event": "span", - "name": "transaction" - }, - "service": { - "environment": "prod", - "name": "apm-a-rum-test-e2e-general-usecase" - }, - "source": { - "ip": "dynamic", - "port": "dynamic" - }, - "span": { - "destination": { - "service": { - "name": "http://localhost:8000", - "resource": "localhost:8000", - "type": "rc" - } - }, - "duration": { - "us": 35060 - }, - "id": "fb8f717930697299", - "name": "http://localhost:8000/test/e2e/general-usecase/app.e2e-bundle.min.js", - "subtype": "script", - "type": "rc" - }, - "timestamp": { - "us": "dynamic" - }, - "trace": { - "id": "286ac3ad697892c406528f13c82e0ce1" - }, - "transaction": { - "id": "ec2e280be8345240" - }, - "url": { - "original": "http://localhost:8000/test/e2e/general-usecase/app.e2e-bundle.min.js?token=REDACTED" - } - }, - { - "@timestamp": "dynamic", - "agent": { - "name": "js-base", - "version": "4.8.1" - }, - "client": "dynamic", - "data_stream.dataset": "apm.rum", - "data_stream.namespace": "default", - "data_stream.type": "traces", - "event": { - "agent_id_status": "missing", - "ingested": "dynamic", - "outcome": "unknown" - }, - "labels": { - "testTagKey": "testTagValue" - }, - "network": { - "connection": { - "type": "5G" - } - }, - "observer": { - "hostname": "dynamic", - "type": "apm-server", - "version": "dynamic" - }, - "parent": { - "id": "ec2e280be8345240" - }, - "processor": { - "event": "span", - "name": "transaction" - }, - "service": { - "environment": "prod", - "name": "apm-a-rum-test-e2e-general-usecase" - }, - "source": { - "ip": "dynamic", - "port": "dynamic" - }, - "span": { - "duration": { - "us": 106000 - }, - "id": "fc546e87a90a774f", - "name": "Parsing the document, executing sy. scripts", - "subtype": "browser-timing", - "type": "hard-navigation" - }, - "timestamp": { - "us": "dynamic" - }, - "trace": { - "id": "286ac3ad697892c406528f13c82e0ce1" - }, - "transaction": { - "id": "ec2e280be8345240" - } - }, - { - "@timestamp": "dynamic", - "agent": { - "name": "js-base", - "version": "4.8.1" - }, - "client": "dynamic", - "data_stream.dataset": "apm.rum", - "data_stream.namespace": "default", - "data_stream.type": "traces", - "event": { - "agent_id_status": "missing", - "ingested": "dynamic", - "outcome": "success" - }, - "http": { - "request": { - "headers": { - "Accept": [ - "application/json" - ] - }, - "method": "GET", - "referrer": "http://localhost:8000/test/e2e/" - }, - "response": { - "decoded_body_size": 690, - "encoded_body_size": 690, - "headers": { - "Content-Type": [ - "application/json" - ] - }, - "status_code": 200, - "transfer_size": 983 - }, - "version": "1.1" - }, - "labels": { - "testTagKey": "testTagValue" - }, - "network": { - "connection": { - "type": "5G" - } - }, - "observer": { - "hostname": "dynamic", - "type": "apm-server", - "version": "dynamic" - }, - "parent": { - "id": "1ef08ac234fca23b455d9e27c660f1ab" - }, - "processor": { - "event": "transaction", - "name": "transaction" - }, - "service": { - "environment": "prod", - "framework": { - "name": "angular", - "version": "2" - }, - "language": { - "name": "javascript", - "version": "6" - }, - "name": "apm-a-rum-test-e2e-general-usecase", - "runtime": { - "name": "v8", - "version": "8.0" - }, - "version": "0.0.1" - }, - "source": { - "ip": "dynamic", - "port": "dynamic" - }, - "timestamp": { - "us": "dynamic" - }, - "trace": { - "id": "286ac3ad697892c406528f13c82e0ce1" - }, - "transaction": { - "custom": { - "testContext": "testContext" - }, - "duration": { - "us": 295000 - }, - "experience": { - "cls": 1, - "fid": 2, - "longtask": { - "count": 3, - "max": 1, - "sum": 2.5 - }, - "tbt": 3.4 - }, - "id": "ec2e280be8345240", - "marks": { - "agent": { - "domComplete": 138, - "domContentLoadedEventEnd": 110, - "domContentLoadedEventStart": 100, - "domInteractive": 120, - "firstContentfulPaint": 70.82500003930181, - "largestContentfulPaint": 131.03000004775822, - "timeToFirstByte": 5 - }, - "navigationTiming": { - "connectEnd": 0, - "connectStart": 0, - "domComplete": 138, - "domContentLoadedEventEnd": 122, - "domContentLoadedEventStart": 120, - "domInteractive": 120, - "domLoading": 14, - "domainLookupEnd": 0, - "domainLookupStart": 0, - "fetchStart": 0, - "loadEventEnd": 138, - "loadEventStart": 138, - "requestStart": 4, - "responseEnd": 6, - "responseStart": 5 - } - }, - "name": "general-usecase-initial-p-load", - "representative_count": 1, - "sampled": true, - "span_count": { - "dropped": 1, - "started": 8 - }, - "type": "p-load" - }, - "url": { - "domain": "localhost", - "full": "http://localhost:8000/test/e2e/general-usecase/", - "original": "http://localhost:8000/test/e2e/general-usecase/", - "path": "/test/e2e/general-usecase/", - "port": 8000, - "scheme": "http" - }, - "user": { - "email": "em", - "id": "uId", - "name": "un" - }, - "user_agent": { - "device": { - "name": "Other" - }, - "name": "Go-http-client", - "original": "Go-http-client/1.1", - "version": "1.1" - } - } - ] -} From 7df5419d5b066160455277213e8ecb5595b1bf2a Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Sat, 21 Jan 2023 19:45:59 +0100 Subject: [PATCH 056/123] lint: remove unused methods --- internal/sourcemap/fetcher.go | 5 ----- internal/sourcemap/search.go | 4 ---- 2 files changed, 9 deletions(-) diff --git a/internal/sourcemap/fetcher.go b/internal/sourcemap/fetcher.go index 7036f0d9c17..6053bb8cfe0 100644 --- a/internal/sourcemap/fetcher.go +++ b/internal/sourcemap/fetcher.go @@ -39,11 +39,6 @@ type Identifier struct { path string } -type Metadata struct { - id Identifier - contentHash string -} - func GetAliases(name string, version string, bundleFilepath string) []Identifier { urlPath, err := url.Parse(bundleFilepath) if err != nil { diff --git a/internal/sourcemap/search.go b/internal/sourcemap/search.go index ef521be4ae7..0235d6ee90d 100644 --- a/internal/sourcemap/search.go +++ b/internal/sourcemap/search.go @@ -79,10 +79,6 @@ func boolean(clause map[string]interface{}) map[string]interface{} { return wrap("bool", clause) } -func must(clauses ...map[string]interface{}) map[string]interface{} { - return map[string]interface{}{"must": clauses} -} - func should(clauses ...map[string]interface{}) map[string]interface{} { return map[string]interface{}{"should": clauses} } From e1865120ac3a67f4229d757ee0ccde1eeedc769c Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Sat, 21 Jan 2023 21:49:18 +0100 Subject: [PATCH 057/123] fix: ignore error when sourcemapping ES config is missing --- internal/beater/config/rum.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/beater/config/rum.go b/internal/beater/config/rum.go index 6cce6808940..b7402e71344 100644 --- a/internal/beater/config/rum.go +++ b/internal/beater/config/rum.go @@ -25,6 +25,7 @@ import ( "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/go-ucfg" "github.com/elastic/apm-server/internal/elasticsearch" ) @@ -112,7 +113,8 @@ func (s *SourceMapping) Unpack(inp *config.C) error { } s.esConfigured = inp.HasField("elasticsearch") var err error - if s.es, err = inp.Child("elasticsearch", -1); err != nil { + var e ucfg.Error + if s.es, err = inp.Child("elasticsearch", -1); err != nil && (!errors.As(err, &e) || e.Reason() != ucfg.ErrMissing) { return errors.Wrap(err, "error storing sourcemap elasticsearch config") } return nil From c194073af5b31eebe103bec34b1f3709a9991100 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Sun, 22 Jan 2023 02:18:40 +0100 Subject: [PATCH 058/123] test: update config test for sourcemap indexpattern changes --- internal/beater/config/config_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/beater/config/config_test.go b/internal/beater/config/config_test.go index 913e74b6eb8..b73f7f2bca5 100644 --- a/internal/beater/config/config_test.go +++ b/internal/beater/config/config_test.go @@ -453,7 +453,7 @@ func TestUnpackConfig(t *testing.T) { Cache: Cache{ Expiration: 7 * time.Second, }, - IndexPattern: "apm-*-sourcemap*", + IndexPattern: ".apm-source-map", ESConfig: elasticsearch.DefaultConfig(), Metadata: []SourceMapMetadata{ { From b56a1ea6faff655221ec6ffba7539770805ac1e6 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Sun, 22 Jan 2023 20:16:03 +0100 Subject: [PATCH 059/123] refactor: improve metadata fetcher --- internal/sourcemap/metadata.go | 70 ++++++++++++++++++++-------------- 1 file changed, 42 insertions(+), 28 deletions(-) diff --git a/internal/sourcemap/metadata.go b/internal/sourcemap/metadata.go index a579cf43547..1b4ed45677c 100644 --- a/internal/sourcemap/metadata.go +++ b/internal/sourcemap/metadata.go @@ -24,6 +24,7 @@ import ( "fmt" "io" "net/http" + "net/url" "sync" "time" @@ -67,59 +68,72 @@ func NewMetadataCachingFetcher( } func (s *MetadataCachingFetcher) Fetch(ctx context.Context, name, version, path string) (*sourcemap.Consumer, error) { - var initPending bool original := Identifier{name: name, version: version, path: path} select { case <-s.init: - // try to minimize lock contention so that update can finish faster - // avoid defer since later down we are waiting for the update routine to - // finish and that would create a deadlock - s.mu.RLock() - if _, found := s.set[original]; found { - s.mu.RUnlock() + // the mutex is shared by the update goroutine, we need to release it + // as soon as possible to avoid blocking updates. + if s.hasID(original) { // Only fetch from ES if the sourcemap id exists return s.fetch(ctx, &original) } - s.mu.RUnlock() default: s.logger.Debugf("Metadata cache not populated. Falling back to backend fetcher for id: %s, %s, %s", name, version, path) // init is in progress, ignore the metadata cache and fetch the sourcemap directly - // return if we get a result or an error + // return if we get a valid sourcemap or an error if c, err := s.backend.Fetch(ctx, original.name, original.version, original.path); c != nil || err != nil { return c, err } - initPending = true - } - keys := GetAliases(name, version, path) - if len(keys) != 0 && initPending { - s.logger.Debug("Found aliases. Blocking until init is completed") - // aliases only work after init + s.logger.Debug("Blocking until init is completed") + + // Aliases are only available after init is completed. <-s.init } - s.mu.RLock() - defer s.mu.RUnlock() + // path is missing from the metadata cache (and ES). + // Is it an alias ? + // Try to retrieve the sourcemap from the alias map + // Only fetch from ES if the sourcemap alias exists + if id, found := s.getAlias(original); found { + return s.fetch(ctx, id) + } - for _, key := range keys { - // Try again from the original cache - // This is because you might be storing the sourcemap in ES with an alias - // or original request id might not be using a clear url. - if _, found := s.set[key]; found { - return s.fetch(ctx, &key) - } + // As a last resort, try to clean the path + if urlPath, err := url.Parse(path); err == nil { + urlPath.RawQuery = "" + urlPath.Fragment = "" + urlPath = urlPath.JoinPath() + + cleanPath := urlPath.String() - // Try to retrieve the sourcemap from alias - // Only fetch from ES if the sourcemap alias exists - if id, found := s.alias[key]; found { - return s.fetch(ctx, id) + if cleanPath != path { + s.logger.Debugf("original filepath %s converted to %s", path, cleanPath) + // we got a different result + return s.Fetch(ctx, name, version, cleanPath) } } return nil, nil } +func (s *MetadataCachingFetcher) hasID(key Identifier) bool { + s.mu.RLock() + defer s.mu.RUnlock() + + _, ok := s.set[key] + return ok +} + +func (s *MetadataCachingFetcher) getAlias(key Identifier) (*Identifier, bool) { + s.mu.RLock() + defer s.mu.RUnlock() + + i, ok := s.alias[key] + return i, ok +} + func (s *MetadataCachingFetcher) fetch(ctx context.Context, key *Identifier) (*sourcemap.Consumer, error) { c, err := s.backend.Fetch(ctx, key.name, key.version, key.path) From 7757b55dea138902085fb1d114a81b97f6318473 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Mon, 23 Jan 2023 01:37:17 +0100 Subject: [PATCH 060/123] fix: document edge cases and try to maximize cache hits --- internal/sourcemap/metadata.go | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/internal/sourcemap/metadata.go b/internal/sourcemap/metadata.go index 1b4ed45677c..8ec4daeb41b 100644 --- a/internal/sourcemap/metadata.go +++ b/internal/sourcemap/metadata.go @@ -100,17 +100,32 @@ func (s *MetadataCachingFetcher) Fetch(ctx context.Context, name, version, path return s.fetch(ctx, id) } - // As a last resort, try to clean the path if urlPath, err := url.Parse(path); err == nil { + // The sourcemap coule be stored in ES with a relative + // bundle filepath but the request came in with an + // absolute path + original.path = urlPath.Path + if urlPath.Path != path && s.hasID(original) { + return s.fetch(ctx, &original) + } + + // The sourcemap could be stored on ES under a certain host + // but a request came in from a different host. + // Look for an alias to the url path to retrieve the correct + // host and fetch the sourcemap + if id, found := s.getAlias(original); found { + return s.fetch(ctx, id) + } + + // Clean the url and try again if the result is different from + // the original bundle filepath urlPath.RawQuery = "" urlPath.Fragment = "" urlPath = urlPath.JoinPath() - cleanPath := urlPath.String() if cleanPath != path { s.logger.Debugf("original filepath %s converted to %s", path, cleanPath) - // we got a different result return s.Fetch(ctx, name, version, cleanPath) } } From 0b38f0d9d4e72ee77b811f462bf7e6ab71a562c3 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Mon, 23 Jan 2023 01:40:15 +0100 Subject: [PATCH 061/123] refactor: improve sourcemap es query --- internal/sourcemap/elasticsearch.go | 16 +++----------- internal/sourcemap/search.go | 33 ++--------------------------- 2 files changed, 5 insertions(+), 44 deletions(-) diff --git a/internal/sourcemap/elasticsearch.go b/internal/sourcemap/elasticsearch.go index ec2f05bf96d..b57a13f8557 100644 --- a/internal/sourcemap/elasticsearch.go +++ b/internal/sourcemap/elasticsearch.go @@ -165,27 +165,17 @@ func parse(body io.ReadCloser, name, version, path string, logger *logp.Logger) } func requestBody(name, version, path string) map[string]interface{} { - aliases := GetAliases(name, version, path) - - m := make([]map[string]interface{}, 0, 1+len(aliases)) - - m = append(m, boostedTerm("_id", name+"-"+version+"-"+path, 2.0)) - - for _, k := range aliases { - id := k.name + "-" + k.version + "-" + k.path - m = append(m, term("_id", id)) - } + id := name + "-" + version + "-" + path return search( size(1), source("content"), query( boolean( - should( - m..., + must( + term("_id", id), ), ), ), - sort(desc("_score")), ) } diff --git a/internal/sourcemap/search.go b/internal/sourcemap/search.go index 0235d6ee90d..a340835c262 100644 --- a/internal/sourcemap/search.go +++ b/internal/sourcemap/search.go @@ -47,24 +47,6 @@ func size(i int) searchOption { } } -func sort(opts ...searchOption) searchOption { - return func(m map[string]interface{}) { - s := make(map[string]interface{}, len(opts)) - - for _, opt := range opts { - opt(s) - } - - m["sort"] = s - } -} - -func desc(k string) searchOption { - return func(m map[string]interface{}) { - m[k] = map[string]interface{}{"order": "desc"} - } -} - func query(q map[string]interface{}) searchOption { return func(m map[string]interface{}) { m["query"] = q @@ -79,21 +61,10 @@ func boolean(clause map[string]interface{}) map[string]interface{} { return wrap("bool", clause) } -func should(clauses ...map[string]interface{}) map[string]interface{} { - return map[string]interface{}{"should": clauses} +func must(clauses ...map[string]interface{}) map[string]interface{} { + return map[string]interface{}{"must": clauses} } func term(k, v string) map[string]interface{} { return map[string]interface{}{"term": map[string]interface{}{k: v}} } - -func boostedTerm(k, v string, boost float32) map[string]interface{} { - return map[string]interface{}{ - "term": map[string]map[string]interface{}{ - k: { - "value": v, - "boost": boost, - }, - }, - } -} From a890a2f9242b342354a05fbdb0d173bbb75cb62b Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Mon, 23 Jan 2023 03:50:41 +0100 Subject: [PATCH 062/123] feat: add back kibana fetcher as fallback --- internal/beater/beater.go | 11 ++- internal/sourcemap/chained.go | 47 ++++++++++ internal/sourcemap/kibana.go | 97 ++++++++++++++++++++ internal/sourcemap/kibana_test.go | 148 ++++++++++++++++++++++++++++++ 4 files changed, 302 insertions(+), 1 deletion(-) create mode 100644 internal/sourcemap/chained.go create mode 100644 internal/sourcemap/kibana.go create mode 100644 internal/sourcemap/kibana_test.go diff --git a/internal/beater/beater.go b/internal/beater/beater.go index 01194d88743..a1c2adc62a7 100644 --- a/internal/beater/beater.go +++ b/internal/beater/beater.go @@ -818,6 +818,9 @@ func newSourcemapFetcher( return nil, err } + // For standalone, we query both Kibana and Elasticsearch for backwards compatibility. + var chained sourcemap.ChainedFetcher + size := 128 invalidationChan := make(chan sourcemap.Identifier, size) @@ -831,7 +834,13 @@ func newSourcemapFetcher( metadataCachingFetcher := sourcemap.NewMetadataCachingFetcher(esClient, cachingFetcher, index, invalidationChan) metadataCachingFetcher.StartBackgroundSync() - return metadataCachingFetcher, nil + chained = append(chained, metadataCachingFetcher) + + if kibanaClient != nil { + chained = append(chained, sourcemap.NewKibanaFetcher(kibanaClient)) + } + + return chained, nil } // TODO: This is copying behavior from libbeat: diff --git a/internal/sourcemap/chained.go b/internal/sourcemap/chained.go new file mode 100644 index 00000000000..daeb11f4104 --- /dev/null +++ b/internal/sourcemap/chained.go @@ -0,0 +1,47 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package sourcemap + +import ( + "context" + + "github.com/go-sourcemap/sourcemap" +) + +// ChainedFetcher is a Fetcher that attempts fetching from each Fetcher in sequence. +type ChainedFetcher []Fetcher + +// Fetch fetches a source map from Kibana. +// +// Fetch calls Fetch on each Fetcher in the chain, in sequence, until one returns +// a non-nil Consumer and nil error. If no Fetch call succeeds, then the last error +// will be returned. +func (c ChainedFetcher) Fetch(ctx context.Context, name, version, path string) (*sourcemap.Consumer, error) { + var lastErr error + for _, f := range c { + consumer, err := f.Fetch(ctx, name, version, path) + if err != nil { + lastErr = err + continue + } + if consumer != nil { + return consumer, nil + } + } + return nil, lastErr +} diff --git a/internal/sourcemap/kibana.go b/internal/sourcemap/kibana.go new file mode 100644 index 00000000000..961a3b0d5a3 --- /dev/null +++ b/internal/sourcemap/kibana.go @@ -0,0 +1,97 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package sourcemap + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + + "github.com/go-sourcemap/sourcemap" + + "github.com/elastic/elastic-agent-libs/logp" + + "github.com/elastic/apm-server/internal/kibana" + "github.com/elastic/apm-server/internal/logs" +) + +const sourcemapArtifactType = "sourcemap" + +type kibanaFetcher struct { + client *kibana.Client + logger *logp.Logger +} + +type kibanaSourceMapArtifact struct { + Type string `json:"type"` + Body struct { + ServiceName string `json:"serviceName"` + ServiceVersion string `json:"serviceVersion"` + BundleFilepath string `json:"bundleFilepath"` + SourceMap json.RawMessage `json:"sourceMap"` + } `json:"body"` +} + +// NewKibanaFetcher returns a Fetcher that fetches source maps stored by Kibana. +func NewKibanaFetcher(c *kibana.Client) Fetcher { + logger := logp.NewLogger(logs.Sourcemap) + return &kibanaFetcher{c, logger} +} + +// Fetch fetches a source map from Kibana. +func (s *kibanaFetcher) Fetch(ctx context.Context, name, version, path string) (*sourcemap.Consumer, error) { + resp, err := s.client.Send(ctx, "GET", "/api/apm/sourcemaps", nil, nil, nil) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("failed to query source maps (%s): %s", resp.Status, body) + } + + var result struct { + Artifacts []kibanaSourceMapArtifact `json:"artifacts"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, err + } + path = maybeParseURLPath(path) + for _, a := range result.Artifacts { + if a.Type != sourcemapArtifactType { + continue + } + if a.Body.ServiceName == name && a.Body.ServiceVersion == version && maybeParseURLPath(a.Body.BundleFilepath) == path { + return parseSourceMap(string(a.Body.SourceMap)) + } + } + return nil, nil +} + +// maybeParseURLPath attempts to parse s as a URL, returning its path component +// if successful. If s cannot be parsed as a URL, s is returned. +func maybeParseURLPath(s string) string { + url, err := url.Parse(s) + if err != nil { + return s + } + return url.Path +} diff --git a/internal/sourcemap/kibana_test.go b/internal/sourcemap/kibana_test.go new file mode 100644 index 00000000000..2565d255a0d --- /dev/null +++ b/internal/sourcemap/kibana_test.go @@ -0,0 +1,148 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package sourcemap + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + libkibana "github.com/elastic/elastic-agent-libs/kibana" + + "github.com/elastic/apm-server/internal/kibana" +) + +func TestKibanaFetcher(t *testing.T) { + fetcher := newTestKibanaFetcher(t, func(w http.ResponseWriter, r *http.Request) { + result := map[string]interface{}{ + "artifacts": []interface{}{ + map[string]interface{}{ + "type": "not_a_sourcemap", + "body": map[string]interface{}{ + "serviceName": "service_name", + "serviceVersion": "service_version", + "bundleFilepath": "http://another_host:456/path", + "sourceMap": "invalid_sourcemap", + }, + }, + map[string]interface{}{ + "type": "sourcemap", + "body": map[string]interface{}{ + "serviceName": "non_matching_service_name", + "serviceVersion": "service_version", + "bundleFilepath": "http://another_host:456/path", + "sourceMap": "invalid_sourcemap", + }, + }, + map[string]interface{}{ + "type": "sourcemap", + "body": map[string]interface{}{ + "serviceName": "service_name", + "serviceVersion": "non_matching_service_version", + "bundleFilepath": "http://another_host:456/path", + "sourceMap": "invalid_sourcemap", + }, + }, + map[string]interface{}{ + "type": "sourcemap", + "body": map[string]interface{}{ + "serviceName": "service_name", + "serviceVersion": "service_version", + "bundleFilepath": "http://another_host:456/non_matching_path", + "sourceMap": "invalid_sourcemap", + }, + }, + map[string]interface{}{ + "type": "sourcemap", + "body": map[string]interface{}{ + "serviceName": "service_name", + "serviceVersion": "service_version", + "bundleFilepath": "http://another_host:456/path", + "sourceMap": json.RawMessage(validSourcemap), + }, + }, + }, + } + json.NewEncoder(w).Encode(result) + }) + consumer, err := fetcher.Fetch(context.Background(), "service_name", "service_version", "http://host:123/path") + require.NoError(t, err) + assert.NotNil(t, consumer) +} + +func TestKibanaFetcherInvalidSourcemap(t *testing.T) { + fetcher := newTestKibanaFetcher(t, func(w http.ResponseWriter, r *http.Request) { + result := map[string]interface{}{ + "artifacts": []interface{}{ + map[string]interface{}{ + "type": "sourcemap", + "body": map[string]interface{}{ + "serviceName": "service_name", + "serviceVersion": "service_version", + "bundleFilepath": "http://another_host:456/path", + "sourceMap": "invalid_sourcemap", + }, + }, + }, + } + json.NewEncoder(w).Encode(result) + }) + consumer, err := fetcher.Fetch(context.Background(), "service_name", "service_version", "http://host:123/path") + require.Error(t, err) + assert.EqualError(t, err, "Could not parse Sourcemap: json: cannot unmarshal string into Go value of type sourcemap.v3") + assert.Nil(t, consumer) +} + +func TestKibanaFetcherNotFound(t *testing.T) { + fetcher := newTestKibanaFetcher(t, func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, `{"artifacts":[]}`) + }) + consumer, err := fetcher.Fetch(context.Background(), "service_name", "service_version", "http://host:123/path") + require.NoError(t, err) + assert.Nil(t, consumer) +} + +func TestKibanaFetcherServerError(t *testing.T) { + fetcher := newTestKibanaFetcher(t, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte("terrible things")) + }) + consumer, err := fetcher.Fetch(context.Background(), "service_name", "service_version", "http://host:123/path") + require.Error(t, err) + assert.EqualError(t, err, "failed to query source maps (500 Internal Server Error): terrible things") + assert.Nil(t, consumer) +} + +func newTestKibanaFetcher(t testing.TB, h http.HandlerFunc) Fetcher { + mux := http.NewServeMux() + mux.HandleFunc("/api/apm/sourcemaps", h) + srv := httptest.NewServer(mux) + t.Cleanup(srv.Close) + + kibanaClient, err := kibana.NewClient(libkibana.ClientConfig{ + Host: srv.Listener.Addr().String(), + }) + require.NoError(t, err) + return NewKibanaFetcher(kibanaClient) +} From 174e87e3deb2f42f33ffae68b15751129ac8786b Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Mon, 23 Jan 2023 03:51:01 +0100 Subject: [PATCH 063/123] test: add systemtest for kibana fetcher --- systemtest/apmservertest/config.go | 5 +++-- systemtest/sourcemap_test.go | 27 +++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/systemtest/apmservertest/config.go b/systemtest/apmservertest/config.go index a87b78f758a..220065a3e68 100644 --- a/systemtest/apmservertest/config.go +++ b/systemtest/apmservertest/config.go @@ -192,8 +192,9 @@ type RUMConfig struct { // RUMSourcemapConfig holds APM Server RUM sourcemap configuration. type RUMSourcemapConfig struct { - Enabled bool `json:"enabled,omitempty"` - Cache *RUMSourcemapCacheConfig `json:"cache,omitempty"` + Enabled bool `json:"enabled,omitempty"` + IndexPattern string `json:"index_pattern"` + Cache *RUMSourcemapCacheConfig `json:"cache,omitempty"` } // RUMSourcemapCacheConfig holds sourcemap cache expiration. diff --git a/systemtest/sourcemap_test.go b/systemtest/sourcemap_test.go index 6361fd8fe59..a486f60c1c8 100644 --- a/systemtest/sourcemap_test.go +++ b/systemtest/sourcemap_test.go @@ -179,6 +179,33 @@ func TestSourcemapElasticsearch(t *testing.T) { assertSourcemapUpdated(t, result, true) } +func TestSourcemapKibana(t *testing.T) { + systemtest.CleanupElasticsearch(t) + + sourcemap, err := os.ReadFile("../testdata/sourcemap/bundle.js.map") + require.NoError(t, err) + systemtest.CreateSourceMap(t, string(sourcemap), "apm-agent-js", "1.0.1", + "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", + ) + + srv := apmservertest.NewUnstartedServerTB(t) + srv.Config.RUM = &apmservertest.RUMConfig{ + Enabled: true, + Sourcemap: &apmservertest.RUMSourcemapConfig{ + // Use the wrong index pattern so that the ES fetcher + // will fail and apm server will fall back to kibana + IndexPattern: "example", + }, + } + err = srv.Start() + require.NoError(t, err) + + // Index an error, applying source mapping and caching the source map in the process. + systemtest.SendRUMEventsPayload(t, srv.URL, "../testdata/intake-v2/errors_rum.ndjson") + result := systemtest.Elasticsearch.ExpectDocs(t, "logs-apm.error-*", nil) + assertSourcemapUpdated(t, result, true) +} + func deleteIndex(t *testing.T, name string) { resp, err := systemtest.Elasticsearch.Indices.Delete([]string{name}) require.NoError(t, err) From d12ebdfb16c3877c8b934955c2dad93965529b86 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Mon, 23 Jan 2023 04:39:19 +0100 Subject: [PATCH 064/123] fix: do not leak sourcemap sync goroutine --- internal/beater/beater.go | 15 +++++++++------ internal/beater/beater_test.go | 6 ++++-- internal/sourcemap/metadata.go | 22 ++++++++++++++-------- 3 files changed, 27 insertions(+), 16 deletions(-) diff --git a/internal/beater/beater.go b/internal/beater/beater.go index a1c2adc62a7..f29adafb317 100644 --- a/internal/beater/beater.go +++ b/internal/beater/beater.go @@ -325,13 +325,14 @@ func (s *Runner) Run(ctx context.Context) error { var sourcemapFetcher sourcemap.Fetcher if s.config.RumConfig.Enabled && s.config.RumConfig.SourceMapping.Enabled { - fetcher, err := newSourcemapFetcher( + fetcher, cleanup, err := newSourcemapFetcher( s.config.RumConfig.SourceMapping, s.fleetConfig, kibanaClient, newElasticsearchClient, ) if err != nil { return err } + defer cleanup() sourcemapFetcher = fetcher } @@ -812,10 +813,10 @@ func newSourcemapFetcher( fleetCfg *config.Fleet, kibanaClient *kibana.Client, newElasticsearchClient func(*elasticsearch.Config) (*elasticsearch.Client, error), -) (sourcemap.Fetcher, error) { +) (sourcemap.Fetcher, context.CancelFunc, error) { esClient, err := newElasticsearchClient(cfg.ESConfig) if err != nil { - return nil, err + return nil, nil, err } // For standalone, we query both Kibana and Elasticsearch for backwards compatibility. @@ -829,10 +830,12 @@ func newSourcemapFetcher( esFetcher := sourcemap.NewElasticsearchFetcher(esClient, index) cachingFetcher, err := sourcemap.NewCachingFetcher(esFetcher, invalidationChan, size) if err != nil { - return nil, err + return nil, nil, err } metadataCachingFetcher := sourcemap.NewMetadataCachingFetcher(esClient, cachingFetcher, index, invalidationChan) - metadataCachingFetcher.StartBackgroundSync() + + ctx, cancel := context.WithCancel(context.Background()) + metadataCachingFetcher.StartBackgroundSync(ctx) chained = append(chained, metadataCachingFetcher) @@ -840,7 +843,7 @@ func newSourcemapFetcher( chained = append(chained, sourcemap.NewKibanaFetcher(kibanaClient)) } - return chained, nil + return chained, cancel, nil } // TODO: This is copying behavior from libbeat: diff --git a/internal/beater/beater_test.go b/internal/beater/beater_test.go index 6c88fd6d156..d1a1130ee7e 100644 --- a/internal/beater/beater_test.go +++ b/internal/beater/beater_test.go @@ -57,11 +57,12 @@ func TestSourcemapIndexPattern(t *testing.T) { cfg.RumConfig.SourceMapping.IndexPattern = indexPattern } - fetcher, err := newSourcemapFetcher( + fetcher, cleanup, err := newSourcemapFetcher( cfg.RumConfig.SourceMapping, nil, nil, elasticsearch.NewClient, ) require.NoError(t, err) + defer cleanup() fetcher.Fetch(context.Background(), "name", "version", "path") require.Len(t, requestPaths, 1) @@ -95,11 +96,12 @@ func TestStoreUsesRUMElasticsearchConfig(t *testing.T) { cfg.RumConfig.SourceMapping.ESConfig = elasticsearch.DefaultConfig() cfg.RumConfig.SourceMapping.ESConfig.Hosts = []string{ts.URL} - fetcher, err := newSourcemapFetcher( + fetcher, cleanup, err := newSourcemapFetcher( cfg.RumConfig.SourceMapping, nil, nil, elasticsearch.NewClient, ) require.NoError(t, err) + defer cleanup() // Check that the provided rum elasticsearch config was used and // Fetch() goes to the test server. _, err = fetcher.Fetch(context.Background(), "app", "1.0", "/bundle/path") diff --git a/internal/sourcemap/metadata.go b/internal/sourcemap/metadata.go index 8ec4daeb41b..93f39815b35 100644 --- a/internal/sourcemap/metadata.go +++ b/internal/sourcemap/metadata.go @@ -216,10 +216,10 @@ func (s *MetadataCachingFetcher) update(ctx context.Context, updates map[Identif s.logger.Debugf("Metadata cache now has %d entries.", len(s.set)) } -func (s *MetadataCachingFetcher) StartBackgroundSync() { +func (s *MetadataCachingFetcher) StartBackgroundSync(parent context.Context) { go func() { // First run, populate cache - ctx, cleanup := context.WithTimeout(context.Background(), 10*time.Second) + ctx, cleanup := context.WithTimeout(parent, 10*time.Second) defer cleanup() defer close(s.init) @@ -236,14 +236,20 @@ func (s *MetadataCachingFetcher) StartBackgroundSync() { t := time.NewTicker(30 * time.Second) defer t.Stop() - for range t.C { - ctx, cleanup := context.WithTimeout(context.Background(), 10*time.Second) + for { + select { + case <-t.C: + ctx, cleanup := context.WithTimeout(parent, 10*time.Second) - if err := s.sync(ctx); err != nil { - s.logger.Errorf("failed to sync sourcemaps metadata: %v", err) - } + if err := s.sync(ctx); err != nil { + s.logger.Errorf("failed to sync sourcemaps metadata: %v", err) + } - cleanup() + cleanup() + case <-parent.Done(): + s.logger.Info("update routine done") + return + } } }() } From 90bc4238c70937b16567c436a19fe87b7ee6d3b9 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Mon, 23 Jan 2023 10:03:58 +0100 Subject: [PATCH 065/123] fix: use scroll search to populate metadata cache --- internal/sourcemap/metadata.go | 120 +++++++++++++++++++++++++++++---- internal/sourcemap/search.go | 6 ++ 2 files changed, 114 insertions(+), 12 deletions(-) diff --git a/internal/sourcemap/metadata.go b/internal/sourcemap/metadata.go index 93f39815b35..8e756dd7d47 100644 --- a/internal/sourcemap/metadata.go +++ b/internal/sourcemap/metadata.go @@ -255,31 +255,64 @@ func (s *MetadataCachingFetcher) StartBackgroundSync(parent context.Context) { } func (s *MetadataCachingFetcher) sync(ctx context.Context) error { + updates := make(map[Identifier]string) + + scrollID, err := s.initialSearch(ctx, updates) + if err != nil { + return err + } + + if scrollID == "" { + return nil + } + + for { + before := len(updates) + + id, err := s.scrollsearch(ctx, scrollID, updates) + if err != nil { + return err + } + + if id != "" { + scrollID = id + } + + if before == len(updates) { + break + } + } + + s.update(ctx, updates) + + return nil +} + +func (s *MetadataCachingFetcher) initialSearch(ctx context.Context, updates map[Identifier]string) (string, error) { resp, err := s.runSearchQuery(ctx) if err != nil { - return errors.Wrap(err, errMsgESFailure) + return "", errors.Wrap(err, errMsgESFailure) } defer resp.Body.Close() // handle error response if resp.StatusCode >= http.StatusMultipleChoices { if resp.StatusCode == http.StatusNotFound { - return nil + return "", nil } b, err := io.ReadAll(resp.Body) if err != nil { - return errors.Wrap(err, errMsgParseSourcemap) + return "", errors.Wrap(err, errMsgParseSourcemap) } - return errors.New(fmt.Sprintf("%s %s", errMsgParseSourcemap, b)) + return "", errors.New(fmt.Sprintf("%s %s", errMsgParseSourcemap, b)) } // parse response body, err := parseResponse(resp.Body, s.logger) if err != nil { - return err + return "", err } - updates := make(map[Identifier]string, len(body.Hits.Hits)) for _, v := range body.Hits.Hits { id := Identifier{ name: v.Source.Service.Name, @@ -290,9 +323,7 @@ func (s *MetadataCachingFetcher) sync(ctx context.Context) error { updates[id] = v.Source.ContentHash } - // Update cache - s.update(ctx, updates) - return nil + return body.ScrollID, nil } func (s *MetadataCachingFetcher) runSearchQuery(ctx context.Context) (*esapi.Response, error) { @@ -305,6 +336,7 @@ func (s *MetadataCachingFetcher) runSearchQuery(ctx context.Context) (*esapi.Res Index: []string{s.index}, Body: &buf, TrackTotalHits: true, + Scroll: time.Minute, } return req.Do(ctx, s.esClient) } @@ -315,13 +347,18 @@ func queryMetadata() map[string]interface{} { ) } -func parseResponse(body io.ReadCloser, logger *logp.Logger) (esSourcemapResponse, error) { +type esSearchSourcemapResponse struct { + ScrollID string `json:"_scroll_id"` + esSourcemapResponse +} + +func parseResponse(body io.ReadCloser, logger *logp.Logger) (esSearchSourcemapResponse, error) { b, err := io.ReadAll(body) if err != nil { - return esSourcemapResponse{}, err + return esSearchSourcemapResponse{}, err } - var esSourcemapResponse esSourcemapResponse + var esSourcemapResponse esSearchSourcemapResponse if err := json.Unmarshal(b, &esSourcemapResponse); err != nil { return esSourcemapResponse, err } @@ -332,3 +369,62 @@ func parseResponse(body io.ReadCloser, logger *logp.Logger) (esSourcemapResponse return esSourcemapResponse, nil } + +func (s *MetadataCachingFetcher) scrollsearch(ctx context.Context, scrollID string, updates map[Identifier]string) (string, error) { + resp, err := s.runScrollSearchQuery(ctx, scrollID) + if err != nil { + return "", errors.Wrap(err, errMsgESFailure) + } + defer resp.Body.Close() + + // handle error response + if resp.StatusCode >= http.StatusMultipleChoices { + if resp.StatusCode == http.StatusNotFound { + return "", nil + } + b, err := io.ReadAll(resp.Body) + if err != nil { + return "", errors.Wrap(err, errMsgParseSourcemap) + } + return "", errors.New(fmt.Sprintf("%s %s", errMsgParseSourcemap, b)) + } + + // parse response + body, err := parseResponse(resp.Body, s.logger) + if err != nil { + return "", err + } + + for _, v := range body.Hits.Hits { + id := Identifier{ + name: v.Source.Service.Name, + version: v.Source.Service.Version, + path: v.Source.File.BundleFilepath, + } + + updates[id] = v.Source.ContentHash + } + + return scrollID, nil + +} + +func (s *MetadataCachingFetcher) runScrollSearchQuery(ctx context.Context, id string) (*esapi.Response, error) { + var buf bytes.Buffer + if err := json.NewEncoder(&buf).Encode(scrollQueryMetadata(id)); err != nil { + return nil, err + } + + req := esapi.ScrollRequest{ + ScrollID: id, + Scroll: time.Minute, + Body: &buf, + } + return req.Do(ctx, s.esClient) +} + +func scrollQueryMetadata(id string) map[string]interface{} { + return search( + scrollID(id), + ) +} diff --git a/internal/sourcemap/search.go b/internal/sourcemap/search.go index a340835c262..f55fd8a3472 100644 --- a/internal/sourcemap/search.go +++ b/internal/sourcemap/search.go @@ -29,6 +29,12 @@ func search(opts ...searchOption) map[string]interface{} { return m } +func scrollID(s string) searchOption { + return func(m map[string]interface{}) { + m["scroll_id"] = s + } +} + func source(s string) searchOption { return func(m map[string]interface{}) { m["_source"] = s From 88f78a0db8e05ee0aa013f7cf1a2a62d66fe25b7 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Mon, 23 Jan 2023 10:05:28 +0100 Subject: [PATCH 066/123] refactor: cleanup search query functions --- internal/sourcemap/search.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/sourcemap/search.go b/internal/sourcemap/search.go index f55fd8a3472..6c078f01e01 100644 --- a/internal/sourcemap/search.go +++ b/internal/sourcemap/search.go @@ -59,7 +59,7 @@ func query(q map[string]interface{}) searchOption { } } -func wrap(k string, v map[string]interface{}) map[string]interface{} { +func wrap(k string, v interface{}) map[string]interface{} { return map[string]interface{}{k: v} } @@ -68,9 +68,9 @@ func boolean(clause map[string]interface{}) map[string]interface{} { } func must(clauses ...map[string]interface{}) map[string]interface{} { - return map[string]interface{}{"must": clauses} + return wrap("must", clauses) } func term(k, v string) map[string]interface{} { - return map[string]interface{}{"term": map[string]interface{}{k: v}} + return wrap("term", wrap(k, v)) } From 873b4c6a33c816d211c915b85d2ea2d8be569e1c Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Mon, 23 Jan 2023 10:45:08 +0100 Subject: [PATCH 067/123] refactor: reduce duplicate code --- internal/sourcemap/metadata.go | 85 ++++++++++++---------------------- 1 file changed, 30 insertions(+), 55 deletions(-) diff --git a/internal/sourcemap/metadata.go b/internal/sourcemap/metadata.go index 8e756dd7d47..d5305aa61f7 100644 --- a/internal/sourcemap/metadata.go +++ b/internal/sourcemap/metadata.go @@ -295,35 +295,7 @@ func (s *MetadataCachingFetcher) initialSearch(ctx context.Context, updates map[ } defer resp.Body.Close() - // handle error response - if resp.StatusCode >= http.StatusMultipleChoices { - if resp.StatusCode == http.StatusNotFound { - return "", nil - } - b, err := io.ReadAll(resp.Body) - if err != nil { - return "", errors.Wrap(err, errMsgParseSourcemap) - } - return "", errors.New(fmt.Sprintf("%s %s", errMsgParseSourcemap, b)) - } - - // parse response - body, err := parseResponse(resp.Body, s.logger) - if err != nil { - return "", err - } - - for _, v := range body.Hits.Hits { - id := Identifier{ - name: v.Source.Service.Name, - version: v.Source.Service.Version, - path: v.Source.File.BundleFilepath, - } - - updates[id] = v.Source.ContentHash - } - - return body.ScrollID, nil + return s.handleUpdateRequest(resp, updates) } func (s *MetadataCachingFetcher) runSearchQuery(ctx context.Context) (*esapi.Response, error) { @@ -352,31 +324,7 @@ type esSearchSourcemapResponse struct { esSourcemapResponse } -func parseResponse(body io.ReadCloser, logger *logp.Logger) (esSearchSourcemapResponse, error) { - b, err := io.ReadAll(body) - if err != nil { - return esSearchSourcemapResponse{}, err - } - - var esSourcemapResponse esSearchSourcemapResponse - if err := json.Unmarshal(b, &esSourcemapResponse); err != nil { - return esSourcemapResponse, err - } - hits := esSourcemapResponse.Hits.Total.Value - if hits == 0 || len(esSourcemapResponse.Hits.Hits) == 0 { - return esSourcemapResponse, nil - } - - return esSourcemapResponse, nil -} - -func (s *MetadataCachingFetcher) scrollsearch(ctx context.Context, scrollID string, updates map[Identifier]string) (string, error) { - resp, err := s.runScrollSearchQuery(ctx, scrollID) - if err != nil { - return "", errors.Wrap(err, errMsgESFailure) - } - defer resp.Body.Close() - +func (s *MetadataCachingFetcher) handleUpdateRequest(resp *esapi.Response, updates map[Identifier]string) (string, error) { // handle error response if resp.StatusCode >= http.StatusMultipleChoices { if resp.StatusCode == http.StatusNotFound { @@ -405,8 +353,35 @@ func (s *MetadataCachingFetcher) scrollsearch(ctx context.Context, scrollID stri updates[id] = v.Source.ContentHash } - return scrollID, nil + return body.ScrollID, nil +} + +func parseResponse(body io.ReadCloser, logger *logp.Logger) (esSearchSourcemapResponse, error) { + b, err := io.ReadAll(body) + if err != nil { + return esSearchSourcemapResponse{}, err + } + + var esSourcemapResponse esSearchSourcemapResponse + if err := json.Unmarshal(b, &esSourcemapResponse); err != nil { + return esSourcemapResponse, err + } + hits := esSourcemapResponse.Hits.Total.Value + if hits == 0 || len(esSourcemapResponse.Hits.Hits) == 0 { + return esSourcemapResponse, nil + } + + return esSourcemapResponse, nil +} + +func (s *MetadataCachingFetcher) scrollsearch(ctx context.Context, scrollID string, updates map[Identifier]string) (string, error) { + resp, err := s.runScrollSearchQuery(ctx, scrollID) + if err != nil { + return "", errors.Wrap(err, errMsgESFailure) + } + defer resp.Body.Close() + return s.handleUpdateRequest(resp, updates) } func (s *MetadataCachingFetcher) runScrollSearchQuery(ctx context.Context, id string) (*esapi.Response, error) { From c9199966f3ff96df5c635d6cdc8610ec53f4c517 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Mon, 23 Jan 2023 10:50:57 +0100 Subject: [PATCH 068/123] docs: add documentation on why we need to renew scrollID --- internal/sourcemap/metadata.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/sourcemap/metadata.go b/internal/sourcemap/metadata.go index d5305aa61f7..b305b0a54c8 100644 --- a/internal/sourcemap/metadata.go +++ b/internal/sourcemap/metadata.go @@ -274,17 +274,21 @@ func (s *MetadataCachingFetcher) sync(ctx context.Context) error { return err } + // From the docs: The initial search request and each subsequent scroll + // request each return a _scroll_id. While the _scroll_id may change between + // requests, it doesn’t always change — in any case, only the most recently + // received _scroll_id should be used. if id != "" { scrollID = id } + // Stop if there are no new updates if before == len(updates) { break } } s.update(ctx, updates) - return nil } From e13870b8d2ff7dae6c75aed6a32153055b23e046 Mon Sep 17 00:00:00 2001 From: kruskall <99559985+kruskall@users.noreply.github.com> Date: Wed, 25 Jan 2023 17:59:46 +0100 Subject: [PATCH 069/123] docs: fix typo Co-authored-by: Carson Ip --- internal/sourcemap/metadata.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/sourcemap/metadata.go b/internal/sourcemap/metadata.go index b305b0a54c8..01cbb2cd036 100644 --- a/internal/sourcemap/metadata.go +++ b/internal/sourcemap/metadata.go @@ -172,11 +172,11 @@ func (s *MetadataCachingFetcher) update(ctx context.Context, updates map[Identif // content hash changed, invalidate the sourcemap cache if contentHash != updatedHash { - s.logger.Debugf("Hash changed: %s -> %s: invaliding %v", contentHash, updatedHash, id) + s.logger.Debugf("Hash changed: %s -> %s: invalidating %v", contentHash, updatedHash, id) select { case s.invalidationChan <- id: case <-ctx.Done(): - s.logger.Errorf("ctx finished while invaliding id: %v", ctx.Err()) + s.logger.Errorf("ctx finished while invalidating id: %v", ctx.Err()) return } From 3066ce0a46e2e1bad172518afc2b340cedc47729 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Wed, 25 Jan 2023 17:57:26 +0100 Subject: [PATCH 070/123] refactor: do not encode scroll id in the body manually --- internal/sourcemap/metadata.go | 12 ------------ internal/sourcemap/search.go | 6 ------ 2 files changed, 18 deletions(-) diff --git a/internal/sourcemap/metadata.go b/internal/sourcemap/metadata.go index 01cbb2cd036..42e2b99d3f3 100644 --- a/internal/sourcemap/metadata.go +++ b/internal/sourcemap/metadata.go @@ -389,21 +389,9 @@ func (s *MetadataCachingFetcher) scrollsearch(ctx context.Context, scrollID stri } func (s *MetadataCachingFetcher) runScrollSearchQuery(ctx context.Context, id string) (*esapi.Response, error) { - var buf bytes.Buffer - if err := json.NewEncoder(&buf).Encode(scrollQueryMetadata(id)); err != nil { - return nil, err - } - req := esapi.ScrollRequest{ ScrollID: id, Scroll: time.Minute, - Body: &buf, } return req.Do(ctx, s.esClient) } - -func scrollQueryMetadata(id string) map[string]interface{} { - return search( - scrollID(id), - ) -} diff --git a/internal/sourcemap/search.go b/internal/sourcemap/search.go index 6c078f01e01..39593d9ecad 100644 --- a/internal/sourcemap/search.go +++ b/internal/sourcemap/search.go @@ -29,12 +29,6 @@ func search(opts ...searchOption) map[string]interface{} { return m } -func scrollID(s string) searchOption { - return func(m map[string]interface{}) { - m["scroll_id"] = s - } -} - func source(s string) searchOption { return func(m map[string]interface{}) { m["_source"] = s From 42bc926ae5278cd49e61140c64980d65146fd3a8 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Wed, 25 Jan 2023 18:05:59 +0100 Subject: [PATCH 071/123] refactor: use result hits instead of manual comparison --- internal/sourcemap/metadata.go | 44 +++++++++++++++++----------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/internal/sourcemap/metadata.go b/internal/sourcemap/metadata.go index 42e2b99d3f3..c9dff8bd844 100644 --- a/internal/sourcemap/metadata.go +++ b/internal/sourcemap/metadata.go @@ -257,19 +257,19 @@ func (s *MetadataCachingFetcher) StartBackgroundSync(parent context.Context) { func (s *MetadataCachingFetcher) sync(ctx context.Context) error { updates := make(map[Identifier]string) - scrollID, err := s.initialSearch(ctx, updates) + result, err := s.initialSearch(ctx, updates) if err != nil { return err } + scrollID := result.ScrollID + if scrollID == "" { return nil } for { - before := len(updates) - - id, err := s.scrollsearch(ctx, scrollID, updates) + result, err = s.scrollsearch(ctx, scrollID, updates) if err != nil { return err } @@ -278,12 +278,12 @@ func (s *MetadataCachingFetcher) sync(ctx context.Context) error { // request each return a _scroll_id. While the _scroll_id may change between // requests, it doesn’t always change — in any case, only the most recently // received _scroll_id should be used. - if id != "" { - scrollID = id + if result.ScrollID != "" { + scrollID = result.ScrollID } // Stop if there are no new updates - if before == len(updates) { + if len(result.Hits.Hits) == 0 { break } } @@ -292,10 +292,10 @@ func (s *MetadataCachingFetcher) sync(ctx context.Context) error { return nil } -func (s *MetadataCachingFetcher) initialSearch(ctx context.Context, updates map[Identifier]string) (string, error) { +func (s *MetadataCachingFetcher) initialSearch(ctx context.Context, updates map[Identifier]string) (*esSearchSourcemapResponse, error) { resp, err := s.runSearchQuery(ctx) if err != nil { - return "", errors.Wrap(err, errMsgESFailure) + return nil, errors.Wrap(err, errMsgESFailure) } defer resp.Body.Close() @@ -328,23 +328,23 @@ type esSearchSourcemapResponse struct { esSourcemapResponse } -func (s *MetadataCachingFetcher) handleUpdateRequest(resp *esapi.Response, updates map[Identifier]string) (string, error) { +func (s *MetadataCachingFetcher) handleUpdateRequest(resp *esapi.Response, updates map[Identifier]string) (*esSearchSourcemapResponse, error) { // handle error response if resp.StatusCode >= http.StatusMultipleChoices { if resp.StatusCode == http.StatusNotFound { - return "", nil + return nil, nil } b, err := io.ReadAll(resp.Body) if err != nil { - return "", errors.Wrap(err, errMsgParseSourcemap) + return nil, errors.Wrap(err, errMsgParseSourcemap) } - return "", errors.New(fmt.Sprintf("%s %s", errMsgParseSourcemap, b)) + return nil, errors.New(fmt.Sprintf("%s %s", errMsgParseSourcemap, b)) } // parse response body, err := parseResponse(resp.Body, s.logger) if err != nil { - return "", err + return nil, err } for _, v := range body.Hits.Hits { @@ -357,31 +357,31 @@ func (s *MetadataCachingFetcher) handleUpdateRequest(resp *esapi.Response, updat updates[id] = v.Source.ContentHash } - return body.ScrollID, nil + return body, nil } -func parseResponse(body io.ReadCloser, logger *logp.Logger) (esSearchSourcemapResponse, error) { +func parseResponse(body io.ReadCloser, logger *logp.Logger) (*esSearchSourcemapResponse, error) { b, err := io.ReadAll(body) if err != nil { - return esSearchSourcemapResponse{}, err + return nil, err } var esSourcemapResponse esSearchSourcemapResponse if err := json.Unmarshal(b, &esSourcemapResponse); err != nil { - return esSourcemapResponse, err + return nil, err } hits := esSourcemapResponse.Hits.Total.Value if hits == 0 || len(esSourcemapResponse.Hits.Hits) == 0 { - return esSourcemapResponse, nil + return &esSourcemapResponse, nil } - return esSourcemapResponse, nil + return &esSourcemapResponse, nil } -func (s *MetadataCachingFetcher) scrollsearch(ctx context.Context, scrollID string, updates map[Identifier]string) (string, error) { +func (s *MetadataCachingFetcher) scrollsearch(ctx context.Context, scrollID string, updates map[Identifier]string) (*esSearchSourcemapResponse, error) { resp, err := s.runScrollSearchQuery(ctx, scrollID) if err != nil { - return "", errors.Wrap(err, errMsgESFailure) + return nil, errors.Wrap(err, errMsgESFailure) } defer resp.Body.Close() From f2588155c60242f0eddbb061812f15ce071ede13 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Wed, 25 Jan 2023 18:06:46 +0100 Subject: [PATCH 072/123] docs: fix unicode chars --- internal/sourcemap/metadata.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/sourcemap/metadata.go b/internal/sourcemap/metadata.go index c9dff8bd844..8d791199ea9 100644 --- a/internal/sourcemap/metadata.go +++ b/internal/sourcemap/metadata.go @@ -276,7 +276,7 @@ func (s *MetadataCachingFetcher) sync(ctx context.Context) error { // From the docs: The initial search request and each subsequent scroll // request each return a _scroll_id. While the _scroll_id may change between - // requests, it doesn’t always change — in any case, only the most recently + // requests, it doesn't always change - in any case, only the most recently // received _scroll_id should be used. if result.ScrollID != "" { scrollID = result.ScrollID From 0c34f3c328b1eb52d33efc26e2daf3446d1efcb5 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Wed, 25 Jan 2023 18:10:16 +0100 Subject: [PATCH 073/123] fix: close invalidation channel --- internal/sourcemap/metadata.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/sourcemap/metadata.go b/internal/sourcemap/metadata.go index 8d791199ea9..d84d8ae0348 100644 --- a/internal/sourcemap/metadata.go +++ b/internal/sourcemap/metadata.go @@ -248,6 +248,8 @@ func (s *MetadataCachingFetcher) StartBackgroundSync(parent context.Context) { cleanup() case <-parent.Done(): s.logger.Info("update routine done") + // close invalidation channel + close(s.invalidationChan) return } } From 444c55bee913ef83c64092863c2ffc1454c8944d Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Wed, 25 Jan 2023 18:12:18 +0100 Subject: [PATCH 074/123] refactor: add missing json struct tag to essourcemapresponse fields --- internal/sourcemap/elasticsearch.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/sourcemap/elasticsearch.go b/internal/sourcemap/elasticsearch.go index b57a13f8557..f4991a7eba7 100644 --- a/internal/sourcemap/elasticsearch.go +++ b/internal/sourcemap/elasticsearch.go @@ -55,8 +55,8 @@ type esFetcher struct { type esSourcemapResponse struct { Hits struct { Total struct { - Value int - } + Value int `json:"value"` + } `json:"total"` Hits []struct { Source struct { Service struct { From b7389a65e6660c4404b82b778bb3aca2d95f90ed Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Wed, 25 Jan 2023 18:14:17 +0100 Subject: [PATCH 075/123] refactor: remove unused fleetcfg parameter --- internal/beater/beater.go | 3 +-- internal/beater/beater_test.go | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/internal/beater/beater.go b/internal/beater/beater.go index 038599d7a94..3824cce2cbb 100644 --- a/internal/beater/beater.go +++ b/internal/beater/beater.go @@ -326,7 +326,7 @@ func (s *Runner) Run(ctx context.Context) error { var sourcemapFetcher sourcemap.Fetcher if s.config.RumConfig.Enabled && s.config.RumConfig.SourceMapping.Enabled { fetcher, cleanup, err := newSourcemapFetcher( - s.config.RumConfig.SourceMapping, s.fleetConfig, + s.config.RumConfig.SourceMapping, kibanaClient, newElasticsearchClient, ) if err != nil { @@ -810,7 +810,6 @@ func (s *Runner) newLibbeatFinalBatchProcessor( func newSourcemapFetcher( cfg config.SourceMapping, - fleetCfg *config.Fleet, kibanaClient *kibana.Client, newElasticsearchClient func(*elasticsearch.Config) (*elasticsearch.Client, error), ) (sourcemap.Fetcher, context.CancelFunc, error) { diff --git a/internal/beater/beater_test.go b/internal/beater/beater_test.go index d1a1130ee7e..576b5618ab5 100644 --- a/internal/beater/beater_test.go +++ b/internal/beater/beater_test.go @@ -58,7 +58,7 @@ func TestSourcemapIndexPattern(t *testing.T) { } fetcher, cleanup, err := newSourcemapFetcher( - cfg.RumConfig.SourceMapping, nil, + cfg.RumConfig.SourceMapping, nil, elasticsearch.NewClient, ) require.NoError(t, err) @@ -97,7 +97,7 @@ func TestStoreUsesRUMElasticsearchConfig(t *testing.T) { cfg.RumConfig.SourceMapping.ESConfig.Hosts = []string{ts.URL} fetcher, cleanup, err := newSourcemapFetcher( - cfg.RumConfig.SourceMapping, nil, + cfg.RumConfig.SourceMapping, nil, elasticsearch.NewClient, ) require.NoError(t, err) From 4610931dcd2d127011a8d1b0b5ff533944371907 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Wed, 25 Jan 2023 18:16:59 +0100 Subject: [PATCH 076/123] refactor: rename sourcemapfetcher cancel func for clarity --- internal/beater/beater.go | 4 ++-- internal/beater/beater_test.go | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/beater/beater.go b/internal/beater/beater.go index 3824cce2cbb..9478e816234 100644 --- a/internal/beater/beater.go +++ b/internal/beater/beater.go @@ -325,14 +325,14 @@ func (s *Runner) Run(ctx context.Context) error { var sourcemapFetcher sourcemap.Fetcher if s.config.RumConfig.Enabled && s.config.RumConfig.SourceMapping.Enabled { - fetcher, cleanup, err := newSourcemapFetcher( + fetcher, cancel, err := newSourcemapFetcher( s.config.RumConfig.SourceMapping, kibanaClient, newElasticsearchClient, ) if err != nil { return err } - defer cleanup() + defer cancel() sourcemapFetcher = fetcher } diff --git a/internal/beater/beater_test.go b/internal/beater/beater_test.go index 576b5618ab5..cd6951a2874 100644 --- a/internal/beater/beater_test.go +++ b/internal/beater/beater_test.go @@ -57,12 +57,12 @@ func TestSourcemapIndexPattern(t *testing.T) { cfg.RumConfig.SourceMapping.IndexPattern = indexPattern } - fetcher, cleanup, err := newSourcemapFetcher( + fetcher, cancel, err := newSourcemapFetcher( cfg.RumConfig.SourceMapping, nil, elasticsearch.NewClient, ) require.NoError(t, err) - defer cleanup() + defer cancel() fetcher.Fetch(context.Background(), "name", "version", "path") require.Len(t, requestPaths, 1) @@ -96,12 +96,12 @@ func TestStoreUsesRUMElasticsearchConfig(t *testing.T) { cfg.RumConfig.SourceMapping.ESConfig = elasticsearch.DefaultConfig() cfg.RumConfig.SourceMapping.ESConfig.Hosts = []string{ts.URL} - fetcher, cleanup, err := newSourcemapFetcher( + fetcher, cancel, err := newSourcemapFetcher( cfg.RumConfig.SourceMapping, nil, elasticsearch.NewClient, ) require.NoError(t, err) - defer cleanup() + defer cancel() // Check that the provided rum elasticsearch config was used and // Fetch() goes to the test server. _, err = fetcher.Fetch(context.Background(), "app", "1.0", "/bundle/path") From ed99eb5a3c8f8dea00306c75414262b4caefc9b9 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Thu, 26 Jan 2023 02:46:17 +0100 Subject: [PATCH 077/123] feat: remove sourcemapping metadata option --- internal/beater/config/config_test.go | 17 ----------------- internal/beater/config/rum.go | 8 -------- 2 files changed, 25 deletions(-) diff --git a/internal/beater/config/config_test.go b/internal/beater/config/config_test.go index ff0b3fb73dd..3427fc54219 100644 --- a/internal/beater/config/config_test.go +++ b/internal/beater/config/config_test.go @@ -281,7 +281,6 @@ func TestUnpackConfig(t *testing.T) { CompressionLevel: 5, Backoff: elasticsearch.DefaultBackoffConfig, }, - Metadata: []SourceMapMetadata{}, Timeout: 2 * time.Second, esConfigured: true, }, @@ -373,14 +372,6 @@ func TestUnpackConfig(t *testing.T) { "rum": map[string]interface{}{ "enabled": true, "source_mapping": map[string]interface{}{ - "metadata": []map[string]string{ - { - "service.name": "opbeans-rum", - "service.version": "1.2.3", - "bundle.filepath": "/test/e2e/general-usecase/bundle.js.map", - "sourcemap.url": "http://somewhere.com/bundle.js.map", - }, - }, "cache": map[string]interface{}{ "expiration": 7, }, @@ -455,14 +446,6 @@ func TestUnpackConfig(t *testing.T) { }, IndexPattern: ".apm-source-map", ESConfig: elasticsearch.DefaultConfig(), - Metadata: []SourceMapMetadata{ - { - ServiceName: "opbeans-rum", - ServiceVersion: "1.2.3", - BundleFilepath: "/test/e2e/general-usecase/bundle.js.map", - SourceMapURL: "http://somewhere.com/bundle.js.map", - }, - }, Timeout: 5 * time.Second, }, LibraryPattern: "rum", diff --git a/internal/beater/config/rum.go b/internal/beater/config/rum.go index b7402e71344..6b32111d8fd 100644 --- a/internal/beater/config/rum.go +++ b/internal/beater/config/rum.go @@ -56,7 +56,6 @@ type SourceMapping struct { Enabled bool `config:"enabled"` IndexPattern string `config:"index_pattern"` ESConfig *elasticsearch.Config `config:"elasticsearch"` - Metadata []SourceMapMetadata `config:"metadata"` Timeout time.Duration `config:"timeout" validate:"positive"` esConfigured bool es *config.C @@ -74,12 +73,6 @@ func (c *RumConfig) setup(log *logp.Logger, outputESCfg *config.C) error { return errors.Wrapf(err, "Invalid regex for `exclude_from_grouping`: ") } - if len(c.SourceMapping.Metadata) > 0 { - // We don't have the fleet fetcher anymore. - // Ignore metadata and setup the elasticsearch config. - log.Warn("Ignoring sourcemap metadata") - } - if outputESCfg == nil { log.Info("Unable to determine sourcemap storage, sourcemaps will not be applied") return nil @@ -126,7 +119,6 @@ func defaultSourcemapping() SourceMapping { Cache: Cache{Expiration: defaultSourcemapCacheExpiration}, IndexPattern: defaultSourcemapIndexPattern, ESConfig: elasticsearch.DefaultConfig(), - Metadata: []SourceMapMetadata{}, Timeout: defaultSourcemapTimeout, } } From 29d329d1c196bc5eacc38c04cf3a5955b4034828 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Thu, 26 Jan 2023 02:51:30 +0100 Subject: [PATCH 078/123] lint: fix linter issues --- internal/beater/config/config_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/beater/config/config_test.go b/internal/beater/config/config_test.go index 3427fc54219..f781bdc06ee 100644 --- a/internal/beater/config/config_test.go +++ b/internal/beater/config/config_test.go @@ -446,7 +446,7 @@ func TestUnpackConfig(t *testing.T) { }, IndexPattern: ".apm-source-map", ESConfig: elasticsearch.DefaultConfig(), - Timeout: 5 * time.Second, + Timeout: 5 * time.Second, }, LibraryPattern: "rum", ExcludeFromGrouping: "^/webpack", From cd6ae96771560628645164b7339ea7d4918e5e70 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Thu, 26 Jan 2023 11:17:23 +0100 Subject: [PATCH 079/123] test: improve TestStoreUsesRUMElasticsearchConfig to make sure the correct sourcemap is returned --- internal/beater/beater_test.go | 40 ++++++++++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/internal/beater/beater_test.go b/internal/beater/beater_test.go index cd6951a2874..95ec20e6aa7 100644 --- a/internal/beater/beater_test.go +++ b/internal/beater/beater_test.go @@ -18,7 +18,11 @@ package beater import ( + "bytes" + "compress/zlib" "context" + "encoding/base64" + "encoding/json" "fmt" "net/http" "net/http/httptest" @@ -86,7 +90,13 @@ func TestStoreUsesRUMElasticsearchConfig(t *testing.T) { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { called = true w.Header().Set("X-Elastic-Product", "Elasticsearch") - w.Write(validSourcemap) + + m := sourcemapHit(validSourcemap) + + b, err := json.Marshal(m) + require.NoError(t, err) + + w.Write(b) })) defer ts.Close() @@ -104,12 +114,38 @@ func TestStoreUsesRUMElasticsearchConfig(t *testing.T) { defer cancel() // Check that the provided rum elasticsearch config was used and // Fetch() goes to the test server. - _, err = fetcher.Fetch(context.Background(), "app", "1.0", "/bundle/path") + c, err := fetcher.Fetch(context.Background(), "app", "1.0", "/bundle/path") require.NoError(t, err) + require.NotNil(t, c) assert.True(t, called) } +func sourcemapHit(sourcemap []byte) map[string]interface{} { + b := &bytes.Buffer{} + + z := zlib.NewWriter(b) + z.Write(sourcemap) + z.Close() + + s := base64.StdEncoding.EncodeToString(b.Bytes()) + + hits := map[string]interface{}{ + "_source": map[string]interface{}{ + "content": s, + }, + } + + resultHits := map[string]interface{}{ + "total": map[string]interface{}{ + "value": 1, + }, + } + resultHits["hits"] = []map[string]interface{}{hits} + result := map[string]interface{}{"hits": resultHits} + return result +} + func TestQueryClusterUUIDRegistriesExist(t *testing.T) { stateRegistry := monitoring.GetNamespace("state").GetRegistry() stateRegistry.Clear() From e2b24a0a3b3a21773158062d27d70fc0a08039ba Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Thu, 26 Jan 2023 11:34:18 +0100 Subject: [PATCH 080/123] feat: return an error if sourcemap is missing --- internal/sourcemap/metadata.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/sourcemap/metadata.go b/internal/sourcemap/metadata.go index d84d8ae0348..ea5a38925cc 100644 --- a/internal/sourcemap/metadata.go +++ b/internal/sourcemap/metadata.go @@ -130,7 +130,7 @@ func (s *MetadataCachingFetcher) Fetch(ctx context.Context, name, version, path } } - return nil, nil + return nil, fmt.Errorf("unable to find sourcemap.url for service.name=%s service.version=%s bundle.path=%s", name, version, path) } func (s *MetadataCachingFetcher) hasID(key Identifier) bool { @@ -155,7 +155,7 @@ func (s *MetadataCachingFetcher) fetch(ctx context.Context, key *Identifier) (*s // log a message if the sourcemap is present in the cache but the backend fetcher did not // find it. if err == nil && c == nil { - s.logger.Debugf("Backend fetcher failed to retrieve sourcemap: %v", key) + return nil, fmt.Errorf("unable to find sourcemap for service.name=%s service.version=%s bundle.path=%s", key.name, key.version, key.path) } return c, err From 0c3eef47aefafe1f1b8c28256f47e06a0029379c Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Thu, 26 Jan 2023 11:35:05 +0100 Subject: [PATCH 081/123] test: update approvals document --- systemtest/approvals/TestNoMatchingSourcemap.approved.json | 6 ++++++ .../approvals/TestRUMRoutingIntegration.approved.json | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/systemtest/approvals/TestNoMatchingSourcemap.approved.json b/systemtest/approvals/TestNoMatchingSourcemap.approved.json index b975d6c1b5f..9e0217a1de0 100644 --- a/systemtest/approvals/TestNoMatchingSourcemap.approved.json +++ b/systemtest/approvals/TestNoMatchingSourcemap.approved.json @@ -51,6 +51,9 @@ "line": { "column": 18, "number": 1 + }, + "sourcemap": { + "error": "unable to find sourcemap.url for service.name=apm-agent-js service.version=1.0.0 bundle.path=http://subdomain1.localhost:8000/test/e2e/general-usecase/bundle.js.map" } }, { @@ -62,6 +65,9 @@ "line": { "column": 18, "number": 1 + }, + "sourcemap": { + "error": "unable to find sourcemap.url for service.name=apm-agent-js service.version=1.0.0 bundle.path=http://subdomain2.localhost:8000/test/e2e/general-usecase/bundle.js.map" } } ], diff --git a/systemtest/approvals/TestRUMRoutingIntegration.approved.json b/systemtest/approvals/TestRUMRoutingIntegration.approved.json index 1ec96e24e9d..791dc75026e 100644 --- a/systemtest/approvals/TestRUMRoutingIntegration.approved.json +++ b/systemtest/approvals/TestRUMRoutingIntegration.approved.json @@ -437,6 +437,9 @@ "line": { "column": 9, "number": 7662 + }, + "sourcemap": { + "error": "unable to find sourcemap.url for service.name=apm-a-rum-test-e2e-general-usecase service.version=0.0.1 bundle.path=http://localhost:8000/test/e2e/general-usecase/app.e2e-bundle.min.js" } }, { @@ -447,6 +450,9 @@ "line": { "column": 3, "number": 7666 + }, + "sourcemap": { + "error": "unable to find sourcemap.url for service.name=apm-a-rum-test-e2e-general-usecase service.version=0.0.1 bundle.path=http://localhost:8000/test/e2e/general-usecase/app.e2e-bundle.min.js" } } ], From 1d4d26191d803f9fee9ae514aab2feb0e3fe11a1 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Fri, 27 Jan 2023 03:17:56 +0100 Subject: [PATCH 082/123] refactor: rework sync logic and decouple fetchers rename caching fetchers for clarity move sync logic to a separate file --- internal/beater/beater.go | 17 +- .../sourcemap/{caching.go => body_caching.go} | 29 +- .../{caching_test.go => body_caching_test.go} | 10 +- internal/sourcemap/metadata.go | 399 ------------------ internal/sourcemap/metadata_caching.go | 218 ++++++++++ internal/sourcemap/processor_test.go | 3 +- internal/sourcemap/sync.go | 242 +++++++++++ 7 files changed, 493 insertions(+), 425 deletions(-) rename internal/sourcemap/{caching.go => body_caching.go} (75%) rename internal/sourcemap/{caching_test.go => body_caching_test.go} (96%) delete mode 100644 internal/sourcemap/metadata.go create mode 100644 internal/sourcemap/metadata_caching.go create mode 100644 internal/sourcemap/sync.go diff --git a/internal/beater/beater.go b/internal/beater/beater.go index 9478e816234..7fb3135ee81 100644 --- a/internal/beater/beater.go +++ b/internal/beater/beater.go @@ -821,20 +821,21 @@ func newSourcemapFetcher( // For standalone, we query both Kibana and Elasticsearch for backwards compatibility. var chained sourcemap.ChainedFetcher - size := 128 - invalidationChan := make(chan sourcemap.Identifier, size) - index := strings.ReplaceAll(cfg.IndexPattern, "%{[observer.version]}", version.Version) + // start background sync job + sync, updateChan := sourcemap.NewSyncWorker(esClient, index) + ctx, cancel := context.WithCancel(context.Background()) + sync.Run(ctx) + esFetcher := sourcemap.NewElasticsearchFetcher(esClient, index) - cachingFetcher, err := sourcemap.NewCachingFetcher(esFetcher, invalidationChan, size) + size := 128 + cachingFetcher, invalidateChan, err := sourcemap.NewBodyCachingFetcher(esFetcher, size) if err != nil { + cancel() return nil, nil, err } - metadataCachingFetcher := sourcemap.NewMetadataCachingFetcher(esClient, cachingFetcher, index, invalidationChan) - - ctx, cancel := context.WithCancel(context.Background()) - metadataCachingFetcher.StartBackgroundSync(ctx) + metadataCachingFetcher := sourcemap.NewMetadataCachingFetcher(cachingFetcher, updateChan, invalidateChan) chained = append(chained, metadataCachingFetcher) diff --git a/internal/sourcemap/caching.go b/internal/sourcemap/body_caching.go similarity index 75% rename from internal/sourcemap/caching.go rename to internal/sourcemap/body_caching.go index 89d06af6ab9..1af7420d6e9 100644 --- a/internal/sourcemap/caching.go +++ b/internal/sourcemap/body_caching.go @@ -34,19 +34,18 @@ var ( errMsgFailure = "failure querying" ) -// CachingFetcher wraps a Fetcher, caching source maps in memory and fetching from the wrapped Fetcher on cache misses. -type CachingFetcher struct { +// BodyCachingFetcher wraps a Fetcher, caching source maps in memory and fetching from the wrapped Fetcher on cache misses. +type BodyCachingFetcher struct { cache *lru.Cache backend Fetcher logger *logp.Logger } -// NewCachingFetcher returns a CachingFetcher that wraps backend, caching results for the configured cacheExpiration. -func NewCachingFetcher( +// NewBodyCachingFetcher returns a CachingFetcher that wraps backend, caching results for the configured cacheExpiration. +func NewBodyCachingFetcher( backend Fetcher, - invalidationChan <-chan Identifier, cacheSize int, -) (*CachingFetcher, error) { +) (*BodyCachingFetcher, chan<- []Identifier, error) { logger := logp.NewLogger(logs.Sourcemap) lruCache, err := lru.NewWithEvict(cacheSize, func(key, value interface{}) { @@ -57,24 +56,28 @@ func NewCachingFetcher( }) if err != nil { - return nil, fmt.Errorf("failed to create lru cache for caching fetcher: %w", err) + return nil, nil, fmt.Errorf("failed to create lru cache for caching fetcher: %w", err) } + ch := make(chan []Identifier) + go func() { - for identifier := range invalidationChan { - lruCache.Remove(identifier) + for arr := range ch { + for _, id := range arr { + lruCache.Remove(id) + } } }() - return &CachingFetcher{ + return &BodyCachingFetcher{ cache: lruCache, backend: backend, logger: logger, - }, nil + }, ch, nil } // Fetch fetches a source map from the cache or wrapped backend. -func (s *CachingFetcher) Fetch(ctx context.Context, name, version, path string) (*sourcemap.Consumer, error) { +func (s *BodyCachingFetcher) Fetch(ctx context.Context, name, version, path string) (*sourcemap.Consumer, error) { key := Identifier{ name: name, version: version, @@ -99,7 +102,7 @@ func (s *CachingFetcher) Fetch(ctx context.Context, name, version, path string) return consumer, nil } -func (s *CachingFetcher) add(key Identifier, consumer *sourcemap.Consumer) { +func (s *BodyCachingFetcher) add(key Identifier, consumer *sourcemap.Consumer) { s.cache.Add(key, consumer) if !s.logger.IsDebug() { return diff --git a/internal/sourcemap/caching_test.go b/internal/sourcemap/body_caching_test.go similarity index 96% rename from internal/sourcemap/caching_test.go rename to internal/sourcemap/body_caching_test.go index efd37865a54..194d49b863e 100644 --- a/internal/sourcemap/caching_test.go +++ b/internal/sourcemap/body_caching_test.go @@ -40,11 +40,12 @@ var unsupportedVersionSourcemap = `{ }` func Test_NewCachingFetcher(t *testing.T) { - _, err := NewCachingFetcher(nil, nil, -1) + _, _, err := NewBodyCachingFetcher(nil, -1) require.Error(t, err) - f, err := NewCachingFetcher(nil, nil, 100) + f, ch, err := NewBodyCachingFetcher(nil, 100) require.NoError(t, err) + close(ch) assert.NotNil(t, f.cache) } @@ -159,9 +160,10 @@ func TestStore_Fetch(t *testing.T) { }) } -func testCachingFetcher(t *testing.T, client *elasticsearch.Client) *CachingFetcher { +func testCachingFetcher(t *testing.T, client *elasticsearch.Client) *BodyCachingFetcher { esFetcher := NewElasticsearchFetcher(client, "apm-*sourcemap*") - cachingFetcher, err := NewCachingFetcher(esFetcher, nil, 100) + cachingFetcher, ch, err := NewBodyCachingFetcher(esFetcher, 100) require.NoError(t, err) + close(ch) return cachingFetcher } diff --git a/internal/sourcemap/metadata.go b/internal/sourcemap/metadata.go deleted file mode 100644 index ea5a38925cc..00000000000 --- a/internal/sourcemap/metadata.go +++ /dev/null @@ -1,399 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package sourcemap - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "net/url" - "sync" - "time" - - "github.com/go-sourcemap/sourcemap" - "github.com/pkg/errors" - - "github.com/elastic/apm-server/internal/elasticsearch" - "github.com/elastic/apm-server/internal/logs" - "github.com/elastic/elastic-agent-libs/logp" - "github.com/elastic/go-elasticsearch/v8/esapi" -) - -type MetadataCachingFetcher struct { - esClient *elasticsearch.Client - set map[Identifier]string - alias map[Identifier]*Identifier - mu sync.RWMutex - backend Fetcher - logger *logp.Logger - index string - init chan struct{} - invalidationChan chan<- Identifier -} - -func NewMetadataCachingFetcher( - c *elasticsearch.Client, - backend Fetcher, - index string, - invalidationChan chan<- Identifier, -) *MetadataCachingFetcher { - return &MetadataCachingFetcher{ - esClient: c, - index: index, - set: make(map[Identifier]string), - alias: make(map[Identifier]*Identifier), - backend: backend, - logger: logp.NewLogger(logs.Sourcemap), - init: make(chan struct{}), - invalidationChan: invalidationChan, - } -} - -func (s *MetadataCachingFetcher) Fetch(ctx context.Context, name, version, path string) (*sourcemap.Consumer, error) { - original := Identifier{name: name, version: version, path: path} - - select { - case <-s.init: - // the mutex is shared by the update goroutine, we need to release it - // as soon as possible to avoid blocking updates. - if s.hasID(original) { - // Only fetch from ES if the sourcemap id exists - return s.fetch(ctx, &original) - } - default: - s.logger.Debugf("Metadata cache not populated. Falling back to backend fetcher for id: %s, %s, %s", name, version, path) - // init is in progress, ignore the metadata cache and fetch the sourcemap directly - // return if we get a valid sourcemap or an error - if c, err := s.backend.Fetch(ctx, original.name, original.version, original.path); c != nil || err != nil { - return c, err - } - - s.logger.Debug("Blocking until init is completed") - - // Aliases are only available after init is completed. - <-s.init - } - - // path is missing from the metadata cache (and ES). - // Is it an alias ? - // Try to retrieve the sourcemap from the alias map - // Only fetch from ES if the sourcemap alias exists - if id, found := s.getAlias(original); found { - return s.fetch(ctx, id) - } - - if urlPath, err := url.Parse(path); err == nil { - // The sourcemap coule be stored in ES with a relative - // bundle filepath but the request came in with an - // absolute path - original.path = urlPath.Path - if urlPath.Path != path && s.hasID(original) { - return s.fetch(ctx, &original) - } - - // The sourcemap could be stored on ES under a certain host - // but a request came in from a different host. - // Look for an alias to the url path to retrieve the correct - // host and fetch the sourcemap - if id, found := s.getAlias(original); found { - return s.fetch(ctx, id) - } - - // Clean the url and try again if the result is different from - // the original bundle filepath - urlPath.RawQuery = "" - urlPath.Fragment = "" - urlPath = urlPath.JoinPath() - cleanPath := urlPath.String() - - if cleanPath != path { - s.logger.Debugf("original filepath %s converted to %s", path, cleanPath) - return s.Fetch(ctx, name, version, cleanPath) - } - } - - return nil, fmt.Errorf("unable to find sourcemap.url for service.name=%s service.version=%s bundle.path=%s", name, version, path) -} - -func (s *MetadataCachingFetcher) hasID(key Identifier) bool { - s.mu.RLock() - defer s.mu.RUnlock() - - _, ok := s.set[key] - return ok -} - -func (s *MetadataCachingFetcher) getAlias(key Identifier) (*Identifier, bool) { - s.mu.RLock() - defer s.mu.RUnlock() - - i, ok := s.alias[key] - return i, ok -} - -func (s *MetadataCachingFetcher) fetch(ctx context.Context, key *Identifier) (*sourcemap.Consumer, error) { - c, err := s.backend.Fetch(ctx, key.name, key.version, key.path) - - // log a message if the sourcemap is present in the cache but the backend fetcher did not - // find it. - if err == nil && c == nil { - return nil, fmt.Errorf("unable to find sourcemap for service.name=%s service.version=%s bundle.path=%s", key.name, key.version, key.path) - } - - return c, err -} - -func (s *MetadataCachingFetcher) update(ctx context.Context, updates map[Identifier]string) { - s.mu.Lock() - defer s.mu.Unlock() - - for id, contentHash := range s.set { - if updatedHash, ok := updates[id]; ok { - // already in the cache, remove from the updates. - delete(updates, id) - - // content hash changed, invalidate the sourcemap cache - if contentHash != updatedHash { - s.logger.Debugf("Hash changed: %s -> %s: invalidating %v", contentHash, updatedHash, id) - select { - case s.invalidationChan <- id: - case <-ctx.Done(): - s.logger.Errorf("ctx finished while invalidating id: %v", ctx.Err()) - return - } - - } - } else { - // the sourcemap no longer exists in ES. - // invalidate the sourcemap cache. - select { - case s.invalidationChan <- id: - case <-ctx.Done(): - s.logger.Errorf("ctx finished while invaliding id: %v", ctx.Err()) - return - } - - // remove from metadata cache - delete(s.set, id) - // remove alias - for _, k := range GetAliases(id.name, id.version, id.path) { - delete(s.alias, k) - } - } - } - // add new sourcemaps to the metadata cache. - for id, contentHash := range updates { - s.set[id] = contentHash - s.logger.Debugf("Added metadata id %v", id) - // store aliases with a pointer to the original id. - // The id is then passed over to the backend fetcher - // to minimize the size of the lru cache and - // and increase cache hits. - for _, k := range GetAliases(id.name, id.version, id.path) { - s.logger.Debugf("Added metadata alias %v -> %v", k, id) - s.alias[k] = &id - } - } - - s.logger.Debugf("Metadata cache now has %d entries.", len(s.set)) -} - -func (s *MetadataCachingFetcher) StartBackgroundSync(parent context.Context) { - go func() { - // First run, populate cache - ctx, cleanup := context.WithTimeout(parent, 10*time.Second) - defer cleanup() - - defer close(s.init) - - if err := s.sync(ctx); err != nil { - s.logger.Errorf("failed to fetch sourcemaps metadata: %v", err) - } - - s.logger.Info("init routine completed") - }() - - go func() { - // TODO make this a config option ? - t := time.NewTicker(30 * time.Second) - defer t.Stop() - - for { - select { - case <-t.C: - ctx, cleanup := context.WithTimeout(parent, 10*time.Second) - - if err := s.sync(ctx); err != nil { - s.logger.Errorf("failed to sync sourcemaps metadata: %v", err) - } - - cleanup() - case <-parent.Done(): - s.logger.Info("update routine done") - // close invalidation channel - close(s.invalidationChan) - return - } - } - }() -} - -func (s *MetadataCachingFetcher) sync(ctx context.Context) error { - updates := make(map[Identifier]string) - - result, err := s.initialSearch(ctx, updates) - if err != nil { - return err - } - - scrollID := result.ScrollID - - if scrollID == "" { - return nil - } - - for { - result, err = s.scrollsearch(ctx, scrollID, updates) - if err != nil { - return err - } - - // From the docs: The initial search request and each subsequent scroll - // request each return a _scroll_id. While the _scroll_id may change between - // requests, it doesn't always change - in any case, only the most recently - // received _scroll_id should be used. - if result.ScrollID != "" { - scrollID = result.ScrollID - } - - // Stop if there are no new updates - if len(result.Hits.Hits) == 0 { - break - } - } - - s.update(ctx, updates) - return nil -} - -func (s *MetadataCachingFetcher) initialSearch(ctx context.Context, updates map[Identifier]string) (*esSearchSourcemapResponse, error) { - resp, err := s.runSearchQuery(ctx) - if err != nil { - return nil, errors.Wrap(err, errMsgESFailure) - } - defer resp.Body.Close() - - return s.handleUpdateRequest(resp, updates) -} - -func (s *MetadataCachingFetcher) runSearchQuery(ctx context.Context) (*esapi.Response, error) { - var buf bytes.Buffer - if err := json.NewEncoder(&buf).Encode(queryMetadata()); err != nil { - return nil, err - } - - req := esapi.SearchRequest{ - Index: []string{s.index}, - Body: &buf, - TrackTotalHits: true, - Scroll: time.Minute, - } - return req.Do(ctx, s.esClient) -} - -func queryMetadata() map[string]interface{} { - return search( - sources([]string{"service.*", "file.path", "content_sha256"}), - ) -} - -type esSearchSourcemapResponse struct { - ScrollID string `json:"_scroll_id"` - esSourcemapResponse -} - -func (s *MetadataCachingFetcher) handleUpdateRequest(resp *esapi.Response, updates map[Identifier]string) (*esSearchSourcemapResponse, error) { - // handle error response - if resp.StatusCode >= http.StatusMultipleChoices { - if resp.StatusCode == http.StatusNotFound { - return nil, nil - } - b, err := io.ReadAll(resp.Body) - if err != nil { - return nil, errors.Wrap(err, errMsgParseSourcemap) - } - return nil, errors.New(fmt.Sprintf("%s %s", errMsgParseSourcemap, b)) - } - - // parse response - body, err := parseResponse(resp.Body, s.logger) - if err != nil { - return nil, err - } - - for _, v := range body.Hits.Hits { - id := Identifier{ - name: v.Source.Service.Name, - version: v.Source.Service.Version, - path: v.Source.File.BundleFilepath, - } - - updates[id] = v.Source.ContentHash - } - - return body, nil -} - -func parseResponse(body io.ReadCloser, logger *logp.Logger) (*esSearchSourcemapResponse, error) { - b, err := io.ReadAll(body) - if err != nil { - return nil, err - } - - var esSourcemapResponse esSearchSourcemapResponse - if err := json.Unmarshal(b, &esSourcemapResponse); err != nil { - return nil, err - } - hits := esSourcemapResponse.Hits.Total.Value - if hits == 0 || len(esSourcemapResponse.Hits.Hits) == 0 { - return &esSourcemapResponse, nil - } - - return &esSourcemapResponse, nil -} - -func (s *MetadataCachingFetcher) scrollsearch(ctx context.Context, scrollID string, updates map[Identifier]string) (*esSearchSourcemapResponse, error) { - resp, err := s.runScrollSearchQuery(ctx, scrollID) - if err != nil { - return nil, errors.Wrap(err, errMsgESFailure) - } - defer resp.Body.Close() - - return s.handleUpdateRequest(resp, updates) -} - -func (s *MetadataCachingFetcher) runScrollSearchQuery(ctx context.Context, id string) (*esapi.Response, error) { - req := esapi.ScrollRequest{ - ScrollID: id, - Scroll: time.Minute, - } - return req.Do(ctx, s.esClient) -} diff --git a/internal/sourcemap/metadata_caching.go b/internal/sourcemap/metadata_caching.go new file mode 100644 index 00000000000..34dd06f90c0 --- /dev/null +++ b/internal/sourcemap/metadata_caching.go @@ -0,0 +1,218 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package sourcemap + +import ( + "context" + "fmt" + "net/url" + "sync" + + "github.com/go-sourcemap/sourcemap" + + "github.com/elastic/apm-server/internal/logs" + "github.com/elastic/elastic-agent-libs/logp" +) + +type MetadataCachingFetcher struct { + set map[Identifier]string + alias map[Identifier]*Identifier + mu sync.RWMutex + backend Fetcher + logger *logp.Logger + init chan struct{} + updateChan <-chan map[Identifier]string + invalidateChan chan<- []Identifier +} + +func NewMetadataCachingFetcher(backend Fetcher, in <-chan map[Identifier]string, out chan<- []Identifier) *MetadataCachingFetcher { + s := &MetadataCachingFetcher{ + set: make(map[Identifier]string), + alias: make(map[Identifier]*Identifier), + backend: backend, + logger: logp.NewLogger(logs.Sourcemap), + init: make(chan struct{}), + updateChan: in, + invalidateChan: out, + } + + go s.handleUpdates() + + return s +} + +func (s *MetadataCachingFetcher) Fetch(ctx context.Context, name, version, path string) (*sourcemap.Consumer, error) { + original := Identifier{name: name, version: version, path: path} + + select { + case <-s.init: + // the mutex is shared by the update goroutine, we need to release it + // as soon as possible to avoid blocking updates. + if s.hasID(original) { + // Only fetch from ES if the sourcemap id exists + return s.fetch(ctx, &original) + } + default: + s.logger.Debugf("Metadata cache not populated. Falling back to backend fetcher for id: %s, %s, %s", name, version, path) + // init is in progress, ignore the metadata cache and fetch the sourcemap directly + // return if we get a valid sourcemap or an error + if c, err := s.backend.Fetch(ctx, original.name, original.version, original.path); c != nil || err != nil { + return c, err + } + + s.logger.Debug("Blocking until init is completed") + + // Aliases are only available after init is completed. + select { + case <-s.init: + case <-ctx.Done(): + return nil, ctx.Err() + } + } + + // path is missing from the metadata cache (and ES). + // Is it an alias ? + // Try to retrieve the sourcemap from the alias map + // Only fetch from ES if the sourcemap alias exists + if id, found := s.getAlias(original); found { + return s.fetch(ctx, id) + } + + if urlPath, err := url.Parse(path); err == nil { + // The sourcemap coule be stored in ES with a relative + // bundle filepath but the request came in with an + // absolute path + original.path = urlPath.Path + if urlPath.Path != path && s.hasID(original) { + return s.fetch(ctx, &original) + } + + // The sourcemap could be stored on ES under a certain host + // but a request came in from a different host. + // Look for an alias to the url path to retrieve the correct + // host and fetch the sourcemap + if id, found := s.getAlias(original); found { + return s.fetch(ctx, id) + } + + // Clean the url and try again if the result is different from + // the original bundle filepath + urlPath.RawQuery = "" + urlPath.Fragment = "" + urlPath = urlPath.JoinPath() + cleanPath := urlPath.String() + + if cleanPath != path { + s.logger.Debugf("original filepath %s converted to %s", path, cleanPath) + return s.Fetch(ctx, name, version, cleanPath) + } + } + + return nil, fmt.Errorf("unable to find sourcemap.url for service.name=%s service.version=%s bundle.path=%s", name, version, path) +} + +func (s *MetadataCachingFetcher) hasID(key Identifier) bool { + s.mu.RLock() + defer s.mu.RUnlock() + + _, ok := s.set[key] + return ok +} + +func (s *MetadataCachingFetcher) getAlias(key Identifier) (*Identifier, bool) { + s.mu.RLock() + defer s.mu.RUnlock() + + i, ok := s.alias[key] + return i, ok +} + +func (s *MetadataCachingFetcher) fetch(ctx context.Context, key *Identifier) (*sourcemap.Consumer, error) { + c, err := s.backend.Fetch(ctx, key.name, key.version, key.path) + + // log a message if the sourcemap is present in the cache but the backend fetcher did not + // find it. + if err == nil && c == nil { + return nil, fmt.Errorf("unable to find sourcemap for service.name=%s service.version=%s bundle.path=%s", key.name, key.version, key.path) + } + + return c, err +} + +func (s *MetadataCachingFetcher) handleUpdates() { + // run once for init + s.update(<-s.updateChan) + close(s.init) + + // wait for updates + for updates := range s.updateChan { + s.update(updates) + } + close(s.invalidateChan) +} + +func (s *MetadataCachingFetcher) update(updates map[Identifier]string) { + s.mu.Lock() + defer s.mu.Unlock() + + var invalidation []Identifier + + for id, contentHash := range s.set { + if updatedHash, ok := updates[id]; ok { + // already in the cache, remove from the updates. + delete(updates, id) + + // content hash changed, invalidate the sourcemap cache + if contentHash != updatedHash { + s.logger.Debugf("Hash changed: %s -> %s: invalidating %v", contentHash, updatedHash, id) + invalidation = append(invalidation, id) + } + } else { + // the sourcemap no longer exists in ES. + // invalidate the sourcemap cache. + invalidation = append(invalidation, id) + + // the sourcemap no longer exists in ES. + // remove from metadata cache + delete(s.set, id) + + // remove alias + for _, k := range GetAliases(id.name, id.version, id.path) { + delete(s.alias, k) + } + } + } + + s.invalidateChan <- invalidation + + // add new sourcemaps to the metadata cache. + for id, contentHash := range updates { + s.set[id] = contentHash + s.logger.Debugf("Added metadata id %v", id) + // store aliases with a pointer to the original id. + // The id is then passed over to the backend fetcher + // to minimize the size of the lru cache and + // and increase cache hits. + for _, k := range GetAliases(id.name, id.version, id.path) { + s.logger.Debugf("Added metadata alias %v -> %v", k, id) + s.alias[k] = &id + } + } + + s.logger.Debugf("Metadata cache now has %d entries.", len(s.set)) +} diff --git a/internal/sourcemap/processor_test.go b/internal/sourcemap/processor_test.go index a31ce7197d3..436f4e778b4 100644 --- a/internal/sourcemap/processor_test.go +++ b/internal/sourcemap/processor_test.go @@ -37,8 +37,9 @@ func TestBatchProcessor(t *testing.T) { sourcemapSearchResponseBody(1, []map[string]interface{}{sourcemapHit(string(validSourcemap))}), ) esFetcher := NewElasticsearchFetcher(client, "index") - fetcher, err := NewCachingFetcher(esFetcher, nil, 100) + fetcher, ch, err := NewBodyCachingFetcher(esFetcher, 100) require.NoError(t, err) + close(ch) originalLinenoWithFilename := 1 originalColnoWithFilename := 7 diff --git a/internal/sourcemap/sync.go b/internal/sourcemap/sync.go new file mode 100644 index 00000000000..a75601c4172 --- /dev/null +++ b/internal/sourcemap/sync.go @@ -0,0 +1,242 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package sourcemap + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "time" + + "github.com/pkg/errors" + + "github.com/elastic/apm-server/internal/elasticsearch" + "github.com/elastic/apm-server/internal/logs" + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/go-elasticsearch/v8/esapi" +) + +const ( + syncTimeout = 10 * time.Second +) + +type SyncWorker struct { + esClient *elasticsearch.Client + logger *logp.Logger + index string + updateChan chan<- map[Identifier]string +} + +func NewSyncWorker(esClient *elasticsearch.Client, index string) (*SyncWorker, <-chan map[Identifier]string) { + ch := make(chan map[Identifier]string) + return &SyncWorker{ + esClient: esClient, + index: index, + logger: logp.NewLogger(logs.Sourcemap), + updateChan: ch, + }, ch +} + +func (s *SyncWorker) Run(parent context.Context) { + go func() { + // First run, populate cache + ctx, cleanup := context.WithTimeout(parent, syncTimeout) + defer cleanup() + + if err := s.sync(ctx); err != nil { + s.logger.Errorf("failed to fetch sourcemaps metadata: %v", err) + // send nil to the update channel so that listeners don't block forever + s.updateChan <- nil + } + + s.logger.Info("init routine completed") + }() + + go func() { + // TODO make this a config option ? + t := time.NewTicker(30 * time.Second) + defer t.Stop() + + for { + select { + case <-t.C: + ctx, cleanup := context.WithTimeout(parent, syncTimeout) + + if err := s.sync(ctx); err != nil { + s.logger.Errorf("failed to sync sourcemaps metadata: %v", err) + } + + cleanup() + case <-parent.Done(): + s.logger.Info("update routine done") + // close update channel + close(s.updateChan) + return + } + } + }() +} + +func (s *SyncWorker) sync(ctx context.Context) error { + updates := make(map[Identifier]string) + + result, err := s.initialSearch(ctx, updates) + if err != nil { + return err + } + + scrollID := result.ScrollID + + if scrollID == "" { + return nil + } + + for { + result, err = s.scrollsearch(ctx, scrollID, updates) + if err != nil { + return err + } + + // From the docs: The initial search request and each subsequent scroll + // request each return a _scroll_id. While the _scroll_id may change between + // requests, it doesn't always change - in any case, only the most recently + // received _scroll_id should be used. + if result.ScrollID != "" { + scrollID = result.ScrollID + } + + // Stop if there are no new updates + if len(result.Hits.Hits) == 0 { + break + } + } + + select { + case s.updateChan <- updates: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func (s *SyncWorker) initialSearch(ctx context.Context, updates map[Identifier]string) (*esSearchSourcemapResponse, error) { + resp, err := s.runSearchQuery(ctx) + if err != nil { + return nil, errors.Wrap(err, errMsgESFailure) + } + defer resp.Body.Close() + + return s.handleUpdateRequest(resp, updates) +} + +func (s *SyncWorker) runSearchQuery(ctx context.Context) (*esapi.Response, error) { + var buf bytes.Buffer + if err := json.NewEncoder(&buf).Encode(queryMetadata()); err != nil { + return nil, err + } + + req := esapi.SearchRequest{ + Index: []string{s.index}, + Body: &buf, + TrackTotalHits: true, + Scroll: time.Minute, + } + return req.Do(ctx, s.esClient) +} + +func queryMetadata() map[string]interface{} { + return search( + sources([]string{"service.*", "file.path", "content_sha256"}), + ) +} + +type esSearchSourcemapResponse struct { + ScrollID string `json:"_scroll_id"` + esSourcemapResponse +} + +func (s *SyncWorker) handleUpdateRequest(resp *esapi.Response, updates map[Identifier]string) (*esSearchSourcemapResponse, error) { + // handle error response + if resp.StatusCode >= http.StatusMultipleChoices { + if resp.StatusCode == http.StatusNotFound { + return nil, nil + } + b, err := io.ReadAll(resp.Body) + if err != nil { + return nil, errors.Wrap(err, errMsgParseSourcemap) + } + return nil, errors.New(fmt.Sprintf("%s %s", errMsgParseSourcemap, b)) + } + + // parse response + body, err := parseResponse(resp.Body, s.logger) + if err != nil { + return nil, err + } + + for _, v := range body.Hits.Hits { + id := Identifier{ + name: v.Source.Service.Name, + version: v.Source.Service.Version, + path: v.Source.File.BundleFilepath, + } + + updates[id] = v.Source.ContentHash + } + + return body, nil +} + +func parseResponse(body io.ReadCloser, logger *logp.Logger) (*esSearchSourcemapResponse, error) { + b, err := io.ReadAll(body) + if err != nil { + return nil, err + } + + var esSourcemapResponse esSearchSourcemapResponse + if err := json.Unmarshal(b, &esSourcemapResponse); err != nil { + return nil, err + } + hits := esSourcemapResponse.Hits.Total.Value + if hits == 0 || len(esSourcemapResponse.Hits.Hits) == 0 { + return &esSourcemapResponse, nil + } + + return &esSourcemapResponse, nil +} + +func (s *SyncWorker) scrollsearch(ctx context.Context, scrollID string, updates map[Identifier]string) (*esSearchSourcemapResponse, error) { + resp, err := s.runScrollSearchQuery(ctx, scrollID) + if err != nil { + return nil, errors.Wrap(err, errMsgESFailure) + } + defer resp.Body.Close() + + return s.handleUpdateRequest(resp, updates) +} + +func (s *SyncWorker) runScrollSearchQuery(ctx context.Context, id string) (*esapi.Response, error) { + req := esapi.ScrollRequest{ + ScrollID: id, + Scroll: time.Minute, + } + return req.Do(ctx, s.esClient) +} From 26ecdfecd02ffe798b66fa4fc2f232d5652b6e87 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Fri, 27 Jan 2023 10:03:46 +0100 Subject: [PATCH 083/123] fix: prevent race condition between sync goroutines --- internal/sourcemap/sync.go | 42 +++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/internal/sourcemap/sync.go b/internal/sourcemap/sync.go index a75601c4172..6abdae79120 100644 --- a/internal/sourcemap/sync.go +++ b/internal/sourcemap/sync.go @@ -68,30 +68,30 @@ func (s *SyncWorker) Run(parent context.Context) { } s.logger.Info("init routine completed") - }() - - go func() { - // TODO make this a config option ? - t := time.NewTicker(30 * time.Second) - defer t.Stop() - for { - select { - case <-t.C: - ctx, cleanup := context.WithTimeout(parent, syncTimeout) - - if err := s.sync(ctx); err != nil { - s.logger.Errorf("failed to sync sourcemaps metadata: %v", err) + go func() { + // TODO make this a config option ? + t := time.NewTicker(30 * time.Second) + defer t.Stop() + + for { + select { + case <-t.C: + ctx, cleanup := context.WithTimeout(parent, syncTimeout) + + if err := s.sync(ctx); err != nil { + s.logger.Errorf("failed to sync sourcemaps metadata: %v", err) + } + + cleanup() + case <-parent.Done(): + s.logger.Info("update routine done") + // close update channel + close(s.updateChan) + return } - - cleanup() - case <-parent.Done(): - s.logger.Info("update routine done") - // close update channel - close(s.updateChan) - return } - } + }() }() } From cf948838f4981d54e5c606ba1bf5be24cd033c5a Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Fri, 27 Jan 2023 10:04:30 +0100 Subject: [PATCH 084/123] refactor: abstract alias retrieval --- internal/sourcemap/metadata_caching.go | 51 ++++++++++++-------------- 1 file changed, 24 insertions(+), 27 deletions(-) diff --git a/internal/sourcemap/metadata_caching.go b/internal/sourcemap/metadata_caching.go index 34dd06f90c0..644e760f44a 100644 --- a/internal/sourcemap/metadata_caching.go +++ b/internal/sourcemap/metadata_caching.go @@ -63,9 +63,9 @@ func (s *MetadataCachingFetcher) Fetch(ctx context.Context, name, version, path case <-s.init: // the mutex is shared by the update goroutine, we need to release it // as soon as possible to avoid blocking updates. - if s.hasID(original) { + if i, ok := s.getID(original); ok { // Only fetch from ES if the sourcemap id exists - return s.fetch(ctx, &original) + return s.fetch(ctx, i) } default: s.logger.Debugf("Metadata cache not populated. Falling back to backend fetcher for id: %s, %s, %s", name, version, path) @@ -83,14 +83,14 @@ func (s *MetadataCachingFetcher) Fetch(ctx context.Context, name, version, path case <-ctx.Done(): return nil, ctx.Err() } - } - // path is missing from the metadata cache (and ES). - // Is it an alias ? - // Try to retrieve the sourcemap from the alias map - // Only fetch from ES if the sourcemap alias exists - if id, found := s.getAlias(original); found { - return s.fetch(ctx, id) + // first map lookup will fail but this is not going + // to be performance issue since it only happens if init + // is in progress. + if i, ok := s.getID(original); ok { + // Only fetch from ES if the sourcemap id exists + return s.fetch(ctx, i) + } } if urlPath, err := url.Parse(path); err == nil { @@ -98,16 +98,14 @@ func (s *MetadataCachingFetcher) Fetch(ctx context.Context, name, version, path // bundle filepath but the request came in with an // absolute path original.path = urlPath.Path - if urlPath.Path != path && s.hasID(original) { - return s.fetch(ctx, &original) - } - - // The sourcemap could be stored on ES under a certain host - // but a request came in from a different host. - // Look for an alias to the url path to retrieve the correct - // host and fetch the sourcemap - if id, found := s.getAlias(original); found { - return s.fetch(ctx, id) + if urlPath.Path != path { + // The sourcemap could be stored on ES under a certain host + // but a request came in from a different host. + // Look for an alias to the url path to retrieve the correct + // host and fetch the sourcemap + if i, ok := s.getID(original); ok { + return s.fetch(ctx, i) + } } // Clean the url and try again if the result is different from @@ -126,18 +124,17 @@ func (s *MetadataCachingFetcher) Fetch(ctx context.Context, name, version, path return nil, fmt.Errorf("unable to find sourcemap.url for service.name=%s service.version=%s bundle.path=%s", name, version, path) } -func (s *MetadataCachingFetcher) hasID(key Identifier) bool { +func (s *MetadataCachingFetcher) getID(key Identifier) (*Identifier, bool) { s.mu.RLock() defer s.mu.RUnlock() - _, ok := s.set[key] - return ok -} - -func (s *MetadataCachingFetcher) getAlias(key Identifier) (*Identifier, bool) { - s.mu.RLock() - defer s.mu.RUnlock() + if _, ok := s.set[key]; ok { + return &key, ok + } + // path is missing from the metadata cache (and ES). + // Is it an alias ? + // Try to retrieve the sourcemap from the alias map i, ok := s.alias[key] return i, ok } From f353cd138864387cc64b53aa2980b75d67d53986 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Tue, 31 Jan 2023 14:36:27 +0100 Subject: [PATCH 085/123] refactor: rework fetchers and improve errors --- internal/beater/beater.go | 9 +- internal/sourcemap/body_caching.go | 37 +- internal/sourcemap/body_caching_test.go | 10 +- internal/sourcemap/chained.go | 6 + internal/sourcemap/elasticsearch.go | 38 +-- internal/sourcemap/elasticsearch_test.go | 28 +- internal/sourcemap/fetcher.go | 25 ++ internal/sourcemap/kibana.go | 2 +- internal/sourcemap/kibana_test.go | 2 +- internal/sourcemap/metadata_fetcher.go | 320 ++++++++++++++++++ internal/sourcemap/processor.go | 2 +- internal/sourcemap/processor_test.go | 5 +- ...tadata_caching.go => sourcemap_fetcher.go} | 124 +------ internal/sourcemap/sync.go | 242 ------------- 14 files changed, 420 insertions(+), 430 deletions(-) create mode 100644 internal/sourcemap/metadata_fetcher.go rename internal/sourcemap/{metadata_caching.go => sourcemap_fetcher.go} (51%) delete mode 100644 internal/sourcemap/sync.go diff --git a/internal/beater/beater.go b/internal/beater/beater.go index 7fb3135ee81..2fca11a7aa1 100644 --- a/internal/beater/beater.go +++ b/internal/beater/beater.go @@ -824,20 +824,19 @@ func newSourcemapFetcher( index := strings.ReplaceAll(cfg.IndexPattern, "%{[observer.version]}", version.Version) // start background sync job - sync, updateChan := sourcemap.NewSyncWorker(esClient, index) ctx, cancel := context.WithCancel(context.Background()) - sync.Run(ctx) + metadataFetcher, invalidationChan := sourcemap.NewMetadataFetcher(ctx, esClient, index) esFetcher := sourcemap.NewElasticsearchFetcher(esClient, index) size := 128 - cachingFetcher, invalidateChan, err := sourcemap.NewBodyCachingFetcher(esFetcher, size) + cachingFetcher, err := sourcemap.NewBodyCachingFetcher(esFetcher, size, invalidationChan) if err != nil { cancel() return nil, nil, err } - metadataCachingFetcher := sourcemap.NewMetadataCachingFetcher(cachingFetcher, updateChan, invalidateChan) + sourcemapFetcher := sourcemap.NewSourcemapFetcher(metadataFetcher, cachingFetcher) - chained = append(chained, metadataCachingFetcher) + chained = append(chained, sourcemapFetcher) if kibanaClient != nil { chained = append(chained, sourcemap.NewKibanaFetcher(kibanaClient)) diff --git a/internal/sourcemap/body_caching.go b/internal/sourcemap/body_caching.go index 1af7420d6e9..19a08f2ea7e 100644 --- a/internal/sourcemap/body_caching.go +++ b/internal/sourcemap/body_caching.go @@ -19,21 +19,16 @@ package sourcemap import ( "context" + "errors" "fmt" - "strings" "github.com/go-sourcemap/sourcemap" lru "github.com/hashicorp/golang-lru" - "github.com/pkg/errors" "github.com/elastic/apm-server/internal/logs" "github.com/elastic/elastic-agent-libs/logp" ) -var ( - errMsgFailure = "failure querying" -) - // BodyCachingFetcher wraps a Fetcher, caching source maps in memory and fetching from the wrapped Fetcher on cache misses. type BodyCachingFetcher struct { cache *lru.Cache @@ -45,7 +40,8 @@ type BodyCachingFetcher struct { func NewBodyCachingFetcher( backend Fetcher, cacheSize int, -) (*BodyCachingFetcher, chan<- []Identifier, error) { + invalidationChan <-chan []Identifier, +) (*BodyCachingFetcher, error) { logger := logp.NewLogger(logs.Sourcemap) lruCache, err := lru.NewWithEvict(cacheSize, func(key, value interface{}) { @@ -56,14 +52,18 @@ func NewBodyCachingFetcher( }) if err != nil { - return nil, nil, fmt.Errorf("failed to create lru cache for caching fetcher: %w", err) + return nil, fmt.Errorf("failed to create lru cache for caching fetcher: %w", err) } - ch := make(chan []Identifier) - go func() { - for arr := range ch { + logger.Info("listening for invalidation...") + + for arr := range invalidationChan { for _, id := range arr { + if logger.IsDebug() { + logger.Debugf("Invalidating id %v", id) + } + lruCache.Remove(id) } } @@ -73,7 +73,7 @@ func NewBodyCachingFetcher( cache: lruCache, backend: backend, logger: logger, - }, ch, nil + }, nil } // Fetch fetches a source map from the cache or wrapped backend. @@ -93,7 +93,7 @@ func (s *BodyCachingFetcher) Fetch(ctx context.Context, name, version, path stri // fetch from the store and ensure caching for all non-temporary results consumer, err := s.backend.Fetch(ctx, name, version, path) if err != nil { - if !strings.Contains(err.Error(), errMsgFailure) { + if errors.Is(err, ErrMalformedSourcemap) { s.add(key, nil) } return nil, err @@ -109,14 +109,3 @@ func (s *BodyCachingFetcher) add(key Identifier, consumer *sourcemap.Consumer) { } s.logger.Debugf("Added id %v. Cache now has %v entries.", key, s.cache.Len()) } - -func parseSourceMap(data string) (*sourcemap.Consumer, error) { - if data == "" { - return nil, nil - } - consumer, err := sourcemap.Parse("", []byte(data)) - if err != nil { - return nil, errors.Wrap(err, errMsgParseSourcemap) - } - return consumer, nil -} diff --git a/internal/sourcemap/body_caching_test.go b/internal/sourcemap/body_caching_test.go index 194d49b863e..c06c57fe1ef 100644 --- a/internal/sourcemap/body_caching_test.go +++ b/internal/sourcemap/body_caching_test.go @@ -40,12 +40,11 @@ var unsupportedVersionSourcemap = `{ }` func Test_NewCachingFetcher(t *testing.T) { - _, _, err := NewBodyCachingFetcher(nil, -1) + _, err := NewBodyCachingFetcher(nil, -1, nil) require.Error(t, err) - f, ch, err := NewBodyCachingFetcher(nil, 100) + f, err := NewBodyCachingFetcher(nil, 100, nil) require.NoError(t, err) - close(ch) assert.NotNil(t, f.cache) } @@ -97,7 +96,7 @@ func TestStore_Fetch(t *testing.T) { }) t.Run("notFoundInES", func(t *testing.T) { - store := testCachingFetcher(t, newMockElasticsearchClient(t, http.StatusNotFound, sourcemapSearchResponseBody(0, nil))) + store := testCachingFetcher(t, newMockElasticsearchClient(t, http.StatusOK, sourcemapSearchResponseBody(0, []map[string]interface{}{}))) //not cached cached, found := store.cache.Get(key) require.False(t, found) @@ -162,8 +161,7 @@ func TestStore_Fetch(t *testing.T) { func testCachingFetcher(t *testing.T, client *elasticsearch.Client) *BodyCachingFetcher { esFetcher := NewElasticsearchFetcher(client, "apm-*sourcemap*") - cachingFetcher, ch, err := NewBodyCachingFetcher(esFetcher, 100) + cachingFetcher, err := NewBodyCachingFetcher(esFetcher, 100, nil) require.NoError(t, err) - close(ch) return cachingFetcher } diff --git a/internal/sourcemap/chained.go b/internal/sourcemap/chained.go index daeb11f4104..8e6873d10c1 100644 --- a/internal/sourcemap/chained.go +++ b/internal/sourcemap/chained.go @@ -19,6 +19,7 @@ package sourcemap import ( "context" + "errors" "github.com/go-sourcemap/sourcemap" ) @@ -35,10 +36,15 @@ func (c ChainedFetcher) Fetch(ctx context.Context, name, version, path string) ( var lastErr error for _, f := range c { consumer, err := f.Fetch(ctx, name, version, path) + if !errors.Is(err, ErrFetcherUnvailable) { + return consumer, err + } + if err != nil { lastErr = err continue } + if consumer != nil { return consumer, nil } diff --git a/internal/sourcemap/elasticsearch.go b/internal/sourcemap/elasticsearch.go index f4991a7eba7..bafb673746d 100644 --- a/internal/sourcemap/elasticsearch.go +++ b/internal/sourcemap/elasticsearch.go @@ -28,7 +28,6 @@ import ( "net/http" "github.com/go-sourcemap/sourcemap" - "github.com/pkg/errors" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/go-elasticsearch/v8/esapi" @@ -37,15 +36,6 @@ import ( "github.com/elastic/apm-server/internal/logs" ) -const ( - errMsgParseSourcemap = "Could not parse Sourcemap" -) - -var ( - errMsgESFailure = errMsgFailure + " ES" - errSourcemapWrongFormat = errors.New("Sourcemapping ES Result not in expected format") -) - type esFetcher struct { client *elasticsearch.Client index string @@ -83,20 +73,20 @@ func NewElasticsearchFetcher(c *elasticsearch.Client, index string) Fetcher { func (s *esFetcher) Fetch(ctx context.Context, name, version, path string) (*sourcemap.Consumer, error) { resp, err := s.runSearchQuery(ctx, name, version, path) if err != nil { - return nil, errors.Wrap(err, errMsgESFailure) + return nil, fmt.Errorf("failure querying ES: %w", err) } defer resp.Body.Close() // handle error response if resp.StatusCode >= http.StatusMultipleChoices { - if resp.StatusCode == http.StatusNotFound { - return nil, nil - } b, err := io.ReadAll(resp.Body) if err != nil { - return nil, errors.Wrap(err, errMsgParseSourcemap) + return nil, fmt.Errorf("failed to read ES response body: %w", err) } - return nil, errors.New(fmt.Sprintf("%s %s", errMsgParseSourcemap, b)) + if resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusForbidden { + return nil, fmt.Errorf("%w: %s: %s", ErrFetcherUnvailable, resp.Status(), string(b)) + } + return nil, fmt.Errorf("ES returned unknown status code: %s", resp.Status()) } // parse response @@ -125,13 +115,13 @@ func (s *esFetcher) Fetch(ctx context.Context, name, version, path string) (*sou return nil, fmt.Errorf("failed to read sourcemap content: %w", err) } - return parseSourceMap(string(uncompressedBody)) + return ParseSourceMap(uncompressedBody) } func (s *esFetcher) runSearchQuery(ctx context.Context, name, version, path string) (*esapi.Response, error) { var buf bytes.Buffer if err := json.NewEncoder(&buf).Encode(requestBody(name, version, path)); err != nil { - return nil, err + return nil, fmt.Errorf("failed to encode request body: %w", err) } req := esapi.SearchRequest{ Index: []string{s.index}, @@ -144,22 +134,18 @@ func (s *esFetcher) runSearchQuery(ctx context.Context, name, version, path stri func parse(body io.ReadCloser, name, version, path string, logger *logp.Logger) (string, error) { var esSourcemapResponse esSourcemapResponse if err := json.NewDecoder(body).Decode(&esSourcemapResponse); err != nil { - return "", err + return "", fmt.Errorf("failed to decode sourcemap: %w", err) } + hits := esSourcemapResponse.Hits.Total.Value if hits == 0 || len(esSourcemapResponse.Hits.Hits) == 0 { return "", nil } - var esSourcemap string - if hits > 1 { - logger.Warnf("%d sourcemaps found for service %s version %s and file %s, using the most recent one", - hits, name, version, path) - } - esSourcemap = esSourcemapResponse.Hits.Hits[0].Source.Sourcemap + esSourcemap := esSourcemapResponse.Hits.Hits[0].Source.Sourcemap // until https://github.com/golang/go/issues/19858 is resolved if esSourcemap == "" { - return "", errSourcemapWrongFormat + return "", fmt.Errorf("sourcemap not in the expected format: %w", ErrMalformedSourcemap) } return esSourcemap, nil } diff --git a/internal/sourcemap/elasticsearch_test.go b/internal/sourcemap/elasticsearch_test.go index efc2c1e371f..6b6dcccf4cd 100644 --- a/internal/sourcemap/elasticsearch_test.go +++ b/internal/sourcemap/elasticsearch_test.go @@ -41,17 +41,20 @@ import ( func Test_esFetcher_fetchError(t *testing.T) { for name, tc := range map[string]struct { - statusCode int - clientError bool - responseBody io.Reader - temporary bool + statusCode int + clientError bool + responseBody io.Reader + temporary bool + expectedErrMessage string }{ "es not reachable": { - clientError: true, - temporary: true, + clientError: true, + temporary: true, + expectedErrMessage: "failure querying ES: client error", }, "es bad request": { - statusCode: http.StatusBadRequest, + statusCode: http.StatusBadRequest, + expectedErrMessage: "ES returned unknown status code: 400 Bad Request", }, "empty sourcemap string": { statusCode: http.StatusOK, @@ -62,6 +65,7 @@ func Test_esFetcher_fetchError(t *testing.T) { }, }, }}), + expectedErrMessage: "sourcemap not in the expected format: sourcemap malformed", }, } { t.Run(name, func(t *testing.T) { @@ -73,11 +77,7 @@ func Test_esFetcher_fetchError(t *testing.T) { } consumer, err := testESFetcher(client).Fetch(context.Background(), "abc", "1.0", "/tmp") - if tc.temporary { - assert.Contains(t, err.Error(), errMsgESFailure) - } else { - assert.NotContains(t, err.Error(), errMsgESFailure) - } + assert.Equal(t, tc.expectedErrMessage, err.Error()) assert.Empty(t, consumer) }) } @@ -90,8 +90,8 @@ func Test_esFetcher_fetch(t *testing.T) { filePath string }{ "no sourcemap found": { - statusCode: http.StatusNotFound, - responseBody: sourcemapSearchResponseBody(0, nil), + statusCode: http.StatusOK, + responseBody: sourcemapSearchResponseBody(0, []map[string]interface{}{}), }, "sourcemap indicated but not found": { statusCode: http.StatusOK, diff --git a/internal/sourcemap/fetcher.go b/internal/sourcemap/fetcher.go index 6053bb8cfe0..ecedc132cd4 100644 --- a/internal/sourcemap/fetcher.go +++ b/internal/sourcemap/fetcher.go @@ -19,11 +19,18 @@ package sourcemap import ( "context" + "errors" + "fmt" "net/url" "github.com/go-sourcemap/sourcemap" ) +var ( + ErrFetcherUnvailable = errors.New("fetcher unavailable") + ErrMalformedSourcemap = errors.New("sourcemap malformed") +) + // Fetcher is an interface for fetching a source map with a given service name, service version, // and bundle filepath. type Fetcher interface { @@ -33,6 +40,13 @@ type Fetcher interface { Fetch(ctx context.Context, name string, version string, bundleFilepath string) (*sourcemap.Consumer, error) } +// MetadataFetcher is an interface for fetching metadata +type MetadataFetcher interface { + GetID(id Identifier) (*Identifier, bool) + + Ready() <-chan struct{} +} + type Identifier struct { name string version string @@ -87,3 +101,14 @@ func GetAliases(name string, version string, bundleFilepath string) []Identifier }, } } + +func ParseSourceMap(data []byte) (*sourcemap.Consumer, error) { + if len(data) == 0 { + return nil, nil + } + consumer, err := sourcemap.Parse("", data) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrMalformedSourcemap, err) + } + return consumer, nil +} diff --git a/internal/sourcemap/kibana.go b/internal/sourcemap/kibana.go index 961a3b0d5a3..23373556f50 100644 --- a/internal/sourcemap/kibana.go +++ b/internal/sourcemap/kibana.go @@ -80,7 +80,7 @@ func (s *kibanaFetcher) Fetch(ctx context.Context, name, version, path string) ( continue } if a.Body.ServiceName == name && a.Body.ServiceVersion == version && maybeParseURLPath(a.Body.BundleFilepath) == path { - return parseSourceMap(string(a.Body.SourceMap)) + return ParseSourceMap(a.Body.SourceMap) } } return nil, nil diff --git a/internal/sourcemap/kibana_test.go b/internal/sourcemap/kibana_test.go index 2565d255a0d..f7636000a3e 100644 --- a/internal/sourcemap/kibana_test.go +++ b/internal/sourcemap/kibana_test.go @@ -110,7 +110,7 @@ func TestKibanaFetcherInvalidSourcemap(t *testing.T) { }) consumer, err := fetcher.Fetch(context.Background(), "service_name", "service_version", "http://host:123/path") require.Error(t, err) - assert.EqualError(t, err, "Could not parse Sourcemap: json: cannot unmarshal string into Go value of type sourcemap.v3") + assert.EqualError(t, err, "sourcemap malformed: json: cannot unmarshal string into Go value of type sourcemap.v3") assert.Nil(t, consumer) } diff --git a/internal/sourcemap/metadata_fetcher.go b/internal/sourcemap/metadata_fetcher.go new file mode 100644 index 00000000000..f3aae56e2fd --- /dev/null +++ b/internal/sourcemap/metadata_fetcher.go @@ -0,0 +1,320 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package sourcemap + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "sync" + "time" + + "github.com/elastic/apm-server/internal/elasticsearch" + "github.com/elastic/apm-server/internal/logs" + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/go-elasticsearch/v8/esapi" +) + +const ( + syncTimeout = 10 * time.Second +) + +type MetadataESFetcher struct { + esClient *elasticsearch.Client + index string + set map[Identifier]string + alias map[Identifier]*Identifier + mu sync.RWMutex + logger *logp.Logger + init chan struct{} + invalidationChan chan<- []Identifier +} + +func NewMetadataFetcher(ctx context.Context, esClient *elasticsearch.Client, index string) (MetadataFetcher, <-chan []Identifier) { + invalidationCh := make(chan []Identifier) + + s := &MetadataESFetcher{ + esClient: esClient, + index: index, + set: make(map[Identifier]string), + alias: make(map[Identifier]*Identifier), + logger: logp.NewLogger(logs.Sourcemap), + init: make(chan struct{}), + invalidationChan: invalidationCh, + } + + s.startBackgrounSync(ctx) + + return s, invalidationCh +} + +func (s *MetadataESFetcher) GetID(key Identifier) (*Identifier, bool) { + s.mu.RLock() + defer s.mu.RUnlock() + + if _, ok := s.set[key]; ok { + return &key, ok + } + + // path is missing from the metadata cache (and ES). + // Is it an alias ? + // Try to retrieve the sourcemap from the alias map + i, ok := s.alias[key] + return i, ok +} + +func (s *MetadataESFetcher) Ready() <-chan struct{} { + return s.init +} + +func (s *MetadataESFetcher) startBackgrounSync(parent context.Context) { + go func() { + s.logger.Debug("populating metadata cache") + + // First run, populate cache + ctx, cancel := context.WithTimeout(parent, syncTimeout) + defer cancel() + + if err := s.sync(ctx); err != nil { + s.logger.Errorf("failed to fetch sourcemaps metadata: %v", err) + } else { + // only close the init chan and mark the fetcher as ready if + // sync succeeded + close(s.init) + } + + s.logger.Info("init routine completed") + + go func() { + // TODO make this a config option ? + t := time.NewTicker(30 * time.Second) + defer t.Stop() + + for { + select { + case <-t.C: + ctx, cancel := context.WithTimeout(parent, syncTimeout) + + if err := s.sync(ctx); err != nil { + s.logger.Errorf("failed to sync sourcemaps metadata: %v", err) + } + + cancel() + case <-parent.Done(): + s.logger.Info("update routine done") + // close invalidation channel + close(s.invalidationChan) + return + } + } + }() + }() +} + +func (s *MetadataESFetcher) sync(ctx context.Context) error { + sourcemaps := make(map[Identifier]string) + + result, err := s.initialSearch(ctx, sourcemaps) + if err != nil { + return err + } + + scrollID := result.ScrollID + + if scrollID == "" { + s.update(ctx, sourcemaps) + return nil + } + + for { + result, err = s.scrollsearch(ctx, scrollID, sourcemaps) + if err != nil { + return fmt.Errorf("failed scroll search: %w", err) + } + + // From the docs: The initial search request and each subsequent scroll + // request each return a _scroll_id. While the _scroll_id may change between + // requests, it doesn't always change - in any case, only the most recently + // received _scroll_id should be used. + if result.ScrollID != "" { + scrollID = result.ScrollID + } + + // Stop if there are no new updates + if len(result.Hits.Hits) == 0 { + break + } + } + + s.update(ctx, sourcemaps) + return nil +} + +func (s *MetadataESFetcher) update(ctx context.Context, sopurcemaps map[Identifier]string) { + s.mu.Lock() + defer s.mu.Unlock() + + var invalidation []Identifier + + for id, contentHash := range s.set { + if updatedHash, ok := sopurcemaps[id]; ok { + // already in the cache, remove from the updates. + delete(sopurcemaps, id) + + // content hash changed, invalidate the sourcemap cache + if contentHash != updatedHash { + s.logger.Debugf("Hash changed: %s -> %s: invalidating %v", contentHash, updatedHash, id) + invalidation = append(invalidation, id) + } + } else { + // the sourcemap no longer exists in ES. + // invalidate the sourcemap cache. + invalidation = append(invalidation, id) + + // the sourcemap no longer exists in ES. + // remove from metadata cache + delete(s.set, id) + + // remove aliases + for _, k := range GetAliases(id.name, id.version, id.path) { + delete(s.alias, k) + } + } + } + + s.invalidationChan <- invalidation + + // add new sourcemaps to the metadata cache. + for id, contentHash := range sopurcemaps { + s.set[id] = contentHash + s.logger.Debugf("Added metadata id %v", id) + // store aliases with a pointer to the original id. + // The id is then passed over to the backend fetcher + // to minimize the size of the lru cache and + // and increase cache hits. + for _, k := range GetAliases(id.name, id.version, id.path) { + s.logger.Debugf("Added metadata alias %v -> %v", k, id) + s.alias[k] = &id + } + } + + s.logger.Debugf("Metadata cache now has %d entries.", len(s.set)) +} + +func (s *MetadataESFetcher) initialSearch(ctx context.Context, updates map[Identifier]string) (*esSearchSourcemapResponse, error) { + resp, err := s.runSearchQuery(ctx) + if err != nil { + return nil, fmt.Errorf("failed to run initial search query: %w", err) + } + defer resp.Body.Close() + + return s.handleUpdateRequest(resp, updates) +} + +func (s *MetadataESFetcher) runSearchQuery(ctx context.Context) (*esapi.Response, error) { + var buf bytes.Buffer + if err := json.NewEncoder(&buf).Encode(queryMetadata()); err != nil { + return nil, fmt.Errorf("failed to encode metadata query: %w", err) + } + + req := esapi.SearchRequest{ + Index: []string{s.index}, + Body: &buf, + TrackTotalHits: true, + Scroll: time.Minute, + } + return req.Do(ctx, s.esClient) +} + +func queryMetadata() map[string]interface{} { + return search( + sources([]string{"service.*", "file.path", "content_sha256"}), + ) +} + +type esSearchSourcemapResponse struct { + ScrollID string `json:"_scroll_id"` + esSourcemapResponse +} + +func (s *MetadataESFetcher) handleUpdateRequest(resp *esapi.Response, updates map[Identifier]string) (*esSearchSourcemapResponse, error) { + // handle error response + if resp.StatusCode >= http.StatusMultipleChoices { + b, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read response body: %w", err) + } + if resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusForbidden { + return nil, fmt.Errorf("%w: %s: %s", ErrFetcherUnvailable, resp.Status(), string(b)) + } + return nil, fmt.Errorf("ES returned unknown status code: %s", resp.Status()) + } + + // parse response + body, err := parseResponse(resp.Body, s.logger) + if err != nil { + return nil, fmt.Errorf("failed to parse response: %w", err) + } + + for _, v := range body.Hits.Hits { + id := Identifier{ + name: v.Source.Service.Name, + version: v.Source.Service.Version, + path: v.Source.File.BundleFilepath, + } + + updates[id] = v.Source.ContentHash + } + + return body, nil +} + +func parseResponse(body io.ReadCloser, logger *logp.Logger) (*esSearchSourcemapResponse, error) { + b, err := io.ReadAll(body) + if err != nil { + return nil, err + } + + var esSourcemapResponse esSearchSourcemapResponse + if err := json.Unmarshal(b, &esSourcemapResponse); err != nil { + return nil, err + } + + return &esSourcemapResponse, nil +} + +func (s *MetadataESFetcher) scrollsearch(ctx context.Context, scrollID string, updates map[Identifier]string) (*esSearchSourcemapResponse, error) { + resp, err := s.runScrollSearchQuery(ctx, scrollID) + if err != nil { + return nil, fmt.Errorf("failed to run scroll search query: %w", err) + } + defer resp.Body.Close() + + return s.handleUpdateRequest(resp, updates) +} + +func (s *MetadataESFetcher) runScrollSearchQuery(ctx context.Context, id string) (*esapi.Response, error) { + req := esapi.ScrollRequest{ + ScrollID: id, + Scroll: time.Minute, + } + return req.Do(ctx, s.esClient) +} diff --git a/internal/sourcemap/processor.go b/internal/sourcemap/processor.go index f1c52e2fb71..99b6ad916f6 100644 --- a/internal/sourcemap/processor.go +++ b/internal/sourcemap/processor.go @@ -121,7 +121,7 @@ func (p BatchProcessor) processStacktraceFrame( mapper, err := p.Fetcher.Fetch(ctx, service.Name, service.Version, path) if err != nil { frame.SourcemapError = err.Error() - p.Logger.Debugf("failed to fetch source map with path (%s): %s", path, frame.SourcemapError) + p.Logger.Debugf("failed to fetch sourcemap with path (%s): %s", path, frame.SourcemapError) return false, "" } if mapper == nil { diff --git a/internal/sourcemap/processor_test.go b/internal/sourcemap/processor_test.go index af24ae3555b..0bdb880105a 100644 --- a/internal/sourcemap/processor_test.go +++ b/internal/sourcemap/processor_test.go @@ -38,9 +38,8 @@ func TestBatchProcessor(t *testing.T) { sourcemapSearchResponseBody(1, []map[string]interface{}{sourcemapHit(string(validSourcemap))}), ) esFetcher := NewElasticsearchFetcher(client, "index") - fetcher, ch, err := NewBodyCachingFetcher(esFetcher, 100) + fetcher, err := NewBodyCachingFetcher(esFetcher, 100, nil) require.NoError(t, err) - close(ch) originalLinenoWithFilename := 1 originalColnoWithFilename := 7 @@ -265,7 +264,7 @@ func TestBatchProcessorElasticsearchUnavailable(t *testing.T) { // we are running the processor twice for a batch of two spans with 2 stacktraceframe each entries := logp.ObserverLogs().TakeAll() require.Len(t, entries, 8) - assert.Equal(t, "failed to fetch source map with path (bundle.js): failure querying ES: client error", entries[0].Message) + assert.Equal(t, "failed to fetch sourcemap with path (bundle.js): failure querying ES: client error", entries[0].Message) } func TestBatchProcessorTimeout(t *testing.T) { diff --git a/internal/sourcemap/metadata_caching.go b/internal/sourcemap/sourcemap_fetcher.go similarity index 51% rename from internal/sourcemap/metadata_caching.go rename to internal/sourcemap/sourcemap_fetcher.go index 644e760f44a..e6de76773a6 100644 --- a/internal/sourcemap/metadata_caching.go +++ b/internal/sourcemap/sourcemap_fetcher.go @@ -21,7 +21,6 @@ import ( "context" "fmt" "net/url" - "sync" "github.com/go-sourcemap/sourcemap" @@ -29,41 +28,30 @@ import ( "github.com/elastic/elastic-agent-libs/logp" ) -type MetadataCachingFetcher struct { - set map[Identifier]string - alias map[Identifier]*Identifier - mu sync.RWMutex - backend Fetcher - logger *logp.Logger - init chan struct{} - updateChan <-chan map[Identifier]string - invalidateChan chan<- []Identifier +type SourcemapFetcher struct { + metadata MetadataFetcher + backend Fetcher + logger *logp.Logger } -func NewMetadataCachingFetcher(backend Fetcher, in <-chan map[Identifier]string, out chan<- []Identifier) *MetadataCachingFetcher { - s := &MetadataCachingFetcher{ - set: make(map[Identifier]string), - alias: make(map[Identifier]*Identifier), - backend: backend, - logger: logp.NewLogger(logs.Sourcemap), - init: make(chan struct{}), - updateChan: in, - invalidateChan: out, +func NewSourcemapFetcher(metadata MetadataFetcher, backend Fetcher) *SourcemapFetcher { + s := &SourcemapFetcher{ + metadata: metadata, + backend: backend, + logger: logp.NewLogger(logs.Sourcemap), } - go s.handleUpdates() - return s } -func (s *MetadataCachingFetcher) Fetch(ctx context.Context, name, version, path string) (*sourcemap.Consumer, error) { +func (s *SourcemapFetcher) Fetch(ctx context.Context, name, version, path string) (*sourcemap.Consumer, error) { original := Identifier{name: name, version: version, path: path} select { - case <-s.init: + case <-s.metadata.Ready(): // the mutex is shared by the update goroutine, we need to release it // as soon as possible to avoid blocking updates. - if i, ok := s.getID(original); ok { + if i, ok := s.metadata.GetID(original); ok { // Only fetch from ES if the sourcemap id exists return s.fetch(ctx, i) } @@ -79,15 +67,15 @@ func (s *MetadataCachingFetcher) Fetch(ctx context.Context, name, version, path // Aliases are only available after init is completed. select { - case <-s.init: + case <-s.metadata.Ready(): case <-ctx.Done(): - return nil, ctx.Err() + return nil, fmt.Errorf("error waiting for metadata fetcher to be ready: %w", ctx.Err()) } // first map lookup will fail but this is not going // to be performance issue since it only happens if init // is in progress. - if i, ok := s.getID(original); ok { + if i, ok := s.metadata.GetID(original); ok { // Only fetch from ES if the sourcemap id exists return s.fetch(ctx, i) } @@ -103,7 +91,7 @@ func (s *MetadataCachingFetcher) Fetch(ctx context.Context, name, version, path // but a request came in from a different host. // Look for an alias to the url path to retrieve the correct // host and fetch the sourcemap - if i, ok := s.getID(original); ok { + if i, ok := s.metadata.GetID(original); ok { return s.fetch(ctx, i) } } @@ -124,22 +112,7 @@ func (s *MetadataCachingFetcher) Fetch(ctx context.Context, name, version, path return nil, fmt.Errorf("unable to find sourcemap.url for service.name=%s service.version=%s bundle.path=%s", name, version, path) } -func (s *MetadataCachingFetcher) getID(key Identifier) (*Identifier, bool) { - s.mu.RLock() - defer s.mu.RUnlock() - - if _, ok := s.set[key]; ok { - return &key, ok - } - - // path is missing from the metadata cache (and ES). - // Is it an alias ? - // Try to retrieve the sourcemap from the alias map - i, ok := s.alias[key] - return i, ok -} - -func (s *MetadataCachingFetcher) fetch(ctx context.Context, key *Identifier) (*sourcemap.Consumer, error) { +func (s *SourcemapFetcher) fetch(ctx context.Context, key *Identifier) (*sourcemap.Consumer, error) { c, err := s.backend.Fetch(ctx, key.name, key.version, key.path) // log a message if the sourcemap is present in the cache but the backend fetcher did not @@ -150,66 +123,3 @@ func (s *MetadataCachingFetcher) fetch(ctx context.Context, key *Identifier) (*s return c, err } - -func (s *MetadataCachingFetcher) handleUpdates() { - // run once for init - s.update(<-s.updateChan) - close(s.init) - - // wait for updates - for updates := range s.updateChan { - s.update(updates) - } - close(s.invalidateChan) -} - -func (s *MetadataCachingFetcher) update(updates map[Identifier]string) { - s.mu.Lock() - defer s.mu.Unlock() - - var invalidation []Identifier - - for id, contentHash := range s.set { - if updatedHash, ok := updates[id]; ok { - // already in the cache, remove from the updates. - delete(updates, id) - - // content hash changed, invalidate the sourcemap cache - if contentHash != updatedHash { - s.logger.Debugf("Hash changed: %s -> %s: invalidating %v", contentHash, updatedHash, id) - invalidation = append(invalidation, id) - } - } else { - // the sourcemap no longer exists in ES. - // invalidate the sourcemap cache. - invalidation = append(invalidation, id) - - // the sourcemap no longer exists in ES. - // remove from metadata cache - delete(s.set, id) - - // remove alias - for _, k := range GetAliases(id.name, id.version, id.path) { - delete(s.alias, k) - } - } - } - - s.invalidateChan <- invalidation - - // add new sourcemaps to the metadata cache. - for id, contentHash := range updates { - s.set[id] = contentHash - s.logger.Debugf("Added metadata id %v", id) - // store aliases with a pointer to the original id. - // The id is then passed over to the backend fetcher - // to minimize the size of the lru cache and - // and increase cache hits. - for _, k := range GetAliases(id.name, id.version, id.path) { - s.logger.Debugf("Added metadata alias %v -> %v", k, id) - s.alias[k] = &id - } - } - - s.logger.Debugf("Metadata cache now has %d entries.", len(s.set)) -} diff --git a/internal/sourcemap/sync.go b/internal/sourcemap/sync.go deleted file mode 100644 index 6abdae79120..00000000000 --- a/internal/sourcemap/sync.go +++ /dev/null @@ -1,242 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package sourcemap - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "time" - - "github.com/pkg/errors" - - "github.com/elastic/apm-server/internal/elasticsearch" - "github.com/elastic/apm-server/internal/logs" - "github.com/elastic/elastic-agent-libs/logp" - "github.com/elastic/go-elasticsearch/v8/esapi" -) - -const ( - syncTimeout = 10 * time.Second -) - -type SyncWorker struct { - esClient *elasticsearch.Client - logger *logp.Logger - index string - updateChan chan<- map[Identifier]string -} - -func NewSyncWorker(esClient *elasticsearch.Client, index string) (*SyncWorker, <-chan map[Identifier]string) { - ch := make(chan map[Identifier]string) - return &SyncWorker{ - esClient: esClient, - index: index, - logger: logp.NewLogger(logs.Sourcemap), - updateChan: ch, - }, ch -} - -func (s *SyncWorker) Run(parent context.Context) { - go func() { - // First run, populate cache - ctx, cleanup := context.WithTimeout(parent, syncTimeout) - defer cleanup() - - if err := s.sync(ctx); err != nil { - s.logger.Errorf("failed to fetch sourcemaps metadata: %v", err) - // send nil to the update channel so that listeners don't block forever - s.updateChan <- nil - } - - s.logger.Info("init routine completed") - - go func() { - // TODO make this a config option ? - t := time.NewTicker(30 * time.Second) - defer t.Stop() - - for { - select { - case <-t.C: - ctx, cleanup := context.WithTimeout(parent, syncTimeout) - - if err := s.sync(ctx); err != nil { - s.logger.Errorf("failed to sync sourcemaps metadata: %v", err) - } - - cleanup() - case <-parent.Done(): - s.logger.Info("update routine done") - // close update channel - close(s.updateChan) - return - } - } - }() - }() -} - -func (s *SyncWorker) sync(ctx context.Context) error { - updates := make(map[Identifier]string) - - result, err := s.initialSearch(ctx, updates) - if err != nil { - return err - } - - scrollID := result.ScrollID - - if scrollID == "" { - return nil - } - - for { - result, err = s.scrollsearch(ctx, scrollID, updates) - if err != nil { - return err - } - - // From the docs: The initial search request and each subsequent scroll - // request each return a _scroll_id. While the _scroll_id may change between - // requests, it doesn't always change - in any case, only the most recently - // received _scroll_id should be used. - if result.ScrollID != "" { - scrollID = result.ScrollID - } - - // Stop if there are no new updates - if len(result.Hits.Hits) == 0 { - break - } - } - - select { - case s.updateChan <- updates: - return nil - case <-ctx.Done(): - return ctx.Err() - } -} - -func (s *SyncWorker) initialSearch(ctx context.Context, updates map[Identifier]string) (*esSearchSourcemapResponse, error) { - resp, err := s.runSearchQuery(ctx) - if err != nil { - return nil, errors.Wrap(err, errMsgESFailure) - } - defer resp.Body.Close() - - return s.handleUpdateRequest(resp, updates) -} - -func (s *SyncWorker) runSearchQuery(ctx context.Context) (*esapi.Response, error) { - var buf bytes.Buffer - if err := json.NewEncoder(&buf).Encode(queryMetadata()); err != nil { - return nil, err - } - - req := esapi.SearchRequest{ - Index: []string{s.index}, - Body: &buf, - TrackTotalHits: true, - Scroll: time.Minute, - } - return req.Do(ctx, s.esClient) -} - -func queryMetadata() map[string]interface{} { - return search( - sources([]string{"service.*", "file.path", "content_sha256"}), - ) -} - -type esSearchSourcemapResponse struct { - ScrollID string `json:"_scroll_id"` - esSourcemapResponse -} - -func (s *SyncWorker) handleUpdateRequest(resp *esapi.Response, updates map[Identifier]string) (*esSearchSourcemapResponse, error) { - // handle error response - if resp.StatusCode >= http.StatusMultipleChoices { - if resp.StatusCode == http.StatusNotFound { - return nil, nil - } - b, err := io.ReadAll(resp.Body) - if err != nil { - return nil, errors.Wrap(err, errMsgParseSourcemap) - } - return nil, errors.New(fmt.Sprintf("%s %s", errMsgParseSourcemap, b)) - } - - // parse response - body, err := parseResponse(resp.Body, s.logger) - if err != nil { - return nil, err - } - - for _, v := range body.Hits.Hits { - id := Identifier{ - name: v.Source.Service.Name, - version: v.Source.Service.Version, - path: v.Source.File.BundleFilepath, - } - - updates[id] = v.Source.ContentHash - } - - return body, nil -} - -func parseResponse(body io.ReadCloser, logger *logp.Logger) (*esSearchSourcemapResponse, error) { - b, err := io.ReadAll(body) - if err != nil { - return nil, err - } - - var esSourcemapResponse esSearchSourcemapResponse - if err := json.Unmarshal(b, &esSourcemapResponse); err != nil { - return nil, err - } - hits := esSourcemapResponse.Hits.Total.Value - if hits == 0 || len(esSourcemapResponse.Hits.Hits) == 0 { - return &esSourcemapResponse, nil - } - - return &esSourcemapResponse, nil -} - -func (s *SyncWorker) scrollsearch(ctx context.Context, scrollID string, updates map[Identifier]string) (*esSearchSourcemapResponse, error) { - resp, err := s.runScrollSearchQuery(ctx, scrollID) - if err != nil { - return nil, errors.Wrap(err, errMsgESFailure) - } - defer resp.Body.Close() - - return s.handleUpdateRequest(resp, updates) -} - -func (s *SyncWorker) runScrollSearchQuery(ctx context.Context, id string) (*esapi.Response, error) { - req := esapi.ScrollRequest{ - ScrollID: id, - Scroll: time.Minute, - } - return req.Do(ctx, s.esClient) -} From 824c68f502a08ec5902d34f105b2b546062207c1 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Thu, 2 Feb 2023 10:43:06 +0100 Subject: [PATCH 086/123] refactor: print routine completion message on success --- internal/sourcemap/metadata_fetcher.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/sourcemap/metadata_fetcher.go b/internal/sourcemap/metadata_fetcher.go index f3aae56e2fd..5e4e6164ca4 100644 --- a/internal/sourcemap/metadata_fetcher.go +++ b/internal/sourcemap/metadata_fetcher.go @@ -99,10 +99,9 @@ func (s *MetadataESFetcher) startBackgrounSync(parent context.Context) { // only close the init chan and mark the fetcher as ready if // sync succeeded close(s.init) + s.logger.Info("init routine completed") } - s.logger.Info("init routine completed") - go func() { // TODO make this a config option ? t := time.NewTicker(30 * time.Second) From 9efecb9afeb14faf8c620710934a45837a50be18 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Thu, 2 Feb 2023 10:44:10 +0100 Subject: [PATCH 087/123] lint: fix variable name typo --- internal/sourcemap/metadata_fetcher.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/sourcemap/metadata_fetcher.go b/internal/sourcemap/metadata_fetcher.go index 5e4e6164ca4..9da30f13423 100644 --- a/internal/sourcemap/metadata_fetcher.go +++ b/internal/sourcemap/metadata_fetcher.go @@ -167,16 +167,16 @@ func (s *MetadataESFetcher) sync(ctx context.Context) error { return nil } -func (s *MetadataESFetcher) update(ctx context.Context, sopurcemaps map[Identifier]string) { +func (s *MetadataESFetcher) update(ctx context.Context, sourcemaps map[Identifier]string) { s.mu.Lock() defer s.mu.Unlock() var invalidation []Identifier for id, contentHash := range s.set { - if updatedHash, ok := sopurcemaps[id]; ok { + if updatedHash, ok := sourcemaps[id]; ok { // already in the cache, remove from the updates. - delete(sopurcemaps, id) + delete(sourcemaps, id) // content hash changed, invalidate the sourcemap cache if contentHash != updatedHash { @@ -202,7 +202,7 @@ func (s *MetadataESFetcher) update(ctx context.Context, sopurcemaps map[Identifi s.invalidationChan <- invalidation // add new sourcemaps to the metadata cache. - for id, contentHash := range sopurcemaps { + for id, contentHash := range sourcemaps { s.set[id] = contentHash s.logger.Debugf("Added metadata id %v", id) // store aliases with a pointer to the original id. From d46604534cce721b557269cee282320a52589d91 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Thu, 2 Feb 2023 11:02:34 +0100 Subject: [PATCH 088/123] feat: remove index pattern setting --- internal/beater/beater.go | 9 +++--- internal/beater/beater_test.go | 46 --------------------------- internal/beater/config/config_test.go | 3 -- internal/beater/config/rum.go | 3 -- systemtest/apmservertest/config.go | 1 - systemtest/sourcemap_test.go | 1 - 6 files changed, 4 insertions(+), 59 deletions(-) diff --git a/internal/beater/beater.go b/internal/beater/beater.go index 2fca11a7aa1..03827342ddf 100644 --- a/internal/beater/beater.go +++ b/internal/beater/beater.go @@ -25,7 +25,6 @@ import ( "net/http" "os" "runtime" - "strings" "time" "github.com/dustin/go-humanize" @@ -808,6 +807,8 @@ func (s *Runner) newLibbeatFinalBatchProcessor( return publisher, stop, nil } +const sourcemapIndex = ".apm-source-map" + func newSourcemapFetcher( cfg config.SourceMapping, kibanaClient *kibana.Client, @@ -821,13 +822,11 @@ func newSourcemapFetcher( // For standalone, we query both Kibana and Elasticsearch for backwards compatibility. var chained sourcemap.ChainedFetcher - index := strings.ReplaceAll(cfg.IndexPattern, "%{[observer.version]}", version.Version) - // start background sync job ctx, cancel := context.WithCancel(context.Background()) - metadataFetcher, invalidationChan := sourcemap.NewMetadataFetcher(ctx, esClient, index) + metadataFetcher, invalidationChan := sourcemap.NewMetadataFetcher(ctx, esClient, sourcemapIndex) - esFetcher := sourcemap.NewElasticsearchFetcher(esClient, index) + esFetcher := sourcemap.NewElasticsearchFetcher(esClient, sourcemapIndex) size := 128 cachingFetcher, err := sourcemap.NewBodyCachingFetcher(esFetcher, size, invalidationChan) if err != nil { diff --git a/internal/beater/beater_test.go b/internal/beater/beater_test.go index 95ec20e6aa7..3fc5504c920 100644 --- a/internal/beater/beater_test.go +++ b/internal/beater/beater_test.go @@ -27,7 +27,6 @@ import ( "net/http" "net/http/httptest" "os" - "strings" "testing" "github.com/stretchr/testify/assert" @@ -35,54 +34,9 @@ import ( "github.com/elastic/apm-server/internal/beater/config" "github.com/elastic/apm-server/internal/elasticsearch" - "github.com/elastic/apm-server/internal/version" "github.com/elastic/elastic-agent-libs/monitoring" ) -func TestSourcemapIndexPattern(t *testing.T) { - test := func(t *testing.T, indexPattern, expected string) { - var requestPaths []string - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // Ignore duplicates: we might have two requests since the first one - // will be from the init goroutine - for _, v := range requestPaths { - if v == r.URL.Path { - return - } - } - requestPaths = append(requestPaths, r.URL.Path) - })) - defer srv.Close() - - cfg := config.DefaultConfig() - cfg.RumConfig.Enabled = true - cfg.RumConfig.SourceMapping.ESConfig.Hosts = []string{srv.URL} - if indexPattern != "" { - cfg.RumConfig.SourceMapping.IndexPattern = indexPattern - } - - fetcher, cancel, err := newSourcemapFetcher( - cfg.RumConfig.SourceMapping, - nil, elasticsearch.NewClient, - ) - require.NoError(t, err) - defer cancel() - fetcher.Fetch(context.Background(), "name", "version", "path") - require.Len(t, requestPaths, 1) - - path := requestPaths[0] - path = strings.TrimPrefix(path, "/") - path = strings.TrimSuffix(path, "/_search") - assert.Equal(t, expected, path) - } - t.Run("default-pattern", func(t *testing.T) { - test(t, "", ".apm-source-map") - }) - t.Run("with-observer-version", func(t *testing.T) { - test(t, "blah-%{[observer.version]}-blah", fmt.Sprintf("blah-%s-blah", version.Version)) - }) -} - var validSourcemap, _ = os.ReadFile("../../testdata/sourcemap/bundle.js.map") func TestStoreUsesRUMElasticsearchConfig(t *testing.T) { diff --git a/internal/beater/config/config_test.go b/internal/beater/config/config_test.go index 33e04ee69fe..b488e3b2df9 100644 --- a/internal/beater/config/config_test.go +++ b/internal/beater/config/config_test.go @@ -175,7 +175,6 @@ func TestUnpackConfig(t *testing.T) { "cache": map[string]interface{}{ "expiration": 8 * time.Minute, }, - "index_pattern": "apm-test*", "elasticsearch.hosts": []string{"localhost:9201", "localhost:9202"}, "timeout": "2s", }, @@ -272,7 +271,6 @@ func TestUnpackConfig(t *testing.T) { SourceMapping: SourceMapping{ Enabled: true, Cache: Cache{Expiration: 8 * time.Minute}, - IndexPattern: "apm-test*", ESConfig: &elasticsearch.Config{ Hosts: elasticsearch.Hosts{"localhost:9201", "localhost:9202"}, Protocol: "http", @@ -444,7 +442,6 @@ func TestUnpackConfig(t *testing.T) { Cache: Cache{ Expiration: 7 * time.Second, }, - IndexPattern: ".apm-source-map", ESConfig: elasticsearch.DefaultConfig(), Timeout: 5 * time.Second, }, diff --git a/internal/beater/config/rum.go b/internal/beater/config/rum.go index 6b32111d8fd..a5e33062bc7 100644 --- a/internal/beater/config/rum.go +++ b/internal/beater/config/rum.go @@ -35,7 +35,6 @@ const ( defaultExcludeFromGrouping = "^/webpack" defaultLibraryPattern = "node_modules|bower_components|~" defaultSourcemapCacheExpiration = 5 * time.Minute - defaultSourcemapIndexPattern = ".apm-source-map" defaultSourcemapTimeout = 5 * time.Second ) @@ -54,7 +53,6 @@ type RumConfig struct { type SourceMapping struct { Cache Cache `config:"cache"` Enabled bool `config:"enabled"` - IndexPattern string `config:"index_pattern"` ESConfig *elasticsearch.Config `config:"elasticsearch"` Timeout time.Duration `config:"timeout" validate:"positive"` esConfigured bool @@ -117,7 +115,6 @@ func defaultSourcemapping() SourceMapping { return SourceMapping{ Enabled: true, Cache: Cache{Expiration: defaultSourcemapCacheExpiration}, - IndexPattern: defaultSourcemapIndexPattern, ESConfig: elasticsearch.DefaultConfig(), Timeout: defaultSourcemapTimeout, } diff --git a/systemtest/apmservertest/config.go b/systemtest/apmservertest/config.go index edfc79655b9..0ecb6e3ffef 100644 --- a/systemtest/apmservertest/config.go +++ b/systemtest/apmservertest/config.go @@ -208,7 +208,6 @@ type RUMConfig struct { // RUMSourcemapConfig holds APM Server RUM sourcemap configuration. type RUMSourcemapConfig struct { Enabled bool `json:"enabled,omitempty"` - IndexPattern string `json:"index_pattern"` Cache *RUMSourcemapCacheConfig `json:"cache,omitempty"` } diff --git a/systemtest/sourcemap_test.go b/systemtest/sourcemap_test.go index a486f60c1c8..7bd5c475214 100644 --- a/systemtest/sourcemap_test.go +++ b/systemtest/sourcemap_test.go @@ -194,7 +194,6 @@ func TestSourcemapKibana(t *testing.T) { Sourcemap: &apmservertest.RUMSourcemapConfig{ // Use the wrong index pattern so that the ES fetcher // will fail and apm server will fall back to kibana - IndexPattern: "example", }, } err = srv.Start() From ed6c474cb803292f6dd6db53c4effbcc4f09274f Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Thu, 2 Feb 2023 11:02:55 +0100 Subject: [PATCH 089/123] docs remove outdated comment --- internal/beater/beater.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/beater/beater.go b/internal/beater/beater.go index 03827342ddf..50554f80781 100644 --- a/internal/beater/beater.go +++ b/internal/beater/beater.go @@ -819,7 +819,6 @@ func newSourcemapFetcher( return nil, nil, err } - // For standalone, we query both Kibana and Elasticsearch for backwards compatibility. var chained sourcemap.ChainedFetcher // start background sync job From ffaef7fb3c1c028ce18f8aedb2a2a2e8face730b Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Thu, 2 Feb 2023 11:08:56 +0100 Subject: [PATCH 090/123] refactor: reduce data type visibility --- internal/sourcemap/body_caching.go | 8 +++--- internal/sourcemap/body_caching_test.go | 2 +- internal/sourcemap/chained.go | 2 +- internal/sourcemap/elasticsearch.go | 6 ++-- internal/sourcemap/fetcher.go | 20 ++++++------- internal/sourcemap/kibana.go | 2 +- internal/sourcemap/metadata_fetcher.go | 38 ++++++++++++------------- internal/sourcemap/sourcemap_fetcher.go | 14 ++++----- 8 files changed, 46 insertions(+), 46 deletions(-) diff --git a/internal/sourcemap/body_caching.go b/internal/sourcemap/body_caching.go index 19a08f2ea7e..3e7f1d7d891 100644 --- a/internal/sourcemap/body_caching.go +++ b/internal/sourcemap/body_caching.go @@ -40,7 +40,7 @@ type BodyCachingFetcher struct { func NewBodyCachingFetcher( backend Fetcher, cacheSize int, - invalidationChan <-chan []Identifier, + invalidationChan <-chan []identifier, ) (*BodyCachingFetcher, error) { logger := logp.NewLogger(logs.Sourcemap) @@ -78,7 +78,7 @@ func NewBodyCachingFetcher( // Fetch fetches a source map from the cache or wrapped backend. func (s *BodyCachingFetcher) Fetch(ctx context.Context, name, version, path string) (*sourcemap.Consumer, error) { - key := Identifier{ + key := identifier{ name: name, version: version, path: path, @@ -93,7 +93,7 @@ func (s *BodyCachingFetcher) Fetch(ctx context.Context, name, version, path stri // fetch from the store and ensure caching for all non-temporary results consumer, err := s.backend.Fetch(ctx, name, version, path) if err != nil { - if errors.Is(err, ErrMalformedSourcemap) { + if errors.Is(err, errMalformedSourcemap) { s.add(key, nil) } return nil, err @@ -102,7 +102,7 @@ func (s *BodyCachingFetcher) Fetch(ctx context.Context, name, version, path stri return consumer, nil } -func (s *BodyCachingFetcher) add(key Identifier, consumer *sourcemap.Consumer) { +func (s *BodyCachingFetcher) add(key identifier, consumer *sourcemap.Consumer) { s.cache.Add(key, consumer) if !s.logger.IsDebug() { return diff --git a/internal/sourcemap/body_caching_test.go b/internal/sourcemap/body_caching_test.go index c06c57fe1ef..5ea66cba796 100644 --- a/internal/sourcemap/body_caching_test.go +++ b/internal/sourcemap/body_caching_test.go @@ -50,7 +50,7 @@ func Test_NewCachingFetcher(t *testing.T) { func TestStore_Fetch(t *testing.T) { serviceName, serviceVersion, path := "foo", "1.0.1", "/tmp" - key := Identifier{ + key := identifier{ name: "foo", version: "1.0.1", path: "/tmp", diff --git a/internal/sourcemap/chained.go b/internal/sourcemap/chained.go index 8e6873d10c1..f5fc5558d60 100644 --- a/internal/sourcemap/chained.go +++ b/internal/sourcemap/chained.go @@ -36,7 +36,7 @@ func (c ChainedFetcher) Fetch(ctx context.Context, name, version, path string) ( var lastErr error for _, f := range c { consumer, err := f.Fetch(ctx, name, version, path) - if !errors.Is(err, ErrFetcherUnvailable) { + if !errors.Is(err, errFetcherUnvailable) { return consumer, err } diff --git a/internal/sourcemap/elasticsearch.go b/internal/sourcemap/elasticsearch.go index bafb673746d..beee37f527a 100644 --- a/internal/sourcemap/elasticsearch.go +++ b/internal/sourcemap/elasticsearch.go @@ -84,7 +84,7 @@ func (s *esFetcher) Fetch(ctx context.Context, name, version, path string) (*sou return nil, fmt.Errorf("failed to read ES response body: %w", err) } if resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusForbidden { - return nil, fmt.Errorf("%w: %s: %s", ErrFetcherUnvailable, resp.Status(), string(b)) + return nil, fmt.Errorf("%w: %s: %s", errFetcherUnvailable, resp.Status(), string(b)) } return nil, fmt.Errorf("ES returned unknown status code: %s", resp.Status()) } @@ -115,7 +115,7 @@ func (s *esFetcher) Fetch(ctx context.Context, name, version, path string) (*sou return nil, fmt.Errorf("failed to read sourcemap content: %w", err) } - return ParseSourceMap(uncompressedBody) + return parseSourceMap(uncompressedBody) } func (s *esFetcher) runSearchQuery(ctx context.Context, name, version, path string) (*esapi.Response, error) { @@ -145,7 +145,7 @@ func parse(body io.ReadCloser, name, version, path string, logger *logp.Logger) esSourcemap := esSourcemapResponse.Hits.Hits[0].Source.Sourcemap // until https://github.com/golang/go/issues/19858 is resolved if esSourcemap == "" { - return "", fmt.Errorf("sourcemap not in the expected format: %w", ErrMalformedSourcemap) + return "", fmt.Errorf("sourcemap not in the expected format: %w", errMalformedSourcemap) } return esSourcemap, nil } diff --git a/internal/sourcemap/fetcher.go b/internal/sourcemap/fetcher.go index ecedc132cd4..82b6733b99a 100644 --- a/internal/sourcemap/fetcher.go +++ b/internal/sourcemap/fetcher.go @@ -27,8 +27,8 @@ import ( ) var ( - ErrFetcherUnvailable = errors.New("fetcher unavailable") - ErrMalformedSourcemap = errors.New("sourcemap malformed") + errFetcherUnvailable = errors.New("fetcher unavailable") + errMalformedSourcemap = errors.New("sourcemap malformed") ) // Fetcher is an interface for fetching a source map with a given service name, service version, @@ -42,18 +42,18 @@ type Fetcher interface { // MetadataFetcher is an interface for fetching metadata type MetadataFetcher interface { - GetID(id Identifier) (*Identifier, bool) + getID(id identifier) (*identifier, bool) - Ready() <-chan struct{} + ready() <-chan struct{} } -type Identifier struct { +type identifier struct { name string version string path string } -func GetAliases(name string, version string, bundleFilepath string) []Identifier { +func getAliases(name string, version string, bundleFilepath string) []identifier { urlPath, err := url.Parse(bundleFilepath) if err != nil { // bundleFilepath is not an url so it @@ -76,7 +76,7 @@ func GetAliases(name string, version string, bundleFilepath string) []Identifier // bundleFilepath is a valid url and it is // already clean. // Only return the url path as an alias - return []Identifier{ + return []identifier{ { name: name, version: version, @@ -85,7 +85,7 @@ func GetAliases(name string, version string, bundleFilepath string) []Identifier } } - return []Identifier{ + return []identifier{ // first try to match the full url { name: name, @@ -102,13 +102,13 @@ func GetAliases(name string, version string, bundleFilepath string) []Identifier } } -func ParseSourceMap(data []byte) (*sourcemap.Consumer, error) { +func parseSourceMap(data []byte) (*sourcemap.Consumer, error) { if len(data) == 0 { return nil, nil } consumer, err := sourcemap.Parse("", data) if err != nil { - return nil, fmt.Errorf("%w: %v", ErrMalformedSourcemap, err) + return nil, fmt.Errorf("%w: %v", errMalformedSourcemap, err) } return consumer, nil } diff --git a/internal/sourcemap/kibana.go b/internal/sourcemap/kibana.go index 23373556f50..1e7e1660cea 100644 --- a/internal/sourcemap/kibana.go +++ b/internal/sourcemap/kibana.go @@ -80,7 +80,7 @@ func (s *kibanaFetcher) Fetch(ctx context.Context, name, version, path string) ( continue } if a.Body.ServiceName == name && a.Body.ServiceVersion == version && maybeParseURLPath(a.Body.BundleFilepath) == path { - return ParseSourceMap(a.Body.SourceMap) + return parseSourceMap(a.Body.SourceMap) } } return nil, nil diff --git a/internal/sourcemap/metadata_fetcher.go b/internal/sourcemap/metadata_fetcher.go index 9da30f13423..2c3cf97aecf 100644 --- a/internal/sourcemap/metadata_fetcher.go +++ b/internal/sourcemap/metadata_fetcher.go @@ -40,22 +40,22 @@ const ( type MetadataESFetcher struct { esClient *elasticsearch.Client index string - set map[Identifier]string - alias map[Identifier]*Identifier + set map[identifier]string + alias map[identifier]*identifier mu sync.RWMutex logger *logp.Logger init chan struct{} - invalidationChan chan<- []Identifier + invalidationChan chan<- []identifier } -func NewMetadataFetcher(ctx context.Context, esClient *elasticsearch.Client, index string) (MetadataFetcher, <-chan []Identifier) { - invalidationCh := make(chan []Identifier) +func NewMetadataFetcher(ctx context.Context, esClient *elasticsearch.Client, index string) (MetadataFetcher, <-chan []identifier) { + invalidationCh := make(chan []identifier) s := &MetadataESFetcher{ esClient: esClient, index: index, - set: make(map[Identifier]string), - alias: make(map[Identifier]*Identifier), + set: make(map[identifier]string), + alias: make(map[identifier]*identifier), logger: logp.NewLogger(logs.Sourcemap), init: make(chan struct{}), invalidationChan: invalidationCh, @@ -66,7 +66,7 @@ func NewMetadataFetcher(ctx context.Context, esClient *elasticsearch.Client, ind return s, invalidationCh } -func (s *MetadataESFetcher) GetID(key Identifier) (*Identifier, bool) { +func (s *MetadataESFetcher) getID(key identifier) (*identifier, bool) { s.mu.RLock() defer s.mu.RUnlock() @@ -81,7 +81,7 @@ func (s *MetadataESFetcher) GetID(key Identifier) (*Identifier, bool) { return i, ok } -func (s *MetadataESFetcher) Ready() <-chan struct{} { +func (s *MetadataESFetcher) ready() <-chan struct{} { return s.init } @@ -129,7 +129,7 @@ func (s *MetadataESFetcher) startBackgrounSync(parent context.Context) { } func (s *MetadataESFetcher) sync(ctx context.Context) error { - sourcemaps := make(map[Identifier]string) + sourcemaps := make(map[identifier]string) result, err := s.initialSearch(ctx, sourcemaps) if err != nil { @@ -167,11 +167,11 @@ func (s *MetadataESFetcher) sync(ctx context.Context) error { return nil } -func (s *MetadataESFetcher) update(ctx context.Context, sourcemaps map[Identifier]string) { +func (s *MetadataESFetcher) update(ctx context.Context, sourcemaps map[identifier]string) { s.mu.Lock() defer s.mu.Unlock() - var invalidation []Identifier + var invalidation []identifier for id, contentHash := range s.set { if updatedHash, ok := sourcemaps[id]; ok { @@ -193,7 +193,7 @@ func (s *MetadataESFetcher) update(ctx context.Context, sourcemaps map[Identifie delete(s.set, id) // remove aliases - for _, k := range GetAliases(id.name, id.version, id.path) { + for _, k := range getAliases(id.name, id.version, id.path) { delete(s.alias, k) } } @@ -209,7 +209,7 @@ func (s *MetadataESFetcher) update(ctx context.Context, sourcemaps map[Identifie // The id is then passed over to the backend fetcher // to minimize the size of the lru cache and // and increase cache hits. - for _, k := range GetAliases(id.name, id.version, id.path) { + for _, k := range getAliases(id.name, id.version, id.path) { s.logger.Debugf("Added metadata alias %v -> %v", k, id) s.alias[k] = &id } @@ -218,7 +218,7 @@ func (s *MetadataESFetcher) update(ctx context.Context, sourcemaps map[Identifie s.logger.Debugf("Metadata cache now has %d entries.", len(s.set)) } -func (s *MetadataESFetcher) initialSearch(ctx context.Context, updates map[Identifier]string) (*esSearchSourcemapResponse, error) { +func (s *MetadataESFetcher) initialSearch(ctx context.Context, updates map[identifier]string) (*esSearchSourcemapResponse, error) { resp, err := s.runSearchQuery(ctx) if err != nil { return nil, fmt.Errorf("failed to run initial search query: %w", err) @@ -254,7 +254,7 @@ type esSearchSourcemapResponse struct { esSourcemapResponse } -func (s *MetadataESFetcher) handleUpdateRequest(resp *esapi.Response, updates map[Identifier]string) (*esSearchSourcemapResponse, error) { +func (s *MetadataESFetcher) handleUpdateRequest(resp *esapi.Response, updates map[identifier]string) (*esSearchSourcemapResponse, error) { // handle error response if resp.StatusCode >= http.StatusMultipleChoices { b, err := io.ReadAll(resp.Body) @@ -262,7 +262,7 @@ func (s *MetadataESFetcher) handleUpdateRequest(resp *esapi.Response, updates ma return nil, fmt.Errorf("failed to read response body: %w", err) } if resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusForbidden { - return nil, fmt.Errorf("%w: %s: %s", ErrFetcherUnvailable, resp.Status(), string(b)) + return nil, fmt.Errorf("%w: %s: %s", errFetcherUnvailable, resp.Status(), string(b)) } return nil, fmt.Errorf("ES returned unknown status code: %s", resp.Status()) } @@ -274,7 +274,7 @@ func (s *MetadataESFetcher) handleUpdateRequest(resp *esapi.Response, updates ma } for _, v := range body.Hits.Hits { - id := Identifier{ + id := identifier{ name: v.Source.Service.Name, version: v.Source.Service.Version, path: v.Source.File.BundleFilepath, @@ -300,7 +300,7 @@ func parseResponse(body io.ReadCloser, logger *logp.Logger) (*esSearchSourcemapR return &esSourcemapResponse, nil } -func (s *MetadataESFetcher) scrollsearch(ctx context.Context, scrollID string, updates map[Identifier]string) (*esSearchSourcemapResponse, error) { +func (s *MetadataESFetcher) scrollsearch(ctx context.Context, scrollID string, updates map[identifier]string) (*esSearchSourcemapResponse, error) { resp, err := s.runScrollSearchQuery(ctx, scrollID) if err != nil { return nil, fmt.Errorf("failed to run scroll search query: %w", err) diff --git a/internal/sourcemap/sourcemap_fetcher.go b/internal/sourcemap/sourcemap_fetcher.go index e6de76773a6..03b99dfa57a 100644 --- a/internal/sourcemap/sourcemap_fetcher.go +++ b/internal/sourcemap/sourcemap_fetcher.go @@ -45,13 +45,13 @@ func NewSourcemapFetcher(metadata MetadataFetcher, backend Fetcher) *SourcemapFe } func (s *SourcemapFetcher) Fetch(ctx context.Context, name, version, path string) (*sourcemap.Consumer, error) { - original := Identifier{name: name, version: version, path: path} + original := identifier{name: name, version: version, path: path} select { - case <-s.metadata.Ready(): + case <-s.metadata.ready(): // the mutex is shared by the update goroutine, we need to release it // as soon as possible to avoid blocking updates. - if i, ok := s.metadata.GetID(original); ok { + if i, ok := s.metadata.getID(original); ok { // Only fetch from ES if the sourcemap id exists return s.fetch(ctx, i) } @@ -67,7 +67,7 @@ func (s *SourcemapFetcher) Fetch(ctx context.Context, name, version, path string // Aliases are only available after init is completed. select { - case <-s.metadata.Ready(): + case <-s.metadata.ready(): case <-ctx.Done(): return nil, fmt.Errorf("error waiting for metadata fetcher to be ready: %w", ctx.Err()) } @@ -75,7 +75,7 @@ func (s *SourcemapFetcher) Fetch(ctx context.Context, name, version, path string // first map lookup will fail but this is not going // to be performance issue since it only happens if init // is in progress. - if i, ok := s.metadata.GetID(original); ok { + if i, ok := s.metadata.getID(original); ok { // Only fetch from ES if the sourcemap id exists return s.fetch(ctx, i) } @@ -91,7 +91,7 @@ func (s *SourcemapFetcher) Fetch(ctx context.Context, name, version, path string // but a request came in from a different host. // Look for an alias to the url path to retrieve the correct // host and fetch the sourcemap - if i, ok := s.metadata.GetID(original); ok { + if i, ok := s.metadata.getID(original); ok { return s.fetch(ctx, i) } } @@ -112,7 +112,7 @@ func (s *SourcemapFetcher) Fetch(ctx context.Context, name, version, path string return nil, fmt.Errorf("unable to find sourcemap.url for service.name=%s service.version=%s bundle.path=%s", name, version, path) } -func (s *SourcemapFetcher) fetch(ctx context.Context, key *Identifier) (*sourcemap.Consumer, error) { +func (s *SourcemapFetcher) fetch(ctx context.Context, key *identifier) (*sourcemap.Consumer, error) { c, err := s.backend.Fetch(ctx, key.name, key.version, key.path) // log a message if the sourcemap is present in the cache but the backend fetcher did not From c4e94072a5b81c453944ec18ebd26c1b993cac55 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Thu, 2 Feb 2023 13:26:28 +0100 Subject: [PATCH 091/123] lint: fix linter issues --- internal/beater/config/config_test.go | 8 ++++---- internal/beater/config/rum.go | 8 ++++---- systemtest/apmservertest/config.go | 4 ++-- systemtest/sourcemap_test.go | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/internal/beater/config/config_test.go b/internal/beater/config/config_test.go index b488e3b2df9..ebec6ca5e76 100644 --- a/internal/beater/config/config_test.go +++ b/internal/beater/config/config_test.go @@ -269,8 +269,8 @@ func TestUnpackConfig(t *testing.T) { AllowOrigins: []string{"example*"}, AllowHeaders: []string{"Authorization"}, SourceMapping: SourceMapping{ - Enabled: true, - Cache: Cache{Expiration: 8 * time.Minute}, + Enabled: true, + Cache: Cache{Expiration: 8 * time.Minute}, ESConfig: &elasticsearch.Config{ Hosts: elasticsearch.Hosts{"localhost:9201", "localhost:9202"}, Protocol: "http", @@ -442,8 +442,8 @@ func TestUnpackConfig(t *testing.T) { Cache: Cache{ Expiration: 7 * time.Second, }, - ESConfig: elasticsearch.DefaultConfig(), - Timeout: 5 * time.Second, + ESConfig: elasticsearch.DefaultConfig(), + Timeout: 5 * time.Second, }, LibraryPattern: "rum", ExcludeFromGrouping: "^/webpack", diff --git a/internal/beater/config/rum.go b/internal/beater/config/rum.go index a5e33062bc7..53ab5d3d6e3 100644 --- a/internal/beater/config/rum.go +++ b/internal/beater/config/rum.go @@ -113,10 +113,10 @@ func (s *SourceMapping) Unpack(inp *config.C) error { func defaultSourcemapping() SourceMapping { return SourceMapping{ - Enabled: true, - Cache: Cache{Expiration: defaultSourcemapCacheExpiration}, - ESConfig: elasticsearch.DefaultConfig(), - Timeout: defaultSourcemapTimeout, + Enabled: true, + Cache: Cache{Expiration: defaultSourcemapCacheExpiration}, + ESConfig: elasticsearch.DefaultConfig(), + Timeout: defaultSourcemapTimeout, } } diff --git a/systemtest/apmservertest/config.go b/systemtest/apmservertest/config.go index 0ecb6e3ffef..3c4de573d92 100644 --- a/systemtest/apmservertest/config.go +++ b/systemtest/apmservertest/config.go @@ -207,8 +207,8 @@ type RUMConfig struct { // RUMSourcemapConfig holds APM Server RUM sourcemap configuration. type RUMSourcemapConfig struct { - Enabled bool `json:"enabled,omitempty"` - Cache *RUMSourcemapCacheConfig `json:"cache,omitempty"` + Enabled bool `json:"enabled,omitempty"` + Cache *RUMSourcemapCacheConfig `json:"cache,omitempty"` } // RUMSourcemapCacheConfig holds sourcemap cache expiration. diff --git a/systemtest/sourcemap_test.go b/systemtest/sourcemap_test.go index 7bd5c475214..3704c27e5c8 100644 --- a/systemtest/sourcemap_test.go +++ b/systemtest/sourcemap_test.go @@ -190,7 +190,7 @@ func TestSourcemapKibana(t *testing.T) { srv := apmservertest.NewUnstartedServerTB(t) srv.Config.RUM = &apmservertest.RUMConfig{ - Enabled: true, + Enabled: true, Sourcemap: &apmservertest.RUMSourcemapConfig{ // Use the wrong index pattern so that the ES fetcher // will fail and apm server will fall back to kibana From 6d1472cda1e48533f850ebc4318180e32b677c17 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Thu, 2 Feb 2023 13:32:53 +0100 Subject: [PATCH 092/123] fix: update sourcemap hash if changed --- internal/sourcemap/metadata_fetcher.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/sourcemap/metadata_fetcher.go b/internal/sourcemap/metadata_fetcher.go index 2c3cf97aecf..397d71166ca 100644 --- a/internal/sourcemap/metadata_fetcher.go +++ b/internal/sourcemap/metadata_fetcher.go @@ -178,8 +178,10 @@ func (s *MetadataESFetcher) update(ctx context.Context, sourcemaps map[identifie // already in the cache, remove from the updates. delete(sourcemaps, id) - // content hash changed, invalidate the sourcemap cache + // content hash changed, invalidate the sourcemap cache and update hash if contentHash != updatedHash { + sourcemaps[id] = updatedHash + s.logger.Debugf("Hash changed: %s -> %s: invalidating %v", contentHash, updatedHash, id) invalidation = append(invalidation, id) } From ab147c5280d8d37c3614af301c4303291e1786c4 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Thu, 2 Feb 2023 15:43:07 +0100 Subject: [PATCH 093/123] test: add es unavailable sourcemap test --- systemtest/apmservertest/config.go | 5 +++-- systemtest/sourcemap_test.go | 28 ++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/systemtest/apmservertest/config.go b/systemtest/apmservertest/config.go index 3c4de573d92..4b478c918f8 100644 --- a/systemtest/apmservertest/config.go +++ b/systemtest/apmservertest/config.go @@ -207,8 +207,9 @@ type RUMConfig struct { // RUMSourcemapConfig holds APM Server RUM sourcemap configuration. type RUMSourcemapConfig struct { - Enabled bool `json:"enabled,omitempty"` - Cache *RUMSourcemapCacheConfig `json:"cache,omitempty"` + Enabled bool `json:"enabled,omitempty"` + Cache *RUMSourcemapCacheConfig `json:"cache,omitempty"` + ESConfig *ElasticsearchOutputConfig `json:"elasticsearch"` } // RUMSourcemapCacheConfig holds sourcemap cache expiration. diff --git a/systemtest/sourcemap_test.go b/systemtest/sourcemap_test.go index 3704c27e5c8..ff0b0d13620 100644 --- a/systemtest/sourcemap_test.go +++ b/systemtest/sourcemap_test.go @@ -179,6 +179,34 @@ func TestSourcemapElasticsearch(t *testing.T) { assertSourcemapUpdated(t, result, true) } +func TestSourcemapElasticsearchUnreachable(t *testing.T) { + systemtest.CleanupElasticsearch(t) + + sourcemap, err := os.ReadFile("../testdata/sourcemap/bundle.js.map") + require.NoError(t, err) + systemtest.CreateSourceMap(t, string(sourcemap), "apm-agent-js", "1.0.1", + "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", + ) + + srv := apmservertest.NewUnstartedServerTB(t) + srv.Config.RUM = &apmservertest.RUMConfig{ + Enabled: true, + Sourcemap: &apmservertest.RUMSourcemapConfig{ + ESConfig: &apmservertest.ElasticsearchOutputConfig{ + // Use an unreachable address + Hosts: []string{"127.0.0.1:12345"}, + }, + }, + } + err = srv.Start() + require.NoError(t, err) + + // Index an error, applying source mapping and caching the source map in the process. + systemtest.SendRUMEventsPayload(t, srv.URL, "../testdata/intake-v2/errors_rum.ndjson") + result := systemtest.Elasticsearch.ExpectDocs(t, "logs-apm.error-*", nil) + assertSourcemapUpdated(t, result, true) +} + func TestSourcemapKibana(t *testing.T) { systemtest.CleanupElasticsearch(t) From dccaec4a825a9444de7095f2365208498b294955 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Thu, 2 Feb 2023 15:43:38 +0100 Subject: [PATCH 094/123] fix: handle es unreachable and clarify status code handling code --- internal/sourcemap/elasticsearch.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/internal/sourcemap/elasticsearch.go b/internal/sourcemap/elasticsearch.go index beee37f527a..17d93da07b2 100644 --- a/internal/sourcemap/elasticsearch.go +++ b/internal/sourcemap/elasticsearch.go @@ -23,8 +23,10 @@ import ( "context" "encoding/base64" "encoding/json" + "errors" "fmt" "io" + "net" "net/http" "github.com/go-sourcemap/sourcemap" @@ -73,6 +75,10 @@ func NewElasticsearchFetcher(c *elasticsearch.Client, index string) Fetcher { func (s *esFetcher) Fetch(ctx context.Context, name, version, path string) (*sourcemap.Consumer, error) { resp, err := s.runSearchQuery(ctx, name, version, path) if err != nil { + var networkErr net.Error + if errors.As(err, &networkErr) { + return nil, fmt.Errorf("failed to reach elasticsearch: %w, %v: ", errFetcherUnvailable, err) + } return nil, fmt.Errorf("failure querying ES: %w", err) } defer resp.Body.Close() @@ -84,6 +90,10 @@ func (s *esFetcher) Fetch(ctx context.Context, name, version, path string) (*sou return nil, fmt.Errorf("failed to read ES response body: %w", err) } if resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusForbidden { + // http.StatusNotFound -> the index is missing + // http.StatusForbidden -> we don't have permission to read from the index + // In both cases we consider the fetcher unavailable so that APM Server can + // fallback to other fetchers return nil, fmt.Errorf("%w: %s: %s", errFetcherUnvailable, resp.Status(), string(b)) } return nil, fmt.Errorf("ES returned unknown status code: %s", resp.Status()) From 067202bc9a9ce5e73fd6e9dcf635a8cd16f20bfd Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Thu, 2 Feb 2023 15:46:51 +0100 Subject: [PATCH 095/123] test: fix kibana sourcemap test --- systemtest/sourcemap_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/systemtest/sourcemap_test.go b/systemtest/sourcemap_test.go index ff0b0d13620..4b1fc1e96b2 100644 --- a/systemtest/sourcemap_test.go +++ b/systemtest/sourcemap_test.go @@ -220,8 +220,11 @@ func TestSourcemapKibana(t *testing.T) { srv.Config.RUM = &apmservertest.RUMConfig{ Enabled: true, Sourcemap: &apmservertest.RUMSourcemapConfig{ - // Use the wrong index pattern so that the ES fetcher + // Use the wrong credentials so that the ES fetcher // will fail and apm server will fall back to kibana + ESConfig: &apmservertest.ElasticsearchOutputConfig{ + APIKey: "example", + }, }, } err = srv.Start() From 5ff04ac33be958a867208004da70675c3dd17342 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Thu, 2 Feb 2023 15:47:10 +0100 Subject: [PATCH 096/123] fix: handle http status 401 correctly --- internal/sourcemap/elasticsearch.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/sourcemap/elasticsearch.go b/internal/sourcemap/elasticsearch.go index 17d93da07b2..9a9aa3ea2e5 100644 --- a/internal/sourcemap/elasticsearch.go +++ b/internal/sourcemap/elasticsearch.go @@ -89,7 +89,7 @@ func (s *esFetcher) Fetch(ctx context.Context, name, version, path string) (*sou if err != nil { return nil, fmt.Errorf("failed to read ES response body: %w", err) } - if resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusForbidden { + if resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden { // http.StatusNotFound -> the index is missing // http.StatusForbidden -> we don't have permission to read from the index // In both cases we consider the fetcher unavailable so that APM Server can From e64ddb3c3f363b93d9f66523d7afe24805e277ba Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Thu, 2 Feb 2023 16:02:45 +0100 Subject: [PATCH 097/123] refactor: simplify chained fetcher logic --- internal/sourcemap/chained.go | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/internal/sourcemap/chained.go b/internal/sourcemap/chained.go index f5fc5558d60..61b5d761aa2 100644 --- a/internal/sourcemap/chained.go +++ b/internal/sourcemap/chained.go @@ -36,18 +36,16 @@ func (c ChainedFetcher) Fetch(ctx context.Context, name, version, path string) ( var lastErr error for _, f := range c { consumer, err := f.Fetch(ctx, name, version, path) + // if there are no errors or the error is not errFetcherUnvailable + // then the fetcher is working/available. + // Return the result: error and consumer. if !errors.Is(err, errFetcherUnvailable) { return consumer, err } - if err != nil { - lastErr = err - continue - } - - if consumer != nil { - return consumer, nil - } + // err is errFetcherUnvailable + // store it in a tmp variable and try the next fetcher + lastErr = err } return nil, lastErr } From 9c19470ae1391e91ddc42bd7aef3886c8e9a360f Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Mon, 6 Feb 2023 10:02:07 +0100 Subject: [PATCH 098/123] refactor: update invalidation goroutine log message level --- internal/sourcemap/body_caching.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/sourcemap/body_caching.go b/internal/sourcemap/body_caching.go index 3e7f1d7d891..ff7971b110b 100644 --- a/internal/sourcemap/body_caching.go +++ b/internal/sourcemap/body_caching.go @@ -56,7 +56,7 @@ func NewBodyCachingFetcher( } go func() { - logger.Info("listening for invalidation...") + logger.Debug("listening for invalidation...") for arr := range invalidationChan { for _, id := range arr { From 55f95c9945cd563261837d1e5e1327297888dceb Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Mon, 6 Feb 2023 14:21:21 +0100 Subject: [PATCH 099/123] fix: create a new context when falling back in the chained fetcher --- internal/sourcemap/chained.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/internal/sourcemap/chained.go b/internal/sourcemap/chained.go index 61b5d761aa2..408db500759 100644 --- a/internal/sourcemap/chained.go +++ b/internal/sourcemap/chained.go @@ -20,6 +20,7 @@ package sourcemap import ( "context" "errors" + "time" "github.com/go-sourcemap/sourcemap" ) @@ -43,6 +44,13 @@ func (c ChainedFetcher) Fetch(ctx context.Context, name, version, path string) ( return consumer, err } + // previous fetcher is unavailable but the deadline expired so we cannot reuse that + if t, _ := ctx.Deadline(); t.Before(time.Now()) { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(context.Background(), 500 * time.Millisecond) + defer cancel() + } + // err is errFetcherUnvailable // store it in a tmp variable and try the next fetcher lastErr = err From 868eb55f0c5261761070465e697b95987eca4790 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Mon, 6 Feb 2023 14:21:45 +0100 Subject: [PATCH 100/123] refactor: update elasticsearch fetcher error message --- internal/sourcemap/elasticsearch.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/sourcemap/elasticsearch.go b/internal/sourcemap/elasticsearch.go index 9a9aa3ea2e5..11253a10ad4 100644 --- a/internal/sourcemap/elasticsearch.go +++ b/internal/sourcemap/elasticsearch.go @@ -77,7 +77,7 @@ func (s *esFetcher) Fetch(ctx context.Context, name, version, path string) (*sou if err != nil { var networkErr net.Error if errors.As(err, &networkErr) { - return nil, fmt.Errorf("failed to reach elasticsearch: %w, %v: ", errFetcherUnvailable, err) + return nil, fmt.Errorf("failed to reach elasticsearch: %w: %v ", errFetcherUnvailable, err) } return nil, fmt.Errorf("failure querying ES: %w", err) } From be3ea36d7e486755931d488e7fbdc90a58a8f24a Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Mon, 6 Feb 2023 15:15:02 +0100 Subject: [PATCH 101/123] lint: fix method name typo --- internal/sourcemap/metadata_fetcher.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/sourcemap/metadata_fetcher.go b/internal/sourcemap/metadata_fetcher.go index 397d71166ca..a731b6477ae 100644 --- a/internal/sourcemap/metadata_fetcher.go +++ b/internal/sourcemap/metadata_fetcher.go @@ -61,7 +61,7 @@ func NewMetadataFetcher(ctx context.Context, esClient *elasticsearch.Client, ind invalidationChan: invalidationCh, } - s.startBackgrounSync(ctx) + s.startBackgroundSync(ctx) return s, invalidationCh } @@ -85,7 +85,7 @@ func (s *MetadataESFetcher) ready() <-chan struct{} { return s.init } -func (s *MetadataESFetcher) startBackgrounSync(parent context.Context) { +func (s *MetadataESFetcher) startBackgroundSync(parent context.Context) { go func() { s.logger.Debug("populating metadata cache") From 54be00c24785edfb2eaa2d056ad4a0b67b10bd09 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Mon, 6 Feb 2023 15:17:03 +0100 Subject: [PATCH 102/123] refactor: avoid additional gorutine --- internal/sourcemap/metadata_fetcher.go | 40 ++++++++++++-------------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/internal/sourcemap/metadata_fetcher.go b/internal/sourcemap/metadata_fetcher.go index a731b6477ae..efd4b355159 100644 --- a/internal/sourcemap/metadata_fetcher.go +++ b/internal/sourcemap/metadata_fetcher.go @@ -102,29 +102,27 @@ func (s *MetadataESFetcher) startBackgroundSync(parent context.Context) { s.logger.Info("init routine completed") } - go func() { - // TODO make this a config option ? - t := time.NewTicker(30 * time.Second) - defer t.Stop() - - for { - select { - case <-t.C: - ctx, cancel := context.WithTimeout(parent, syncTimeout) - - if err := s.sync(ctx); err != nil { - s.logger.Errorf("failed to sync sourcemaps metadata: %v", err) - } - - cancel() - case <-parent.Done(): - s.logger.Info("update routine done") - // close invalidation channel - close(s.invalidationChan) - return + // TODO make this a config option ? + t := time.NewTicker(30 * time.Second) + defer t.Stop() + + for { + select { + case <-t.C: + ctx, cancel := context.WithTimeout(parent, syncTimeout) + + if err := s.sync(ctx); err != nil { + s.logger.Errorf("failed to sync sourcemaps metadata: %v", err) } + + cancel() + case <-parent.Done(): + s.logger.Info("update routine done") + // close invalidation channel + close(s.invalidationChan) + return } - }() + } }() } From 17b485b29a2711231958ced503adf590b08642e3 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Mon, 6 Feb 2023 15:19:43 +0100 Subject: [PATCH 103/123] refactor: remove isdebug check --- internal/sourcemap/body_caching.go | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/internal/sourcemap/body_caching.go b/internal/sourcemap/body_caching.go index ff7971b110b..75eacee1f60 100644 --- a/internal/sourcemap/body_caching.go +++ b/internal/sourcemap/body_caching.go @@ -45,9 +45,6 @@ func NewBodyCachingFetcher( logger := logp.NewLogger(logs.Sourcemap) lruCache, err := lru.NewWithEvict(cacheSize, func(key, value interface{}) { - if !logger.IsDebug() { - return - } logger.Debugf("Removed id %v", key) }) @@ -60,10 +57,7 @@ func NewBodyCachingFetcher( for arr := range invalidationChan { for _, id := range arr { - if logger.IsDebug() { - logger.Debugf("Invalidating id %v", id) - } - + logger.Debugf("Invalidating id %v", id) lruCache.Remove(id) } } @@ -104,8 +98,5 @@ func (s *BodyCachingFetcher) Fetch(ctx context.Context, name, version, path stri func (s *BodyCachingFetcher) add(key identifier, consumer *sourcemap.Consumer) { s.cache.Add(key, consumer) - if !s.logger.IsDebug() { - return - } s.logger.Debugf("Added id %v. Cache now has %v entries.", key, s.cache.Len()) } From 557e4a3aaa3c2a04c523d13ae525e2e3afb3a8f4 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Mon, 6 Feb 2023 17:11:42 +0100 Subject: [PATCH 104/123] refactor: update es fetcher to use get api --- internal/sourcemap/elasticsearch.go | 72 +++++++------------------- internal/sourcemap/metadata_fetcher.go | 21 ++++++++ 2 files changed, 41 insertions(+), 52 deletions(-) diff --git a/internal/sourcemap/elasticsearch.go b/internal/sourcemap/elasticsearch.go index 11253a10ad4..72302a3b191 100644 --- a/internal/sourcemap/elasticsearch.go +++ b/internal/sourcemap/elasticsearch.go @@ -44,25 +44,19 @@ type esFetcher struct { logger *logp.Logger } -type esSourcemapResponse struct { - Hits struct { - Total struct { - Value int `json:"value"` - } `json:"total"` - Hits []struct { - Source struct { - Service struct { - Name string `json:"name"` - Version string `json:"version"` - } `json:"service"` - File struct { - BundleFilepath string `json:"path"` - } `json:"file"` - Sourcemap string `json:"content"` - ContentHash string `json:"content_sha256"` - } `json:"_source"` - } `json:"hits"` - } `json:"hits"` +type esGetSourcemapResponse struct { + Found bool `json:"found"` + Source struct { + Service struct { + Name string `json:"name"` + Version string `json:"version"` + } `json:"service"` + File struct { + BundleFilepath string `json:"path"` + } `json:"file"` + Sourcemap string `json:"content"` + ContentHash string `json:"content_sha256"` + } `json:"_source"` } // NewElasticsearchFetcher returns a Fetcher for fetching source maps stored in Elasticsearch. @@ -129,49 +123,23 @@ func (s *esFetcher) Fetch(ctx context.Context, name, version, path string) (*sou } func (s *esFetcher) runSearchQuery(ctx context.Context, name, version, path string) (*esapi.Response, error) { - var buf bytes.Buffer - if err := json.NewEncoder(&buf).Encode(requestBody(name, version, path)); err != nil { - return nil, fmt.Errorf("failed to encode request body: %w", err) - } - req := esapi.SearchRequest{ - Index: []string{s.index}, - Body: &buf, - TrackTotalHits: true, + id := name + "-" + version + "-" + path + req := esapi.GetRequest{ + Index: s.index, + DocumentID: id, } return req.Do(ctx, s.client) } func parse(body io.ReadCloser, name, version, path string, logger *logp.Logger) (string, error) { - var esSourcemapResponse esSourcemapResponse + var esSourcemapResponse esGetSourcemapResponse if err := json.NewDecoder(body).Decode(&esSourcemapResponse); err != nil { return "", fmt.Errorf("failed to decode sourcemap: %w", err) } - hits := esSourcemapResponse.Hits.Total.Value - if hits == 0 || len(esSourcemapResponse.Hits.Hits) == 0 { + if !esSourcemapResponse.Found { return "", nil } - esSourcemap := esSourcemapResponse.Hits.Hits[0].Source.Sourcemap - // until https://github.com/golang/go/issues/19858 is resolved - if esSourcemap == "" { - return "", fmt.Errorf("sourcemap not in the expected format: %w", errMalformedSourcemap) - } - return esSourcemap, nil -} - -func requestBody(name, version, path string) map[string]interface{} { - id := name + "-" + version + "-" + path - - return search( - size(1), - source("content"), - query( - boolean( - must( - term("_id", id), - ), - ), - ), - ) + return esSourcemapResponse.Source.Sourcemap, nil } diff --git a/internal/sourcemap/metadata_fetcher.go b/internal/sourcemap/metadata_fetcher.go index efd4b355159..7f2bc87396c 100644 --- a/internal/sourcemap/metadata_fetcher.go +++ b/internal/sourcemap/metadata_fetcher.go @@ -254,6 +254,27 @@ type esSearchSourcemapResponse struct { esSourcemapResponse } +type esSourcemapResponse struct { + Hits struct { + Total struct { + Value int `json:"value"` + } `json:"total"` + Hits []struct { + Source struct { + Service struct { + Name string `json:"name"` + Version string `json:"version"` + } `json:"service"` + File struct { + BundleFilepath string `json:"path"` + } `json:"file"` + Sourcemap string `json:"content"` + ContentHash string `json:"content_sha256"` + } `json:"_source"` + } `json:"hits"` + } `json:"hits"` +} + func (s *MetadataESFetcher) handleUpdateRequest(resp *esapi.Response, updates map[identifier]string) (*esSearchSourcemapResponse, error) { // handle error response if resp.StatusCode >= http.StatusMultipleChoices { From af66eda5dec7b23be8ca5934060035d91e507bca Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Mon, 6 Feb 2023 18:20:12 +0100 Subject: [PATCH 105/123] refactor: use es library source field instead of encoding the request body manually --- internal/sourcemap/metadata_fetcher.go | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/internal/sourcemap/metadata_fetcher.go b/internal/sourcemap/metadata_fetcher.go index 7f2bc87396c..cbf219cf50e 100644 --- a/internal/sourcemap/metadata_fetcher.go +++ b/internal/sourcemap/metadata_fetcher.go @@ -18,7 +18,6 @@ package sourcemap import ( - "bytes" "context" "encoding/json" "fmt" @@ -229,26 +228,15 @@ func (s *MetadataESFetcher) initialSearch(ctx context.Context, updates map[ident } func (s *MetadataESFetcher) runSearchQuery(ctx context.Context) (*esapi.Response, error) { - var buf bytes.Buffer - if err := json.NewEncoder(&buf).Encode(queryMetadata()); err != nil { - return nil, fmt.Errorf("failed to encode metadata query: %w", err) - } - req := esapi.SearchRequest{ Index: []string{s.index}, - Body: &buf, + Source: []string{"service.*", "file.path", "content_sha256"}, TrackTotalHits: true, Scroll: time.Minute, } return req.Do(ctx, s.esClient) } -func queryMetadata() map[string]interface{} { - return search( - sources([]string{"service.*", "file.path", "content_sha256"}), - ) -} - type esSearchSourcemapResponse struct { ScrollID string `json:"_scroll_id"` esSourcemapResponse From 6eb77468085ccaa1d9b4d6291702d0305285da42 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Mon, 6 Feb 2023 18:20:45 +0100 Subject: [PATCH 106/123] refactor: remove unused search.go file --- internal/sourcemap/search.go | 70 ------------------------------------ 1 file changed, 70 deletions(-) delete mode 100644 internal/sourcemap/search.go diff --git a/internal/sourcemap/search.go b/internal/sourcemap/search.go deleted file mode 100644 index 39593d9ecad..00000000000 --- a/internal/sourcemap/search.go +++ /dev/null @@ -1,70 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package sourcemap - -type searchOption func(map[string]interface{}) - -func search(opts ...searchOption) map[string]interface{} { - m := make(map[string]interface{}, len(opts)) - - for _, opt := range opts { - opt(m) - } - - return m -} - -func source(s string) searchOption { - return func(m map[string]interface{}) { - m["_source"] = s - } -} - -func sources(s []string) searchOption { - return func(m map[string]interface{}) { - m["_source"] = s - } -} - -func size(i int) searchOption { - return func(m map[string]interface{}) { - m["size"] = 1 - } -} - -func query(q map[string]interface{}) searchOption { - return func(m map[string]interface{}) { - m["query"] = q - } -} - -func wrap(k string, v interface{}) map[string]interface{} { - return map[string]interface{}{k: v} -} - -func boolean(clause map[string]interface{}) map[string]interface{} { - return wrap("bool", clause) -} - -func must(clauses ...map[string]interface{}) map[string]interface{} { - return wrap("must", clauses) -} - -func term(k, v string) map[string]interface{} { - return wrap("term", wrap(k, v)) -} From 2ac90e09c23e2dcd1ec5da53e871033adce0118a Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Mon, 6 Feb 2023 18:21:17 +0100 Subject: [PATCH 107/123] fix: encode document id when building the search query --- internal/sourcemap/elasticsearch.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/sourcemap/elasticsearch.go b/internal/sourcemap/elasticsearch.go index 72302a3b191..d0a1139b285 100644 --- a/internal/sourcemap/elasticsearch.go +++ b/internal/sourcemap/elasticsearch.go @@ -28,6 +28,7 @@ import ( "io" "net" "net/http" + "net/url" "github.com/go-sourcemap/sourcemap" @@ -126,7 +127,7 @@ func (s *esFetcher) runSearchQuery(ctx context.Context, name, version, path stri id := name + "-" + version + "-" + path req := esapi.GetRequest{ Index: s.index, - DocumentID: id, + DocumentID: url.PathEscape(id), } return req.Do(ctx, s.client) } From a44b82a2d5ac9c5fa0b3afc33357000608bb515d Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Tue, 7 Feb 2023 01:09:02 +0100 Subject: [PATCH 108/123] feat: block until the metadata cache is populated --- internal/sourcemap/fetcher.go | 2 + internal/sourcemap/metadata_fetcher.go | 54 ++++++++++++++++++++----- internal/sourcemap/sourcemap_fetcher.go | 38 +++++------------ 3 files changed, 57 insertions(+), 37 deletions(-) diff --git a/internal/sourcemap/fetcher.go b/internal/sourcemap/fetcher.go index 82b6733b99a..ea9c6495d94 100644 --- a/internal/sourcemap/fetcher.go +++ b/internal/sourcemap/fetcher.go @@ -45,6 +45,8 @@ type MetadataFetcher interface { getID(id identifier) (*identifier, bool) ready() <-chan struct{} + + err() error } type identifier struct { diff --git a/internal/sourcemap/metadata_fetcher.go b/internal/sourcemap/metadata_fetcher.go index cbf219cf50e..44a21bd2289 100644 --- a/internal/sourcemap/metadata_fetcher.go +++ b/internal/sourcemap/metadata_fetcher.go @@ -20,6 +20,7 @@ package sourcemap import ( "context" "encoding/json" + "errors" "fmt" "io" "net/http" @@ -44,6 +45,7 @@ type MetadataESFetcher struct { mu sync.RWMutex logger *logp.Logger init chan struct{} + initErr error invalidationChan chan<- []identifier } @@ -84,23 +86,47 @@ func (s *MetadataESFetcher) ready() <-chan struct{} { return s.init } +func (s *MetadataESFetcher) err() error { + select { + case <-s.ready(): + s.mu.RLock() + defer s.mu.RUnlock() + return s.initErr + default: + return errors.New("metadata es fetcher not ready") + } +} + func (s *MetadataESFetcher) startBackgroundSync(parent context.Context) { go func() { s.logger.Debug("populating metadata cache") - // First run, populate cache - ctx, cancel := context.WithTimeout(parent, syncTimeout) - defer cancel() + ctx, cancel := context.WithTimeout(parent, 1*time.Second) + err := s.ping(ctx) + cancel() - if err := s.sync(ctx); err != nil { - s.logger.Errorf("failed to fetch sourcemaps metadata: %v", err) + if err != nil { + // it is fine to not lock here since err will not access + // initErr until the init channel is closed. + s.initErr = fmt.Errorf("failed to ping es cluster: %w: %v", errFetcherUnvailable, err) + s.logger.Error(s.initErr) } else { - // only close the init chan and mark the fetcher as ready if - // sync succeeded - close(s.init) - s.logger.Info("init routine completed") + // First run, populate cache + ctx, cancel = context.WithTimeout(parent, syncTimeout) + err := s.sync(ctx) + cancel() + + if err != nil { + s.logger.Errorf("failed to fetch sourcemaps metadata: %v", err) + } else { + // only close the init chan and mark the fetcher as ready if + // sync succeeded + s.logger.Info("init routine completed") + } } + close(s.init) + // TODO make this a config option ? t := time.NewTicker(30 * time.Second) defer t.Stop() @@ -125,6 +151,16 @@ func (s *MetadataESFetcher) startBackgroundSync(parent context.Context) { }() } +func (s *MetadataESFetcher) ping(ctx context.Context) error { + // we cannot use PingRequest because the library is + // building a broken url and the request is timing out. + req := esapi.IndicesGetRequest{ + Index: []string{s.index}, + } + _, err := req.Do(ctx, s.esClient) + return err +} + func (s *MetadataESFetcher) sync(ctx context.Context) error { sourcemaps := make(map[identifier]string) diff --git a/internal/sourcemap/sourcemap_fetcher.go b/internal/sourcemap/sourcemap_fetcher.go index 03b99dfa57a..689ad393269 100644 --- a/internal/sourcemap/sourcemap_fetcher.go +++ b/internal/sourcemap/sourcemap_fetcher.go @@ -49,36 +49,18 @@ func (s *SourcemapFetcher) Fetch(ctx context.Context, name, version, path string select { case <-s.metadata.ready(): - // the mutex is shared by the update goroutine, we need to release it - // as soon as possible to avoid blocking updates. - if i, ok := s.metadata.getID(original); ok { - // Only fetch from ES if the sourcemap id exists - return s.fetch(ctx, i) - } - default: - s.logger.Debugf("Metadata cache not populated. Falling back to backend fetcher for id: %s, %s, %s", name, version, path) - // init is in progress, ignore the metadata cache and fetch the sourcemap directly - // return if we get a valid sourcemap or an error - if c, err := s.backend.Fetch(ctx, original.name, original.version, original.path); c != nil || err != nil { - return c, err - } - - s.logger.Debug("Blocking until init is completed") - - // Aliases are only available after init is completed. - select { - case <-s.metadata.ready(): - case <-ctx.Done(): - return nil, fmt.Errorf("error waiting for metadata fetcher to be ready: %w", ctx.Err()) + if err := s.metadata.err(); err != nil { + return nil, err } + case <-ctx.Done(): + return nil, fmt.Errorf("error waiting for metadata fetcher to be ready: %w", ctx.Err()) + } - // first map lookup will fail but this is not going - // to be performance issue since it only happens if init - // is in progress. - if i, ok := s.metadata.getID(original); ok { - // Only fetch from ES if the sourcemap id exists - return s.fetch(ctx, i) - } + // the mutex is shared by the update goroutine, we need to release it + // as soon as possible to avoid blocking updates. + if i, ok := s.metadata.getID(original); ok { + // Only fetch from ES if the sourcemap id exists + return s.fetch(ctx, i) } if urlPath, err := url.Parse(path); err == nil { From ee25e373b4d83effc46c35e97eee833b97c5be6d Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Tue, 7 Feb 2023 01:09:56 +0100 Subject: [PATCH 109/123] fix: handle 401 unauthorized in metadata fetcher --- internal/sourcemap/metadata_fetcher.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/sourcemap/metadata_fetcher.go b/internal/sourcemap/metadata_fetcher.go index 44a21bd2289..63cb6f8fe77 100644 --- a/internal/sourcemap/metadata_fetcher.go +++ b/internal/sourcemap/metadata_fetcher.go @@ -306,7 +306,7 @@ func (s *MetadataESFetcher) handleUpdateRequest(resp *esapi.Response, updates ma if err != nil { return nil, fmt.Errorf("failed to read response body: %w", err) } - if resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusForbidden { + if resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusForbidden || resp.StatusCode == http.StatusUnauthorized { return nil, fmt.Errorf("%w: %s: %s", errFetcherUnvailable, resp.Status(), string(b)) } return nil, fmt.Errorf("ES returned unknown status code: %s", resp.Status()) From 11c097be65c7b9f2832f2c6f1f9a1341f19c731c Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Tue, 7 Feb 2023 02:00:22 +0100 Subject: [PATCH 110/123] fix: set the init error during init --- internal/sourcemap/metadata_fetcher.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/sourcemap/metadata_fetcher.go b/internal/sourcemap/metadata_fetcher.go index 63cb6f8fe77..56c04c96d91 100644 --- a/internal/sourcemap/metadata_fetcher.go +++ b/internal/sourcemap/metadata_fetcher.go @@ -116,6 +116,8 @@ func (s *MetadataESFetcher) startBackgroundSync(parent context.Context) { err := s.sync(ctx) cancel() + s.initErr = err + if err != nil { s.logger.Errorf("failed to fetch sourcemaps metadata: %v", err) } else { From 3530b493ffb5e55d6c816adc42430298d9a4e648 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Tue, 7 Feb 2023 02:08:15 +0100 Subject: [PATCH 111/123] refactor: cleanup metadata updating to avoid useless map operations --- internal/sourcemap/metadata_fetcher.go | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/internal/sourcemap/metadata_fetcher.go b/internal/sourcemap/metadata_fetcher.go index 56c04c96d91..b5c40ecc8d6 100644 --- a/internal/sourcemap/metadata_fetcher.go +++ b/internal/sourcemap/metadata_fetcher.go @@ -210,13 +210,11 @@ func (s *MetadataESFetcher) update(ctx context.Context, sourcemaps map[identifie for id, contentHash := range s.set { if updatedHash, ok := sourcemaps[id]; ok { - // already in the cache, remove from the updates. - delete(sourcemaps, id) - - // content hash changed, invalidate the sourcemap cache and update hash - if contentHash != updatedHash { - sourcemaps[id] = updatedHash - + if contentHash == updatedHash { + // already in the cache, remove from the updates. + delete(sourcemaps, id) + } else { + // content hash changed, invalidate the sourcemap cache s.logger.Debugf("Hash changed: %s -> %s: invalidating %v", contentHash, updatedHash, id) invalidation = append(invalidation, id) } From 14d230a8f7c0943e0d63c7f7d6a8d3650d8f265a Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Tue, 7 Feb 2023 02:10:58 +0100 Subject: [PATCH 112/123] fix: do not block forever when invalidating sourcemaps --- internal/sourcemap/metadata_fetcher.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/sourcemap/metadata_fetcher.go b/internal/sourcemap/metadata_fetcher.go index b5c40ecc8d6..9dcaf9c467d 100644 --- a/internal/sourcemap/metadata_fetcher.go +++ b/internal/sourcemap/metadata_fetcher.go @@ -234,7 +234,11 @@ func (s *MetadataESFetcher) update(ctx context.Context, sourcemaps map[identifie } } - s.invalidationChan <- invalidation + select { + case s.invalidationChan <- invalidation: + case <-ctx.Done(): + s.logger.Debug("timed out while invalidating soucemaps") + } // add new sourcemaps to the metadata cache. for id, contentHash := range sourcemaps { From d8924ccf2dfa539e9e91a7f589a293cdbe26aed0 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Tue, 7 Feb 2023 02:11:47 +0100 Subject: [PATCH 113/123] lint: fix linter issues --- internal/sourcemap/chained.go | 2 +- systemtest/sourcemap_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/sourcemap/chained.go b/internal/sourcemap/chained.go index 408db500759..af055c2b04a 100644 --- a/internal/sourcemap/chained.go +++ b/internal/sourcemap/chained.go @@ -47,7 +47,7 @@ func (c ChainedFetcher) Fetch(ctx context.Context, name, version, path string) ( // previous fetcher is unavailable but the deadline expired so we cannot reuse that if t, _ := ctx.Deadline(); t.Before(time.Now()) { var cancel context.CancelFunc - ctx, cancel = context.WithTimeout(context.Background(), 500 * time.Millisecond) + ctx, cancel = context.WithTimeout(context.Background(), 500*time.Millisecond) defer cancel() } diff --git a/systemtest/sourcemap_test.go b/systemtest/sourcemap_test.go index 4b1fc1e96b2..0332c9ce0a3 100644 --- a/systemtest/sourcemap_test.go +++ b/systemtest/sourcemap_test.go @@ -218,7 +218,7 @@ func TestSourcemapKibana(t *testing.T) { srv := apmservertest.NewUnstartedServerTB(t) srv.Config.RUM = &apmservertest.RUMConfig{ - Enabled: true, + Enabled: true, Sourcemap: &apmservertest.RUMSourcemapConfig{ // Use the wrong credentials so that the ES fetcher // will fail and apm server will fall back to kibana From 3ef10e589ee84747eb72c7f4094a9a95fcff989f Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Tue, 7 Feb 2023 02:24:24 +0100 Subject: [PATCH 114/123] test: refactor sourcemap test for clarity and duplicate code --- systemtest/sourcemap_test.go | 107 ++++++++++++++--------------------- 1 file changed, 42 insertions(+), 65 deletions(-) diff --git a/systemtest/sourcemap_test.go b/systemtest/sourcemap_test.go index 0332c9ce0a3..42022ed21c6 100644 --- a/systemtest/sourcemap_test.go +++ b/systemtest/sourcemap_test.go @@ -159,81 +159,58 @@ func TestSourcemapCaching(t *testing.T) { assertSourcemapUpdated(t, result, true) } -func TestSourcemapElasticsearch(t *testing.T) { - systemtest.CleanupElasticsearch(t) - - sourcemap, err := os.ReadFile("../testdata/sourcemap/bundle.js.map") - require.NoError(t, err) - systemtest.CreateSourceMap(t, string(sourcemap), "apm-agent-js", "1.0.1", - "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", - ) - - srv := apmservertest.NewUnstartedServerTB(t) - srv.Config.RUM = &apmservertest.RUMConfig{Enabled: true} - err = srv.Start() - require.NoError(t, err) - - // Index an error, applying source mapping and caching the source map in the process. - systemtest.SendRUMEventsPayload(t, srv.URL, "../testdata/intake-v2/errors_rum.ndjson") - result := systemtest.Elasticsearch.ExpectDocs(t, "logs-apm.error-*", nil) - assertSourcemapUpdated(t, result, true) -} - -func TestSourcemapElasticsearchUnreachable(t *testing.T) { - systemtest.CleanupElasticsearch(t) - - sourcemap, err := os.ReadFile("../testdata/sourcemap/bundle.js.map") - require.NoError(t, err) - systemtest.CreateSourceMap(t, string(sourcemap), "apm-agent-js", "1.0.1", - "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", - ) - - srv := apmservertest.NewUnstartedServerTB(t) - srv.Config.RUM = &apmservertest.RUMConfig{ - Enabled: true, - Sourcemap: &apmservertest.RUMSourcemapConfig{ - ESConfig: &apmservertest.ElasticsearchOutputConfig{ +func TestSourcemapFetcher(t *testing.T) { + testCases := []struct { + name string + disableKibana bool + rumESConfig *apmservertest.ElasticsearchOutputConfig + }{ + { + name: "elasticsearch", + disableKibana: true, + }, { + name: "kibana fallback with es unreachable", + rumESConfig: &apmservertest.ElasticsearchOutputConfig{ // Use an unreachable address Hosts: []string{"127.0.0.1:12345"}, }, + }, { + name: "kibana fallback with es credentials unauthorized", + rumESConfig: &apmservertest.ElasticsearchOutputConfig{ + // Use the wrong credentials so that the ES fetcher + // will fail and apm server will fall back to kiban + APIKey: "example", + }, }, } - err = srv.Start() - require.NoError(t, err) - // Index an error, applying source mapping and caching the source map in the process. - systemtest.SendRUMEventsPayload(t, srv.URL, "../testdata/intake-v2/errors_rum.ndjson") - result := systemtest.Elasticsearch.ExpectDocs(t, "logs-apm.error-*", nil) - assertSourcemapUpdated(t, result, true) -} + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + systemtest.CleanupElasticsearch(t) -func TestSourcemapKibana(t *testing.T) { - systemtest.CleanupElasticsearch(t) + sourcemap, err := os.ReadFile("../testdata/sourcemap/bundle.js.map") + require.NoError(t, err) + systemtest.CreateSourceMap(t, string(sourcemap), "apm-agent-js", "1.0.1", + "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", + ) - sourcemap, err := os.ReadFile("../testdata/sourcemap/bundle.js.map") - require.NoError(t, err) - systemtest.CreateSourceMap(t, string(sourcemap), "apm-agent-js", "1.0.1", - "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", - ) + srv := apmservertest.NewUnstartedServerTB(t) + srv.Config.Kibana.Enabled = !tc.disableKibana + srv.Config.RUM = &apmservertest.RUMConfig{ + Enabled: true, + Sourcemap: &apmservertest.RUMSourcemapConfig{ + ESConfig: tc.rumESConfig, + }, + } + err = srv.Start() + require.NoError(t, err) - srv := apmservertest.NewUnstartedServerTB(t) - srv.Config.RUM = &apmservertest.RUMConfig{ - Enabled: true, - Sourcemap: &apmservertest.RUMSourcemapConfig{ - // Use the wrong credentials so that the ES fetcher - // will fail and apm server will fall back to kibana - ESConfig: &apmservertest.ElasticsearchOutputConfig{ - APIKey: "example", - }, - }, + // Index an error, applying source mapping and caching the source map in the process. + systemtest.SendRUMEventsPayload(t, srv.URL, "../testdata/intake-v2/errors_rum.ndjson") + result := systemtest.Elasticsearch.ExpectDocs(t, "logs-apm.error-*", nil) + assertSourcemapUpdated(t, result, true) + }) } - err = srv.Start() - require.NoError(t, err) - - // Index an error, applying source mapping and caching the source map in the process. - systemtest.SendRUMEventsPayload(t, srv.URL, "../testdata/intake-v2/errors_rum.ndjson") - result := systemtest.Elasticsearch.ExpectDocs(t, "logs-apm.error-*", nil) - assertSourcemapUpdated(t, result, true) } func deleteIndex(t *testing.T, name string) { From bbebe6a059396d0be87aed55dc015d0103aefbe3 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Tue, 7 Feb 2023 03:13:36 +0100 Subject: [PATCH 115/123] test: update unit test for new es response format --- internal/beater/beater_test.go | 87 +++++++++++++++++------- internal/sourcemap/body_caching_test.go | 18 ++--- internal/sourcemap/elasticsearch_test.go | 43 +++--------- internal/sourcemap/processor_test.go | 4 +- 4 files changed, 79 insertions(+), 73 deletions(-) diff --git a/internal/beater/beater_test.go b/internal/beater/beater_test.go index 3fc5504c920..8ded9e68068 100644 --- a/internal/beater/beater_test.go +++ b/internal/beater/beater_test.go @@ -42,20 +42,30 @@ var validSourcemap, _ = os.ReadFile("../../testdata/sourcemap/bundle.js.map") func TestStoreUsesRUMElasticsearchConfig(t *testing.T) { var called bool ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - called = true w.Header().Set("X-Elastic-Product", "Elasticsearch") - - m := sourcemapHit(validSourcemap) - - b, err := json.Marshal(m) - require.NoError(t, err) - - w.Write(b) + switch r.URL.Path { + case "/.apm-source-map": + // ping request from the metadata fetcher. + // Send a status ok + w.WriteHeader(http.StatusOK) + case "/.apm-source-map/_search": + // search request from the metadata fetcher + m := sourcemapSearchResponseBody("app", "1.0", "/bundle/path") + w.Write(m) + case "/.apm-source-map/_doc/app-1.0-/bundle/path": + m := sourcemapGetResponseBody(true, validSourcemap) + w.Write(m) + called = true + default: + w.WriteHeader(http.StatusTeapot) + t.Fatalf("unhandled request path: %s", r.URL.Path) + } })) defer ts.Close() cfg := config.DefaultConfig() cfg.RumConfig.Enabled = true + cfg.Kibana.Enabled = false cfg.RumConfig.SourceMapping.Enabled = true cfg.RumConfig.SourceMapping.ESConfig = elasticsearch.DefaultConfig() cfg.RumConfig.SourceMapping.ESConfig.Hosts = []string{ts.URL} @@ -75,29 +85,58 @@ func TestStoreUsesRUMElasticsearchConfig(t *testing.T) { assert.True(t, called) } -func sourcemapHit(sourcemap []byte) map[string]interface{} { - b := &bytes.Buffer{} - - z := zlib.NewWriter(b) - z.Write(sourcemap) - z.Close() +func sourcemapSearchResponseBody(name string, version string, bundlePath string) []byte { + result := map[string]interface{}{ + "hits": map[string]interface{}{ + "total": map[string]interface{}{ + "value": 1, + }, + "hits": []map[string]interface{}{ + { + "_source": map[string]interface{}{ + "service": map[string]interface{}{ + "name": name, + "version": version, + }, + "file": map[string]interface{}{ + "path": bundlePath, + }, + }, + }, + }, + }, + } - s := base64.StdEncoding.EncodeToString(b.Bytes()) + data, err := json.Marshal(result) + if err != nil { + panic(err) + } + return data +} - hits := map[string]interface{}{ +func sourcemapGetResponseBody(found bool, b []byte) []byte { + result := map[string]interface{}{ + "found": found, "_source": map[string]interface{}{ - "content": s, + "content": encodeSourcemap(b), }, } - resultHits := map[string]interface{}{ - "total": map[string]interface{}{ - "value": 1, - }, + data, err := json.Marshal(result) + if err != nil { + panic(err) } - resultHits["hits"] = []map[string]interface{}{hits} - result := map[string]interface{}{"hits": resultHits} - return result + return data +} + +func encodeSourcemap(sourcemap []byte) string { + b := &bytes.Buffer{} + + z := zlib.NewWriter(b) + z.Write(sourcemap) + z.Close() + + return base64.StdEncoding.EncodeToString(b.Bytes()) } func TestQueryClusterUUIDRegistriesExist(t *testing.T) { diff --git a/internal/sourcemap/body_caching_test.go b/internal/sourcemap/body_caching_test.go index 5ea66cba796..c88d83e4f8a 100644 --- a/internal/sourcemap/body_caching_test.go +++ b/internal/sourcemap/body_caching_test.go @@ -59,9 +59,7 @@ func TestStore_Fetch(t *testing.T) { t.Run("cache", func(t *testing.T) { t.Run("nil", func(t *testing.T) { var nilConsumer *sourcemap.Consumer - store := testCachingFetcher(t, newMockElasticsearchClient(t, http.StatusOK, - sourcemapSearchResponseBody(1, []map[string]interface{}{sourcemapHit(validSourcemap)}), - )) + store := testCachingFetcher(t, newMockElasticsearchClient(t, http.StatusOK, sourcemapESResponseBody(true, validSourcemap))) store.add(key, nilConsumer) mapper, err := store.Fetch(context.Background(), serviceName, serviceVersion, path) @@ -82,9 +80,7 @@ func TestStore_Fetch(t *testing.T) { }) t.Run("validFromES", func(t *testing.T) { - store := testCachingFetcher(t, newMockElasticsearchClient(t, http.StatusOK, - sourcemapSearchResponseBody(1, []map[string]interface{}{sourcemapHit(validSourcemap)}), - )) + store := testCachingFetcher(t, newMockElasticsearchClient(t, http.StatusOK, sourcemapESResponseBody(true, validSourcemap))) mapper, err := store.Fetch(context.Background(), serviceName, serviceVersion, path) require.NoError(t, err) require.NotNil(t, mapper) @@ -96,7 +92,7 @@ func TestStore_Fetch(t *testing.T) { }) t.Run("notFoundInES", func(t *testing.T) { - store := testCachingFetcher(t, newMockElasticsearchClient(t, http.StatusOK, sourcemapSearchResponseBody(0, []map[string]interface{}{}))) + store := testCachingFetcher(t, newMockElasticsearchClient(t, http.StatusOK, sourcemapESResponseBody(false, ""))) //not cached cached, found := store.cache.Get(key) require.False(t, found) @@ -115,12 +111,8 @@ func TestStore_Fetch(t *testing.T) { t.Run("invalidFromES", func(t *testing.T) { for name, client := range map[string]*elasticsearch.Client{ - "invalid": newMockElasticsearchClient(t, http.StatusOK, - sourcemapSearchResponseBody(1, []map[string]interface{}{sourcemapHit("foo")}), - ), - "unsupportedVersion": newMockElasticsearchClient(t, http.StatusOK, - sourcemapSearchResponseBody(1, []map[string]interface{}{sourcemapHit(unsupportedVersionSourcemap)}), - ), + "invalid": newMockElasticsearchClient(t, http.StatusOK, sourcemapESResponseBody(true, "foo")), + "unsupportedVersion": newMockElasticsearchClient(t, http.StatusOK, sourcemapESResponseBody(true, unsupportedVersionSourcemap)), } { t.Run(name, func(t *testing.T) { store := testCachingFetcher(t, client) diff --git a/internal/sourcemap/elasticsearch_test.go b/internal/sourcemap/elasticsearch_test.go index 6b6dcccf4cd..b99b93e7450 100644 --- a/internal/sourcemap/elasticsearch_test.go +++ b/internal/sourcemap/elasticsearch_test.go @@ -56,17 +56,6 @@ func Test_esFetcher_fetchError(t *testing.T) { statusCode: http.StatusBadRequest, expectedErrMessage: "ES returned unknown status code: 400 Bad Request", }, - "empty sourcemap string": { - statusCode: http.StatusOK, - responseBody: sourcemapSearchResponseBody(1, []map[string]interface{}{{ - "_source": map[string]interface{}{ - "sourcemap": map[string]interface{}{ - "sourcemap": "", - }, - }, - }}), - expectedErrMessage: "sourcemap not in the expected format: sourcemap malformed", - }, } { t.Run(name, func(t *testing.T) { var client *elasticsearch.Client @@ -91,15 +80,11 @@ func Test_esFetcher_fetch(t *testing.T) { }{ "no sourcemap found": { statusCode: http.StatusOK, - responseBody: sourcemapSearchResponseBody(0, []map[string]interface{}{}), - }, - "sourcemap indicated but not found": { - statusCode: http.StatusOK, - responseBody: sourcemapSearchResponseBody(1, []map[string]interface{}{}), + responseBody: sourcemapESResponseBody(false, ""), }, "valid sourcemap found": { statusCode: http.StatusOK, - responseBody: sourcemapSearchResponseBody(1, []map[string]interface{}{sourcemapHit(validSourcemap)}), + responseBody: sourcemapESResponseBody(true, validSourcemap), filePath: "bundle.js", }, } { @@ -122,16 +107,14 @@ func testESFetcher(client *elasticsearch.Client) *esFetcher { return &esFetcher{client: client, index: "apm-sourcemap", logger: logp.NewLogger(logs.Sourcemap)} } -func sourcemapSearchResponseBody(hitsTotal int, hits []map[string]interface{}) io.Reader { - resultHits := map[string]interface{}{ - "total": map[string]interface{}{ - "value": hitsTotal, +func sourcemapESResponseBody(found bool, s string) io.Reader { + result := map[string]interface{}{ + "found": found, + "_source": map[string]interface{}{ + "content": encodeSourcemap(s), }, } - if hits != nil { - resultHits["hits"] = hits - } - result := map[string]interface{}{"hits": resultHits} + data, err := json.Marshal(result) if err != nil { panic(err) @@ -139,20 +122,14 @@ func sourcemapSearchResponseBody(hitsTotal int, hits []map[string]interface{}) i return bytes.NewReader(data) } -func sourcemapHit(sourcemap string) map[string]interface{} { +func encodeSourcemap(sourcemap string) string { b := &bytes.Buffer{} z := zlib.NewWriter(b) z.Write([]byte(sourcemap)) z.Close() - s := base64.StdEncoding.EncodeToString(b.Bytes()) - - return map[string]interface{}{ - "_source": map[string]interface{}{ - "content": s, - }, - } + return base64.StdEncoding.EncodeToString(b.Bytes()) } // newUnavailableElasticsearchClient returns an elasticsearch.Client configured diff --git a/internal/sourcemap/processor_test.go b/internal/sourcemap/processor_test.go index 0bdb880105a..95feff1a6c2 100644 --- a/internal/sourcemap/processor_test.go +++ b/internal/sourcemap/processor_test.go @@ -34,9 +34,7 @@ import ( ) func TestBatchProcessor(t *testing.T) { - client := newMockElasticsearchClient(t, http.StatusOK, - sourcemapSearchResponseBody(1, []map[string]interface{}{sourcemapHit(string(validSourcemap))}), - ) + client := newMockElasticsearchClient(t, http.StatusOK, sourcemapESResponseBody(true, validSourcemap)) esFetcher := NewElasticsearchFetcher(client, "index") fetcher, err := NewBodyCachingFetcher(esFetcher, 100, nil) require.NoError(t, err) From c24db8dfef26341ca8071f0a0d2f6aa6ddb509f1 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Tue, 7 Feb 2023 03:18:28 +0100 Subject: [PATCH 116/123] docs: remove stale comment --- internal/sourcemap/sourcemap_fetcher.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/internal/sourcemap/sourcemap_fetcher.go b/internal/sourcemap/sourcemap_fetcher.go index 689ad393269..6f5e7f350db 100644 --- a/internal/sourcemap/sourcemap_fetcher.go +++ b/internal/sourcemap/sourcemap_fetcher.go @@ -56,8 +56,6 @@ func (s *SourcemapFetcher) Fetch(ctx context.Context, name, version, path string return nil, fmt.Errorf("error waiting for metadata fetcher to be ready: %w", ctx.Err()) } - // the mutex is shared by the update goroutine, we need to release it - // as soon as possible to avoid blocking updates. if i, ok := s.metadata.getID(original); ok { // Only fetch from ES if the sourcemap id exists return s.fetch(ctx, i) From 08ae6025bba27bdfb33b1ac2ac8f544148aae7e8 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Tue, 7 Feb 2023 03:19:51 +0100 Subject: [PATCH 117/123] fix: close ping request response body --- internal/sourcemap/metadata_fetcher.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/internal/sourcemap/metadata_fetcher.go b/internal/sourcemap/metadata_fetcher.go index 9dcaf9c467d..4f507bffe3c 100644 --- a/internal/sourcemap/metadata_fetcher.go +++ b/internal/sourcemap/metadata_fetcher.go @@ -159,7 +159,10 @@ func (s *MetadataESFetcher) ping(ctx context.Context) error { req := esapi.IndicesGetRequest{ Index: []string{s.index}, } - _, err := req.Do(ctx, s.esClient) + resp, err := req.Do(ctx, s.esClient) + if err == nil { + resp.Body.Close() + } return err } From 2e814f4527ac965803f33afabaa5ae8c17f6ee73 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Tue, 7 Feb 2023 12:23:10 +0100 Subject: [PATCH 118/123] fix: do not send empty invalidations --- internal/sourcemap/metadata_fetcher.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/internal/sourcemap/metadata_fetcher.go b/internal/sourcemap/metadata_fetcher.go index 4f507bffe3c..db8474e0a27 100644 --- a/internal/sourcemap/metadata_fetcher.go +++ b/internal/sourcemap/metadata_fetcher.go @@ -237,10 +237,12 @@ func (s *MetadataESFetcher) update(ctx context.Context, sourcemaps map[identifie } } - select { - case s.invalidationChan <- invalidation: - case <-ctx.Done(): - s.logger.Debug("timed out while invalidating soucemaps") + if len(invalidation) != 0 { + select { + case s.invalidationChan <- invalidation: + case <-ctx.Done(): + s.logger.Debug("timed out while invalidating soucemaps") + } } // add new sourcemaps to the metadata cache. From 53c7d97addc5dd93525ec350d9bcea053cf6d0ad Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Tue, 7 Feb 2023 12:51:40 +0100 Subject: [PATCH 119/123] test: add metadata fetcher tests --- internal/sourcemap/metadata_fetcher_test.go | 331 ++++++++++++++++++++ 1 file changed, 331 insertions(+) create mode 100644 internal/sourcemap/metadata_fetcher_test.go diff --git a/internal/sourcemap/metadata_fetcher_test.go b/internal/sourcemap/metadata_fetcher_test.go new file mode 100644 index 00000000000..c0771d628a8 --- /dev/null +++ b/internal/sourcemap/metadata_fetcher_test.go @@ -0,0 +1,331 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package sourcemap + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/elastic/apm-server/internal/elasticsearch" + "github.com/elastic/elastic-agent-libs/logp" +) + +func TestMetadataFetcher(t *testing.T) { + defaultID := metadata{ + identifier: identifier{ + name: "app", + version: "1.0", + path: "/bundle/path", + }, + contentHash: "foo", + } + + defaultSearchResponse := func(w http.ResponseWriter, r *http.Request) { + m := sourcemapSearchResponseBody([]metadata{defaultID}) + w.Write(m) + } + + testCases := []struct { + name string + pingStatus int + pingUnreachable bool + searchUnreachable bool + searchReponse func(http.ResponseWriter, *http.Request) + expectErr bool + expectID bool + }{ + { + name: "200", + pingStatus: http.StatusOK, + searchReponse: defaultSearchResponse, + expectID: true, + }, { + name: "ping unreachable", + pingUnreachable: true, + expectErr: true, + expectID: false, + }, { + name: "search unreachable", + pingStatus: http.StatusOK, + searchUnreachable: true, + expectErr: true, + expectID: false, + }, { + name: "init error", + pingStatus: http.StatusOK, + searchReponse: func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusGatewayTimeout) + }, + expectErr: true, + expectID: false, + }, { + name: "malformed response", + pingStatus: http.StatusOK, + searchReponse: func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("foo")) + }, + expectErr: true, + expectID: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + waitCh := make(chan struct{}) + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Elastic-Product", "Elasticsearch") + switch r.URL.Path { + case "/.apm-source-map": + if tc.pingUnreachable { + <-waitCh + return + } + // ping request from the metadata fetcher. + w.WriteHeader(tc.pingStatus) + case "/.apm-source-map/_search": + if tc.searchUnreachable { + <-waitCh + return + } + // search request from the metadata fetcher + tc.searchReponse(w, r) + default: + w.WriteHeader(http.StatusTeapot) + t.Fatalf("unhandled request path: %s", r.URL.Path) + } + })) + defer ts.Close() + + esConfig := elasticsearch.DefaultConfig() + esConfig.Hosts = []string{ts.URL} + + esClient, err := elasticsearch.NewClient(esConfig) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + fetcher, _ := NewMetadataFetcher(ctx, esClient, ".apm-source-map") + + <-fetcher.ready() + if tc.expectErr { + assert.Error(t, fetcher.err()) + } else { + assert.NoError(t, fetcher.err()) + } + + _, ok := fetcher.getID(defaultID.identifier) + assert.Equal(t, tc.expectID, ok) + + close(waitCh) + }) + } +} + +type metadata struct { + identifier + contentHash string +} + +func TestInvalidation(t *testing.T) { + defaultID := metadata{ + identifier: identifier{ + name: "app", + version: "1.0", + path: "/bundle/path", + }, + contentHash: "foo", + } + + defaultSearchResponse := func(w http.ResponseWriter, r *http.Request) { + m := sourcemapSearchResponseBody([]metadata{defaultID}) + w.Write(m) + } + + testCases := []struct { + name string + set map[identifier]string + alias map[identifier]*identifier + searchReponse func(http.ResponseWriter, *http.Request) + expectedInvalidation []identifier + expectedset map[identifier]string + expectedalias map[identifier]*identifier + }{ + { + name: "hash changed", + set: map[identifier]string{defaultID.identifier: "bar"}, + searchReponse: defaultSearchResponse, + expectedInvalidation: []identifier{defaultID.identifier}, + expectedset: map[identifier]string{defaultID.identifier: "foo"}, + expectedalias: map[identifier]*identifier{}, + }, { + name: "sourcemap deleted", + set: map[identifier]string{defaultID.identifier: "bar"}, + searchReponse: func(w http.ResponseWriter, r *http.Request) { + m := sourcemapSearchResponseBody([]metadata{}) + w.Write(m) + }, + expectedInvalidation: []identifier{defaultID.identifier}, + expectedset: map[identifier]string{}, + expectedalias: map[identifier]*identifier{}, + }, { + name: "update ok", + set: map[identifier]string{{name: "example", version: "1.0", path: "/"}: "bar"}, + searchReponse: func(w http.ResponseWriter, r *http.Request) { + bar := metadata{ + identifier: identifier{ + name: "example", + version: "1.0", + path: "/", + }, + contentHash: "bar", + } + m := sourcemapSearchResponseBody([]metadata{defaultID, bar}) + w.Write(m) + }, + expectedset: map[identifier]string{defaultID.identifier: "foo", {name: "example", version: "1.0", path: "/"}: "bar"}, + expectedalias: map[identifier]*identifier{}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + require.NoError(t, logp.DevelopmentSetup(logp.ToObserverOutput())) + t.Cleanup(func() { + if t.Failed() { + for _, le := range logp.ObserverLogs().All() { + t.Log(le) + } + } + }) + c := make(chan struct{}) + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + select { + case <-c: + case <-time.After(1 * time.Second): + t.Fatalf("timeout out waiting for channel") + } + w.Header().Set("X-Elastic-Product", "Elasticsearch") + switch r.URL.Path { + case "/.apm-source-map": + // ping request from the metadata fetcher. + // Send a status ok + w.WriteHeader(http.StatusOK) + case "/.apm-source-map/_search": + // search request from the metadata fetcher + tc.searchReponse(w, r) + default: + w.WriteHeader(http.StatusTeapot) + t.Fatalf("unhandled request path: %s", r.URL.Path) + } + })) + defer ts.Close() + + esConfig := elasticsearch.DefaultConfig() + esConfig.Hosts = []string{ts.URL} + + esClient, err := elasticsearch.NewClient(esConfig) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + fetcher, invalidationChan := NewMetadataFetcher(ctx, esClient, ".apm-source-map") + + invCh := make(chan struct{}) + go func() { + defer close(invCh) + i, ok := <-invalidationChan + if !ok { + return + } + if tc.expectedInvalidation == nil { + t.Errorf("unexpected invalidation: %v", i) + } else { + assert.Equal(t, tc.expectedInvalidation, i) + } + }() + + esf := fetcher.(*MetadataESFetcher) + if tc.set != nil { + esf.set = tc.set + } + if tc.alias != nil { + esf.alias = tc.alias + } + close(c) + + <-fetcher.ready() + + assert.NoError(t, fetcher.err()) + + assert.Equal(t, esf.set, tc.expectedset) + assert.Equal(t, esf.alias, tc.expectedalias) + + if tc.expectedInvalidation != nil { + select { + case <-invCh: + case <-time.After(50 * time.Millisecond): + t.Fatal("timed out waiting for invalidations") + } + } + }) + } +} + +func sourcemapSearchResponseBody(ids []metadata) []byte { + m := make([]map[string]interface{}, 0, len(ids)) + for _, id := range ids { + m = append(m, map[string]interface{}{ + "_source": map[string]interface{}{ + "service": map[string]interface{}{ + "name": id.name, + "version": id.version, + }, + "file": map[string]interface{}{ + "path": id.path, + }, + "content_sha256": id.contentHash, + }, + }) + } + + result := map[string]interface{}{ + "hits": map[string]interface{}{ + "total": map[string]interface{}{ + "value": len(ids), + }, + "hits": m, + }, + } + + data, err := json.Marshal(result) + if err != nil { + panic(err) + } + return data +} From 8028c46293a5af4e6cbcd26d7ba66131d53666ad Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Tue, 7 Feb 2023 15:33:58 +0100 Subject: [PATCH 120/123] test: add sourcemap fetcher tests --- internal/sourcemap/sourcemap_fetcher_test.go | 135 +++++++++++++++++++ 1 file changed, 135 insertions(+) create mode 100644 internal/sourcemap/sourcemap_fetcher_test.go diff --git a/internal/sourcemap/sourcemap_fetcher_test.go b/internal/sourcemap/sourcemap_fetcher_test.go new file mode 100644 index 00000000000..1d69e8f6a55 --- /dev/null +++ b/internal/sourcemap/sourcemap_fetcher_test.go @@ -0,0 +1,135 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package sourcemap + +import ( + "context" + "fmt" + "testing" + + "github.com/go-sourcemap/sourcemap" + "github.com/stretchr/testify/assert" +) + +func TestSourcemapFetcher(t *testing.T) { + defaultID := metadata{ + identifier: identifier{ + name: "app", + version: "1.0", + path: "/bundle/path", + }, + contentHash: "foo", + } + + absPathID := defaultID + absPathID.path = "http://example.com" + defaultID.path + + testCases := []struct { + name string + fetchedID identifier + expectedID identifier + set map[identifier]string + alias map[identifier]*identifier + }{ + { + name: "fetch relative path from main set", + fetchedID: defaultID.identifier, + expectedID: defaultID.identifier, + set: map[identifier]string{defaultID.identifier: "foo"}, + }, { + name: "fetch relative path from alias", + fetchedID: defaultID.identifier, + expectedID: absPathID.identifier, + alias: map[identifier]*identifier{defaultID.identifier: &absPathID.identifier}, + }, { + name: "fetch absolute path from main set", + fetchedID: absPathID.identifier, + expectedID: absPathID.identifier, + set: map[identifier]string{absPathID.identifier: "foo"}, + }, { + name: "fetch absolute path from alias", + fetchedID: absPathID.identifier, + expectedID: defaultID.identifier, + alias: map[identifier]*identifier{absPathID.identifier: &defaultID.identifier}, + }, { + name: "fetch path with url query", + fetchedID: identifier{name: absPathID.name, version: absPathID.version, path: absPathID.path + "?foo=bar"}, + expectedID: absPathID.identifier, + set: map[identifier]string{absPathID.identifier: "foo"}, + }, { + name: "fetch path with url fragment", + fetchedID: identifier{name: absPathID.name, version: absPathID.version, path: absPathID.path + "#foo"}, + expectedID: absPathID.identifier, + set: map[identifier]string{absPathID.identifier: "foo"}, + }, { + name: "fetch path not cleaned", + fetchedID: identifier{name: absPathID.name, version: absPathID.version, path: "http://example.com/.././bundle/././path/../path"}, + expectedID: absPathID.identifier, + set: map[identifier]string{absPathID.identifier: "foo"}, + }, { + name: "fetch path not cleaned with query and fragment from alias", + fetchedID: identifier{name: absPathID.name, version: absPathID.version, path: "http://example.com/.././bundle/././path/../path#foo?foo=bar"}, + expectedID: defaultID.identifier, + alias: map[identifier]*identifier{absPathID.identifier: &defaultID.identifier}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + mFetcher := MetadataESFetcher{ + set: map[identifier]string{}, + alias: map[identifier]*identifier{}, + init: make(chan struct{}), + } + if tc.set != nil { + mFetcher.set = tc.set + } + if tc.alias != nil { + mFetcher.alias = tc.alias + } + close(mFetcher.init) + + monitor := &monitoredFetcher{matchID: tc.expectedID} + f := NewSourcemapFetcher(&mFetcher, monitor) + _, err := f.Fetch(context.Background(), tc.fetchedID.name, tc.fetchedID.version, tc.fetchedID.path) + assert.NoError(t, err) + // make sure we are forwarding to the backend fetcher once + assert.Equal(t, 1, monitor.called) + }) + } +} + +type monitoredFetcher struct { + called int + matchID identifier +} + +func (s *monitoredFetcher) Fetch(ctx context.Context, name string, version string, bundleFilepath string) (*sourcemap.Consumer, error) { + s.called++ + if s.matchID.name != name { + return nil, fmt.Errorf("mismatched name: expected %s but got %s", s.matchID.name, name) + } + if s.matchID.version != version { + return nil, fmt.Errorf("mismatched version: expected %s but got %s", s.matchID.version, version) + } + if s.matchID.path != bundleFilepath { + return nil, fmt.Errorf("mismatched path: expected %s but got %s", s.matchID.path, bundleFilepath) + } + + return &sourcemap.Consumer{}, nil +} From 4de127a22b600d144dd520db499046079e7dae31 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Tue, 7 Feb 2023 16:13:43 +0100 Subject: [PATCH 121/123] docs: remove todo --- internal/sourcemap/metadata_fetcher.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/sourcemap/metadata_fetcher.go b/internal/sourcemap/metadata_fetcher.go index db8474e0a27..1ae6189adef 100644 --- a/internal/sourcemap/metadata_fetcher.go +++ b/internal/sourcemap/metadata_fetcher.go @@ -129,7 +129,6 @@ func (s *MetadataESFetcher) startBackgroundSync(parent context.Context) { close(s.init) - // TODO make this a config option ? t := time.NewTicker(30 * time.Second) defer t.Stop() From 7b7544640cda19fb7b36b1496b82f8aa3f36fbc3 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Tue, 7 Feb 2023 16:16:55 +0100 Subject: [PATCH 122/123] feat: remove cache expiration option --- internal/beater/config/config_test.go | 4 ---- internal/beater/config/rum.go | 3 --- systemtest/apmservertest/config.go | 6 ------ 3 files changed, 13 deletions(-) diff --git a/internal/beater/config/config_test.go b/internal/beater/config/config_test.go index ebec6ca5e76..189c19bf2dc 100644 --- a/internal/beater/config/config_test.go +++ b/internal/beater/config/config_test.go @@ -270,7 +270,6 @@ func TestUnpackConfig(t *testing.T) { AllowHeaders: []string{"Authorization"}, SourceMapping: SourceMapping{ Enabled: true, - Cache: Cache{Expiration: 8 * time.Minute}, ESConfig: &elasticsearch.Config{ Hosts: elasticsearch.Hosts{"localhost:9201", "localhost:9202"}, Protocol: "http", @@ -439,9 +438,6 @@ func TestUnpackConfig(t *testing.T) { AllowHeaders: []string{}, SourceMapping: SourceMapping{ Enabled: true, - Cache: Cache{ - Expiration: 7 * time.Second, - }, ESConfig: elasticsearch.DefaultConfig(), Timeout: 5 * time.Second, }, diff --git a/internal/beater/config/rum.go b/internal/beater/config/rum.go index 53ab5d3d6e3..2599e6b1d0c 100644 --- a/internal/beater/config/rum.go +++ b/internal/beater/config/rum.go @@ -34,7 +34,6 @@ const ( allowAllOrigins = "*" defaultExcludeFromGrouping = "^/webpack" defaultLibraryPattern = "node_modules|bower_components|~" - defaultSourcemapCacheExpiration = 5 * time.Minute defaultSourcemapTimeout = 5 * time.Second ) @@ -51,7 +50,6 @@ type RumConfig struct { // SourceMapping holds sourcemap config information type SourceMapping struct { - Cache Cache `config:"cache"` Enabled bool `config:"enabled"` ESConfig *elasticsearch.Config `config:"elasticsearch"` Timeout time.Duration `config:"timeout" validate:"positive"` @@ -114,7 +112,6 @@ func (s *SourceMapping) Unpack(inp *config.C) error { func defaultSourcemapping() SourceMapping { return SourceMapping{ Enabled: true, - Cache: Cache{Expiration: defaultSourcemapCacheExpiration}, ESConfig: elasticsearch.DefaultConfig(), Timeout: defaultSourcemapTimeout, } diff --git a/systemtest/apmservertest/config.go b/systemtest/apmservertest/config.go index 4b478c918f8..927977e6382 100644 --- a/systemtest/apmservertest/config.go +++ b/systemtest/apmservertest/config.go @@ -208,15 +208,9 @@ type RUMConfig struct { // RUMSourcemapConfig holds APM Server RUM sourcemap configuration. type RUMSourcemapConfig struct { Enabled bool `json:"enabled,omitempty"` - Cache *RUMSourcemapCacheConfig `json:"cache,omitempty"` ESConfig *ElasticsearchOutputConfig `json:"elasticsearch"` } -// RUMSourcemapCacheConfig holds sourcemap cache expiration. -type RUMSourcemapCacheConfig struct { - Expiration time.Duration `json:"expiration,omitempty"` -} - // APIKeyConfig holds agent auth configuration. type AgentAuthConfig struct { SecretToken string `json:"secret_token,omitempty"` From 1d74a28c6536fb7ab5f38e8b0333e4f0fff7bce9 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Tue, 7 Feb 2023 17:07:49 +0100 Subject: [PATCH 123/123] lint: fix linter issues --- internal/beater/config/config_test.go | 2 +- internal/beater/config/rum.go | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/beater/config/config_test.go b/internal/beater/config/config_test.go index 189c19bf2dc..9c73b18d6a1 100644 --- a/internal/beater/config/config_test.go +++ b/internal/beater/config/config_test.go @@ -437,7 +437,7 @@ func TestUnpackConfig(t *testing.T) { AllowOrigins: []string{"*"}, AllowHeaders: []string{}, SourceMapping: SourceMapping{ - Enabled: true, + Enabled: true, ESConfig: elasticsearch.DefaultConfig(), Timeout: 5 * time.Second, }, diff --git a/internal/beater/config/rum.go b/internal/beater/config/rum.go index 2599e6b1d0c..6ce4d474abb 100644 --- a/internal/beater/config/rum.go +++ b/internal/beater/config/rum.go @@ -31,10 +31,10 @@ import ( ) const ( - allowAllOrigins = "*" - defaultExcludeFromGrouping = "^/webpack" - defaultLibraryPattern = "node_modules|bower_components|~" - defaultSourcemapTimeout = 5 * time.Second + allowAllOrigins = "*" + defaultExcludeFromGrouping = "^/webpack" + defaultLibraryPattern = "node_modules|bower_components|~" + defaultSourcemapTimeout = 5 * time.Second ) // RumConfig holds config information related to the RUM endpoint