Skip to content

Commit 1a20fb2

Browse files
rubvsrubvs
committed
remove pre-downloading geoip database
1 parent ea1286b commit 1a20fb2

File tree

4 files changed

+106
-105
lines changed

4 files changed

+106
-105
lines changed

‎docker-compose.yml‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,6 @@ services:
3333
- "xpack.security.authc.api_key.enabled=true"
3434
- "logger.org.elasticsearch=${ES_LOG_LEVEL:-error}"
3535
- "action.destructive_requires_name=false"
36-
- "ingest.geoip.downloader.enabled=true"
37-
- "ingest.geoip.downloader.eager.download=true"
3836
volumes:
3937
- "./testing/docker/elasticsearch/roles.yml:/usr/share/elasticsearch/config/roles.yml"
4038
- "./testing/docker/elasticsearch/users:/usr/share/elasticsearch/config/users"

‎systemtest/geoip.go‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,3 +16,87 @@
1616
// under the License.
1717

1818
package systemtest
19+
20+
import (
21+
"encoding/json"
22+
"fmt"
23+
"io"
24+
"net/http"
25+
"strings"
26+
"testing"
27+
"time"
28+
29+
"github.com/stretchr/testify/require"
30+
)
31+
32+
var stats struct {
33+
GeoIPStats `json:"stats"`
34+
}
35+
36+
type GeoIPStats struct {
37+
DatabasesCount int `json:"databases_count"`
38+
}
39+
40+
const requestBody = `{"metadata":{"service":{"name":"rum-js-test","agent":{"name":"rum-js","version":"5.5.0"}}}}
41+
{"transaction":{"trace_id":"611f4fa950f04631aaaaaaaaaaaaaaaa","id":"611f4fa950f04631","type":"page-load","duration":643,"span_count":{"started":0}}}
42+
{"metricset":{"samples":{"transaction.breakdown.count":{"value":12},"transaction.duration.sum.us":{"value":12},"transaction.duration.count":{"value":2},"transaction.self_time.sum.us":{"value":10},"transaction.self_time.count":{"value":2},"span.self_time.count":{"value":1},"span.self_time.sum.us":{"value":633.288}},"transaction":{"type":"request","name":"GET /"},"span":{"type":"external","subtype":"http"},"timestamp": 1496170422281000}}
43+
`
44+
45+
func GeoIpLazyDownload(t *testing.T, url string) {
46+
t.Helper()
47+
48+
req, err := http.NewRequest("POST", url, strings.NewReader(requestBody))
49+
require.NoError(t, err)
50+
51+
req.Header.Set("Content-Type", "application/x-ndjson")
52+
req.Header.Set("X-Forwarded-For", "8.8.8.8")
53+
54+
resp, err := http.DefaultClient.Do(req)
55+
require.NoError(t, err)
56+
require.Equal(t, http.StatusAccepted, resp.StatusCode)
57+
58+
io.Copy(io.Discard, resp.Body)
59+
resp.Body.Close()
60+
61+
err = waitGeoIPDownload(t)
62+
require.NoError(t, err)
63+
64+
// Clean all datastreams containing tags.
65+
CleanupElasticsearch(t)
66+
}
67+
68+
func waitGeoIPDownload(t *testing.T) error {
69+
t.Helper()
70+
71+
timer := time.NewTimer(30 * time.Second)
72+
ticker := time.NewTicker(500 * time.Millisecond)
73+
defer ticker.Stop()
74+
75+
first := true
76+
for {
77+
select {
78+
case <-ticker.C:
79+
resp, err := Elasticsearch.Ingest.GeoIPStats()
80+
require.NoError(t, err)
81+
82+
body, err := io.ReadAll(resp.Body)
83+
require.NoError(t, err)
84+
85+
err = json.Unmarshal([]byte(body), &stats)
86+
require.NoError(t, err)
87+
88+
// [GeoLite2-ASN.mmdb, GeoLite2-City.mmdb, GeoLite2-Country.mmdb]
89+
if stats.DatabasesCount == 3 {
90+
t.Log("GeoIp database downloaded")
91+
return nil
92+
}
93+
case <-timer.C:
94+
return fmt.Errorf("download timeout exceeded")
95+
}
96+
97+
if first {
98+
t.Log("waiting for GeoIP database to download")
99+
first = false
100+
}
101+
}
102+
}

‎systemtest/main_test.go‎

Lines changed: 0 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -18,77 +18,21 @@
1818
package systemtest
1919

2020
import (
21-
"encoding/json"
22-
"fmt"
23-
"io"
2421
"log"
2522
"os"
2623
"testing"
27-
"time"
2824
)
2925

30-
var stats struct {
31-
GeoIPStats `json:"stats"`
32-
}
33-
34-
type GeoIPStats struct {
35-
DatabasesCount int `json:"databases_count"`
36-
}
37-
3826
func TestMain(m *testing.M) {
3927
log.Println("INFO: starting stack containers...")
4028
initContainers()
4129
if err := StartStackContainers(); err != nil {
4230
log.Fatalf("failed to start stack containers: %v", err)
4331
}
4432
initElasticSearch()
45-
if err := waitGeoIPDownload(); err != nil {
46-
log.Fatalf("failed to download GeoIp database: %v", err)
47-
}
4833
initKibana()
4934
initSettings()
5035
initOTEL()
5136
log.Println("INFO: running system tests...")
5237
os.Exit(m.Run())
5338
}
54-
55-
func waitGeoIPDownload() error {
56-
timer := time.NewTimer(30 * time.Second)
57-
58-
ticker := time.NewTicker(500 * time.Millisecond)
59-
defer ticker.Stop()
60-
61-
first := true
62-
for {
63-
select {
64-
case <-ticker.C:
65-
resp, err := Elasticsearch.Ingest.GeoIPStats()
66-
if err != nil {
67-
return err
68-
}
69-
70-
body, err := io.ReadAll(resp.Body)
71-
if err != nil {
72-
return err
73-
}
74-
75-
err = json.Unmarshal([]byte(body), &stats)
76-
if err != nil {
77-
return err
78-
}
79-
80-
// [GeoLite2-ASN.mmdb, GeoLite2-City.mmdb, GeoLite2-Country.mmdb]
81-
if stats.DatabasesCount == 3 {
82-
log.Println("GeoIp database downloaded")
83-
return nil
84-
}
85-
case <-timer.C:
86-
return fmt.Errorf("download timeout exceeded")
87-
}
88-
89-
if first {
90-
log.Println("waiting for GeoIP database to download")
91-
first = false
92-
}
93-
}
94-
}

‎systemtest/rum_test.go‎

Lines changed: 22 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@ import (
2727
"path/filepath"
2828
"strings"
2929
"testing"
30-
"time"
3130

3231
"github.com/stretchr/testify/assert"
3332
"github.com/stretchr/testify/require"
@@ -39,24 +38,36 @@ import (
3938
"github.com/elastic/apm-tools/pkg/espoll"
4039
)
4140

42-
const requestBody = `{"metadata":{"service":{"name":"rum-js-test","agent":{"name":"rum-js","version":"5.5.0"}}}}
41+
func TestRUMXForwardedFor(t *testing.T) {
42+
systemtest.CleanupElasticsearch(t)
43+
44+
srv := apmservertest.NewUnstartedServerTB(t)
45+
srv.Config.RUM = &apmservertest.RUMConfig{Enabled: true}
46+
err := srv.Start()
47+
require.NoError(t, err)
48+
49+
serverURL, err := url.Parse(srv.URL)
50+
require.NoError(t, err)
51+
serverURL.Path = "/intake/v2/rum/events"
52+
53+
systemtest.GeoIpLazyDownload(t, serverURL.String())
54+
55+
const body = `{"metadata":{"service":{"name":"rum-js-test","agent":{"name":"rum-js","version":"5.5.0"}}}}
4356
{"transaction":{"trace_id":"611f4fa950f04631aaaaaaaaaaaaaaaa","id":"611f4fa950f04631","type":"page-load","duration":643,"span_count":{"started":0}}}
4457
{"metricset":{"samples":{"transaction.breakdown.count":{"value":12},"transaction.duration.sum.us":{"value":12},"transaction.duration.count":{"value":2},"transaction.self_time.sum.us":{"value":10},"transaction.self_time.count":{"value":2},"span.self_time.count":{"value":1},"span.self_time.sum.us":{"value":633.288}},"transaction":{"type":"request","name":"GET /"},"span":{"type":"external","subtype":"http"},"timestamp": 1496170422281000}}
4558
`
4659

47-
func lazyDownload(t *testing.T, url string) ([]espoll.SearchHit, bool) {
48-
t.Helper()
49-
50-
systemtest.CleanupElasticsearch(t)
51-
52-
req, err := http.NewRequest("POST", url, strings.NewReader(requestBody))
60+
req, err := http.NewRequest("POST", serverURL.String(), strings.NewReader(body))
5361
require.NoError(t, err)
5462

5563
req.Header.Set("Content-Type", "application/x-ndjson")
5664
req.Header.Set("X-Forwarded-For", "220.244.41.16")
5765

5866
resp, err := http.DefaultClient.Do(req)
5967
require.NoError(t, err)
68+
69+
_, err = io.ReadAll(resp.Body)
70+
require.NoError(t, err)
6071
require.Equal(t, http.StatusAccepted, resp.StatusCode)
6172

6273
io.Copy(io.Discard, resp.Body)
@@ -75,48 +86,12 @@ func lazyDownload(t *testing.T, url string) ([]espoll.SearchHit, bool) {
7586
Field: "processor.event",
7687
Values: []any{"transaction", "metric"},
7788
},
78-
espoll.WithTimeout(30*time.Second),
79-
)
80-
81-
for _, hit := range result.Hits.Hits {
82-
tags, ok := hit.Source["tags"]
83-
if ok {
84-
t.Logf("unexpect tags field: %v", tags)
85-
return nil, false
86-
}
87-
}
88-
89-
return result.Hits.Hits, true
90-
}
91-
92-
func TestRUMXForwardedFor(t *testing.T) {
93-
systemtest.CleanupElasticsearch(t)
94-
95-
srv := apmservertest.NewUnstartedServerTB(t)
96-
srv.Config.RUM = &apmservertest.RUMConfig{Enabled: true}
97-
err := srv.Start()
98-
require.NoError(t, err)
99-
100-
serverURL, err := url.Parse(srv.URL)
101-
require.NoError(t, err)
102-
serverURL.Path = "/intake/v2/rum/events"
103-
104-
var (
105-
ok bool
106-
hits []espoll.SearchHit
107-
)
108-
assert.Eventually(t,
109-
func() bool {
110-
hits, ok = lazyDownload(t, serverURL.String())
111-
return ok
112-
},
113-
3*time.Minute,
114-
30*time.Second,
115-
"failed to lazily download geoip database",
89+
// espoll.WithTimeout(30*time.Second),
11690
)
11791

92+
// Also checks for the absence of `tags` fields.
11893
approvaltest.ApproveFields(
119-
t, t.Name(), hits,
94+
t, t.Name(), result.Hits.Hits,
12095
// RUM timestamps are set by the server based on the time the payload is received.
12196
"@timestamp", "timestamp.us",
12297
// RUM events have the source port recorded, and in the tests it will be dynamic

0 commit comments

Comments
 (0)