Skip to content

Commit

Permalink
Cut 0.30.2 (thanos-io#6081)
Browse files Browse the repository at this point in the history
* tracing: fixed panic because of nil sampler (thanos-io#6066)

* fixed panic because of nil sampler

Signed-off-by: Vasiliy Rumyantsev <4119114+xBazilio@users.noreply.github.com>

* added CHANGELOG entry

Signed-off-by: Vasiliy Rumyantsev <4119114+xBazilio@users.noreply.github.com>

Signed-off-by: Vasiliy Rumyantsev <4119114+xBazilio@users.noreply.github.com>

* bump version to 0.30.2

Signed-off-by: Ben Ye <benye@amazon.com>

* Updates busybox SHA (thanos-io#6046)

Signed-off-by: GitHub <noreply@github.com>

Signed-off-by: GitHub <noreply@github.com>
Co-authored-by: yeya24 <yeya24@users.noreply.github.com>

* Use `e2edb.NewMinio` to disable SSE-S3 in e2e tests (thanos-io#6055)

* Use e2edb.NewMinio to disable SSE

Signed-off-by: Saswata Mukherjee <saswataminsta@yahoo.com>

* Use temp fork for TLS

Signed-off-by: Saswata Mukherjee <saswataminsta@yahoo.com>

* Fix broken rules api fanout test

Signed-off-by: Saswata Mukherjee <saswataminsta@yahoo.com>

* Fix broken query compatibility test

Signed-off-by: Saswata Mukherjee <saswataminsta@yahoo.com>

* Remove fork

Signed-off-by: Saswata Mukherjee <saswataminsta@yahoo.com>

Signed-off-by: Saswata Mukherjee <saswataminsta@yahoo.com>

---------

Signed-off-by: Vasiliy Rumyantsev <4119114+xBazilio@users.noreply.github.com>
Signed-off-by: Ben Ye <benye@amazon.com>
Signed-off-by: GitHub <noreply@github.com>
Signed-off-by: Saswata Mukherjee <saswataminsta@yahoo.com>
Co-authored-by: Vasiliy Rumyantsev <4119114+xBazilio@users.noreply.github.com>
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: yeya24 <yeya24@users.noreply.github.com>
Co-authored-by: Saswata Mukherjee <saswataminsta@yahoo.com>
  • Loading branch information
5 people authored and Nathaniel Graham committed Apr 17, 2023
1 parent 1327498 commit 3af74d6
Show file tree
Hide file tree
Showing 13 changed files with 74 additions and 188 deletions.
10 changes: 5 additions & 5 deletions .busybox-versions
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# Auto generated by busybox-updater.sh. DO NOT EDIT
amd64=21ab7a8218a3eded24d4f86d50cd2b8917e8d4c33be731eeb97e0e7568b52f82
arm64=bfba906739665ef14b021ee7d421275bcab47af8025a61c9b220e635c7b9c810
arm=16f4b2b8e017d87963d1f8daa331badcf0316b4adc7a8f8f8ee5c59ea32208f2
ppc64le=7cc6a566a2b427a724fee63403a30a571bd73c4e9e05a8e84c10d75debd57e9d
s390x=df395ffb2bd9086be276a4eba37be1edac468097afa7aa875807dc65a561e7a7
amd64=e6cbef04eb81ffc90238a69e5b82af7e21ac5654ed6db0fc8bfe0250cea2b6fc
arm64=97989df9593527c9360ada8bcda4144795a31629792b487317b49da1d2b9816d
arm=609c4b635550f51f103912dac4bf7ce66df444380278cd0592e9d01938ea8632
ppc64le=0fb57914d52f70c5d8095bed79107acf0363a4f1404268a28eecefa8cb5eef78
s390x=dbe837ced07bd54874578ab39cd80c45f15666d0698c0f5e3ab8d37fff5c45f7
6 changes: 6 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,12 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re

## Unreleased

## [v0.30.2](https://github.com/thanos-io/thanos/tree/release-0.30) - 28.01.2023

### Fixed

- [#6066](https://github.com/thanos-io/thanos/pull/6066) Tracing: fixed panic because of nil sampler

## [v0.30.1](https://github.com/thanos-io/thanos/tree/release-0.30) - 4.01.2023

### Fixed
Expand Down
2 changes: 1 addition & 1 deletion VERSION
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0.30.1
0.30.2
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ require (
github.com/chromedp/chromedp v0.8.2
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dustin/go-humanize v1.0.0
github.com/efficientgo/e2e v0.13.1-0.20220923082810-8fa9daa8af8a
github.com/efficientgo/e2e v0.14.1-0.20230119090947-fa7ceb0197c5
github.com/efficientgo/tools/extkingpin v0.0.0-20220817170617-6c25e3b627dd
github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb
github.com/fatih/structtag v1.2.0
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -255,8 +255,8 @@ github.com/edsrzf/mmap-go v1.1.0 h1:6EUwBLQ/Mcr1EYLE4Tn1VdW1A4ckqCQWZBw8Hr0kjpQ=
github.com/edsrzf/mmap-go v1.1.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8EIth78Q=
github.com/efficientgo/core v1.0.0-rc.0.0.20221201130417-ba593f67d2a4 h1:rydBwnBoywKQMjWF0z8SriYtQ+uUcaFsxuijMjJr5PI=
github.com/efficientgo/core v1.0.0-rc.0.0.20221201130417-ba593f67d2a4/go.mod h1:kQa0V74HNYMfuJH6jiPiwNdpWXl4xd/K4tzlrcvYDQI=
github.com/efficientgo/e2e v0.13.1-0.20220923082810-8fa9daa8af8a h1:cnJajqeh/HjvJLhI3wPvWG9OQ4gU79+4pELRD5Pkih8=
github.com/efficientgo/e2e v0.13.1-0.20220923082810-8fa9daa8af8a/go.mod h1:Hi+sz0REtlhVZ8zcdeTC3j6LUEEpJpPtNjOaOKuNcgI=
github.com/efficientgo/e2e v0.14.1-0.20230119090947-fa7ceb0197c5 h1:N1fHVcNEPMJNB93sxT6icl5yvoFxa2sp3/1NnVlrAFc=
github.com/efficientgo/e2e v0.14.1-0.20230119090947-fa7ceb0197c5/go.mod h1:plsKU0YHE9uX+7utvr7SiDtVBSHJyEfHRO4UnUgDmts=
github.com/efficientgo/tools/extkingpin v0.0.0-20220817170617-6c25e3b627dd h1:VaYzzXeUbC5fVheskcKVNOyJMEYD+HgrJNzIAg/mRIM=
github.com/efficientgo/tools/extkingpin v0.0.0-20220817170617-6c25e3b627dd/go.mod h1:ZV0utlglOczUWv3ih2AbqPSoLoFzdplUYxwV62eZi6Q=
github.com/elastic/go-sysinfo v1.1.1/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0=
Expand Down
23 changes: 13 additions & 10 deletions pkg/tracing/jaeger/config_yaml.go
Original file line number Diff line number Diff line change
Expand Up @@ -129,25 +129,28 @@ func getSampler(config Config) tracesdk.Sampler {
case "remote":
remoteOptions := getRemoteOptions(config)
sampler = jaegerremote.New(config.ServiceName, remoteOptions...)
// Fallback always to default (rate limiting).
case "ratelimiting":
fallthrough
default:
// The same config options are applicable to both remote and rate-limiting samplers.
remoteOptions := getRemoteOptions(config)
sampler = jaegerremote.New(config.ServiceName, remoteOptions...)
sampler, ok := sampler.(*rateLimitingSampler)
if ok {
sampler.Update(config.SamplerParam)
}
default:
var root tracesdk.Sampler
var parentOptions []tracesdk.ParentBasedSamplerOption
if config.SamplerParentConfig.LocalParentSampled {
parentOptions = append(parentOptions, tracesdk.WithLocalParentSampled(root))
}
if config.SamplerParentConfig.RemoteParentSampled {
parentOptions = append(parentOptions, tracesdk.WithRemoteParentSampled(root))
}
sampler = tracesdk.ParentBased(root, parentOptions...)
}

var parentOptions []tracesdk.ParentBasedSamplerOption
if config.SamplerParentConfig.LocalParentSampled {
parentOptions = append(parentOptions, tracesdk.WithLocalParentSampled(sampler))
}
if config.SamplerParentConfig.RemoteParentSampled {
parentOptions = append(parentOptions, tracesdk.WithRemoteParentSampled(sampler))
}
sampler = tracesdk.ParentBased(sampler, parentOptions...)

return sampler
}

Expand Down
7 changes: 4 additions & 3 deletions test/e2e/compact_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (
"time"

"github.com/efficientgo/e2e"
e2edb "github.com/efficientgo/e2e/db"
e2emon "github.com/efficientgo/e2e/monitoring"
"github.com/efficientgo/e2e/monitoring/matchers"
"github.com/go-kit/log"
Expand Down Expand Up @@ -347,11 +348,11 @@ func testCompactWithStoreGateway(t *testing.T, penaltyDedup bool) {
testutil.Ok(t, os.MkdirAll(dir, os.ModePerm))

const bucket = "compact-test"
m := e2ethanos.NewMinio(e, "minio", bucket)
m := e2edb.NewMinio(e, "minio", bucket, e2edb.WithMinioTLS())
testutil.Ok(t, e2e.StartAndWaitReady(m))

bkt, err := s3.NewBucketWithConfig(logger,
e2ethanos.NewS3Config(bucket, m.Endpoint("https"), m.Dir()), "test-feed")
e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed")
testutil.Ok(t, err)

ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
Expand Down Expand Up @@ -454,7 +455,7 @@ func testCompactWithStoreGateway(t *testing.T, penaltyDedup bool) {

bktConfig := client.BucketConfig{
Type: client.S3,
Config: e2ethanos.NewS3Config(bucket, m.InternalEndpoint("https"), m.InternalDir()),
Config: e2ethanos.NewS3Config(bucket, m.InternalEndpoint("http"), m.InternalDir()),
}

// Crank down the deletion mark delay since deduplication can miss blocks in the presence of replica labels it doesn't know about.
Expand Down
129 changes: 1 addition & 128 deletions test/e2e/e2ethanos/services.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,8 @@
package e2ethanos

import (
"crypto/rand"
"crypto/rsa"
"crypto/x509"
"encoding/json"
"encoding/pem"
"fmt"
"math/big"
"net"
"os"
"path/filepath"
"strconv"
Expand All @@ -31,9 +25,9 @@ import (
"gopkg.in/yaml.v2"

"github.com/thanos-io/objstore/client"
"github.com/thanos-io/objstore/exthttp"
"github.com/thanos-io/objstore/providers/s3"

"github.com/thanos-io/objstore/exthttp"
"github.com/thanos-io/thanos/pkg/alert"
"github.com/thanos-io/thanos/pkg/httpconfig"

Expand Down Expand Up @@ -945,57 +939,6 @@ http {
}), "http")
}

// NewMinio returns minio server, used as a local replacement for S3.
// TODO(@matej-g): This is a temporary workaround for https://github.com/efficientgo/e2e/issues/11;
// after this is addresses fixed all calls should be replaced with e2edb.NewMinio.
func NewMinio(e e2e.Environment, name, bktName string) *e2emon.InstrumentedRunnable {
image := "minio/minio:RELEASE.2022-07-30T05-21-40Z"
minioKESGithubContent := "https://raw.githubusercontent.com/minio/kes/master"

httpsPort := 8090
consolePort := 8080
f := e.Runnable(fmt.Sprintf("minio-%s", name)).
WithPorts(map[string]int{"https": httpsPort, "console": consolePort}).
Future()

if err := os.MkdirAll(filepath.Join(f.Dir(), "certs", "CAs"), 0750); err != nil {
return &e2emon.InstrumentedRunnable{Runnable: e2e.NewFailedRunnable(name, errors.Wrap(err, "create certs dir"))}
}

if err := genCerts(
filepath.Join(f.Dir(), "certs", "public.crt"),
filepath.Join(f.Dir(), "certs", "private.key"),
filepath.Join(f.Dir(), "certs", "CAs", "ca.crt"),
fmt.Sprintf("%s-minio-%s", e.Name(), name),
); err != nil {
return &e2emon.InstrumentedRunnable{Runnable: e2e.NewFailedRunnable(name, errors.Wrap(err, "fail to generate certs"))}
}

commands := []string{
fmt.Sprintf("curl -sSL --tlsv1.2 -O '%s/root.key' -O '%s/root.cert'", minioKESGithubContent, minioKESGithubContent),
fmt.Sprintf("mkdir -p /data/%s && minio server --certs-dir %s/certs --address :%v --console-address :%v /data", bktName, f.InternalDir(), httpsPort, consolePort),
}

minio := e2emon.AsInstrumented(f.Init(e2e.StartOptions{
Image: image,
// Create the required bucket before starting minio.
Command: e2e.NewCommandWithoutEntrypoint("sh", "-c", strings.Join(commands, " && ")),
Readiness: e2e.NewHTTPSReadinessProbe("console", "/", 200, 200),
EnvVars: map[string]string{
"MINIO_ROOT_USER": e2edb.MinioAccessKey,
"MINIO_ROOT_PASSWORD": e2edb.MinioSecretKey,
"MINIO_BROWSER": "on",
"ENABLE_HTTPS": "1",
// https://docs.min.io/docs/minio-kms-quickstart-guide.html
"MINIO_KMS_KES_ENDPOINT": "https://play.min.io:7373",
"MINIO_KMS_KES_KEY_FILE": "root.key",
"MINIO_KMS_KES_CERT_FILE": "root.cert",
"MINIO_KMS_KES_KEY_NAME": "my-minio-key",
},
}), "https")
return minio
}

func NewMemcached(e e2e.Environment, name string) *e2emon.InstrumentedRunnable {
return e2emon.AsInstrumented(e.Runnable(fmt.Sprintf("memcached-%s", name)).
WithPorts(map[string]int{"memcached": 11211}).
Expand Down Expand Up @@ -1061,76 +1004,6 @@ func NewToolsBucketWeb(
})), "http")
}

// genCerts generates certificates and writes those to the provided paths.
func genCerts(certPath, privkeyPath, caPath, serverName string) error {
var caRoot = &x509.Certificate{
SerialNumber: big.NewInt(2019),
NotAfter: time.Now().AddDate(10, 0, 0),
IsCA: true,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
BasicConstraintsValid: true,
}

var cert = &x509.Certificate{
SerialNumber: big.NewInt(1658),
DNSNames: []string{serverName},
IPAddresses: []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP("::1")},
NotAfter: time.Now().AddDate(10, 0, 0),
SubjectKeyId: []byte{1, 2, 3},
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
KeyUsage: x509.KeyUsageDigitalSignature,
}

caPrivKey, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
return err
}

certPrivKey, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
return err
}
// Generate CA cert.
caBytes, err := x509.CreateCertificate(rand.Reader, caRoot, caRoot, &caPrivKey.PublicKey, caPrivKey)
if err != nil {
return err
}
caPEM := pem.EncodeToMemory(&pem.Block{
Type: "CERTIFICATE",
Bytes: caBytes,
})
err = os.WriteFile(caPath, caPEM, 0644)
if err != nil {
return err
}

// Sign the cert with the CA private key.
certBytes, err := x509.CreateCertificate(rand.Reader, cert, caRoot, &certPrivKey.PublicKey, caPrivKey)
if err != nil {
return err
}
certPEM := pem.EncodeToMemory(&pem.Block{
Type: "CERTIFICATE",
Bytes: certBytes,
})
err = os.WriteFile(certPath, certPEM, 0644)
if err != nil {
return err
}

certPrivKeyPEM := pem.EncodeToMemory(&pem.Block{
Type: "RSA PRIVATE KEY",
Bytes: x509.MarshalPKCS1PrivateKey(certPrivKey),
})
err = os.WriteFile(privkeyPath, certPrivKeyPEM, 0644)
if err != nil {
return err
}

return nil
}

func NewS3Config(bucket, endpoint, basePath string) s3.Config {
httpDefaultConf := s3.DefaultConfig.HTTPConfig
httpDefaultConf.TLSConfig = exthttp.TLSConfig{
Expand Down
5 changes: 3 additions & 2 deletions test/e2e/info_api_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import (
"github.com/thanos-io/objstore/client"

"github.com/efficientgo/core/testutil"
e2edb "github.com/efficientgo/e2e/db"
"github.com/thanos-io/thanos/pkg/query"
"github.com/thanos-io/thanos/pkg/runutil"
"github.com/thanos-io/thanos/test/e2e/e2ethanos"
Expand All @@ -37,14 +38,14 @@ func TestInfo(t *testing.T) {
testutil.Ok(t, e2e.StartAndWaitReady(prom1, sidecar1, prom2, sidecar2, prom3, sidecar3))

const bucket = "info-api-test"
m := e2ethanos.NewMinio(e, "thanos-minio", bucket)
m := e2edb.NewMinio(e, "thanos-minio", bucket, e2edb.WithMinioTLS())
testutil.Ok(t, e2e.StartAndWaitReady(m))
store := e2ethanos.NewStoreGW(
e,
"1",
client.BucketConfig{
Type: client.S3,
Config: e2ethanos.NewS3Config(bucket, m.InternalEndpoint("https"), m.InternalDir()),
Config: e2ethanos.NewS3Config(bucket, m.InternalEndpoint("http"), m.InternalDir()),
},
"",
nil,
Expand Down
14 changes: 7 additions & 7 deletions test/e2e/query_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -542,7 +542,7 @@ config:
ruleAndAssert(t, ctx, q.Endpoint("http"), "", []*rulespb.RuleGroup{
{
Name: "example_abort",
File: "/shared/data/querier-1/rules/rules.yaml",
File: q.Dir() + "/rules/rules.yaml",
Rules: []*rulespb.Rule{
rulespb.NewAlertingRule(&rulespb.Alert{
Name: "TestAlert_AbortOnPartialResponse",
Expand Down Expand Up @@ -591,11 +591,11 @@ func TestQueryStoreMetrics(t *testing.T) {
t.Cleanup(cancel)

bucket := "store-gw-test"
minio := e2ethanos.NewMinio(e, "thanos-minio", bucket)
minio := e2edb.NewMinio(e, "thanos-minio", bucket, e2edb.WithMinioTLS())
testutil.Ok(t, e2e.StartAndWaitReady(minio))

l := log.NewLogfmtLogger(os.Stdout)
bkt, err := s3.NewBucketWithConfig(l, e2ethanos.NewS3Config(bucket, minio.Endpoint("https"), minio.Dir()), "test")
bkt, err := s3.NewBucketWithConfig(l, e2ethanos.NewS3Config(bucket, minio.Endpoint("http"), minio.Dir()), "test")
testutil.Ok(t, err)

// Preparing 2 different blocks for the tests.
Expand Down Expand Up @@ -639,7 +639,7 @@ func TestQueryStoreMetrics(t *testing.T) {
"s1",
client.BucketConfig{
Type: client.S3,
Config: e2ethanos.NewS3Config(bucket, minio.InternalEndpoint("https"), minio.InternalDir()),
Config: e2ethanos.NewS3Config(bucket, minio.InternalEndpoint("http"), minio.InternalDir()),
},
"",
nil,
Expand Down Expand Up @@ -719,7 +719,7 @@ func TestSidecarStorePushdown(t *testing.T) {
testutil.Ok(t, e2e.StartAndWaitReady(prom1, sidecar1))

const bucket = "store-gateway-test"
m := e2ethanos.NewMinio(e, "thanos-minio", bucket)
m := e2edb.NewMinio(e, "thanos-minio", bucket, e2edb.WithMinioTLS())
testutil.Ok(t, e2e.StartAndWaitReady(m))

dir := filepath.Join(e.SharedDir(), "tmp")
Expand All @@ -736,7 +736,7 @@ func TestSidecarStorePushdown(t *testing.T) {
testutil.Ok(t, err)

l := log.NewLogfmtLogger(os.Stdout)
bkt, err := s3.NewBucketWithConfig(l, e2ethanos.NewS3Config(bucket, m.Endpoint("https"), m.Dir()), "test")
bkt, err := s3.NewBucketWithConfig(l, e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test")
testutil.Ok(t, err)
testutil.Ok(t, objstore.UploadDir(ctx, l, bkt, path.Join(dir, id1.String()), id1.String()))

Expand All @@ -745,7 +745,7 @@ func TestSidecarStorePushdown(t *testing.T) {
"1",
client.BucketConfig{
Type: client.S3,
Config: e2ethanos.NewS3Config(bucket, m.InternalEndpoint("https"), m.InternalDir()),
Config: e2ethanos.NewS3Config(bucket, m.InternalEndpoint("http"), m.InternalDir()),
},
"",
nil,
Expand Down
8 changes: 4 additions & 4 deletions test/e2e/rules_api_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ func TestRulesAPI_Fanout(t *testing.T) {
ruleAndAssert(t, ctx, q.Endpoint("http"), "", []*rulespb.RuleGroup{
{
Name: "example_abort",
File: "/shared/data/querier-query/rules/rules.yaml",
File: q.Dir() + "/rules/rules.yaml",
Rules: []*rulespb.Rule{
rulespb.NewAlertingRule(&rulespb.Alert{
Name: "TestAlert_AbortOnPartialResponse",
Expand All @@ -112,7 +112,7 @@ func TestRulesAPI_Fanout(t *testing.T) {
},
{
Name: "example_abort",
File: "/shared/data/querier-query/thanos-rules/rules-0.yaml",
File: q.Dir() + "/thanos-rules/rules-0.yaml",
Rules: []*rulespb.Rule{
rulespb.NewAlertingRule(&rulespb.Alert{
Name: "TestAlert_AbortOnPartialResponse",
Expand All @@ -127,7 +127,7 @@ func TestRulesAPI_Fanout(t *testing.T) {
},
{
Name: "example_warn",
File: "/shared/data/querier-query/thanos-rules/rules-1.yaml",
File: q.Dir() + "/thanos-rules/rules-1.yaml",
Rules: []*rulespb.Rule{
rulespb.NewAlertingRule(&rulespb.Alert{
Name: "TestAlert_WarnOnPartialResponse",
Expand All @@ -142,7 +142,7 @@ func TestRulesAPI_Fanout(t *testing.T) {
},
{
Name: "example_with_limit",
File: "/shared/data/querier-query/thanos-rules/rules-with-limit.yaml",
File: q.Dir() + "/thanos-rules/rules-with-limit.yaml",
Limit: 1,
Rules: []*rulespb.Rule{
rulespb.NewAlertingRule(&rulespb.Alert{
Expand Down
Loading

0 comments on commit 3af74d6

Please sign in to comment.