diff --git a/.circleci/config.yml b/.circleci/config.yml index 5676f44d66..69077958e6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -42,7 +42,8 @@ jobs: echo "Skipping SWIFT tests." export THANOS_SKIP_TENCENT_COS_TESTS="true" echo "Skipping TENCENT COS tests." - + export THANOS_SKIP_ALIYUN_OSS_TESTS="true" + echo "Skipping ALIYUN OSS tests." make test # Cross build is needed for publish_release but needs to be done outside of docker. diff --git a/CHANGELOG.md b/CHANGELOG.md index bf3ebe3dae..9f21153000 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel ## Unreleased. - [#1338](https://github.com/thanos-io/thanos/pull/1338) Querier still warns on store API duplicate, but allows a single one from duplicated set. This is gracefully warn about the problematic logic and not disrupt immediately. +- [#1234](https://github.com/thanos-io/thanos/pull/1234) `AliYun OSS` object storage, see [documents](docs/storage.md#aliyun-oss-configuration) for further information. ### Fixed @@ -23,6 +24,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel ### Added + - [#1097](https://github.com/thanos-io/thanos/pull/1097) Added `thanos check rules` linter for Thanos rule rules files. - [#1253](https://github.com/thanos-io/thanos/pull/1253) Add support for specifying a maximum amount of retries when using Azure Blob storage (default: no retries). diff --git a/Makefile b/Makefile index aba7962ef2..f7545825f1 100644 --- a/Makefile +++ b/Makefile @@ -149,8 +149,8 @@ docs: $(EMBEDMD) build .PHONY: check-docs check-docs: $(EMBEDMD) $(LICHE) build @EMBEDMD_BIN="$(EMBEDMD)" scripts/genflagdocs.sh check - @$(LICHE) --recursive docs --exclude "cloud.tencent.com" --document-root . - @$(LICHE) --exclude "cloud.tencent.com" --document-root . *.md + @$(LICHE) --recursive docs --exclude "(cloud.tencent.com|alibabacloud.com)" --document-root . + @$(LICHE) --exclude "(cloud.tencent.com|alibabacloud.com)" --document-root . *.md # format formats the code (including imports format). .PHONY: format diff --git a/docs/storage.md b/docs/storage.md index b9050801e4..4e08f32324 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -22,6 +22,7 @@ Current object storage client implementations: | Azure Storage Account | Stable (production usage) | yes | @vglafirov | | OpenStack Swift | Beta (working PoCs, testing usage) | no | @sudhi-vm | | Tencent COS | Beta (testing usage) | no | @jojohappy | +| AliYun OSS | Beta (testing usage) | no | @shaulboozhiao,@wujinhu | NOTE: Currently Thanos requires strong consistency (write-read) for object store implementation. @@ -308,3 +309,20 @@ config: ``` Set the flags `--objstore.config-file` to reference to the configuration file. + +## AliYun OSS Configuration +In order to use AliYun OSS object storage, you should first create a bucket with proper Storage Class , ACLs and get the access key on the AliYun cloud. Go to [https://www.alibabacloud.com/product/oss](https://www.alibabacloud.com/product/oss) for more detail. + +To use AliYun OSS object storage, please specify following yaml configuration file in `objstore.config*` flag. + +[embedmd]:# (flags/config_aliyunoss.txt $) +```$ +type: ALIYUNOSS +config: + endpoint: "" + bucket: "" + access_key_id: "" + access_key_secret: "" +``` + +Use --objstore.config-file to reference to this configuration file. diff --git a/go.mod b/go.mod index 9873a47487..57eef80a65 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,9 @@ require ( cloud.google.com/go v0.34.0 github.com/Azure/azure-storage-blob-go v0.7.0 github.com/NYTimes/gziphandler v1.1.1 + github.com/aliyun/aliyun-oss-go-sdk v1.9.8 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da + github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f // indirect github.com/cespare/xxhash v1.1.0 github.com/fatih/structtag v1.0.0 github.com/fortytw2/leaktest v1.3.0 @@ -15,6 +17,7 @@ require ( github.com/google/martian v2.1.0+incompatible // indirect github.com/googleapis/gax-go v2.0.2+incompatible // indirect github.com/gophercloud/gophercloud v0.0.0-20190301152420-fca40860790e + github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 github.com/grpc-ecosystem/go-grpc-prometheus v0.0.0-20181025070259-68e3a13e4117 github.com/hashicorp/golang-lru v0.5.1 @@ -33,18 +36,23 @@ require ( github.com/pkg/errors v0.8.1 github.com/prometheus/client_golang v1.0.0 github.com/prometheus/common v0.6.0 + github.com/prometheus/procfs v0.0.3 // indirect github.com/prometheus/prometheus v2.9.2+incompatible github.com/prometheus/tsdb v0.8.0 github.com/uber-go/atomic v1.4.0 // indirect github.com/uber/jaeger-client-go v2.16.0+incompatible github.com/uber/jaeger-lib v2.0.0+incompatible go.uber.org/atomic v1.4.0 // indirect + golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 // indirect + golang.org/x/net v0.0.0-20190628185345-da137c7871d7 // indirect golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 golang.org/x/sync v0.0.0-20190423024810-112230192c58 + golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb // indirect golang.org/x/text v0.3.2 google.golang.org/api v0.3.2 google.golang.org/grpc v1.19.1 gopkg.in/alecthomas/kingpin.v2 v2.2.6 gopkg.in/fsnotify.v1 v1.4.7 + gopkg.in/fsnotify/fsnotify.v1 v1.4.7 // indirect gopkg.in/yaml.v2 v2.2.2 ) diff --git a/go.sum b/go.sum index 24b25203f3..21377ca7e8 100644 --- a/go.sum +++ b/go.sum @@ -26,6 +26,8 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5Vpd github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/aliyun/aliyun-oss-go-sdk v1.9.8 h1:BOflvK0Zs/zGmoabyFIzTg5c3kguktWTXEwewwbuba0= +github.com/aliyun/aliyun-oss-go-sdk v1.9.8/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I= @@ -33,6 +35,8 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/aws/aws-sdk-go v0.0.0-20180507225419-00862f899353 h1:qFKf58XUUvHaEz0zFkLJsQ4dzoAyrQ8QyhK4nHGHBI4= github.com/aws/aws-sdk-go v0.0.0-20180507225419-00862f899353/go.mod h1:ZRmQr0FajVIyZ4ZzBYKG5P3ZqPz9IHG41ZoMu1ADI3k= +github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f h1:ZNv7On9kyUzm7fvRZumSyy/IUiSC7AzL0I1jKKtwooA= +github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f/go.mod h1:AuiFmCCPBSrqvVMvuqFuk0qogytodnVFVSN5CeJB8Gc= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= @@ -108,6 +112,8 @@ github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCy github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v0.0.0-20150304233714-bbcb9da2d746 h1:M6d2zDTA4cKXT6OwFsJxlo5tWrAukj3KfvJ1zcBatnA= @@ -123,6 +129,8 @@ github.com/gophercloud/gophercloud v0.0.0-20190301152420-fca40860790e h1:hQpY0g0 github.com/gophercloud/gophercloud v0.0.0-20190301152420-fca40860790e/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e h1:JKmoR8x90Iww1ks85zJ1lfDGgIiMDuIptTOhJq+zKyg= +github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 h1:pdN6V1QBWetyv/0+wjACpqVH+eVULgEjkurDLq3goeM= @@ -284,6 +292,8 @@ github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1 h1:/K3IL0Z1quvmJ github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.3 h1:CTwfnzjQ+8dS6MhHHu4YswVAD99sL2wjPqP+VkURmKE= +github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/prometheus v2.9.2+incompatible h1:5QVnXpkSsbbG59TyZ99clRfaHQy2QuIlTv6dEgS66C4= github.com/prometheus/prometheus v2.9.2+incompatible/go.mod h1:vdLuLLM0uqhLSofrQ7Nev2b/rQUyZ+pkT3zF7LB/i1g= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= @@ -298,6 +308,7 @@ github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFo github.com/samuel/go-zookeeper v0.0.0-20161028232340-1d7be4effb13 h1:4AQBn5RJY4WH8t8TLEMZUsWeXHAUcoao42TCAfpEJJE= github.com/samuel/go-zookeeper v0.0.0-20161028232340-1d7be4effb13/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/sasha-s/go-deadlock v0.0.0-20161201235124-341000892f3d/go.mod h1:StQn567HiB1fF2yJ44N9au7wOhrPS3iZqiDbRupzT10= +github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= @@ -337,6 +348,8 @@ golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f h1:R423Cnkcp5JABoeemiGEPlt9tHXFfw5kvc0yqlxRPWo= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc= +golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -357,6 +370,8 @@ golang.org/x/net v0.0.0-20190522155817-f3200d17e092 h1:4QSRKanuywn15aTZvI/mIDEgP golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 h1:dfGZHvZk057jK2MCeWus/TowKpJ8y4AmooUzdBSR9GU= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7 h1:rTIdg5QFRR7XCaK4LCjBiPbx8j4DQRpdYMnGn/bJUEU= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 h1:Wo7BWFiOk0QRFMLYMqJGFMd9CgUAcGx7V+qEg/h5IBI= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -382,6 +397,8 @@ golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb h1:fgwFCsaw9buMuxNd6+DQfAuSFqbNiQZpcgJQAgJsK6k= +golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -416,6 +433,8 @@ gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/fsnotify/fsnotify.v1 v1.3.1 h1:2fkCHbPQZNYRAyRyIV9VX0bpRkxIorlQDiYRmufHnhA= gopkg.in/fsnotify/fsnotify.v1 v1.3.1/go.mod h1:Fyux9zXlo4rWoMSIzpn9fDAYjalPqJ/K1qJ27s+7ltE= +gopkg.in/fsnotify/fsnotify.v1 v1.4.7 h1:XNNYLJHt73EyYiCZi6+xjupS9CpvmiDgjPTAjrBlQbo= +gopkg.in/fsnotify/fsnotify.v1 v1.4.7/go.mod h1:Fyux9zXlo4rWoMSIzpn9fDAYjalPqJ/K1qJ27s+7ltE= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.42.0 h1:7N3gPTt50s8GuLortA00n8AqRTk75qOP98+mTPpgzRk= diff --git a/pkg/objstore/client/factory.go b/pkg/objstore/client/factory.go index 2a4edc86b0..50caf4d8e7 100644 --- a/pkg/objstore/client/factory.go +++ b/pkg/objstore/client/factory.go @@ -13,6 +13,7 @@ import ( "github.com/thanos-io/thanos/pkg/objstore/azure" "github.com/thanos-io/thanos/pkg/objstore/cos" "github.com/thanos-io/thanos/pkg/objstore/gcs" + "github.com/thanos-io/thanos/pkg/objstore/oss" "github.com/thanos-io/thanos/pkg/objstore/s3" "github.com/thanos-io/thanos/pkg/objstore/swift" yaml "gopkg.in/yaml.v2" @@ -21,11 +22,12 @@ import ( type ObjProvider string const ( - GCS ObjProvider = "GCS" - S3 ObjProvider = "S3" - AZURE ObjProvider = "AZURE" - SWIFT ObjProvider = "SWIFT" - COS ObjProvider = "COS" + GCS ObjProvider = "GCS" + S3 ObjProvider = "S3" + AZURE ObjProvider = "AZURE" + SWIFT ObjProvider = "SWIFT" + COS ObjProvider = "COS" + ALIYUNOSS ObjProvider = "ALIYUNOSS" ) type BucketConfig struct { @@ -59,6 +61,8 @@ func NewBucket(logger log.Logger, confContentYaml []byte, reg prometheus.Registe bucket, err = swift.NewContainer(logger, config) case string(COS): bucket, err = cos.NewBucket(logger, config, component) + case string(ALIYUNOSS): + bucket, err = oss.NewBucket(logger, config, component) default: return nil, errors.Errorf("bucket with type %s is not supported", bucketConf.Type) } diff --git a/pkg/objstore/objtesting/foreach.go b/pkg/objstore/objtesting/foreach.go index 66dc47b24f..d92e45d9fd 100644 --- a/pkg/objstore/objtesting/foreach.go +++ b/pkg/objstore/objtesting/foreach.go @@ -11,6 +11,7 @@ import ( "github.com/thanos-io/thanos/pkg/objstore/cos" "github.com/thanos-io/thanos/pkg/objstore/gcs" "github.com/thanos-io/thanos/pkg/objstore/inmem" + "github.com/thanos-io/thanos/pkg/objstore/oss" "github.com/thanos-io/thanos/pkg/objstore/s3" "github.com/thanos-io/thanos/pkg/objstore/swift" "github.com/thanos-io/thanos/pkg/testutil" @@ -117,4 +118,20 @@ func ForeachStore(t *testing.T, testFn func(t testing.TB, bkt objstore.Bucket)) } else { t.Log("THANOS_SKIP_TENCENT_COS_TESTS envvar present. Skipping test against Tencent COS.") } + + // Optional OSS. + if _, ok := os.LookupEnv("THANOS_SKIP_ALIYUN_OSS_TESTS"); !ok { + bkt, closeFn, err := oss.NewTestBucket(t) + testutil.Ok(t, err) + + ok := t.Run("AliYun oss", func(t *testing.T) { + testFn(t, bkt) + }) + closeFn() + if !ok { + return + } + } else { + t.Log("THANOS_SKIP_ALIYUN_OSS_TESTS envvar present. Skipping test against AliYun OSS.") + } } diff --git a/pkg/objstore/oss/oss.go b/pkg/objstore/oss/oss.go new file mode 100644 index 0000000000..7294fb18c4 --- /dev/null +++ b/pkg/objstore/oss/oss.go @@ -0,0 +1,362 @@ +package oss + +import ( + "context" + "fmt" + "io" + "io/ioutil" + "math" + "math/rand" + "net/http" + "os" + "strings" + "testing" + "time" + + alioss "github.com/aliyun/aliyun-oss-go-sdk/oss" + "github.com/go-kit/kit/log" + "github.com/pkg/errors" + "github.com/thanos-io/thanos/pkg/objstore" + "github.com/thanos-io/thanos/pkg/runutil" + yaml "gopkg.in/yaml.v2" +) + +type Config struct { + Endpoint string `yaml:"endpoint"` + Bucket string `yaml:"bucket"` + AccessKeyID string `yaml:"access_key_id"` + AccessKeySecret string `yaml:"access_key_secret"` +} + +type Bucket struct { + name string + logger log.Logger + client *alioss.Client + config Config + bucket *alioss.Bucket +} + +func NewTestBucket(t testing.TB) (objstore.Bucket, func(), error) { + c := configFromEnv() + err := func(conf Config) error { + if conf.Endpoint == "" { + return errors.New("no aliyun oss endpoint in config file") + } + if conf.AccessKeyID == "" { + return errors.New("access_key_id is not present in config file") + } + if conf.AccessKeySecret == "" { + return errors.New("access_key_secret is not present in config file") + } + return nil + }(c) + if err != nil { + return nil, nil, err + } + if c.Bucket != "" && os.Getenv("THANOS_ALLOW_EXISTING_BUCKET_USE") == "true" { + t.Log("ALIYUNOSS_BUCKET is defined. Normally this tests will create temporary bucket " + + "and delete it after test. Unset ALIYUNOSS_BUCKET env variable to use default logic. If you really want to run " + + "tests against provided (NOT USED!) bucket, set THANOS_ALLOW_EXISTING_BUCKET_USE=true.") + return NewTestBucketFromConfig(t, c, true) + } + return NewTestBucketFromConfig(t, c, false) +} + +func calculateChunks(name string, r io.Reader) (int, int64, error) { + switch r.(type) { + case *os.File: + f, _ := r.(*os.File) + if fileInfo, err := f.Stat(); err == nil { + s := fileInfo.Size() + return int(math.Floor(float64(s) / alioss.MaxPartSize)), s % alioss.MaxPartSize, nil + } + case *strings.Reader: + f, _ := r.(*strings.Reader) + return int(math.Floor(float64(f.Size()) / alioss.MaxPartSize)), f.Size() % alioss.MaxPartSize, nil + } + return -1, 0, errors.New("unsupported implement of io.Reader") +} + +// Upload the contents of the reader as an object into the bucket. +func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { + chunksnum, lastslice, err := calculateChunks(name, r) + if err != nil { + return err + } + + ncloser := ioutil.NopCloser(r) + if chunksnum == 0 { + if err := b.bucket.PutObject(name, ncloser); err != nil { + return errors.Wrap(err, "failed to upload oss object") + } + } else { + init, err := b.bucket.InitiateMultipartUpload(name) + if err != nil { + return errors.Wrap(err, "failed to initiate multi-part upload") + } + chunk := 0 + uploadEveryPart := func(everypartsize int64, cnk int) (alioss.UploadPart, error) { + prt, err := b.bucket.UploadPart(init, ncloser, everypartsize, cnk) + if err != nil { + if err2 := b.bucket.AbortMultipartUpload(init); err2 != nil { + return prt, errors.Wrap(err2, "failed to abort multi-part upload") + } + return prt, errors.Wrap(err, "failed to upload multi-part chunk") + } + return prt, nil + } + var parts []alioss.UploadPart + for ; chunk < chunksnum; chunk++ { + part, err := uploadEveryPart(alioss.MaxPartSize, chunk+1) + if err != nil { + return err + } + parts = append(parts, part) + } + if lastslice != 0 { + part, err := uploadEveryPart(lastslice, chunksnum+1) + if err != nil { + return errors.Wrap(err, "failed to upload the last chunk") + } + parts = append(parts, part) + } + _, err = b.bucket.CompleteMultipartUpload(init, parts) + if err != nil { + return errors.Wrap(err, "failed to set multi-part upload completive") + } + } + return nil +} + +// Delete removes the object with the given name. +func (b *Bucket) Delete(ctx context.Context, name string) error { + if err := b.bucket.DeleteObject(name); err != nil { + return errors.Wrap(err, "delete oss object") + } + return nil +} + +func configFromEnv() Config { + c := Config{ + Endpoint: os.Getenv("ALIYUNOSS_ENDPOINT"), + Bucket: os.Getenv("ALIYUNOSS_BUCKET"), + AccessKeyID: os.Getenv("ALIYUNOSS_ACCESS_KEY_ID"), + AccessKeySecret: os.Getenv("ALIYUNOSS_ACCESS_KEY_SECRET"), + } + return c +} + +func NewBucket(logger log.Logger, conf []byte, component string) (*Bucket, error) { + config, err := func(conf []byte) (Config, error) { + var config Config + if err := yaml.Unmarshal(conf, &config); err != nil { + return Config{}, err + } + + return config, nil + }(conf) + + if err != nil { + return nil, errors.Wrap(err, "parse aliyun oss config file failed") + } + if err := validate(config); err != nil { + return nil, errors.Wrap(err, "validate aliyun oss config file failed") + } + + client, err := alioss.New(config.Endpoint, config.AccessKeyID, config.AccessKeySecret) + if err != nil { + return nil, errors.Wrap(err, "create aliyun oss client failed") + } + bk, err := client.Bucket(config.Bucket) + if err != nil { + return nil, errors.Wrapf(err, "use aliyun oss bucket %s failed", config.Bucket) + } + + bkt := &Bucket{ + logger: logger, + client: client, + name: config.Bucket, + config: config, + bucket: bk, + } + return bkt, nil +} + +// Iter calls f for each entry in the given directory (not recursive). The argument to f is the full +// object name including the prefix of the inspected directory. +func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error) error { + if dir != "" { + dir = strings.TrimSuffix(dir, objstore.DirDelim) + objstore.DirDelim + } + + marker := alioss.Marker("") + for { + if err := ctx.Err(); err != nil { + return errors.Wrap(err, "context closed while iterating bucket") + } + objects, err := b.bucket.ListObjects(alioss.Prefix(dir), alioss.Delimiter(objstore.DirDelim), marker) + if err != nil { + return errors.Wrap(err, "listing aliyun oss bucket failed") + } + marker = alioss.Marker(objects.NextMarker) + + for _, object := range objects.Objects { + if err := f(object.Key); err != nil { + return errors.Wrapf(err, "callback func invoke for object %s failed ", object.Key) + } + } + + for _, object := range objects.CommonPrefixes { + if err := f(object); err != nil { + return errors.Wrapf(err, "callback func invoke for directory %s failed", object) + } + } + if !objects.IsTruncated { + break + } + } + + return nil +} + +func (b *Bucket) Name() string { + return b.name +} + +// validate checks to see the config options are set. +func validate(conf Config) error { + if conf.Endpoint == "" { + return errors.New("no aliyun oss endpoint in config file") + } + if conf.Bucket == "" { + return errors.New("no aliyun oss bucket in config file") + } + + if conf.AccessKeyID == "" { + return errors.New("access_key_id is not present in config file") + } + if conf.AccessKeySecret == "" { + return errors.New("access_key_secret is not present in config file") + } + return nil +} + +func NewTestBucketFromConfig(t testing.TB, c Config, reuseBucket bool) (objstore.Bucket, func(), error) { + if c.Bucket == "" { + src := rand.NewSource(time.Now().UnixNano()) + + bktToCreate := strings.Replace(fmt.Sprintf("test_%s_%x", strings.ToLower(t.Name()), src.Int63()), "_", "-", -1) + if len(bktToCreate) >= 63 { + bktToCreate = bktToCreate[:63] + } + testclient, err := alioss.New(c.Endpoint, c.AccessKeyID, c.AccessKeySecret) + if err != nil { + return nil, nil, errors.Wrap(err, "create aliyun oss client failed") + } + + if err := testclient.CreateBucket(bktToCreate); err != nil { + return nil, nil, errors.Wrapf(err, "create aliyun oss bucket %s failed", bktToCreate) + } + c.Bucket = bktToCreate + } + + bc, err := yaml.Marshal(c) + if err != nil { + return nil, nil, err + } + + b, err := NewBucket(log.NewNopLogger(), bc, "thanos-aliyun-oss-test") + if err != nil { + return nil, nil, err + } + + if reuseBucket { + if err := b.Iter(context.Background(), "", func(f string) error { + return errors.Errorf("bucket %s is not empty", c.Bucket) + }); err != nil { + return nil, nil, errors.Wrapf(err, "oss check bucket %s", c.Bucket) + } + + t.Log("WARNING. Reusing", c.Bucket, "Aliyun OSS bucket for OSS tests. Manual cleanup afterwards is required") + return b, func() {}, nil + } + + return b, func() { + objstore.EmptyBucket(t, context.Background(), b) + if err := b.client.DeleteBucket(c.Bucket); err != nil { + t.Logf("deleting bucket %s failed: %s", c.Bucket, err) + } + }, nil +} + +func (b *Bucket) Close() error { return nil } + +func setRange(start, end int64) (alioss.Option, error) { + var opt alioss.Option + if 0 <= start && start <= end { + opt = alioss.Range(start, end) + } else { + return nil, errors.Errorf("Invalid range specified: start=%d end=%d", start, end) + } + return opt, nil +} + +func (b *Bucket) getRange(ctx context.Context, name string, off, length int64) (io.ReadCloser, error) { + if len(name) == 0 { + return nil, errors.New("given object name should not empty") + } + + var opts []alioss.Option + if length != -1 { + opt, err := setRange(off, off+length-1) + if err != nil { + return nil, err + } + opts = append(opts, opt) + } + + resp, err := b.bucket.GetObject(name, opts...) + if err != nil { + return nil, err + } + + if _, err := resp.Read(nil); err != nil { + runutil.CloseWithLogOnErr(b.logger, resp, "oss get range obj close") + return nil, err + } + + return resp, nil +} + +// Get returns a reader for the given object name. +func (b *Bucket) Get(ctx context.Context, name string) (io.ReadCloser, error) { + return b.getRange(ctx, name, 0, -1) +} + +func (b *Bucket) GetRange(ctx context.Context, name string, off, length int64) (io.ReadCloser, error) { + return b.getRange(ctx, name, off, length) +} + +// Exists checks if the given object exists in the bucket. +func (b *Bucket) Exists(ctx context.Context, name string) (bool, error) { + exists, err := b.bucket.IsObjectExist(name) + if err != nil { + if b.IsObjNotFoundErr(err) { + return false, nil + } + return false, errors.Wrap(err, "cloud not check if object exists") + } + + return exists, nil +} + +// IsObjNotFoundErr returns true if error means that object is not found. Relevant to Get operations. +func (b *Bucket) IsObjNotFoundErr(err error) bool { + switch aliErr := err.(type) { + case alioss.ServiceError: + if aliErr.StatusCode == http.StatusNotFound { + return true + } + } + return false +} diff --git a/pkg/query/api/v1_test.go b/pkg/query/api/v1_test.go index 36fcf06644..ce43f1c392 100644 --- a/pkg/query/api/v1_test.go +++ b/pkg/query/api/v1_test.go @@ -38,7 +38,9 @@ import ( "github.com/prometheus/prometheus/promql" "github.com/prometheus/prometheus/storage" "github.com/thanos-io/thanos/pkg/compact" + extpromhttp "github.com/thanos-io/thanos/pkg/extprom/http" + "github.com/thanos-io/thanos/pkg/query" "github.com/thanos-io/thanos/pkg/testutil" ) diff --git a/pkg/query/storeset_test.go b/pkg/query/storeset_test.go index 4dd59b42b8..2ff65776cb 100644 --- a/pkg/query/storeset_test.go +++ b/pkg/query/storeset_test.go @@ -12,8 +12,10 @@ import ( "sort" "github.com/fortytw2/leaktest" + "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" + "github.com/thanos-io/thanos/pkg/store/storepb" "github.com/thanos-io/thanos/pkg/testutil" "google.golang.org/grpc" diff --git a/pkg/receive/handler.go b/pkg/receive/handler.go index 5e16bdd3ba..9f72599406 100644 --- a/pkg/receive/handler.go +++ b/pkg/receive/handler.go @@ -23,6 +23,7 @@ import ( "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/prometheus/common/route" promtsdb "github.com/prometheus/prometheus/storage/tsdb" + terrors "github.com/prometheus/tsdb/errors" "github.com/thanos-io/thanos/pkg/runutil" "github.com/thanos-io/thanos/pkg/store/prompb" diff --git a/scripts/bucketcfggen/main.go b/scripts/bucketcfggen/main.go index 72fa0a361d..00f7f9d9e8 100644 --- a/scripts/bucketcfggen/main.go +++ b/scripts/bucketcfggen/main.go @@ -11,11 +11,13 @@ import ( "github.com/fatih/structtag" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" + "github.com/pkg/errors" "github.com/thanos-io/thanos/pkg/objstore/azure" "github.com/thanos-io/thanos/pkg/objstore/client" "github.com/thanos-io/thanos/pkg/objstore/cos" "github.com/thanos-io/thanos/pkg/objstore/gcs" + "github.com/thanos-io/thanos/pkg/objstore/oss" "github.com/thanos-io/thanos/pkg/objstore/s3" "github.com/thanos-io/thanos/pkg/objstore/swift" kingpin "gopkg.in/alecthomas/kingpin.v2" @@ -24,11 +26,12 @@ import ( var ( configs = map[client.ObjProvider]interface{}{ - client.AZURE: azure.Config{}, - client.GCS: gcs.Config{}, - client.S3: s3.Config{}, - client.SWIFT: swift.SwiftConfig{}, - client.COS: cos.Config{}, + client.AZURE: azure.Config{}, + client.GCS: gcs.Config{}, + client.S3: s3.Config{}, + client.SWIFT: swift.SwiftConfig{}, + client.COS: cos.Config{}, + client.ALIYUNOSS: oss.Config{}, } )