From 81f2951b8ac933623a077acaed0b2a9aba69763a Mon Sep 17 00:00:00 2001 From: soulbird Date: Mon, 28 Nov 2022 12:18:02 +0800 Subject: [PATCH 1/9] feat: add vault common components --- .github/workflows/build.yml | 3 +- .github/workflows/centos7-ci.yml | 3 +- apisix/kms/vault.lua | 94 ++++++++++++++++++ ci/init-common-test-service.sh | 20 ++++ ci/init-plugin-test-service.sh | 3 - ci/pod/docker-compose.common.yml | 17 ++++ ci/pod/docker-compose.plugin.yml | 16 ---- t/kms/vault.t | 158 +++++++++++++++++++++++++++++++ 8 files changed, 293 insertions(+), 21 deletions(-) create mode 100644 apisix/kms/vault.lua create mode 100644 ci/init-common-test-service.sh create mode 100644 t/kms/vault.t diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9031b2dadcf6..d67b3276b046 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -33,7 +33,7 @@ jobs: - t/plugin/[a-k]* - t/plugin/[l-z]* - t/admin t/cli t/config-center-yaml t/control t/core t/debug t/deployment t/discovery t/error_page t/misc - - t/node t/pubsub t/router t/script t/stream-node t/utils t/wasm t/xds-library t/xrpc + - t/node t/pubsub t/router t/script t/stream-node t/utils t/wasm t/xds-library t/xrpc t/kms runs-on: ${{ matrix.platform }} timeout-minutes: 90 @@ -86,6 +86,7 @@ jobs: - name: Linux launch common services run: | make ci-env-up project_compose_ci=ci/pod/docker-compose.common.yml + sudo ./ci/init-common-test-service.sh - name: Create tarball if: ${{ startsWith(github.ref, 'refs/heads/release/') }} diff --git a/.github/workflows/centos7-ci.yml b/.github/workflows/centos7-ci.yml index ba76759bfc07..5d243bea39fe 100644 --- a/.github/workflows/centos7-ci.yml +++ b/.github/workflows/centos7-ci.yml @@ -31,7 +31,7 @@ jobs: - t/plugin/[a-k]* - t/plugin/[l-z]* - t/admin t/cli t/config-center-yaml t/control t/core t/debug t/deployment t/discovery t/error_page t/misc - - t/node t/pubsub t/router t/script t/stream-node t/utils t/wasm t/xds-library + - t/node t/pubsub t/router t/script t/stream-node t/utils t/wasm t/xds-library t/kms steps: - name: Check out code @@ -72,6 +72,7 @@ jobs: - name: Linux launch common services run: | make ci-env-up project_compose_ci=ci/pod/docker-compose.common.yml + sudo ./ci/init-common-test-service.sh - name: Build rpm package if: ${{ startsWith(github.ref, 'refs/heads/release/') }} diff --git a/apisix/kms/vault.lua b/apisix/kms/vault.lua new file mode 100644 index 000000000000..781390671550 --- /dev/null +++ b/apisix/kms/vault.lua @@ -0,0 +1,94 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +--- Vault Tools. +-- Vault is an identity-based secrets and encryption management system. + +local core = require("apisix.core") +local http = require("resty.http") +local json = require("cjson") + +local norm_path = require("pl.path").normpath +local string = require("apisix.core.string") + +local find = string.find +local sub = string.sub +local reverse = string.reverse + +local _M = {} + + +local function make_request_to_vault(conf, method, key, data) + local httpc = http.new() + -- config timeout or default to 5000 ms + httpc:set_timeout((conf.timeout or 5)*1000) + + local req_addr = conf.uri .. norm_path("/v1/" + .. conf.prefix .. "/" .. key) + + local res, err = httpc:request_uri(req_addr, { + method = method, + headers = { + ["X-Vault-Token"] = conf.token + }, + body = core.json.encode(data or {}, true) + }) + + if not res then + return nil, err + end + + return res.body +end + +-- key is the vault kv engine path +local function get(conf, key) + core.log.info("fetching data from vault for key: ", key) + + local idx = find(reverse(key), "/") + if not idx then + return nil, "error key format, key: " .. key + end + idx = #key - idx + 1 + local main_key = sub(key, 1, idx - 1) + if main_key == "" then + return nil, "can't find main key, key: " .. key + end + local sub_key = sub(key, idx + 1) + if sub_key == "" then + return nil, "can't find sub key, key: " .. key + end + + core.log.info("main: ", main_key, " sub: ", sub_key) + + local res, err = make_request_to_vault(conf, "GET", main_key) + if not res then + return nil, "failed to retrtive data from vault kv engine: " .. err + end + + local ret = json.decode(res) + if not ret then + return nil, "failed to decode result, res: " .. res + end + + return ret[sub_key] +end + +_M.get = get + + +return _M diff --git a/ci/init-common-test-service.sh b/ci/init-common-test-service.sh new file mode 100644 index 000000000000..0008784a66f2 --- /dev/null +++ b/ci/init-common-test-service.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# prepare vault kv engine +docker exec -i vault sh -c "VAULT_TOKEN='root' VAULT_ADDR='http://0.0.0.0:8200' vault secrets enable -path=kv -version=1 kv" diff --git a/ci/init-plugin-test-service.sh b/ci/init-plugin-test-service.sh index 1f973ce36f47..fd090441e3fb 100755 --- a/ci/init-plugin-test-service.sh +++ b/ci/init-plugin-test-service.sh @@ -39,9 +39,6 @@ docker exec -i rmqnamesrv /home/rocketmq/rocketmq-4.6.0/bin/mqadmin updateTopic docker exec -i rmqnamesrv /home/rocketmq/rocketmq-4.6.0/bin/mqadmin updateTopic -n rocketmq_namesrv:9876 -t test3 -c DefaultCluster docker exec -i rmqnamesrv /home/rocketmq/rocketmq-4.6.0/bin/mqadmin updateTopic -n rocketmq_namesrv:9876 -t test4 -c DefaultCluster -# prepare vault kv engine -docker exec -i vault sh -c "VAULT_TOKEN='root' VAULT_ADDR='http://0.0.0.0:8200' vault secrets enable -path=kv -version=1 kv" - # wait for keycloak ready bash -c 'while true; do curl -s localhost:8080 &>/dev/null; ret=$?; [[ $ret -eq 0 ]] && break; sleep 3; done' docker cp ci/kcadm_configure_cas.sh apisix_keycloak_new:/tmp/ diff --git a/ci/pod/docker-compose.common.yml b/ci/pod/docker-compose.common.yml index 9e0394a48bd2..c30ddcc5ad8c 100644 --- a/ci/pod/docker-compose.common.yml +++ b/ci/pod/docker-compose.common.yml @@ -87,3 +87,20 @@ services: - "5004:6382" - "5005:6383" - "5006:6384" + + + ## HashiCorp Vault + vault: + image: vault:1.9.0 + container_name: vault + restart: unless-stopped + ports: + - "8200:8200" + cap_add: + - IPC_LOCK + environment: + VAULT_DEV_ROOT_TOKEN_ID: root + VAULT_DEV_LISTEN_ADDRESS: 0.0.0.0:8200 + command: [ "vault", "server", "-dev" ] + networks: + vault_net: diff --git a/ci/pod/docker-compose.plugin.yml b/ci/pod/docker-compose.plugin.yml index 6715c7a33433..b8bc0a5eb070 100644 --- a/ci/pod/docker-compose.plugin.yml +++ b/ci/pod/docker-compose.plugin.yml @@ -130,22 +130,6 @@ services: networks: skywalk_net: - ## HashiCorp Vault - vault: - image: vault:1.9.0 - container_name: vault - restart: unless-stopped - ports: - - "8200:8200" - cap_add: - - IPC_LOCK - environment: - VAULT_DEV_ROOT_TOKEN_ID: root - VAULT_DEV_LISTEN_ADDRESS: 0.0.0.0:8200 - command: [ "vault", "server", "-dev" ] - networks: - vault_net: - ## OpenLDAP openldap: diff --git a/t/kms/vault.t b/t/kms/vault.t new file mode 100644 index 000000000000..7914699b7dd9 --- /dev/null +++ b/t/kms/vault.t @@ -0,0 +1,158 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +use t::APISIX 'no_plan'; + +repeat_each(1); +no_long_string(); +no_root_location(); +log_level("info"); + +run_tests; + +__DATA__ + +=== TEST 1: check key: error format +--- config + location /t { + content_by_lua_block { + local vault = require("apisix.kms.vault") + local conf = { + prefix = "/kv/prefix", + token = "root", + uri = "http://127.0.0.1:2800" + } + local data, err = vault.get(conf, "apisix") + if err then + return ngx.say(err) + end + + ngx.say("done") + } + } +--- request +GET /t +--- response_body +error key format, key: apisix + + + +=== TEST 2: check key: no main key +--- config + location /t { + content_by_lua_block { + local vault = require("apisix.kms.vault") + local conf = { + prefix = "/kv/prefix", + token = "root", + uri = "http://127.0.0.1:2800" + } + local data, err = vault.get(conf, "/apisix") + if err then + return ngx.say(err) + end + + ngx.say("done") + } + } +--- request +GET /t +--- response_body +can't find main key, key: /apisix + + + +=== TEST 3: check key: no sub key +--- config + location /t { + content_by_lua_block { + local vault = require("apisix.kms.vault") + local conf = { + prefix = "/kv/prefix", + token = "root", + uri = "http://127.0.0.1:2800" + } + local data, err = vault.get(conf, "apisix/") + if err then + return ngx.say(err) + end + + ngx.say("done") + } + } +--- request +GET /t +--- response_body +can't find sub key, key: apisix/ + + + +=== TEST 4: error vault uri +--- config + location /t { + content_by_lua_block { + local vault = require("apisix.kms.vault") + local conf = { + prefix = "/kv/prefix", + token = "root", + uri = "http://127.0.0.2:2800" + } + local data, err = vault.get(conf, "/apisix/sub") + if err then + return ngx.say(err) + end + + ngx.say("done") + } + } +--- request +GET /t +--- response_body +failed to retrtive data from vault kv engine: timeout +--- timeout: 6 + + + +=== TEST 5: store secret into vault +--- exec +VAULT_TOKEN='root' VAULT_ADDR='http://0.0.0.0:8200' vault kv put kv/apisix/apisix-key/jack key=value +--- response_body +Success! Data written to: kv/apisix/apisix-key/jack + + + +=== TEST 6: get value from vault +--- config + location /t { + content_by_lua_block { + local vault = require("apisix.kms.vault") + local conf = { + prefix = "/kv/apisix", + token = "root", + uri = "http://127.0.0.1:8200" + } + local value, err = vault.get(conf, "/apisix-key/jack/key") + if err then + return ngx.say(err) + end + + ngx.say(value) + } + } +--- request +GET /t +--- response_body +value From 14b0f3b36f733bf69b964d0fe7914e2b64504419 Mon Sep 17 00:00:00 2001 From: soulbird Date: Mon, 28 Nov 2022 12:33:46 +0800 Subject: [PATCH 2/9] fix docker compose --- .github/workflows/centos7-ci.yml | 2 +- ci/pod/docker-compose.common.yml | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/centos7-ci.yml b/.github/workflows/centos7-ci.yml index 5d243bea39fe..2b21bdd44419 100644 --- a/.github/workflows/centos7-ci.yml +++ b/.github/workflows/centos7-ci.yml @@ -72,7 +72,7 @@ jobs: - name: Linux launch common services run: | make ci-env-up project_compose_ci=ci/pod/docker-compose.common.yml - sudo ./ci/init-common-test-service.sh + sudo ./ci/init-common-test-service.sh - name: Build rpm package if: ${{ startsWith(github.ref, 'refs/heads/release/') }} diff --git a/ci/pod/docker-compose.common.yml b/ci/pod/docker-compose.common.yml index c30ddcc5ad8c..222dc1e1eed4 100644 --- a/ci/pod/docker-compose.common.yml +++ b/ci/pod/docker-compose.common.yml @@ -102,5 +102,3 @@ services: VAULT_DEV_ROOT_TOKEN_ID: root VAULT_DEV_LISTEN_ADDRESS: 0.0.0.0:8200 command: [ "vault", "server", "-dev" ] - networks: - vault_net: From 9a163272dbccd5e5e7e348218469e6f1fdcd2a11 Mon Sep 17 00:00:00 2001 From: soulbird Date: Mon, 28 Nov 2022 12:38:46 +0800 Subject: [PATCH 3/9] fix init-common-test-service.sh --- ci/init-common-test-service.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 ci/init-common-test-service.sh diff --git a/ci/init-common-test-service.sh b/ci/init-common-test-service.sh old mode 100644 new mode 100755 From 484c8245adf70bad6a660e3961ab3acd00094352 Mon Sep 17 00:00:00 2001 From: soulbird Date: Mon, 28 Nov 2022 13:11:38 +0800 Subject: [PATCH 4/9] fix test case --- t/kms/vault.t | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/t/kms/vault.t b/t/kms/vault.t index 7914699b7dd9..5e0cdb54aceb 100644 --- a/t/kms/vault.t +++ b/t/kms/vault.t @@ -121,7 +121,7 @@ can't find sub key, key: apisix/ --- request GET /t --- response_body -failed to retrtive data from vault kv engine: timeout +failed to retrtive data from vault kv engine: connection refused --- timeout: 6 @@ -140,7 +140,7 @@ Success! Data written to: kv/apisix/apisix-key/jack content_by_lua_block { local vault = require("apisix.kms.vault") local conf = { - prefix = "/kv/apisix", + prefix = "kv/apisix", token = "root", uri = "http://127.0.0.1:8200" } @@ -149,7 +149,7 @@ Success! Data written to: kv/apisix/apisix-key/jack return ngx.say(err) end - ngx.say(value) + ngx.say("value") } } --- request From 4e0a25c3606fb308b33a6a3ddd4212b02d55efd6 Mon Sep 17 00:00:00 2001 From: soulbird Date: Mon, 28 Nov 2022 13:14:14 +0800 Subject: [PATCH 5/9] fix docker compose --- ci/pod/docker-compose.common.yml | 5 +++++ ci/pod/docker-compose.plugin.yml | 1 - 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/ci/pod/docker-compose.common.yml b/ci/pod/docker-compose.common.yml index 222dc1e1eed4..7e43dc4ca93f 100644 --- a/ci/pod/docker-compose.common.yml +++ b/ci/pod/docker-compose.common.yml @@ -102,3 +102,8 @@ services: VAULT_DEV_ROOT_TOKEN_ID: root VAULT_DEV_LISTEN_ADDRESS: 0.0.0.0:8200 command: [ "vault", "server", "-dev" ] + networks: + vault_net: + +networks: + vault_net: \ No newline at end of file diff --git a/ci/pod/docker-compose.plugin.yml b/ci/pod/docker-compose.plugin.yml index b8bc0a5eb070..4b8c2a4c16b4 100644 --- a/ci/pod/docker-compose.plugin.yml +++ b/ci/pod/docker-compose.plugin.yml @@ -268,5 +268,4 @@ networks: kafka_net: skywalk_net: rocketmq_net: - vault_net: opa_net: From ef8a71c1e3f0410d18adad6171e177c4621da2c2 Mon Sep 17 00:00:00 2001 From: soulbird Date: Mon, 28 Nov 2022 13:17:36 +0800 Subject: [PATCH 6/9] fix lint --- ci/pod/docker-compose.common.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/ci/pod/docker-compose.common.yml b/ci/pod/docker-compose.common.yml index 7e43dc4ca93f..222dc1e1eed4 100644 --- a/ci/pod/docker-compose.common.yml +++ b/ci/pod/docker-compose.common.yml @@ -102,8 +102,3 @@ services: VAULT_DEV_ROOT_TOKEN_ID: root VAULT_DEV_LISTEN_ADDRESS: 0.0.0.0:8200 command: [ "vault", "server", "-dev" ] - networks: - vault_net: - -networks: - vault_net: \ No newline at end of file From 9fd7515bec374b258374f1b6b6b57b0dc23da5cd Mon Sep 17 00:00:00 2001 From: soulbird Date: Mon, 28 Nov 2022 13:27:18 +0800 Subject: [PATCH 7/9] fix vault --- apisix/kms/vault.lua | 6 +++--- ci/init-common-test-service.sh | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/apisix/kms/vault.lua b/apisix/kms/vault.lua index 781390671550..708ded9be243 100644 --- a/apisix/kms/vault.lua +++ b/apisix/kms/vault.lua @@ -45,7 +45,7 @@ local function make_request_to_vault(conf, method, key, data) headers = { ["X-Vault-Token"] = conf.token }, - body = core.json.encode(data or {}, true) + body = core.json.encode(data or {}, true) }) if not res then @@ -81,11 +81,11 @@ local function get(conf, key) end local ret = json.decode(res) - if not ret then + if not ret or not ret.data then return nil, "failed to decode result, res: " .. res end - return ret[sub_key] + return ret.data[sub_key] end _M.get = get diff --git a/ci/init-common-test-service.sh b/ci/init-common-test-service.sh index 0008784a66f2..7a54cd49a2b6 100755 --- a/ci/init-common-test-service.sh +++ b/ci/init-common-test-service.sh @@ -17,4 +17,5 @@ # # prepare vault kv engine +sleep 3s docker exec -i vault sh -c "VAULT_TOKEN='root' VAULT_ADDR='http://0.0.0.0:8200' vault secrets enable -path=kv -version=1 kv" From 9a34e401bc6ea589dbb7ffa443dfb207c9f54db0 Mon Sep 17 00:00:00 2001 From: soulbird Date: Mon, 28 Nov 2022 13:47:56 +0800 Subject: [PATCH 8/9] fix luarocks --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index 49468dc57e41..f8668c8d4e16 100644 --- a/Makefile +++ b/Makefile @@ -345,6 +345,9 @@ install: runtime $(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/pubsub $(ENV_INSTALL) apisix/pubsub/*.lua $(ENV_INST_LUADIR)/apisix/pubsub/ + $(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/kms + $(ENV_INSTALL) apisix/kms/*.lua $(ENV_INST_LUADIR)/apisix/kms/ + $(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/plugins/zipkin $(ENV_INSTALL) apisix/plugins/zipkin/*.lua $(ENV_INST_LUADIR)/apisix/plugins/zipkin/ From 49eb385f2fbb756333a09bd48ef692ded8a7091f Mon Sep 17 00:00:00 2001 From: soulbird Date: Mon, 28 Nov 2022 17:35:58 +0800 Subject: [PATCH 9/9] fix comment --- .github/workflows/build.yml | 4 ++-- .github/workflows/centos7-ci.yml | 4 ++-- apisix/kms/vault.lua | 13 +++++-------- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d67b3276b046..0626767ad9ba 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -32,8 +32,8 @@ jobs: test_dir: - t/plugin/[a-k]* - t/plugin/[l-z]* - - t/admin t/cli t/config-center-yaml t/control t/core t/debug t/deployment t/discovery t/error_page t/misc - - t/node t/pubsub t/router t/script t/stream-node t/utils t/wasm t/xds-library t/xrpc t/kms + - t/admin t/cli t/config-center-yaml t/control t/core t/debug t/deployment t/discovery t/error_page t/kms t/misc + - t/node t/pubsub t/router t/script t/stream-node t/utils t/wasm t/xds-library t/xrpc runs-on: ${{ matrix.platform }} timeout-minutes: 90 diff --git a/.github/workflows/centos7-ci.yml b/.github/workflows/centos7-ci.yml index 2b21bdd44419..d5b8facc42e5 100644 --- a/.github/workflows/centos7-ci.yml +++ b/.github/workflows/centos7-ci.yml @@ -30,8 +30,8 @@ jobs: test_dir: - t/plugin/[a-k]* - t/plugin/[l-z]* - - t/admin t/cli t/config-center-yaml t/control t/core t/debug t/deployment t/discovery t/error_page t/misc - - t/node t/pubsub t/router t/script t/stream-node t/utils t/wasm t/xds-library t/kms + - t/admin t/cli t/config-center-yaml t/control t/core t/debug t/deployment t/discovery t/error_page t/kms t/misc + - t/node t/pubsub t/router t/script t/stream-node t/utils t/wasm t/xds-library steps: - name: Check out code diff --git a/apisix/kms/vault.lua b/apisix/kms/vault.lua index 708ded9be243..1343002fcb20 100644 --- a/apisix/kms/vault.lua +++ b/apisix/kms/vault.lua @@ -20,14 +20,11 @@ local core = require("apisix.core") local http = require("resty.http") -local json = require("cjson") local norm_path = require("pl.path").normpath -local string = require("apisix.core.string") -local find = string.find -local sub = string.sub -local reverse = string.reverse +local sub = core.string.sub +local rfind_char = core.string.rfind_char local _M = {} @@ -59,11 +56,11 @@ end local function get(conf, key) core.log.info("fetching data from vault for key: ", key) - local idx = find(reverse(key), "/") + local idx = rfind_char(key, '/') if not idx then return nil, "error key format, key: " .. key end - idx = #key - idx + 1 + local main_key = sub(key, 1, idx - 1) if main_key == "" then return nil, "can't find main key, key: " .. key @@ -80,7 +77,7 @@ local function get(conf, key) return nil, "failed to retrtive data from vault kv engine: " .. err end - local ret = json.decode(res) + local ret = core.json.decode(res) if not ret or not ret.data then return nil, "failed to decode result, res: " .. res end