diff --git a/.gitignore b/.gitignore index 4f05f19ac96..bb59b436031 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,5 @@ cmd/docs/*.1 cmd/docs/*.yaml crossdock/crossdock-* run-crossdock.log + +__pycache__ diff --git a/Makefile b/Makefile index 8e49043d1e9..66da34e4b07 100644 --- a/Makefile +++ b/Makefile @@ -95,6 +95,11 @@ storage-integration-test: go-gen go clean -testcache bash -c "set -e; set -o pipefail; $(GOTEST) $(STORAGE_PKGS) | $(COLORIZE)" +.PHONE: test-compile-es-scripts +test-compile-es-scripts: + docker run --rm -it -v ${PWD}:/tmp/jaeger python:3-alpine /usr/local/bin/python -m py_compile /tmp/jaeger/plugin/storage/es/esRollover.py + docker run --rm -it -v ${PWD}:/tmp/jaeger python:3-alpine /usr/local/bin/python -m py_compile /tmp/jaeger/plugin/storage/es/esCleaner.py + .PHONY: index-cleaner-integration-test index-cleaner-integration-test: docker-images-elastic # Expire tests results for storage integration tests since the environment might change diff --git a/pkg/es/config/config.go b/pkg/es/config/config.go index c9d1a26ad95..9db6be5051c 100644 --- a/pkg/es/config/config.go +++ b/pkg/es/config/config.go @@ -302,6 +302,8 @@ func (c *Configuration) getConfigOptions(logger *zap.Logger) ([]elastic.ClientOp } else { httpTransport := &http.Transport{ Proxy: http.ProxyFromEnvironment, + // #nosec G402 + TLSClientConfig: &tls.Config{InsecureSkipVerify: c.TLS.SkipHostVerify}, } if c.TLS.CaPath != "" { ctls := &TLSConfig{CaPath: c.TLS.CaPath} @@ -309,7 +311,7 @@ func (c *Configuration) getConfigOptions(logger *zap.Logger) ([]elastic.ClientOp if err != nil { return nil, err } - httpTransport.TLSClientConfig = &tls.Config{RootCAs: ca} + httpTransport.TLSClientConfig.RootCAs = ca } token := "" diff --git a/plugin/storage/es/esCleaner.py b/plugin/storage/es/esCleaner.py index 4314c751836..b3fd8357b1d 100755 --- a/plugin/storage/es/esCleaner.py +++ b/plugin/storage/es/esCleaner.py @@ -23,20 +23,10 @@ def main(): print('ES_TLS_CA ... Path to TLS CA file.') print('ES_TLS_CERT ... Path to TLS certificate file.') print('ES_TLS_KEY ... Path to TLS key file.') + print('ES_TLS_SKIP_HOST_VERIFY ... (insecure) Skip server\'s certificate chain and host name verification.') sys.exit(1) - username = os.getenv("ES_USERNAME") - password = os.getenv("ES_PASSWORD") - - if username is not None and password is not None: - client = elasticsearch.Elasticsearch(sys.argv[2:], http_auth=(username, password)) - elif str2bool(os.getenv("ES_TLS", 'false')): - context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH, cafile=os.getenv("ES_TLS_CA")) - context.load_cert_chain(certfile=os.getenv("ES_TLS_CERT"), keyfile=os.getenv("ES_TLS_KEY")) - client = elasticsearch.Elasticsearch(sys.argv[2:], ssl_context=context) - else: - client = elasticsearch.Elasticsearch(sys.argv[2:]) - + client = create_client(os.getenv("ES_USERNAME"), os.getenv("ES_PASSWORD"), str2bool(os.getenv("ES_TLS", 'false')), os.getenv("ES_TLS_CA"), os.getenv("ES_TLS_CERT"), os.getenv("ES_TLS_KEY"), str2bool(os.getenv("ES_TLS_SKIP_HOST_VERIFY", 'false'))) ilo = curator.IndexList(client) empty_list(ilo, 'Elasticsearch has no indices') @@ -102,5 +92,21 @@ def str2bool(v): return v.lower() in ('true', '1') +def create_client(username, password, tls, ca, cert, key, skipHostVerify): + context = ssl.create_default_context() + if ca is not None: + context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH, cafile=ca) + elif skipHostVerify: + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + if username is not None and password is not None: + return elasticsearch.Elasticsearch(sys.argv[2:], http_auth=(username, password), ssl_context=context) + elif tls: + context.load_cert_chain(certfile=cert, keyfile=key) + return elasticsearch.Elasticsearch(sys.argv[2:], ssl_context=context) + else: + return elasticsearch.Elasticsearch(sys.argv[2:], ssl_context=context) + + if __name__ == "__main__": main() diff --git a/plugin/storage/es/esRollover.py b/plugin/storage/es/esRollover.py index 916ef6ce768..30c657d4c65 100755 --- a/plugin/storage/es/esRollover.py +++ b/plugin/storage/es/esRollover.py @@ -35,6 +35,7 @@ def main(): print('ES_TLS_CA ... Path to TLS CA file.') print('ES_TLS_CERT ... Path to TLS certificate file.') print('ES_TLS_KEY ... Path to TLS key file.') + print('ES_TLS_SKIP_HOST_VERIFY ... (insecure) Skip server\'s certificate chain and host name verification.') print('ES_VERSION ... The major Elasticsearch version. If not specified, the value will be auto-detected from Elasticsearch.') print('init configuration:') print('\tSHARDS ... the number of shards per index in Elasticsearch (default {}).'.format(SHARDS)) @@ -46,18 +47,7 @@ def main(): print('\tUNIT_COUNT ... count of UNITs (default {}).'.format(UNIT_COUNT)) sys.exit(1) - username = os.getenv("ES_USERNAME") - password = os.getenv("ES_PASSWORD") - - if username is not None and password is not None: - client = elasticsearch.Elasticsearch(sys.argv[2:], http_auth=(username, password)) - elif str2bool(os.getenv("ES_TLS", 'false')): - context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH, cafile=os.getenv("ES_TLS_CA")) - context.load_cert_chain(certfile=os.getenv("ES_TLS_CERT"), keyfile=os.getenv("ES_TLS_KEY")) - client = elasticsearch.Elasticsearch(sys.argv[2:], ssl_context=context) - else: - client = elasticsearch.Elasticsearch(sys.argv[2:]) - + client = create_client(os.getenv("ES_USERNAME"), os.getenv("ES_PASSWORD"), str2bool(os.getenv("ES_TLS", 'false')), os.getenv("ES_TLS_CA"), os.getenv("ES_TLS_CERT"), os.getenv("ES_TLS_KEY"), str2bool(os.getenv("ES_TLS_SKIP_HOST_VERIFY", 'false'))) prefix = os.getenv('INDEX_PREFIX', '') if prefix != '': prefix += '-' @@ -107,7 +97,7 @@ def perform_action(action, client, write_alias, read_alias, index_to_rollover, t def create_index_template(template, template_name): print('Creating index template {}'.format(template_name)) headers = {'Content-Type': 'application/json'} - s = get_request_session(os.getenv("ES_USERNAME"), os.getenv("ES_PASSWORD"), str2bool(os.getenv("ES_TLS", 'false')), os.getenv("ES_TLS_CA"), os.getenv("ES_TLS_CERT"), os.getenv("ES_TLS_KEY")) + s = get_request_session(os.getenv("ES_USERNAME"), os.getenv("ES_PASSWORD"), str2bool(os.getenv("ES_TLS", 'false')), os.getenv("ES_TLS_CA"), os.getenv("ES_TLS_CERT"), os.getenv("ES_TLS_KEY"), os.getenv("ES_TLS_SKIP_HOST_VERIFY", 'false')) r = s.put(sys.argv[2] + '/_template/' + template_name, headers=headers, data=template) print(r.text) r.raise_for_status() @@ -202,8 +192,12 @@ def empty_list(ilo, error_msg): sys.exit(0) -def get_request_session(username, password, tls, ca, cert, key): +def get_request_session(username, password, tls, ca, cert, key, skipHostVerify): session = requests.Session() + if ca is not None: + session.verify = ca + elif skipHostVerify: + session.verify = False if username is not None and password is not None: session.auth = HTTPBasicAuth(username, password) elif tls: @@ -221,6 +215,22 @@ def get_version(client): return esVersion +def create_client(username, password, tls, ca, cert, key, skipHostVerify): + context = ssl.create_default_context() + if ca is not None: + context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH, cafile=ca) + elif skipHostVerify: + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + if username is not None and password is not None: + return elasticsearch.Elasticsearch(sys.argv[2:], http_auth=(username, password), ssl_context=context) + elif tls: + context.load_cert_chain(certfile=cert, keyfile=key) + return elasticsearch.Elasticsearch(sys.argv[2:], ssl_context=context) + else: + return elasticsearch.Elasticsearch(sys.argv[2:], ssl_context=context) + + if __name__ == "__main__": logging.getLogger().setLevel(logging.DEBUG) main() diff --git a/scripts/travis/es-integration-test.sh b/scripts/travis/es-integration-test.sh index fd68ea1dd36..4e1f5ac28f6 100755 --- a/scripts/travis/es-integration-test.sh +++ b/scripts/travis/es-integration-test.sh @@ -21,6 +21,7 @@ echo "Executing token propatagion test" make build-crossdock-ui-placeholder GOOS=linux make build-query +make test-compile-es-scripts SPAN_STORAGE_TYPE=elasticsearch ./cmd/query/query-linux --es.server-urls=http://127.0.0.1:9200 --es.tls=false --es.version=7 --query.bearer-token-propagation=true & PID=$(echo $!) make token-propagation-integration-test