Merge branch 'master' into commonTlsConfig

jaegertracing · Dec 19, 2019 · 0eaf54c · 0eaf54c
2 parents 15650eb + 46d7f17
commit 0eaf54c
Show file tree

Hide file tree

Showing 120 changed files with 3,722 additions and 1,428 deletions.
diff --git a/.devbots/needs-triage.yml b/.devbots/needs-triage.yml
@@ -0,0 +1,2 @@
+enabled: true
+label: "needs-triage"
diff --git a/.github/stale.yml b/.github/stale.yml
@@ -0,0 +1,4 @@
+daysUntilStale: 60
+daysUntilClose: 7
+onlyLabels:
+  - needs-info
diff --git a/.gitignore b/.gitignore
@@ -29,3 +29,5 @@ cmd/docs/*.1
 cmd/docs/*.yaml
 crossdock/crossdock-*
 run-crossdock.log
+
+__pycache__
diff --git a/.travis.yml b/.travis.yml
@@ -5,7 +5,7 @@ go_import_path: github.com/jaegertracing/jaeger
 
 dist: trusty
 
-jobs:
+matrix:
   include:
   - go: "1.13.x"
     env:
@@ -36,15 +36,6 @@ jobs:
   - go: "1.13.x"
     env:
     - HOTROD=true
-  - stage: Fuzz regression
-    dist: bionic
-    go: "1.13.x"
-    script: ./scripts/travis/fuzzit.sh local-regression
-  - stage: Fuzz
-    if: branch = master AND type IN (push)
-    dist: bionic
-    go: "1.13.x"
-    script: ./scripts/travis/fuzzit.sh fuzzing
 
 services:
   - docker
@@ -79,7 +70,9 @@ script:
   - if [ "$TESTS" == true ]; then make test-ci ; else echo 'skipping tests'; fi
   - if [ "$ALL_IN_ONE" == true ]; then bash ./scripts/travis/build-all-in-one-image.sh ; else echo 'skipping all_in_one'; fi
   - if [ "$CROSSDOCK" == true ]; then bash ./scripts/travis/build-crossdock.sh ; else echo 'skipping crossdock'; fi
-  - if [ "$DOCKER" == true ]; then bash ./scripts/travis/build-docker-images.sh ; else echo 'skipping docker images'; fi
+  - if [ "$DOCKER" == true ]; then bash ./scripts/travis/build-docker-images.sh ; else echo 'skipping build-docker-images'; fi
+  - if [ "$DOCKER" == true ]; then bash ./scripts/travis/upload-all-docker-images.sh ; else echo 'skipping docker upload'; fi
+  - if [ "$DEPLOY" == true ]; then make build-all-platforms ; else echo 'skipping build-all-platforms'; fi
   - if [ "$ES_INTEGRATION_TEST" == true ]; then bash ./scripts/travis/es-integration-test.sh ; else echo 'skipping elastic search integration test'; fi
   - if [ "$KAFKA_INTEGRATION_TEST" == true ]; then bash ./scripts/travis/kafka-integration-test.sh ; else echo 'skipping kafka integration test'; fi
   - if [ "$CASSANDRA_INTEGRATION_TEST" == true ]; then bash ./scripts/travis/cassandra-integration-test.sh ; else echo 'skipping cassandra integration test'; fi
@@ -94,7 +87,7 @@ after_failure:
   - if [ "$CROSSDOCK" == true ]; then make crossdock-logs ; else echo 'skipping crossdock'; fi
 
 before_deploy:
-  - if [ "$DEPLOY" == true ]; then make build-all-platforms && bash ./scripts/travis/package-deploy.sh ; else echo 'skipping deploy'; fi
+  - if [ "$DEPLOY" == true ]; then bash ./scripts/travis/package-deploy.sh ; else echo 'skipping deploying binaries'; fi
 
 deploy:
   provider: releases

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,21 +1,178 @@
 Changes by Version
 ==================
 
-1.15.0 (unreleased)
+1.17.0 (unreleased)
+------------------
+
+### Backend Changes
+
+#### Breaking Changes
+
+#### New Features
+
+#### Bug fixes, Minor Improvements
+
+### UI Changes
+
+1.16.0 (2019-12-17)
+------------------
+
+### Backend Changes
+
+#### Breaking Changes
+
+##### List of service operations can be classified by span kinds ([#1943](https://github.com/jaegertracing/jaeger/pull/1943), [#1942](https://github.com/jaegertracing/jaeger/pull/1942), [#1937](https://github.com/jaegertracing/jaeger/pull/1937), [@guo0693](https://github.com/guo0693))
+
+* Endpoint changes:
+    * Both Http & gRPC servers now take new optional parameter `spanKind` in addition to `service`. When spanKind
+     is absent or empty, operations from all kinds of spans will be returned.
+    * Instead of returning a list of string, both Http & gRPC servers return a list of operation struct. Please 
+    update your client code to process the new response. Example response:
+        ```
+        curl 'http://localhost:6686/api/operations?service=UserService&spanKind=server' | jq
+        {
+            "data": [{
+                "name": "UserService::getExtendedUser",
+                "spanKind": "server"
+            },
+            {
+                "name": "UserService::getUserProfile",
+                "spanKind": "server"
+            }],
+            "total": 2,
+            "limit": 0,
+            "offset": 0,
+            "errors": null
+        }
+        ```
+    * The legacy http endpoint stay untouched:
+        ```
+        /services/{%s}/operations
+        ```    
+* Storage plugin changes:
+    * Memory updated to support spanKind on write & read, no migration is required.
+    * [Badger](https://github.com/jaegertracing/jaeger/issues/1922) & [ElasticSearch](https://github.com/jaegertracing/jaeger/issues/1923) 
+    to be implemented:  
+    For now `spanKind` will be set as empty string during read & write, only `name` will be valid operation name.
+    * Cassandra updated to support spanKind on write & read ([#1937](https://github.com/jaegertracing/jaeger/pull/1937), [@guo0693](https://github.com/guo0693)):  
+        If you don't run the migration script, nothing will break, the system will used the old table 
+        `operation_names` and set empty `spanKind` in the response.  
+        Steps to get the updated functionality:
+        1.  You will need to run below command on the host you can use `cqlsh` to connect the the cassandra contact
+         point
+            ```
+            KEYSPACE=jaeger_v1 CQL_CMD='cqlsh host 9042 -u test_user -p test_password --request-timeout=3000' 
+            bash ./v002tov003.sh
+            ```
+            The script will create new table `operation_names_v2` and migrate data from the old table.  
+            `spanKind` column will be empty for those data.  
+            At the end, it will ask you whether you want to drop the old table or not.
+        2. Restart ingester & query services so that they begin to use the new table
+
+##### Trace and Span IDs are always padded to 32 or 16 hex characters with leading zeros ([#1956](https://github.com/jaegertracing/jaeger/pull/1956), [@yurishkuro](https://github.com/yurishkuro))
+
+Previously, Jaeger backend always rendered trace and span IDs as  the shortest possible hex string, e.g. an ID
+with numeric value 255 would be rendered as a string `ff`. This change makes the IDs to always render as 16 or 32
+characters long hex string, e.g. the same id=255 would render as `00000000000000ff`. It mostly affects how UI
+displays the IDs, the URLs, and the JSON returned from `jaeger-query` service.
+
+Motivation: Among randomly generated and uniformly distributed trace IDs, only 1/16th of them start with 0
+followed by a significant digit, 1/256th start with two 0s, and so on in decreasing geometric progression.
+Therefore, trimming the leading 0s is a very modest optimization on the size of the data being transmitted or stored.
+
+However, trimming 0s leads to ambiguities when the IDs are used as correlations with other monitoring systems,
+such as logging, that treat the IDs as opaque strings and cannot establish the equivalence between padded and
+unpadded IDs. It is also incompatible with W3C Trace Context and Zipkin B3 formats, both of which include all
+leading 0s, so an application instrumented with OpenTelemetry SDKs may be logging different trace ID strings
+than application instrumented with Jaeger SDKs (related issue #1657).
+
+Overall, the change is backward compatible:
+  * links with non-padded IDs in the UI will still work
+  * data stored in Elasticsearch (where IDs are represented as strings) is still readable
+
+However, some custom integration that rely on exact string matches of trace IDs may be broken.
+
+##### Change default rollover conditions to 2 days ([#1963](https://github.com/jaegertracing/jaeger/pull/1963), [@pavolloffay](https://github.com/pavolloffay))
+
+Change default rollover conditions from 7 days to 2 days.
+
+Given that by default Jaeger uses daily indices and some organizations do not keep data longer than 7 days
+the default of 7 days seems unreasonable - it might result in a too big index and
+running curator would immediately remove the old index.
+
+#### New Features
+
+* Support collector tags, similar to agent tags ([#1854](https://github.com/jaegertracing/jaeger/pull/1854), [@radekg](https://github.com/radekg))
+* Support insecure TLS and only CA cert for Elasticsearch ([#1918](https://github.com/jaegertracing/jaeger/pull/1918), [@pavolloffay](https://github.com/pavolloffay))
+* Allow tracer config via env vars ([#1919](https://github.com/jaegertracing/jaeger/pull/1919), [@yurishkuro](https://github.com/yurishkuro))
+* Allow turning off tags/logs indexing in Cassandra ([#1915](https://github.com/jaegertracing/jaeger/pull/1915), [@joe-elliott](https://github.com/joe-elliott))
+* Blacklisting/Whitelisting tags for Cassandra indexing  ([#1904](https://github.com/jaegertracing/jaeger/pull/1904), [@joe-elliott](https://github.com/joe-elliott))
+
+#### Bug fixes, Minor Improvements
+
+* Support custom basepath in HotROD ([#1894](https://github.com/jaegertracing/jaeger/pull/1894), [@jan25](https://github.com/jan25))
+* Deprecate tchannel reporter flags ([#1978](https://github.com/jaegertracing/jaeger/pull/1978), [@objectiser](https://github.com/objectiser))
+* Do not truncate tags in Elasticsearch ([#1970](https://github.com/jaegertracing/jaeger/pull/1970), [@pavolloffay](https://github.com/pavolloffay))
+* Export SaveSpan to enable multiplexing ([#1968](https://github.com/jaegertracing/jaeger/pull/1968), [@albertteoh](https://github.com/albertteoh))
+* Make rollover init step idempotent ([#1964](https://github.com/jaegertracing/jaeger/pull/1964), [@pavolloffay](https://github.com/pavolloffay))
+* Update python urllib3 version required by curator ([#1965](https://github.com/jaegertracing/jaeger/pull/1965), [@pavolloffay](https://github.com/pavolloffay))
+* Allow changing max log level for gRPC storage plugins ([#1962](https://github.com/jaegertracing/jaeger/pull/1962), [@yyyogev](https://github.com/yyyogev))
+* Fix the bug that operation_name table can not be init more than once ([#1961](https://github.com/jaegertracing/jaeger/pull/1961), [@guo0693](https://github.com/guo0693))
+* Improve migration script ([#1946](https://github.com/jaegertracing/jaeger/pull/1946), [@guo0693](https://github.com/guo0693))
+* Fix order of the returned results from badger backend.  ([#1939](https://github.com/jaegertracing/jaeger/pull/1939), [@burmanm](https://github.com/burmanm))
+* Update python pathlib to pathlib2 ([#1930](https://github.com/jaegertracing/jaeger/pull/1930), [@objectiser](https://github.com/objectiser))
+* Use proxy env vars if they're configured ([#1910](https://github.com/jaegertracing/jaeger/pull/1910), [@zoidbergwill](https://github.com/zoidbergwill))
+
+### UI Changes
+
+* UI pinned to version 1.6.0. The changelog is available here [v1.6.0](https://github.com/jaegertracing/jaeger-ui/blob/master/CHANGELOG.md#v160-december-16-2019)
+
+1.15.1 (2019-11-07)
+------------------
+
+##### Bug fixes, Minor Improvements
+
+* Build platform binaries as part of CI ([#1909](https://github.com/jaegertracing/jaeger/pull/1909), [@yurishkuro](https://github.com/yurishkuro))
+* Upgrade and fix dependencies ([#1907](https://github.com/jaegertracing/jaeger/pull/1907), [@yurishkuro](https://github.com/yurishkuro))
+
+
+1.15.0 (2019-11-07)
 ------------------
 
 #### Backend Changes
 
 ##### Breaking Changes
 
-* The default value for the Ingester's flag `ingester.deadlockInterval` has been changed to `0`. With the new default, the ingester won't `panic` if there are no messages for the last minute. To restore the previous behavior, set the flag's value to `1m`.
+* The default value for the Ingester's flag `ingester.deadlockInterval` has been changed to `0` ([#1868](https://github.com/jaegertracing/jaeger/pull/1868), [@jpkrohling](https://github.com/jpkrohling))
+
+  With the new default, the ingester won't `panic` if there are no messages for the last minute. To restore the previous behavior, set the flag's value to `1m`.
+
+* Mark `--collector.grpc.tls.client.ca` flag as deprecated for jaeger-collector. ([#1840](https://github.com/jaegertracing/jaeger/pull/1840), [@yurishkuro](https://github.com/yurishkuro))
+
+  The deprecated flag will still work until being removed, it's recommended to use `--collector.grpc.tls.client-ca` instead.
 
 ##### New Features
 
+* Support TLS for Kafka ([#1414](https://github.com/jaegertracing/jaeger/pull/1414), [@MichaHoffmann](https://github.com/MichaHoffmann))
+* Add ack and compression parameters for Kafka #1359 ([#1712](https://github.com/jaegertracing/jaeger/pull/1712), [@chandresh-pancholi](https://github.com/chandresh-pancholi))
+* Propagate the bearer token to the gRPC plugin server ([#1822](https://github.com/jaegertracing/jaeger/pull/1822), [@radekg](https://github.com/radekg))
+* Add Truncate and ReadOnly options for badger ([#1842](https://github.com/jaegertracing/jaeger/pull/1842), [@burmanm](https://github.com/burmanm))
+
 ##### Bug fixes, Minor Improvements
 
+* Use correct context on ES search methods ([#1850](https://github.com/jaegertracing/jaeger/pull/1850), [@rubenvp8510](https://github.com/rubenvp8510))
+* Handling of expected error codes coming from grpc storage plugins #1741 ([#1814](https://github.com/jaegertracing/jaeger/pull/1814), [@chandresh-pancholi](https://github.com/chandresh-pancholi))
+* Fix ordering of indexScanKeys after TraceID parsing ([#1809](https://github.com/jaegertracing/jaeger/pull/1809), [@burmanm](https://github.com/burmanm))
+* Small memory optimizations in badger write-path ([#1771](https://github.com/jaegertracing/jaeger/pull/1771), [@burmanm](https://github.com/burmanm))
+* Set an empty value when a default env var value is missing ([#1777](https://github.com/jaegertracing/jaeger/pull/1777), [@jpkrohling](https://github.com/jpkrohling))
+* Decouple storage dependencies and bump Go to 1.13.x ([#1886](https://github.com/jaegertracing/jaeger/pull/1886), [@yurishkuro](https://github.com/yurishkuro))
+* Update gopkg.in/yaml.v2 dependency to v2.2.4 ([#1865](https://github.com/jaegertracing/jaeger/pull/1865), [@objectiser](https://github.com/objectiser))
+* Upgrade jaeger-client 2.19 and jaeger-lib 2.2 and prom client 1.x ([#1810](https://github.com/jaegertracing/jaeger/pull/1810), [@yurishkuro](https://github.com/yurishkuro))
+* Unpin grpc version and use serviceConfig to set the load balancer  ([#1786](https://github.com/jaegertracing/jaeger/pull/1786), [@guanw](https://github.com/guanw))
+
 #### UI Changes
 
+* UI pinned to version 1.5.0. The changelog is available here [v1.5.0](https://github.com/jaegertracing/jaeger-ui/blob/master/CHANGELOG.md#v150-november-4-2019)
 
 1.14.0 (2019-09-02)
 ------------------
@@ -26,7 +183,7 @@ Changes by Version
 
 * Create ES index templates instead of indices ([#1627](https://github.com/jaegertracing/jaeger/pull/1627), [@pavolloffay](https://github.com/pavolloffay))
 
-  This can break existing Elasticsearch deployments if security policies are applied. 
+  This can break existing Elasticsearch deployments if security policies are applied.
   For instance Jaeger `X-Pack` configuration now requires permission to create index templates - `manage_index_templates`.
 
 ##### New Features
@@ -35,12 +192,12 @@ Changes by Version
 * Add Elasticsearch version flag ([#1753](https://github.com/jaegertracing/jaeger/pull/1753), [@pavolloffay](https://github.com/pavolloffay))
 * Add Elasticsearch 7 support ([#1690](https://github.com/jaegertracing/jaeger/pull/1690), [@gregoryfranklin](https://github.com/gregoryfranklin))
 
-  The index mappings in Elasticsearch 7 are not backwards compatible with the older versions. 
+  The index mappings in Elasticsearch 7 are not backwards compatible with the older versions.
   Therefore using Elasticsearch 7 with data created with older version would not work.
   Elasticsearch 6.8 supports 7.x, 6.x, 5.x compatible mappings. The upgrade has to be done
   first to ES 6.8, then apply data migration or wait until old daily indices are removed (this requires
   to start Jaeger with `--es.version=7` to force using ES 7.x mappings for newly created indices).
-  
+
   Jaeger by default uses Elasticsearch ping endpoint (`/`) to derive the version which is used
   for index mappings selection. The version can be overridden by flag `--es.version`.
 
@@ -154,15 +311,15 @@ Changes by Version
     --ingester.encoding
     --ingester.topic
     --ingester.group-id
-    ``` 
-    
+    ```
+
     In the Collector, they are replaced by:
     ```
     --kafka.producer.brokers
     --kafka.producer.encoding
     --kafka.producer.topic
     ```
-    
+
     In the Ingester, they are replaced by:
     ```
     --kafka.consumer.brokers
@@ -219,15 +376,15 @@ Changes by Version
     --kafka.brokers
     --kafka.encoding
     --kafka.topic
-    ``` 
-    
+    ```
+
     In the Collector, they are replaced by:
     ```
     --kafka.producer.brokers
     --kafka.producer.encoding
     --kafka.producer.topic
     ```
-    
+
     In the Ingester, they are replaced by:
     ```
     --kafka.consumer.brokers

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -142,3 +142,13 @@ a comment explaining why it is there, for example:
 $ cat ./pkg/cassandra/config/.nocover
 requires connection to Cassandra
 ```
+
+## Merging PRs
+Before merging a PR make sure:
+* the title is descriptive and follows [a good commit message](./CONTRIBUTING_GUIDELINES.md)
+* pull request is assigned to the current release milestone
+* add `changelog:*` and other labels
+
+Merge the PR by using "Squash and merge" option on Github. Avoid creating merge commits.
+After the merge make sure referenced issues were closed.
+