Merge branch 'master' of https://github.com/envoyproxy/envoy

Sync to upstream master HEAD.

Signed-off-by: Xin Zhuang <stevenzzz@google.com>

.circleci/config.yml

@@ -9,6 +9,8 @@ jobs:
        - image: *envoy-build-image
      resource_class: xlarge
      working_directory: /source
+     environment:
+       BAZEL_REMOTE_CACHE: https://storage.googleapis.com/envoy-circleci-bazel-cache/
      steps:
        - run: rm -rf /home/circleci/project/.git # CircleCI git caching is likely broken
        - checkout
@@ -21,6 +23,8 @@ jobs:
        - image: *envoy-build-image
      resource_class: xlarge
      working_directory: /source
+     environment:
+       BAZEL_REMOTE_CACHE: https://storage.googleapis.com/envoy-circleci-bazel-cache/
      steps:
        - run: rm -rf /home/circleci/project/.git # CircleCI git caching is likely broken
        - run: echo $CIRCLE_SHA1
@@ -33,6 +37,8 @@ jobs:
        - image: *envoy-build-image
      resource_class: xlarge
      working_directory: /source
+     environment:
+       BAZEL_REMOTE_CACHE: https://storage.googleapis.com/envoy-circleci-bazel-cache/
      steps:
        - run: rm -rf /home/circleci/project/.git # CircleCI git caching is likely broken
        - checkout
@@ -64,6 +70,8 @@ jobs:
        - run: ci/filter_example_mirror.sh
    ipv6_tests:
      machine: true
+     environment:
+       BAZEL_REMOTE_CACHE: https://storage.googleapis.com/envoy-circleci-bazel-cache/
      steps:
      - run: rm -rf /home/circleci/project/.git # CircleCI git caching is likely broken
      - checkout
@@ -136,6 +144,8 @@ jobs:
    mac:
      macos:
        xcode: "9.3.0"
+     environment:
+       BAZEL_REMOTE_CACHE: https://storage.googleapis.com/envoy-circleci-bazel-cache/
      steps:
        - run: sudo ntpdate -vu time.apple.com
        - run: rm -rf /home/circleci/project/.git # CircleCI git caching is likely broken

api/envoy/api/v2/cds.proto

@@ -46,7 +46,7 @@ service ClusterDiscoveryService {
 // [#protodoc-title: Clusters]
 
 // Configuration for a single upstream cluster.
-// [#comment:next free field: 35]
+// [#comment:next free field: 36]
 message Cluster {
   // Supplies the name of the cluster which must be unique across all clusters.
   // The cluster name is used when emitting
@@ -226,6 +226,12 @@ message Cluster {
   // connections to happen over plain text.
   core.Http2ProtocolOptions http2_protocol_options = 14;
 
+  // The extension_protocol_options field is used to provide extension-specific protocol options
+  // for upstream connections. The key should match the extension filter name, such as
+  // "envoy.filters.network.thrift_proxy". See the extension's documentation for details on
+  // specific options.
+  map<string, google.protobuf.Struct> extension_protocol_options = 35;
+
   reserved 15;
 
   // If the DNS refresh rate is specified and the cluster type is either

api/envoy/config/filter/http/ext_authz/v2alpha/ext_authz.proto

@@ -42,24 +42,31 @@ message ExtAuthz {
 // whether the request is authorized or not.
 //
 // A successful check allows the authorization service adding or overriding headers from the
-// original request before dispatching it to the upstream. This is done by including the headers in
-// the response sent back from the authorization service to the filter. Note that `Status`,
-// `Method`, `Path` and `Content Length` response headers are automatically removed from this
-// response by the filter. If other headers need be deleted, they should be specified in
-// `response_headers_to_remove` field.
+// original request before dispatching it to the upstream. This is done by configuring which headers
+// in the authorization response should be sent to the upstream. See *allowed_authorization_headers*
+// bellow.
 //
 // A failed check will cause this filter to close the HTTP request normally with 403 (Forbidden),
-// unless a different status code has been indicated by the authorization service via response
-// headers. The HTTP service also allows the authorization filter to also pass data from the
-// response body to the downstream client in case of a denied request.
+// unless a different status code has been indicated by the authorization server via response
+// headers. If other headers in the authorization response need to be sent to client, this can also
+// be done by specifying them in *allowed_authorization_headers*.
 message HttpService {
   // Sets the HTTP server URI which the authorization requests must be sent to.
   envoy.api.v2.core.HttpUri server_uri = 1;
 
-  // Sets an optional prefix to the value of authorization request header `path`.
+  // Sets an optional prefix to the value of authorization request header *Path*.
   string path_prefix = 2;
 
-  // Sets a list of headers that should be not be sent *from the authorization server* to the
-  // upstream.
-  repeated string response_headers_to_remove = 3;
+  reserved 3;
+
+  // Sets a list of headers that can be sent from the authorization server to the upstream service,
+  // or to the downstream client when present in the authorization response. Note that a matched
+  // request header will have its value overridden by the ones sent from the authorization server.
+  repeated string allowed_authorization_headers = 4;
+
+  // Sets a list of headers that should be sent *from the filter* to the authorization server
+  // when they are also present in the client request. Note that *Content-Length*, *Authority*,
+  // *Method* and *Path* are always dispatched to the authorization server by default. The message
+  // will not contain body data and the *Content-Length* will be set to zero.
+  repeated string allowed_request_headers = 5;
 }

api/envoy/service/discovery/v2/hds.proto

@@ -77,7 +77,7 @@ message Capability {
     TCP = 1;
     REDIS = 2;
   }
-  repeated Protocol health_check_protocol = 1;
+  repeated Protocol health_check_protocols = 1;
 }
 
 message HealthCheckRequest {
@@ -113,11 +113,11 @@ message LocalityEndpoints {
 message ClusterHealthCheck {
   string cluster_name = 1;
   repeated envoy.api.v2.core.HealthCheck health_checks = 2;
-  repeated LocalityEndpoints endpoints = 3;
+  repeated LocalityEndpoints locality_endpoints = 3;
 }
 
 message HealthCheckSpecifier {
-  repeated ClusterHealthCheck health_check = 1;
+  repeated ClusterHealthCheck cluster_health_checks = 1;
   // The default is 1 second.
   google.protobuf.Duration interval = 2;
 }

bazel/README.md

@@ -27,7 +27,7 @@ up-to-date with the latest security patches. See
 for how to update or override dependencies.
 
 1. Install the latest version of [Bazel](https://bazel.build/versions/master/docs/install.html) in your environment.
-2. Install external dependencies libtool, cmake, ninja, and realpath libraries separately.
+2. Install external dependencies libtool, cmake, ninja, realpath and curl libraries separately.
 On Ubuntu, run the following commands:
 ```
  apt-get install libtool
@@ -36,6 +36,7 @@ On Ubuntu, run the following commands:
  apt-get install clang-format-5.0
  apt-get install automake
  apt-get install ninja-build
+ apt-get install curl
 ```
 
 On Fedora (maybe also other red hat distros), run the following:

ci/build_container/build_container_centos.sh

@@ -14,6 +14,7 @@ yum install -y java-1.8.0-openjdk-devel unzip which openssl rpm-build \
 yum clean all
 
 ln -s /usr/bin/cmake3 /usr/bin/cmake
+ln -s /usr/bin/ninja-build /usr/bin/ninja
 
 # latest bazel installer
 BAZEL_VERSION="$(curl -s https://api.github.com/repos/bazelbuild/bazel/releases/latest |

ci/do_ci.sh

@@ -9,6 +9,7 @@ if [[ "$1" == "fix_format" || "$1" == "check_format" ]]; then
   build_setup_args="-nofetch"
 fi
 
+. "$(dirname "$0")"/setup_gcs_cache.sh
 . "$(dirname "$0")"/build_setup.sh $build_setup_args
 
 echo "building using ${NUM_CPUS} CPUs"

ci/do_circle_ci_ipv6_tests.sh

@@ -22,5 +22,6 @@ echo "disk space at beginning of build:"
 df -h
 
 docker run -t -i -v "$ENVOY_BUILD_DIR":/build -v "$ENVOY_SRCDIR":/source \
+  --env GCP_SERVICE_ACCOUNT_KEY --env BAZEL_REMOTE_CACHE \
   envoyproxy/envoy-build:"$ENVOY_BUILD_SHA" /bin/bash -c "cd /source && ci/do_ci.sh $TEST_TYPE"
 

ci/mac_ci_steps.sh

@@ -2,7 +2,9 @@
 
 set -e
 
-BAZEL_BUILD_OPTIONS="--curses=no --show_task_finish --verbose_failures"
+. "$(dirname "$0")"/setup_gcs_cache.sh
+
+BAZEL_BUILD_OPTIONS="--curses=no --show_task_finish --verbose_failures ${BAZEL_BUILD_EXTRA_OPTIONS}"
 # TODO(zuercher): remove --flaky_test_attempts when https://github.com/envoyproxy/envoy/issues/2428
 # is resolved.
 BAZEL_TEST_OPTIONS="${BAZEL_BUILD_OPTIONS} --test_output=all --flaky_test_attempts=integration@2"

ci/setup_gcs_cache.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+set -e
+
+if [[ ! -z "${BAZEL_REMOTE_CACHE}" ]]; then
+  if [[ ! -z "${GCP_SERVICE_ACCOUNT_KEY}" ]]; then
+    # mktemp will create a tempfile with u+rw permission minus umask, it will not be readable by all
+    # users by default.
+    GCP_SERVICE_ACCOUNT_KEY_FILE=$(mktemp -t gcp_service_account.XXXXXX.json)
+
+    gcp_service_account_cleanup() {
+      echo "Deleting service account key file..."
+      rm -rf "${GCP_SERVICE_ACCOUNT_KEY_FILE}"
+    }
+
+    trap gcp_service_account_cleanup EXIT
+
+    echo "${GCP_SERVICE_ACCOUNT_KEY}" | base64 --decode > "${GCP_SERVICE_ACCOUNT_KEY_FILE}"
+
+    export BAZEL_BUILD_EXTRA_OPTIONS="${BAZEL_BUILD_EXTRA_OPTIONS} \
+      --remote_http_cache=${BAZEL_REMOTE_CACHE} \
+      --google_credentials=${GCP_SERVICE_ACCOUNT_KEY_FILE}"
+    echo "Set up bazel read/write HTTP cache at ${BAZEL_REMOTE_CACHE}."
+  else
+    export BAZEL_BUILD_EXTRA_OPTIONS="${BAZEL_BUILD_EXTRA_OPTIONS} \
+      --remote_http_cache=${BAZEL_REMOTE_CACHE} --noremote_upload_local_results"
+    echo "Set up bazel read only HTTP cache at ${BAZEL_REMOTE_CACHE}."
+  fi
+else
+  echo "No remote cache bucket is set, skipping setup remote cache."
+fi

docs/root/configuration/cluster_manager/cluster_stats.rst

@@ -146,15 +146,19 @@ are rooted at *cluster.<name>.* and contain the following statistics:
   :header: Name, Type, Description
   :widths: 1, 1, 2
 
+  upstream_rq_completed, Counter, "Total upstream requests completed"
   upstream_rq_<\*xx>, Counter, "Aggregate HTTP response codes (e.g., 2xx, 3xx, etc.)"
   upstream_rq_<\*>, Counter, "Specific HTTP response codes (e.g., 201, 302, etc.)"
   upstream_rq_time, Histogram, Request time milliseconds
+  canary.upstream_rq_completed, Counter, "Total upstream canary requests completed"
   canary.upstream_rq_<\*xx>, Counter, Upstream canary aggregate HTTP response codes
   canary.upstream_rq_<\*>, Counter, Upstream canary specific HTTP response codes
   canary.upstream_rq_time, Histogram, Upstream canary request time milliseconds
+  internal.upstream_rq_completed, Counter, "Total internal origin requests completed"
   internal.upstream_rq_<\*xx>, Counter, Internal origin aggregate HTTP response codes
   internal.upstream_rq_<\*>, Counter, Internal origin specific HTTP response codes
   internal.upstream_rq_time, Histogram, Internal origin request time milliseconds
+  external.upstream_rq_completed, Counter, "Total external origin requests completed"
   external.upstream_rq_<\*xx>, Counter, External origin aggregate HTTP response codes
   external.upstream_rq_<\*>, Counter, External origin specific HTTP response codes
   external.upstream_rq_time, Histogram, External origin request time milliseconds

docs/root/configuration/http_conn_man/stats.rst

@@ -45,6 +45,7 @@ statistics:
    downstream_rq_tx_reset, Counter, Total request resets sent
    downstream_rq_non_relative_path, Counter, Total requests with a non-relative HTTP path
    downstream_rq_too_large, Counter, Total requests resulting in a 413 due to buffering an overly large body
+   downstream_rq_completed, Counter, Total requests that resulted in a response (e.g. does not include aborted requests)
    downstream_rq_1xx, Counter, Total 1xx responses
    downstream_rq_2xx, Counter, Total 2xx responses
    downstream_rq_3xx, Counter, Total 3xx responses
@@ -82,6 +83,7 @@ following statistics:
    :header: Name, Type, Description
    :widths: 1, 1, 2
 
+   downstream_rq_completed, Counter, Total responses
    downstream_rq_1xx, Counter, Total 1xx responses
    downstream_rq_2xx, Counter, Total 2xx responses
    downstream_rq_3xx, Counter, Total 3xx responses

docs/root/intro/version_history.rst

@@ -25,6 +25,8 @@ Version history
   defaults to 5 minutes; if you have other timeouts (e.g. connection idle timeout, upstream
   response per-retry) that are longer than this in duration, you may want to consider setting a
   non-default per-stream idle timeout.
+* http: added upstream_rq_completed counter for :ref:`total requests completed <config_cluster_manager_cluster_stats_dynamic_http>` to dynamic HTTP counters.
+* http: added downstream_rq_completed counter for :ref:`total requests completed <config_http_conn_man_stats>`, including on a :ref:`per-listener basis <config_http_conn_man_stats_per_listener>`.
 * http: added support for a :ref:`per-stream idle timeout
   <envoy_api_field_route.RouteAction.idle_timeout>`. This defaults to 5 minutes; if you have
   other timeouts (e.g. connection idle timeout, upstream response per-retry) that are longer than

include/envoy/api/os_sys_calls.h

@@ -17,106 +17,112 @@ namespace Api {
 /**
  * SysCallResult holds the rc and errno values resulting from a system call.
  */
-struct SysCallResult {
+template <typename T> struct SysCallResult {
 
   /**
    * The return code from the system call.
    */
-  int rc_;
+  T rc_;
 
   /**
    * The errno value as captured after the system call.
    */
   int errno_;
 };
 
+typedef SysCallResult<int> SysCallIntResult;
+typedef SysCallResult<ssize_t> SysCallSizeResult;
+typedef SysCallResult<void*> SysCallPtrResult;
+
 class OsSysCalls {
 public:
   virtual ~OsSysCalls() {}
 
   /**
    * @see bind (man 2 bind)
    */
-  virtual int bind(int sockfd, const sockaddr* addr, socklen_t addrlen) PURE;
+  virtual SysCallIntResult bind(int sockfd, const sockaddr* addr, socklen_t addrlen) PURE;
 
   /**
    * @see ioctl (man 2 ioctl)
    */
-  virtual int ioctl(int sockfd, unsigned long int request, void* argp) PURE;
+  virtual SysCallIntResult ioctl(int sockfd, unsigned long int request, void* argp) PURE;
 
   /**
    * Open file by full_path with given flags and mode.
    * @return file descriptor.
    */
-  virtual int open(const std::string& full_path, int flags, int mode) PURE;
+  virtual SysCallIntResult open(const std::string& full_path, int flags, int mode) PURE;
 
   /**
    * Write num_bytes to fd from buffer.
    * @return number of bytes written if non negative, otherwise error code.
    */
-  virtual ssize_t write(int fd, const void* buffer, size_t num_bytes) PURE;
+  virtual SysCallSizeResult write(int fd, const void* buffer, size_t num_bytes) PURE;
 
   /**
    * @see writev (man 2 writev)
    */
-  virtual ssize_t writev(int fd, const iovec* iovec, int num_iovec) PURE;
+  virtual SysCallSizeResult writev(int fd, const iovec* iovec, int num_iovec) PURE;
 
   /**
    * @see readv (man 2 readv)
    */
-  virtual ssize_t readv(int fd, const iovec* iovec, int num_iovec) PURE;
+  virtual SysCallSizeResult readv(int fd, const iovec* iovec, int num_iovec) PURE;
 
   /**
    * @see recv (man 2 recv)
    */
-  virtual ssize_t recv(int socket, void* buffer, size_t length, int flags) PURE;
+  virtual SysCallSizeResult recv(int socket, void* buffer, size_t length, int flags) PURE;
 
   /**
    * Release all resources allocated for fd.
    * @return zero on success, -1 returned otherwise.
    */
-  virtual int close(int fd) PURE;
+  virtual SysCallIntResult close(int fd) PURE;
 
   /**
    * @see shm_open (man 3 shm_open)
    */
-  virtual int shmOpen(const char* name, int oflag, mode_t mode) PURE;
+  virtual SysCallIntResult shmOpen(const char* name, int oflag, mode_t mode) PURE;
 
   /**
    * @see shm_unlink (man 3 shm_unlink)
    */
-  virtual int shmUnlink(const char* name) PURE;
+  virtual SysCallIntResult shmUnlink(const char* name) PURE;
 
   /**
    * @see man 2 ftruncate
    */
-  virtual int ftruncate(int fd, off_t length) PURE;
+  virtual SysCallIntResult ftruncate(int fd, off_t length) PURE;
 
   /**
    * @see man 2 mmap
    */
-  virtual void* mmap(void* addr, size_t length, int prot, int flags, int fd, off_t offset) PURE;
+  virtual SysCallPtrResult mmap(void* addr, size_t length, int prot, int flags, int fd,
+                                off_t offset) PURE;
 
   /**
    * @see man 2 stat
    */
-  virtual int stat(const char* pathname, struct stat* buf) PURE;
+  virtual SysCallIntResult stat(const char* pathname, struct stat* buf) PURE;
 
   /**
    * @see man 2 setsockopt
    */
-  virtual int setsockopt(int sockfd, int level, int optname, const void* optval,
-                         socklen_t optlen) PURE;
+  virtual SysCallIntResult setsockopt(int sockfd, int level, int optname, const void* optval,
+                                      socklen_t optlen) PURE;
 
   /**
    * @see man 2 getsockopt
    */
-  virtual int getsockopt(int sockfd, int level, int optname, void* optval, socklen_t* optlen) PURE;
+  virtual SysCallIntResult getsockopt(int sockfd, int level, int optname, void* optval,
+                                      socklen_t* optlen) PURE;
 
   /**
    * @see man 2 socket
    */
-  virtual int socket(int domain, int type, int protocol) PURE;
+  virtual SysCallIntResult socket(int domain, int type, int protocol) PURE;
 };
 
 typedef std::unique_ptr<OsSysCalls> OsSysCallsPtr;