nginx.conf

load_module modules/ngx_http_opentracing_module.so;

daemon off;

worker_processes ${WORKER_PROCESSES};
worker_rlimit_nofile ${WORKER_RLIMIT_NOFILE};
worker_shutdown_timeout ${WORKER_SHUTDOWN_TIMEOUT};

error_log /dev/stderr warn;
pid /var/run/nginx.pid;

events {
  multi_accept       ${MULTI_ACCEPT};
  worker_connections ${WORKER_CONNECTIONS};
}

http {
  opentracing_load_tracer /usr/local/lib/libjaegertracing.so /etc/jaeger-nginx-config.yaml;

  #include /etc/nginx/mime.types;
  include includes/lua-init.conf;
  default_type application/octet-stream;
  server_names_hash_bucket_size 128;

  map $${PROXY_HOST} $proxy_host {
    default $${PROXY_HOST};
  }

  map $status $loggable {
    ~^[${LOG_LEVEL}] 1;
    default 0;
  }
  # logs
  log_format  main escape=json '{ "timestamp": "$time_iso8601",'
                    ' "type": "nginx.access",'
                    ' "host": "$host",'
                    ' "remote_addr": "$remote_addr",'
                    ' "remote_user": "$remote_user",'
                    ' "request_method": "$request_method",'
                    ' "request_time": "$request_time",'
                    ' "request": "$request",'
                    ' "status": "$status",'
                    ' "body_bytes_sent": "$body_bytes_sent",'
                    ' "http_referer": "$http_referer",'
                    ' "http_user_agent": "$http_user_agent",'
                    ' "http_cf_ray": "$http_cf_ray",'
                    ' "proxy_host": "$proxy_host",'
                    ' "http_x_forwarded_for": "$http_x_forwarded_for"}';

  access_log /dev/stdout main if=$loggable;

  aio                 threads;
  aio_write           on;

  tcp_nopush          on;
  tcp_nodelay         on;

  sendfile on;
  server_tokens off;

  # compression
  gzip on;
  gzip_comp_level 5;
  gzip_min_length 1000;
  gzip_proxied any;
  gzip_types application/atom+xml application/javascript application/json application/hal+json application/rss+xml application/vnd.ms-fontobject application/x-font-ttf application/x-web-app-manifest+json application/xhtml+xml application/xml font/opentype image/svg+xml image/x-icon text/css text/plain text/x-component text/xml application/soap+xml application/xml+rss text/javascript;
  gzip_vary on;

  # request sizes
  client_max_body_size ${CLIENT_MAX_BODY_SIZE};

  # ssl from https://cipherli.st/
  ssl_protocols ${SSL_PROTOCOLS};
  ssl_prefer_server_ciphers on;
  ssl_ciphers "EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH";
  ssl_ecdh_curve secp384r1;
  ssl_session_cache shared:SSL:10m;
  ssl_session_tickets off;
  ssl_stapling on;
  ssl_stapling_verify on;
  resolver 8.8.8.8 8.8.4.4 valid=300s;
  resolver_timeout 5s;

  # set source ip from CF-Connecting-IP header

  # kubernetes hides the cloudflare source ip behind a service, so inspecting those for CF-Connecting-IP header until kubernetes preserves the source ip
  set_real_ip_from 10.0.0.0/8;
  set_real_ip_from 172.16.0.0/12;
  set_real_ip_from 192.168.0.0/16;

  # from https://www.cloudflare.com/ips/
  set_real_ip_from 103.21.244.0/22;
  set_real_ip_from 103.22.200.0/22;
  set_real_ip_from 103.31.4.0/22;
  set_real_ip_from 104.16.0.0/12;
  set_real_ip_from 108.162.192.0/18;
  set_real_ip_from 131.0.72.0/22;
  set_real_ip_from 141.101.64.0/18;
  set_real_ip_from 162.158.0.0/15;
  set_real_ip_from 172.64.0.0/13;
  set_real_ip_from 173.245.48.0/20;
  set_real_ip_from 188.114.96.0/20;
  set_real_ip_from 190.93.240.0/20;
  set_real_ip_from 197.234.240.0/22;
  set_real_ip_from 198.41.128.0/17;
  set_real_ip_from 199.27.128.0/21;

  real_ip_header CF-Connecting-IP;

  # set whitelisting
  allow 10.0.0.0/8;
  allow 172.16.0.0/12;
  allow 192.168.0.0/16;
  ${ALLOW_CIDRS}
  deny all;

  # set proxy defaults
  proxy_set_header Host $proxy_host;
  proxy_set_header X-Real-IP $remote_addr;
  proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
  proxy_set_header X-Forwarded-Host $host;
  proxy_set_header X-Forwarded-Proto $scheme;

  proxy_http_version 1.1;
  proxy_ssl_verify off;

  # allow websocket connections
  map $http_upgrade $connection_upgrade {
      default upgrade;
      ''      close;
  }
  proxy_set_header Upgrade $http_upgrade;
  proxy_set_header Connection $connection_upgrade;

  # set buffer params
  proxy_request_buffering ${PROXY_REQUEST_BUFFERING};
  proxy_buffering ${PROXY_BUFFERING};
  proxy_buffers ${PROXY_BUFFERS_NUMBER} ${PROXY_BUFFERS_SIZE};
  proxy_buffer_size ${PROXY_BUFFER_SIZE};
  large_client_header_buffers ${LARGE_CLIENT_HEADER_BUFFERS};

  # custom timeouts
  client_body_timeout ${CLIENT_BODY_TIMEOUT};
  client_header_timeout ${CLIENT_HEADER_TIMEOUT};
  client_body_buffer_size ${CLIENT_BODY_BUFFER_SIZE};
  keepalive_timeout ${KEEPALIVE_TIMEOUT} ${KEEPALIVE_TIMEOUT_HEADER};
  send_timeout ${SEND_TIMEOUT};
  proxy_connect_timeout ${PROXY_CONNECT_TIMEOUT};
  proxy_send_timeout ${PROXY_SEND_TIMEOUT};
  proxy_read_timeout ${PROXY_READ_TIMEOUT};

  # configure backend
  upstream backend {
    server ${OFFLOAD_TO_HOST}:${OFFLOAD_TO_PORT};

    keepalive ${UPSTREAM_KEEPALIVE_CONNECTIONS};
    keepalive_timeout  ${UPSTREAM_KEEPALIVE_TIMEOUT};
    keepalive_requests ${UPSTREAM_KEEPALIVE_REQUESTS};
  }

  upstream backend_grpc {
    server ${OFFLOAD_TO_HOST_GRPC}:${OFFLOAD_TO_PORT_GRPC};

    keepalive ${UPSTREAM_KEEPALIVE_CONNECTIONS};
    keepalive_timeout  ${UPSTREAM_KEEPALIVE_TIMEOUT};
    keepalive_requests ${UPSTREAM_KEEPALIVE_REQUESTS};
  }

  # http traffic directly to k8s service
  server {
    listen 80;
    server_name ${SERVICE_NAME}
                ${SERVICE_NAME}.${NAMESPACE}
                ${SERVICE_NAME}.${NAMESPACE}.svc
                ${SERVICE_NAME}.${NAMESPACE}.svc.cluster.local
                ${NAMESPACE}-${SERVICE_NAME}-internal.${DNS_ZONE}
                ~^\d+\.\d+\.\d+\.\d+$
                ;

    location = ${HEALT_CHECK_PATH} {
      proxy_pass ${OFFLOAD_TO_PROTO}://backend;
      access_log /dev/null;
      error_log /dev/stderr;


      # headers that get redefined if not specified here
      proxy_set_header Host $proxy_host;
      proxy_set_header Connection $connection_upgrade;

      #cors
    }

    location / {
      opentracing on;
      opentracing_propagate_context;
      opentracing_operation_name "HTTP $request_method $uri";
      opentracing_trace_locations off;
      opentracing_tag upstream_cache_status $upstream_cache_status;
      opentracing_tag upstream_connect_time $upstream_connect_time;
      opentracing_tag upstream_header_time $upstream_header_time;
      opentracing_tag upstream_response_time $upstream_response_time;

      proxy_pass ${OFFLOAD_TO_PROTO}://backend;

      # headers that get redefined if not specified here
      proxy_set_header Host $proxy_host;
      proxy_set_header Upgrade $http_upgrade;
      proxy_set_header Connection $connection_upgrade;

      #cors
    }
  }

  # http to https redirect for all other traffic
  server {
    listen 80 default_server;

    return 301 https://$host$request_uri;
  }

  # https
  server {
    listen 443 ssl http2;

    ssl_certificate /etc/ssl/private/ssl.pem;
    ssl_certificate_key /etc/ssl/private/ssl.key;
    ssl_trusted_certificate /etc/ssl/private/ssl.pem;

    location = ${HEALT_CHECK_PATH} {
      proxy_pass ${OFFLOAD_TO_PROTO}://backend;
      access_log /dev/null;
      error_log /dev/stderr;

      # headers that get redefined if not specified here
      proxy_set_header Host $proxy_host;
      proxy_set_header Connection $connection_upgrade;

      #cors
    }

    location / {
      opentracing on;
      opentracing_propagate_context;
      opentracing_operation_name "HTTP $request_method $uri";
      opentracing_trace_locations off;
      opentracing_tag upstream_cache_status $upstream_cache_status;
      opentracing_tag upstream_connect_time $upstream_connect_time;
      opentracing_tag upstream_header_time $upstream_header_time;
      opentracing_tag upstream_response_time $upstream_response_time;

      # headers that get redefined if not specified here
      proxy_set_header Host $proxy_host;
      proxy_set_header Upgrade $http_upgrade;
      proxy_set_header Connection $connection_upgrade;

      #cors
      if ($content_type = "application/grpc") {
        grpc_pass ${OFFLOAD_TO_PROTO_GRPC}://backend_grpc;
      }

      if ($content_type != "application/grpc") {
        proxy_pass ${OFFLOAD_TO_PROTO}://backend;
      }
    }
  }

  # readiness
  server {
    listen 81;

    location /readiness {
      keepalive_timeout 0;
      access_log /dev/null;
      error_log /dev/stderr;

      proxy_pass ${OFFLOAD_TO_PROTO}://backend${HEALT_CHECK_PATH};

      # headers that get redefined if not specified here
      proxy_set_header Host $proxy_host;
      proxy_set_header Connection $connection_upgrade;
    }
  }

  # liveness
  server {
    listen 82;

    location /liveness {
      keepalive_timeout 0;
      access_log /dev/null;
      error_log /dev/stderr;

      add_header Content-Type text/plain;
      return 200;
    }
  }

  # prometheus
  server {
    listen ${PROMETHEUS_METRICS_PORT};

    location /metrics {
      access_log /dev/null;
      error_log /dev/stderr;

      content_by_lua '
        metric_connections:set(ngx.var.connections_reading, {"reading"})
        metric_connections:set(ngx.var.connections_waiting, {"waiting"})
        metric_connections:set(ngx.var.connections_writing, {"writing"})
        prometheus:collect()
      ';
    }
  }
}