Merge branch 'master' into gatewayapi/route-attachment

docs/en/latest/monitoring.md

@@ -0,0 +1,142 @@
+---
+title: Monitoring APISIX with Helm Chart
+keywords:
+  - APISIX Ingress Controller
+  - Apache APISIX
+  - Prometheus
+  - Grafana
+description: A tutorial on configuring monitoring services for APISIX.
+---
+
+<!--
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+-->
+
+APISIX has detailed telemetry data. With helm chart, we can easily configure the monitoring system.
+
+This tutorial will show how to achieve it.
+
+## Install Prometheus and Grafana
+
+To collect metrics and visualize them, we need to install Prometheus and Grafana first.
+
+The APISIX helm chart we will deploy later also contains a `ServiceMonitor` resource, so we should ensure the cluster has its CRD installed. Installing Prometheus Operator will apply the required CRD.
+
+Run the following command to install Prometheus Operator and Grafana:
+
+```bash
+helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
+helm repo update
+
+helm install -n monitoring prometheus prometheus-community/kube-prometheus-stack \
+  --create-namespace \
+  --set 'prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues=false'
+```
+
+We set option `prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues` to false to force Prometheus to watch all service monitors in the cluster for test purposes.
+
+The default Grafana username and password is `admin` and `prom-operator`.
+
+## Install APISIX
+
+We should enable service monitor to tell Prometheus to collect metrics from APISIX.
+
+Install APISIX via helm chart with `serviceMonitor.enabled=true` option:
+
+```bash
+helm repo add apisix https://charts.apiseven.com
+helm repo update
+
+helm install apisix apisix/apisix --create-namespace --namespace apisix \
+  --set ingress-controller.config.apisix.serviceNamespace=apisix \
+  --set serviceMonitor.enabled=true \
+  --set ingress-controller.enabled=true
+```
+
+## Configure Grafana Dashboard
+
+Import [APISIX Grafana dashboard](https://grafana.com/grafana/dashboards/11719-apache-apisix/) via dashboard ID `11719`.
+
+The dashboard should be able to display some data, including total requests, handled connections, etc. Routing-related panels such as bandwidth and latency will show "No data" because we haven't made any requests yet. Create some routes with the "prometheus" plugin enabled and make some requests to them to generate some data for these panels.
+
+## Manual Configuration and Troubleshooting
+
+If you already have an installation of APISIX and Prometheus Operator, you can manually configure the `ServiceMonitor` resource and the service that exposes APISIX metrics.
+
+### Service Monitor and APISIX Service
+
+The magic behind `serviceMonitor.enabled=true` helm chart option is `ServiceMonitor` resource. Its content is as follows.
+
+```yaml
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+...
+spec:
+  namespaceSelector:
+    matchNames:
+    - apisix
+  selector:
+    matchLabels:
+      helm.sh/chart: apisix-1.1.1
+      app.kubernetes.io/name: apisix
+      app.kubernetes.io/instance: apisix
+      app.kubernetes.io/version: "3.1.1"
+      app.kubernetes.io/managed-by: Helm
+      app.kubernetes.io/service: apisix-gateway
+  endpoints:
+  - scheme: http
+    targetPort: prometheus
+    path: /apisix/prometheus/metrics
+    interval: 15s
+```
+
+The spec uses `namespaceSelector` and `selector` to match the `apisix-gateway` service. The former matches the namespace of APISIX we deployed, and the latter matches the exact service `apisix-gateway`.
+
+The field `endpoints` tells Prometheus where to collect the metrics. Note that the `targetPort` field points to the port of service with the same name. If your service doesn't have a port named `prometheus`, create one.
+
+The helm chart exposes APISIX metrics in the `apisix-gateway` service by default. Change the selector to match your service if needed.
+
+### Prometheus Spec
+
+If everything works fine, the `Status > Targets` page of Prometheus Web UI will show the APISIX service monitor. If you don't see it, you should make sure Prometheus is watching the `ServiceMonitor` resource we created.
+
+By default, the `Prometheus` resource created by the helm chart `kube-prometheus-stack` is as follows.
+
+```yaml
+apiVersion: monitoring.coreos.com/v1
+kind: Prometheus
+...
+spec:
+  ...
+  serviceMonitorNamespaceSelector: {}
+  serviceMonitorSelector:
+    matchLabels:
+      release: prometheus
+```
+
+Thus, we pass the option `prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues=false` to clear the `serviceMonitorSelector` field. Configure this resource to fit your needs.
+
+For example, if you want to set service monitor selector to `prom=watching`, your helm command should be:
+
+```bash
+helm install -n monitoring prometheus prometheus-community/kube-prometheus-stack \
+  --create-namespace \
+  --set 'prometheus.prometheusSpec.serviceMonitorSelector.matchLabels.prom=watching'
+```
+
+Because we set values for `serviceMonitorSelector`, so we don't need to configure `serviceMonitorSelectorNilUsesHelmValues` anymore.

go.mod

@@ -13,16 +13,16 @@ require (
 	github.com/prometheus/client_model v0.3.0
 	github.com/slok/kubewebhook/v2 v2.5.0
 	github.com/spf13/cobra v1.6.1
-	github.com/stretchr/testify v1.8.1
+	github.com/stretchr/testify v1.8.2
 	github.com/xeipuuv/gojsonschema v1.2.0
 	go.uber.org/multierr v1.9.0
 	go.uber.org/zap v1.24.0
 	golang.org/x/net v0.7.0
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1
 	gopkg.in/yaml.v2 v2.4.0
-	k8s.io/api v0.26.1
-	k8s.io/apimachinery v0.26.1
-	k8s.io/client-go v0.26.1
+	k8s.io/api v0.26.2
+	k8s.io/apimachinery v0.26.2
+	k8s.io/client-go v0.26.2
 	k8s.io/code-generator v0.26.2
 	sigs.k8s.io/controller-runtime v0.13.0
 	sigs.k8s.io/gateway-api v0.6.0

go.sum

@@ -325,8 +325,9 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
-github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
+github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
 github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
 github.com/ugorji/go/codec v1.2.9 h1:rmenucSohSTiyL09Y+l2OCk+FrMxGMzho2+tjr5ticU=
@@ -682,13 +683,13 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-k8s.io/api v0.26.1 h1:f+SWYiPd/GsiWwVRz+NbFyCgvv75Pk9NK6dlkZgpCRQ=
-k8s.io/api v0.26.1/go.mod h1:xd/GBNgR0f707+ATNyPmQ1oyKSgndzXij81FzWGsejg=
+k8s.io/api v0.26.2 h1:dM3cinp3PGB6asOySalOZxEG4CZ0IAdJsrYZXE/ovGQ=
+k8s.io/api v0.26.2/go.mod h1:1kjMQsFE+QHPfskEcVNgL3+Hp88B80uj0QtSOlj8itU=
 k8s.io/apiextensions-apiserver v0.26.0 h1:Gy93Xo1eg2ZIkNX/8vy5xviVSxwQulsnUdQ00nEdpDo=
-k8s.io/apimachinery v0.26.1 h1:8EZ/eGJL+hY/MYCNwhmDzVqq2lPl3N3Bo8rvweJwXUQ=
-k8s.io/apimachinery v0.26.1/go.mod h1:tnPmbONNJ7ByJNz9+n9kMjNP8ON+1qoAIIC70lztu74=
-k8s.io/client-go v0.26.1 h1:87CXzYJnAMGaa/IDDfRdhTzxk/wzGZ+/HUQpqgVSZXU=
-k8s.io/client-go v0.26.1/go.mod h1:IWNSglg+rQ3OcvDkhY6+QLeasV4OYHDjdqeWkDQZwGE=
+k8s.io/apimachinery v0.26.2 h1:da1u3D5wfR5u2RpLhE/ZtZS2P7QvDgLZTi9wrNZl/tQ=
+k8s.io/apimachinery v0.26.2/go.mod h1:ats7nN1LExKHvJ9TmwootT00Yz05MuYqPXEXaVeOy5I=
+k8s.io/client-go v0.26.2 h1:s1WkVujHX3kTp4Zn4yGNFK+dlDXy1bAAkIl+cFAiuYI=
+k8s.io/client-go v0.26.2/go.mod h1:u5EjOuSyBa09yqqyY7m3abZeovO/7D/WehVVlZ2qcqU=
 k8s.io/code-generator v0.26.2 h1:QMgN5oXUgQe27uMaqpbT0hg6ti+rvgCWaHEDMHVhox8=
 k8s.io/code-generator v0.26.2/go.mod h1:ryaiIKwfxEJEaywEzx3dhWOydpVctKYbqLajJf0O8dI=
 k8s.io/gengo v0.0.0-20221011193443-fad74ee6edd9 h1:iu3o/SxaHVI7tKPtkGzD3M9IzrE21j+CUKH98NQJ8Ms=

pkg/kube/apisix/apis/config/v2/types.go

@@ -359,6 +359,13 @@ type ApisixConsumer struct {
 
 // ApisixConsumerSpec defines the desired state of ApisixConsumer.
 type ApisixConsumerSpec struct {
+	// IngressClassName is the name of an IngressClass cluster resource.
+	// controller implementations use this field to know whether they should be
+	// serving this ApisixConsumer resource, by a transitive connection
+	// (controller -> IngressClass -> ApisixConsumer resource).
+	// +optional
+	IngressClassName string `json:"ingressClassName,omitempty" yaml:"ingressClassName,omitempty"`
+
 	AuthParameter ApisixConsumerAuthParameter `json:"authParameter" yaml:"authParameter"`
 }
 
@@ -730,6 +737,12 @@ type HostType string
 
 // ApisixTlsSpec is the specification of ApisixSSL.
 type ApisixTlsSpec struct {
+	// IngressClassName is the name of an IngressClass cluster resource.
+	// controller implementations use this field to know whether they should be
+	// serving this ApisixTls resource, by a transitive connection
+	// (controller -> IngressClass -> ApisixTls resource).
+	// +optional
+	IngressClassName string `json:"ingressClassName,omitempty" yaml:"ingressClassName,omitempty"`
 	// +required
 	// +kubebuilder:validation:Required
 	// +kubebuilder:validation:MinItems=1
@@ -774,6 +787,10 @@ type ApisixPluginConfig struct {
 
 // ApisixPluginConfigSpec defines the desired state of ApisixPluginConfigSpec.
 type ApisixPluginConfigSpec struct {
+	// IngressClassName is the name of an IngressClass cluster resource.
+	// The controller uses this field to decide whether the resource should be managed or not.
+	// +optional
+	IngressClassName string `json:"ingressClassName,omitempty" yaml:"ingressClassName,omitempty"`
 	// Plugins contains a list of ApisixRoutePlugin
 	// +required
 	Plugins []ApisixRoutePlugin `json:"plugins" yaml:"plugins"`

pkg/providers/apisix/apisix_consumer.go

@@ -232,6 +232,9 @@ func (c *apisixConsumerController) onAdd(obj interface{}) {
 	if !c.namespaceProvider.IsWatchingNamespace(key) {
 		return
 	}
+	if !c.isEffective(ac) {
+		return
+	}
 	log.Debugw("ApisixConsumer add event arrived",
 		zap.Any("object", obj),
 	)
@@ -269,6 +272,9 @@ func (c *apisixConsumerController) onUpdate(oldObj, newObj interface{}) {
 	if !c.namespaceProvider.IsWatchingNamespace(key) {
 		return
 	}
+	if !c.isEffective(curr) {
+		return
+	}
 	log.Debugw("ApisixConsumer update event arrived",
 		zap.Any("new object", curr),
 		zap.Any("old object", prev),
@@ -308,6 +314,9 @@ func (c *apisixConsumerController) onDelete(obj interface{}) {
 	if !c.namespaceProvider.IsWatchingNamespace(key) {
 		return
 	}
+	if !c.isEffective(ac) {
+		return
+	}
 	log.Debugw("ApisixConsumer delete event arrived",
 		zap.Any("final state", ac),
 	)
@@ -337,7 +346,10 @@ func (c *apisixConsumerController) ResourceSync() {
 		ac, err := kube.NewApisixConsumer(obj)
 		if err != nil {
 			log.Errorw("found ApisixConsumer resource with bad type", zap.String("error", err.Error()))
-			return
+			continue
+		}
+		if !c.isEffective(ac) {
+			continue
 		}
 		c.workqueue.Add(&types.Event{
 			Type: types.EventAdd,
@@ -412,3 +424,11 @@ func (c *apisixConsumerController) recordStatus(at interface{}, reason string, e
 		log.Errorf("unsupported resource record: %s", v)
 	}
 }
+
+func (c *apisixConsumerController) isEffective(ac kube.ApisixConsumer) bool {
+	if ac.GroupVersion() == config.ApisixV2 {
+		return utils.MatchCRDsIngressClass(ac.V2().Spec.IngressClassName, c.Kubernetes.IngressClass)
+	}
+	// Compatible with legacy versions
+	return true
+}

pkg/providers/apisix/apisix_plugin_config.go

@@ -288,13 +288,16 @@ func (c *apisixPluginConfigController) onAdd(obj interface{}) {
 		log.Errorf("found ApisixPluginConfig resource with bad meta namespace key: %s", err)
 		return
 	}
+	apc := kube.MustNewApisixPluginConfig(obj)
+	if !c.isEffective(apc) {
+		return
+	}
 	if !c.namespaceProvider.IsWatchingNamespace(key) {
 		return
 	}
 	log.Debugw("ApisixPluginConfig add event arrived",
 		zap.Any("object", obj))
 
-	apc := kube.MustNewApisixPluginConfig(obj)
 	c.workqueue.Add(&types.Event{
 		Type: types.EventAdd,
 		Object: kube.ApisixPluginConfigEvent{
@@ -317,6 +320,9 @@ func (c *apisixPluginConfigController) onUpdate(oldObj, newObj interface{}) {
 		log.Errorf("found ApisixPluginConfig resource with bad meta namespace key: %s", err)
 		return
 	}
+	if !c.isEffective(curr) {
+		return
+	}
 	if !c.namespaceProvider.IsWatchingNamespace(key) {
 		return
 	}
@@ -350,6 +356,9 @@ func (c *apisixPluginConfigController) onDelete(obj interface{}) {
 		log.Errorf("found ApisixPluginConfig resource with bad meta namesapce key: %s", err)
 		return
 	}
+	if !c.isEffective(apc) {
+		return
+	}
 	if !c.namespaceProvider.IsWatchingNamespace(key) {
 		return
 	}
@@ -376,10 +385,13 @@ func (c *apisixPluginConfigController) ResourceSync() {
 			log.Errorw("ApisixPluginConfig sync failed, found ApisixPluginConfig resource with bad meta namespace key", zap.String("error", err.Error()))
 			continue
 		}
+		apc := kube.MustNewApisixPluginConfig(obj)
+		if !c.isEffective(apc) {
+			continue
+		}
 		if !c.namespaceProvider.IsWatchingNamespace(key) {
 			continue
 		}
-		apc := kube.MustNewApisixPluginConfig(obj)
 		c.workqueue.Add(&types.Event{
 			Type: types.EventAdd,
 			Object: kube.ApisixPluginConfigEvent{
@@ -453,3 +465,20 @@ func (c *apisixPluginConfigController) recordStatus(at interface{}, reason strin
 		log.Errorf("unsupported resource record: %s", v)
 	}
 }
+
+func (c *apisixPluginConfigController) isEffective(apc kube.ApisixPluginConfig) bool {
+	if apc.GroupVersion() == config.ApisixV2 {
+		ingClassName := apc.V2().Spec.IngressClassName
+		ok := utils.MatchCRDsIngressClass(ingClassName, c.Kubernetes.IngressClass)
+		if !ok {
+			log.Debugw("IngressClass: ApisixPluginConfig ignored",
+				zap.String("key", apc.V2().Namespace+"/"+apc.V2().Name),
+				zap.String("ingressClass", apc.V2().Spec.IngressClassName),
+			)
+		}
+
+		return ok
+	}
+	// Compatible with legacy versions
+	return true
+}