diff --git a/.gitignore b/.gitignore index d1994bf0ae09a..cc51711016642 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,8 @@ coverage.txt # emacs .#* + +# terraform +.terraform* +*.tfstate* +*.tfvars \ No newline at end of file diff --git a/scripts/gcplog/README.md b/scripts/gcplog/README.md new file mode 100644 index 0000000000000..f653a7a8f1596 --- /dev/null +++ b/scripts/gcplog/README.md @@ -0,0 +1,60 @@ +# Cloud provisioning for GCP logs + +This document covers how to configure your GCP via Terraform to make cloud logs available for `promtail` to consume. + +To choose what logs need to exported from Google Cloud, we use log filters. Log filters are normal GCP logging queries except the goal is export logs from specific set Google cloud resources + +e.g: Export Google APP Engine logs +```bash +resource.type="gae_app" AND +severity>=ERROR +``` + +e.g: Export Google HTTP Loadbalancer logs +```bash +resource.type="http_load_balancer" AND +httpRequest.status>=500 +``` +You can read more about these log filters in [GCP logging](https://cloud.google.com/logging/docs/view/query-library) + +## Prerequisite +- Terraform >= 0.14.5 +- GCP Service account credentials with following roles/permissions + - "roles/pubsub.editor" + - "roles/logging.configWriter" + +## Usage + +```bash +terraform init +``` + +```bash +terraform plan +``` + +```bash +terraform apply +``` + +Terraform will prompt for following variables. + +1. credentials_file - ServiceAccount credentials file with permissions mentioned in the prerequisite. +2. zone - GCP zone (e.g: `us-central1-b`) +3. region - GCP region (e.g: `us-central1`) +4. project - GCP Project ID +5. logname - Logname is the name we use to create pubsub topics, log router and pubsub subscription. + +you can pass these variables via CLI. + +e.g: +```bash +terraform apply \ +-var="credentials_file=./permissions.json" \ +-var="zone=us-central1-b" \ +-var="region=us-central1" \ +-var="project=grafanalabs-dev" \ +-var="logname=cloud-logs" +``` + +These variables can be passed in multiple ways. For complete reference refer terraform [doc](https://www.terraform.io/docs/configuration/variables.html#assigning-values-to-root-module-variables) diff --git a/scripts/gcplog/main.tf b/scripts/gcplog/main.tf new file mode 100644 index 0000000000000..54a1a6c5e4170 --- /dev/null +++ b/scripts/gcplog/main.tf @@ -0,0 +1,40 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = "3.5.0" + } + } +} + +variable "credentials_file" {} +variable "zone" {} +variable "region" {} +variable "project" {} +variable "logname" { + default = "cloud-logs" +} + +provider "google" { + credentials = file(var.credentials_file) + project = var.project + zone = var.zone + region= var.region + +} + +resource "google_pubsub_topic" "cloud-logs" { + name= var.logname +} + +resource "google_logging_project_sink" "cloud-logs" { + name = var.logname + destination = "pubsub.googleapis.com/projects/personal-226821/topics/${var.logname}" + filter = "resource.type = http_load_balancer AND httpRequest.status >= 200" + unique_writer_identity = true +} + +resource "google_pubsub_subscription" "coud-logs" { + name = var.logname + topic = google_pubsub_topic.cloud-logs.name +}