Merge pull request #113 from NandosUK/feature/NAP-398

Feature/nap 398

experimental/terraform/resource_bigquery.tf

@@ -90,6 +90,44 @@ resource "google_bigquery_table" "view_teams_incidents" {
  ]
 }
 
+resource "google_bigquery_table" "view_repository" {
+ dataset_id = google_bigquery_dataset.four_keys.dataset_id
+ table_id = "repository"
+ view {
+ query = file("../../queries/repository.sql")
+ use_legacy_sql = false
+ }
+ deletion_protection = false
+ depends_on = [
+ google_bigquery_table.events_raw
+ ]
+}
+
+resource "google_bigquery_table" "view_team" {
+ dataset_id = google_bigquery_dataset.four_keys.dataset_id
+ table_id = "team"
+ view {
+ query = file("../../queries/team.sql")
+ use_legacy_sql = false
+ }
+ deletion_protection = false
+ depends_on = [
+ google_bigquery_table.events_raw
+ ]
+}
+resource "google_bigquery_table" "view_repo_team" {
+ dataset_id = google_bigquery_dataset.four_keys.dataset_id
+ table_id = "repo_team"
+ view {
+ query = file("../../queries/repo_team.sql")
+ use_legacy_sql = false
+ }
+ deletion_protection = false
+ depends_on = [
+ google_bigquery_table.events_raw
+ ]
+}
+
 resource "google_project_iam_member" "parser_bq_project_access" {
  role = "roles/bigquery.user"
  member = "serviceAccount:${google_service_account.fourkeys.email}"

queries/repo_team.sql

@@ -0,0 +1,15 @@
+SELECT
+team_id,
+repo_id,
+FROM
+(
+ SELECT
+ source,
+ CASE WHEN source LIKE "github%" THEN JSON_EXTRACT_SCALAR(metadata, '$.team.id') end as team_id,
+ CASE WHEN source LIKE "github%" THEN JSON_EXTRACT_SCALAR(metadata, '$.repository.id') end as repo_id,
+ CASE WHEN source LIKE "github%" THEN JSON_EXTRACT_SCALAR(metadata, '$.action') end as action,
+ FROM `mgt-fourkeys-8d87d44d.four_keys.events_raw` WHERE event_type = "team_add"
+) issue where action in ('injected','created')
+GROUP BY
+team_id,
+repo_id

queries/repository.sql

@@ -0,0 +1,30 @@
+ SELECT
+ id,
+ default_branch,
+ fork,
+ full_name,
+ language,
+ name,
+ owner_id
+ FROM
+ (
+ SELECT
+ source,
+ CASE WHEN source LIKE "github%" THEN JSON_EXTRACT_SCALAR(metadata, '$.repository.id') end as id,
+ CASE WHEN source LIKE "github%" THEN JSON_EXTRACT_SCALAR(metadata, '$.repository.name') end as name,
+ CASE WHEN source LIKE "github%" THEN JSON_EXTRACT_SCALAR(metadata, '$.repository.full_name') end as full_name,
+ CASE WHEN source LIKE "github%" THEN JSON_EXTRACT_SCALAR(metadata, '$.repository.default_branch') end as default_branch,
+ CASE WHEN source LIKE "github%" THEN JSON_EXTRACT_SCALAR(metadata, '$.repository.fork') end as fork,
+ CASE WHEN source LIKE "github%" THEN JSON_EXTRACT_SCALAR(metadata, '$.repository.language') end as language,
+ CASE WHEN source LIKE "github%" THEN JSON_EXTRACT_SCALAR(metadata, '$.repository.owner.id') end as owner_id,
+ CASE WHEN source LIKE "github%" THEN JSON_EXTRACT_SCALAR(metadata, '$.action') end as action,
+ FROM `mgt-fourkeys-8d87d44d.four_keys.events_raw` WHERE event_type LIKE "repository"
+ ) issue where action in ('injected','created')
+ GROUP BY
+ id,
+ default_branch,
+ fork,
+ full_name,
+ language,
+ owner_id,
+ name

queries/team.sql

@@ -0,0 +1,40 @@
+# Incidents Table
+SELECT
+-- distinct(service_id),
+teams_id,
+team_name,
+'NA' as parent_id,
+'NA' as org_id
+FROM
+(
+ SELECT 
+ CASE WHEN source LIKE "pagerduty%" THEN JSON_EXTRACT_SCALAR(metadata, '$.event.data.teams[0].id') end as teams_id,
+ CASE WHEN source LIKE "pagerduty%" THEN JSON_EXTRACT_SCALAR(metadata, '$.event.data.teams[0].summary') end as team_name,
+ FROM `mgt-fourkeys-8d87d44d.four_keys.events_raw`
+ WHERE event_type like "incident%" 
+) issue
+GROUP BY teams_id,team_name,parent_id, org_id
+UNION ALL 
+SELECT
+team_id,
+team_name,
+parent_id,
+org_id
+FROM
+(
+ SELECT
+ source,
+ CASE WHEN source LIKE "github%" THEN JSON_EXTRACT_SCALAR(metadata, '$.team.id') end as team_id,
+ CASE WHEN source LIKE "github%" THEN JSON_EXTRACT_SCALAR(metadata, '$.team.name') end as team_name,
+ CASE WHEN source LIKE "github%" THEN JSON_EXTRACT_SCALAR(metadata, '$.team.slug') end as slug,
+ CASE WHEN source LIKE "github%" THEN JSON_EXTRACT_SCALAR(metadata, '$.team.parent') end as parent_id,
+ CASE WHEN source LIKE "github%" THEN JSON_EXTRACT_SCALAR(metadata, '$.team.privacy') end as privacy,
+ CASE WHEN source LIKE "github%" THEN JSON_EXTRACT_SCALAR(metadata, '$.action') end as action,
+ CASE WHEN source LIKE "github%" THEN JSON_EXTRACT_SCALAR(metadata, '$.organization.id') end as org_id,
+ FROM `mgt-fourkeys-8d87d44d.four_keys.events_raw` WHERE event_type = "team"
+) issue where action in ('injected','created')
+GROUP BY
+team_id,
+team_name,
+parent_id,
+org_id

setup/setup.sh

@@ -367,6 +367,9 @@ schedule_bq_queries(){
  ./schedule.sh --query_file=deployments.sql --table=deployments --project_id=$FOURKEYS_PROJECT
  ./schedule.sh --query_file=incidents.sql --table=incidents --project_id=$FOURKEYS_PROJECT
  ./schedule.sh --query_file=teams_incidents.sql --table=teams_incidents --project_id=$FOURKEYS_PROJECT
+ ./schedule.sh --query_file=team.sql --table=team --project_id=$FOURKEYS_PROJECT
+ ./schedule.sh --query_file=repository.sql --table=team --project_id=$FOURKEYS_PROJECT
+ ./schedule.sh --query_file=repo_team.sql --table=team --project_id=$FOURKEYS_PROJECT
 
  set +x; echo
  cd ${DIR}