Create documentation on testing BigQuery from ingestion-beam.

mozilla · Apr 26, 2019 · 0fc5170 · 0fc5170
1 parent 5bb0aba
commit 0fc5170
Show file tree

Hide file tree

Showing 5 changed files with 125 additions and 6 deletions.
diff --git a/docs/architecture/diagram.svg b/docs/architecture/diagram.svg
diff --git a/docs/diagrams/workflow.mmd b/docs/diagrams/workflow.mmd
@@ -0,0 +1,27 @@
+graph TD
+
+subgraph dataops/sandbox/my-project
+    dataflow
+    bigquery
+    pubsub
+    subscription
+
+    pubsub --> |gcloud pubsub subscriptions create| subscription
+    subscription --> dataflow
+    dataflow --> bigquery
+end
+
+subgraph mozilla-pipeline-schemas
+    mps[repository archive]
+end
+
+subgraph ingestion-beam
+    src[src/]
+    schemas[schemas.tar.gz]
+    bq-schemas[bq-schemas/]
+
+    src --> |mvn compile exec:java| dataflow
+    mps --> |download-schemas| schemas
+    schemas --> |generate-bq-schemas| bq-schemas
+    bq-schemas --> |update-bq-table| bigquery
+end
diff --git a/docs/diagrams/workflow.svg b/docs/diagrams/workflow.svg