From 3df711f15c948a9b13a3ac2df3da49b916a3c5d0 Mon Sep 17 00:00:00 2001
From: Anthony Miyaguchi <amiyaguchi@mozilla.com>
Date: Fri, 26 Apr 2019 16:23:10 -0700
Subject: [PATCH] Create documentation on testing BigQuery from ingestion-beam.

---
 docs/architecture/diagram.svg      | 12 ++---
 docs/diagrams/workflow.mmd         | 27 ++++++++++
 docs/diagrams/workflow.svg         |  4 ++
 docs/ingestion_testing_workflow.md | 85 ++++++++++++++++++++++++++++++
 ingestion-beam/README.md           |  3 ++
 5 files changed, 125 insertions(+), 6 deletions(-)
 create mode 100644 docs/diagrams/workflow.mmd
 create mode 100644 docs/diagrams/workflow.svg
 create mode 100644 docs/ingestion_testing_workflow.md

diff --git a/docs/architecture/diagram.svg b/docs/architecture/diagram.svg
index 4006fd861d..1a48990bbb 100644
--- a/docs/architecture/diagram.svg
+++ b/docs/architecture/diagram.svg
@@ -1,8 +1,8 @@
-<svg id="mermaid-1551364991765" width="100%" xmlns="http://www.w3.org/2000/svg" style="max-width: 1104.75px;" viewBox="0 0 1104.75 632"><style>#mermaid-1551364991765 .label{font-family:trebuchet ms,verdana,arial;color:#333}#mermaid-1551364991765 .node circle,#mermaid-1551364991765 .node ellipse,#mermaid-1551364991765 .node polygon,#mermaid-1551364991765 .node rect{fill:#ececff;stroke:#9370db;stroke-width:1px}#mermaid-1551364991765 .node.clickable{cursor:pointer}#mermaid-1551364991765 .arrowheadPath{fill:#333}#mermaid-1551364991765 .edgePath .path{stroke:#333;stroke-width:1.5px}#mermaid-1551364991765 .edgeLabel{background-color:#e8e8e8}#mermaid-1551364991765 .cluster rect{fill:#ffffde!important;stroke:#aa3!important;stroke-width:1px!important}#mermaid-1551364991765 .cluster text{fill:#333}#mermaid-1551364991765 div.mermaidTooltip{position:absolute;text-align:center;max-width:200px;padding:2px;font-family:trebuchet ms,verdana,arial;font-size:12px;background:#ffffde;border:1px solid #aa3;border-radius:2px;pointer-events:none;z-index:100}#mermaid-1551364991765 .actor{stroke:#ccf;fill:#ececff}#mermaid-1551364991765 text.actor{fill:#000;stroke:none}#mermaid-1551364991765 .actor-line{stroke:grey}#mermaid-1551364991765 .messageLine0{marker-end:"url(#arrowhead)"}#mermaid-1551364991765 .messageLine0,#mermaid-1551364991765 .messageLine1{stroke-width:1.5;stroke-dasharray:"2 2";stroke:#333}#mermaid-1551364991765 #arrowhead{fill:#333}#mermaid-1551364991765 #crosshead path{fill:#333!important;stroke:#333!important}#mermaid-1551364991765 .messageText{fill:#333;stroke:none}#mermaid-1551364991765 .labelBox{stroke:#ccf;fill:#ececff}#mermaid-1551364991765 .labelText,#mermaid-1551364991765 .loopText{fill:#000;stroke:none}#mermaid-1551364991765 .loopLine{stroke-width:2;stroke-dasharray:"2 2";marker-end:"url(#arrowhead)";stroke:#ccf}#mermaid-1551364991765 .note{stroke:#aa3;fill:#fff5ad}#mermaid-1551364991765 .noteText{fill:#000;stroke:none;font-family:trebuchet ms,verdana,arial;font-size:14px}#mermaid-1551364991765 .section{stroke:none;opacity:.2}#mermaid-1551364991765 .section0{fill:rgba(102,102,255,.49)}#mermaid-1551364991765 .section2{fill:#fff400}#mermaid-1551364991765 .section1,#mermaid-1551364991765 .section3{fill:#fff;opacity:.2}#mermaid-1551364991765 .sectionTitle0,#mermaid-1551364991765 .sectionTitle1,#mermaid-1551364991765 .sectionTitle2,#mermaid-1551364991765 .sectionTitle3{fill:#333}#mermaid-1551364991765 .sectionTitle{text-anchor:start;font-size:11px;text-height:14px}#mermaid-1551364991765 .grid .tick{stroke:#d3d3d3;opacity:.3;shape-rendering:crispEdges}#mermaid-1551364991765 .grid path{stroke-width:0}#mermaid-1551364991765 .today{fill:none;stroke:red;stroke-width:2px}#mermaid-1551364991765 .task{stroke-width:2}#mermaid-1551364991765 .taskText{text-anchor:middle;font-size:11px}#mermaid-1551364991765 .taskTextOutsideRight{fill:#000;text-anchor:start;font-size:11px}#mermaid-1551364991765 .taskTextOutsideLeft{fill:#000;text-anchor:end;font-size:11px}#mermaid-1551364991765 .taskText0,#mermaid-1551364991765 .taskText1,#mermaid-1551364991765 .taskText2,#mermaid-1551364991765 .taskText3{fill:#fff}#mermaid-1551364991765 .task0,#mermaid-1551364991765 .task1,#mermaid-1551364991765 .task2,#mermaid-1551364991765 .task3{fill:#8a90dd;stroke:#534fbc}#mermaid-1551364991765 .taskTextOutside0,#mermaid-1551364991765 .taskTextOutside1,#mermaid-1551364991765 .taskTextOutside2,#mermaid-1551364991765 .taskTextOutside3{fill:#000}#mermaid-1551364991765 .active0,#mermaid-1551364991765 .active1,#mermaid-1551364991765 .active2,#mermaid-1551364991765 .active3{fill:#bfc7ff;stroke:#534fbc}#mermaid-1551364991765 .activeText0,#mermaid-1551364991765 .activeText1,#mermaid-1551364991765 .activeText2,#mermaid-1551364991765 .activeText3{fill:#000!important}#mermaid-1551364991765 .done0,#mermaid-1551364991765 .done1,#mermaid-1551364991765 .done2,#mermaid-1551364991765 .done3{stroke:grey;fill:#d3d3d3;stroke-width:2}#mermaid-1551364991765 .doneText0,#mermaid-1551364991765 .doneText1,#mermaid-1551364991765 .doneText2,#mermaid-1551364991765 .doneText3{fill:#000!important}#mermaid-1551364991765 .crit0,#mermaid-1551364991765 .crit1,#mermaid-1551364991765 .crit2,#mermaid-1551364991765 .crit3{stroke:#f88;fill:red;stroke-width:2}#mermaid-1551364991765 .activeCrit0,#mermaid-1551364991765 .activeCrit1,#mermaid-1551364991765 .activeCrit2,#mermaid-1551364991765 .activeCrit3{stroke:#f88;fill:#bfc7ff;stroke-width:2}#mermaid-1551364991765 .doneCrit0,#mermaid-1551364991765 .doneCrit1,#mermaid-1551364991765 .doneCrit2,#mermaid-1551364991765 .doneCrit3{stroke:#f88;fill:#d3d3d3;stroke-width:2;cursor:pointer;shape-rendering:crispEdges}#mermaid-1551364991765 .activeCritText0,#mermaid-1551364991765 .activeCritText1,#mermaid-1551364991765 .activeCritText2,#mermaid-1551364991765 .activeCritText3,#mermaid-1551364991765 .doneCritText0,#mermaid-1551364991765 .doneCritText1,#mermaid-1551364991765 .doneCritText2,#mermaid-1551364991765 .doneCritText3{fill:#000!important}#mermaid-1551364991765 .titleText{text-anchor:middle;font-size:18px;fill:#000}#mermaid-1551364991765 g.classGroup text{fill:#9370db;stroke:none;font-family:trebuchet ms,verdana,arial;font-size:10px}#mermaid-1551364991765 g.classGroup rect{fill:#ececff;stroke:#9370db}#mermaid-1551364991765 g.classGroup line{stroke:#9370db;stroke-width:1}#mermaid-1551364991765 .classLabel .box{stroke:none;stroke-width:0;fill:#ececff;opacity:.5}#mermaid-1551364991765 .classLabel .label{fill:#9370db;font-size:10px}#mermaid-1551364991765 .relation{stroke:#9370db;stroke-width:1;fill:none}#mermaid-1551364991765 #compositionEnd,#mermaid-1551364991765 #compositionStart{fill:#9370db;stroke:#9370db;stroke-width:1}#mermaid-1551364991765 #aggregationEnd,#mermaid-1551364991765 #aggregationStart{fill:#ececff;stroke:#9370db;stroke-width:1}#mermaid-1551364991765 #dependencyEnd,#mermaid-1551364991765 #dependencyStart,#mermaid-1551364991765 #extensionEnd,#mermaid-1551364991765 #extensionStart{fill:#9370db;stroke:#9370db;stroke-width:1}#mermaid-1551364991765 .branch-label,#mermaid-1551364991765 .commit-id,#mermaid-1551364991765 .commit-msg{fill:#d3d3d3;color:#d3d3d3}
-#mermaid-1551364991765 .cyan &gt; * { fill:#eff !important; stroke:#099 !important; }
-#mermaid-1551364991765 .green &gt; * { fill:#efe !important; stroke:#090 !important; }
-#mermaid-1551364991765 .magenta &gt; * { fill:#fef !important; stroke:#909 !important; }
-#mermaid-1551364991765 .orange &gt; * { fill:#fee !important; stroke:#f90 !important; }</style><style>#mermaid-1551364991765 {
+<svg id="mermaid-1556319862364" width="100%" xmlns="http://www.w3.org/2000/svg" style="max-width: 1104.75px;" viewBox="0 0 1104.75 632"><style>#mermaid-1556319862364 .label{font-family:trebuchet ms,verdana,arial;color:#333}#mermaid-1556319862364 .node circle,#mermaid-1556319862364 .node ellipse,#mermaid-1556319862364 .node polygon,#mermaid-1556319862364 .node rect{fill:#ececff;stroke:#9370db;stroke-width:1px}#mermaid-1556319862364 .node.clickable{cursor:pointer}#mermaid-1556319862364 .arrowheadPath{fill:#333}#mermaid-1556319862364 .edgePath .path{stroke:#333;stroke-width:1.5px}#mermaid-1556319862364 .edgeLabel{background-color:#e8e8e8}#mermaid-1556319862364 .cluster rect{fill:#ffffde!important;stroke:#aa3!important;stroke-width:1px!important}#mermaid-1556319862364 .cluster text{fill:#333}#mermaid-1556319862364 div.mermaidTooltip{position:absolute;text-align:center;max-width:200px;padding:2px;font-family:trebuchet ms,verdana,arial;font-size:12px;background:#ffffde;border:1px solid #aa3;border-radius:2px;pointer-events:none;z-index:100}#mermaid-1556319862364 .actor{stroke:#ccf;fill:#ececff}#mermaid-1556319862364 text.actor{fill:#000;stroke:none}#mermaid-1556319862364 .actor-line{stroke:grey}#mermaid-1556319862364 .messageLine0{marker-end:"url(#arrowhead)"}#mermaid-1556319862364 .messageLine0,#mermaid-1556319862364 .messageLine1{stroke-width:1.5;stroke-dasharray:"2 2";stroke:#333}#mermaid-1556319862364 #arrowhead{fill:#333}#mermaid-1556319862364 #crosshead path{fill:#333!important;stroke:#333!important}#mermaid-1556319862364 .messageText{fill:#333;stroke:none}#mermaid-1556319862364 .labelBox{stroke:#ccf;fill:#ececff}#mermaid-1556319862364 .labelText,#mermaid-1556319862364 .loopText{fill:#000;stroke:none}#mermaid-1556319862364 .loopLine{stroke-width:2;stroke-dasharray:"2 2";marker-end:"url(#arrowhead)";stroke:#ccf}#mermaid-1556319862364 .note{stroke:#aa3;fill:#fff5ad}#mermaid-1556319862364 .noteText{fill:#000;stroke:none;font-family:trebuchet ms,verdana,arial;font-size:14px}#mermaid-1556319862364 .section{stroke:none;opacity:.2}#mermaid-1556319862364 .section0{fill:rgba(102,102,255,.49)}#mermaid-1556319862364 .section2{fill:#fff400}#mermaid-1556319862364 .section1,#mermaid-1556319862364 .section3{fill:#fff;opacity:.2}#mermaid-1556319862364 .sectionTitle0,#mermaid-1556319862364 .sectionTitle1,#mermaid-1556319862364 .sectionTitle2,#mermaid-1556319862364 .sectionTitle3{fill:#333}#mermaid-1556319862364 .sectionTitle{text-anchor:start;font-size:11px;text-height:14px}#mermaid-1556319862364 .grid .tick{stroke:#d3d3d3;opacity:.3;shape-rendering:crispEdges}#mermaid-1556319862364 .grid path{stroke-width:0}#mermaid-1556319862364 .today{fill:none;stroke:red;stroke-width:2px}#mermaid-1556319862364 .task{stroke-width:2}#mermaid-1556319862364 .taskText{text-anchor:middle;font-size:11px}#mermaid-1556319862364 .taskTextOutsideRight{fill:#000;text-anchor:start;font-size:11px}#mermaid-1556319862364 .taskTextOutsideLeft{fill:#000;text-anchor:end;font-size:11px}#mermaid-1556319862364 .taskText0,#mermaid-1556319862364 .taskText1,#mermaid-1556319862364 .taskText2,#mermaid-1556319862364 .taskText3{fill:#fff}#mermaid-1556319862364 .task0,#mermaid-1556319862364 .task1,#mermaid-1556319862364 .task2,#mermaid-1556319862364 .task3{fill:#8a90dd;stroke:#534fbc}#mermaid-1556319862364 .taskTextOutside0,#mermaid-1556319862364 .taskTextOutside1,#mermaid-1556319862364 .taskTextOutside2,#mermaid-1556319862364 .taskTextOutside3{fill:#000}#mermaid-1556319862364 .active0,#mermaid-1556319862364 .active1,#mermaid-1556319862364 .active2,#mermaid-1556319862364 .active3{fill:#bfc7ff;stroke:#534fbc}#mermaid-1556319862364 .activeText0,#mermaid-1556319862364 .activeText1,#mermaid-1556319862364 .activeText2,#mermaid-1556319862364 .activeText3{fill:#000!important}#mermaid-1556319862364 .done0,#mermaid-1556319862364 .done1,#mermaid-1556319862364 .done2,#mermaid-1556319862364 .done3{stroke:grey;fill:#d3d3d3;stroke-width:2}#mermaid-1556319862364 .doneText0,#mermaid-1556319862364 .doneText1,#mermaid-1556319862364 .doneText2,#mermaid-1556319862364 .doneText3{fill:#000!important}#mermaid-1556319862364 .crit0,#mermaid-1556319862364 .crit1,#mermaid-1556319862364 .crit2,#mermaid-1556319862364 .crit3{stroke:#f88;fill:red;stroke-width:2}#mermaid-1556319862364 .activeCrit0,#mermaid-1556319862364 .activeCrit1,#mermaid-1556319862364 .activeCrit2,#mermaid-1556319862364 .activeCrit3{stroke:#f88;fill:#bfc7ff;stroke-width:2}#mermaid-1556319862364 .doneCrit0,#mermaid-1556319862364 .doneCrit1,#mermaid-1556319862364 .doneCrit2,#mermaid-1556319862364 .doneCrit3{stroke:#f88;fill:#d3d3d3;stroke-width:2;cursor:pointer;shape-rendering:crispEdges}#mermaid-1556319862364 .activeCritText0,#mermaid-1556319862364 .activeCritText1,#mermaid-1556319862364 .activeCritText2,#mermaid-1556319862364 .activeCritText3,#mermaid-1556319862364 .doneCritText0,#mermaid-1556319862364 .doneCritText1,#mermaid-1556319862364 .doneCritText2,#mermaid-1556319862364 .doneCritText3{fill:#000!important}#mermaid-1556319862364 .titleText{text-anchor:middle;font-size:18px;fill:#000}#mermaid-1556319862364 g.classGroup text{fill:#9370db;stroke:none;font-family:trebuchet ms,verdana,arial;font-size:10px}#mermaid-1556319862364 g.classGroup rect{fill:#ececff;stroke:#9370db}#mermaid-1556319862364 g.classGroup line{stroke:#9370db;stroke-width:1}#mermaid-1556319862364 .classLabel .box{stroke:none;stroke-width:0;fill:#ececff;opacity:.5}#mermaid-1556319862364 .classLabel .label{fill:#9370db;font-size:10px}#mermaid-1556319862364 .relation{stroke:#9370db;stroke-width:1;fill:none}#mermaid-1556319862364 #compositionEnd,#mermaid-1556319862364 #compositionStart{fill:#9370db;stroke:#9370db;stroke-width:1}#mermaid-1556319862364 #aggregationEnd,#mermaid-1556319862364 #aggregationStart{fill:#ececff;stroke:#9370db;stroke-width:1}#mermaid-1556319862364 #dependencyEnd,#mermaid-1556319862364 #dependencyStart,#mermaid-1556319862364 #extensionEnd,#mermaid-1556319862364 #extensionStart{fill:#9370db;stroke:#9370db;stroke-width:1}#mermaid-1556319862364 .branch-label,#mermaid-1556319862364 .commit-id,#mermaid-1556319862364 .commit-msg{fill:#d3d3d3;color:#d3d3d3}
+#mermaid-1556319862364 .cyan &gt; * { fill:#eff !important; stroke:#099 !important; }
+#mermaid-1556319862364 .green &gt; * { fill:#efe !important; stroke:#090 !important; }
+#mermaid-1556319862364 .magenta &gt; * { fill:#fef !important; stroke:#909 !important; }
+#mermaid-1556319862364 .orange &gt; * { fill:#fee !important; stroke:#f90 !important; }</style><style>#mermaid-1556319862364 {
     color: rgb(0, 0, 0);
     font: normal normal 400 normal 16px / normal "Times New Roman";
-  }</style><g transform="translate(-12, -12)"><g class="output"><g class="clusters"><g class="cluster" id="subGraph0" transform="translate(184,240)" style="opacity: 1;"><rect width="328" height="440" x="-164" y="-220"></rect><g class="label"><g transform="translate(0,0)"><foreignObject width="0" height="0"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;"></div></foreignObject></g></g><text x="0" y="-206" fill="black" stroke="none" id="mermaid-1551364991765Text" style="text-anchor: middle;">Colors</text></g></g><g class="edgePaths"><g class="edgePath" style="opacity: 1;"><path class="path" d="M446.5,83L446.5,108L446.5,133" marker-end="url(#arrowhead81)" style="fill:none"></path><defs><marker id="arrowhead81" viewBox="0 0 10 10" refX="9" refY="5" markerUnits="strokeWidth" markerWidth="8" markerHeight="6" orient="auto"><path d="M 0 0 L 10 5 L 0 10 z" class="arrowheadPath" style="stroke-width: 1; stroke-dasharray: 1, 0;"></path></marker></defs></g><g class="edgePath" style="opacity: 1;"><path class="path" d="M446.5,171L446.5,196L446.5,221" marker-end="url(#arrowhead82)" style="fill:none"></path><defs><marker id="arrowhead82" viewBox="0 0 10 10" refX="9" refY="5" markerUnits="strokeWidth" markerWidth="8" markerHeight="6" orient="auto"><path d="M 0 0 L 10 5 L 0 10 z" class="arrowheadPath" style="stroke-width: 1; stroke-dasharray: 1, 0;"></path></marker></defs></g><g class="edgePath" style="opacity: 1;"><path class="path" d="M446.0681818181818,259L445.5,284L445.5,309" marker-end="url(#arrowhead83)" style="fill:none"></path><defs><marker id="arrowhead83" viewBox="0 0 10 10" refX="9" refY="5" markerUnits="strokeWidth" markerWidth="8" markerHeight="6" orient="auto"><path d="M 0 0 L 10 5 L 0 10 z" class="arrowheadPath" style="stroke-width: 1; stroke-dasharray: 1, 0;"></path></marker></defs></g><g class="edgePath" style="opacity: 1;"><path class="path" d="M445.5,347L445.5,372L445.5,397" marker-end="url(#arrowhead84)" style="fill:none"></path><defs><marker id="arrowhead84" viewBox="0 0 10 10" refX="9" refY="5" markerUnits="strokeWidth" markerWidth="8" markerHeight="6" orient="auto"><path d="M 0 0 L 10 5 L 0 10 z" class="arrowheadPath" style="stroke-width: 1; stroke-dasharray: 1, 0;"></path></marker></defs></g><g class="edgePath" style="opacity: 1;"><path class="path" d="M489.0340909090909,259L545,284L590.1704545454545,309" marker-end="url(#arrowhead85)" style="fill:none"></path><defs><marker id="arrowhead85" viewBox="0 0 10 10" refX="9" refY="5" markerUnits="strokeWidth" markerWidth="8" markerHeight="6" orient="auto"><path d="M 0 0 L 10 5 L 0 10 z" class="arrowheadPath" style="stroke-width: 1; stroke-dasharray: 1, 0;"></path></marker></defs></g><g class="edgePath" style="opacity: 1;"><path class="path" d="M882,254.54469854469855L721,284L665,309.53367875647666" marker-end="url(#arrowhead86)" style="fill:none"></path><defs><marker id="arrowhead86" viewBox="0 0 10 10" refX="9" refY="5" markerUnits="strokeWidth" markerWidth="8" markerHeight="6" orient="auto"><path d="M 0 0 L 0 0 L 0 0 z" style="fill: #333"></path></marker></defs></g><g class="edgePath" style="opacity: 1;"><path class="path" d="M624.5,347L624.5,372L624.5,397" marker-end="url(#arrowhead87)" style="fill:none"></path><defs><marker id="arrowhead87" viewBox="0 0 10 10" refX="9" refY="5" markerUnits="strokeWidth" markerWidth="8" markerHeight="6" orient="auto"><path d="M 0 0 L 10 5 L 0 10 z" class="arrowheadPath" style="stroke-width: 1; stroke-dasharray: 1, 0;"></path></marker></defs></g><g class="edgePath" style="opacity: 1;"><path class="path" d="M557,430.0591715976331L413.25,460L413.25,485L413.25,510" marker-end="url(#arrowhead88)" style="fill:none"></path><defs><marker id="arrowhead88" viewBox="0 0 10 10" refX="9" refY="5" markerUnits="strokeWidth" markerWidth="8" markerHeight="6" orient="auto"><path d="M 0 0 L 10 5 L 0 10 z" class="arrowheadPath" style="stroke-width: 1; stroke-dasharray: 1, 0;"></path></marker></defs></g><g class="edgePath" style="opacity: 1;"><path class="path" d="M413.25,548L413.25,573L413.25,598" marker-end="url(#arrowhead89)" style="fill:none"></path><defs><marker id="arrowhead89" viewBox="0 0 10 10" refX="9" refY="5" markerUnits="strokeWidth" markerWidth="8" markerHeight="6" orient="auto"><path d="M 0 0 L 10 5 L 0 10 z" class="arrowheadPath" style="stroke-width: 1; stroke-dasharray: 1, 0;"></path></marker></defs></g><g class="edgePath" style="opacity: 1;"><path class="path" d="M605.3920454545455,435L580.25,460L580.25,485L580.25,510" marker-end="url(#arrowhead90)" style="fill:none"></path><defs><marker id="arrowhead90" viewBox="0 0 10 10" refX="9" refY="5" markerUnits="strokeWidth" markerWidth="8" markerHeight="6" orient="auto"><path d="M 0 0 L 10 5 L 0 10 z" class="arrowheadPath" style="stroke-width: 1; stroke-dasharray: 1, 0;"></path></marker></defs></g><g class="edgePath" style="opacity: 1;"><path class="path" d="M580.25,548L580.25,573L580.25,598" marker-end="url(#arrowhead91)" style="fill:none"></path><defs><marker id="arrowhead91" viewBox="0 0 10 10" refX="9" refY="5" markerUnits="strokeWidth" markerWidth="8" markerHeight="6" orient="auto"><path d="M 0 0 L 10 5 L 0 10 z" class="arrowheadPath" style="stroke-width: 1; stroke-dasharray: 1, 0;"></path></marker></defs></g><g class="edgePath" style="opacity: 1;"><path class="path" d="M666.1704545454545,435L721,460L721,485L908,519.2120582120582" marker-end="url(#arrowhead92)" style="fill:none"></path><defs><marker id="arrowhead92" viewBox="0 0 10 10" refX="9" refY="5" markerUnits="strokeWidth" markerWidth="8" markerHeight="6" orient="auto"><path d="M 0 0 L 10 5 L 0 10 z" class="arrowheadPath" style="stroke-width: 1; stroke-dasharray: 1, 0;"></path></marker></defs></g><g class="edgePath" style="opacity: 1;"><path class="path" d="M961.5,259L961.5,284L961.5,328L961.5,372L961.5,416L961.5,460L961.5,485L961.5,510" marker-end="url(#arrowhead93)" style="fill:none"></path><defs><marker id="arrowhead93" viewBox="0 0 10 10" refX="9" refY="5" markerUnits="strokeWidth" markerWidth="8" markerHeight="6" orient="auto"><path d="M 0 0 L 10 5 L 0 10 z" class="arrowheadPath" style="stroke-width: 1; stroke-dasharray: 1, 0;"></path></marker></defs></g><g class="edgePath" style="opacity: 1;"><path class="path" d="M908,541.5046480743692L773.25,573L773.25,598" marker-end="url(#arrowhead94)" style="fill:none"></path><defs><marker id="arrowhead94" viewBox="0 0 10 10" refX="9" refY="5" markerUnits="strokeWidth" markerWidth="8" markerHeight="6" orient="auto"><path d="M 0 0 L 10 5 L 0 10 z" class="arrowheadPath" style="stroke-width: 1; stroke-dasharray: 1, 0;"></path></marker></defs></g><g class="edgePath" style="opacity: 1;"><path class="path" d="M981.0397727272727,548L1006.75,573L1006.75,598" marker-end="url(#arrowhead95)" style="fill:none"></path><defs><marker id="arrowhead95" viewBox="0 0 10 10" refX="9" refY="5" markerUnits="strokeWidth" markerWidth="8" markerHeight="6" orient="auto"><path d="M 0 0 L 10 5 L 0 10 z" class="arrowheadPath" style="stroke-width: 1; stroke-dasharray: 1, 0;"></path></marker></defs></g><g class="edgePath" style="opacity: 1;"><path class="path" d="M184,83L184,108L184,133" marker-end="url(#arrowhead96)" style="fill:none"></path><defs><marker id="arrowhead96" viewBox="0 0 10 10" refX="9" refY="5" markerUnits="strokeWidth" markerWidth="8" markerHeight="6" orient="auto"><path d="M 0 0 L 0 0 L 0 0 z" style="fill: #333"></path></marker></defs></g><g class="edgePath" style="opacity: 1;"><path class="path" d="M184,171L184,196L184,221" marker-end="url(#arrowhead97)" style="fill:none"></path><defs><marker id="arrowhead97" viewBox="0 0 10 10" refX="9" refY="5" markerUnits="strokeWidth" markerWidth="8" markerHeight="6" orient="auto"><path d="M 0 0 L 0 0 L 0 0 z" style="fill: #333"></path></marker></defs></g><g class="edgePath" style="opacity: 1;"><path class="path" d="M184,259L184,284L184,309" marker-end="url(#arrowhead98)" style="fill:none"></path><defs><marker id="arrowhead98" viewBox="0 0 10 10" refX="9" refY="5" markerUnits="strokeWidth" markerWidth="8" markerHeight="6" orient="auto"><path d="M 0 0 L 0 0 L 0 0 z" style="fill: #333"></path></marker></defs></g><g class="edgePath" style="opacity: 1;"><path class="path" d="M184,347L184,372L184,397" marker-end="url(#arrowhead99)" style="fill:none"></path><defs><marker id="arrowhead99" viewBox="0 0 10 10" refX="9" refY="5" markerUnits="strokeWidth" markerWidth="8" markerHeight="6" orient="auto"><path d="M 0 0 L 0 0 L 0 0 z" style="fill: #333"></path></marker></defs></g></g><g class="edgeLabels"><g class="edgeLabel" transform="" style="opacity: 1;"><g transform="translate(0,0)" class="label"><foreignObject width="0" height="0"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;"><span class="edgeLabel"></span></div></foreignObject></g></g><g class="edgeLabel" transform="" style="opacity: 1;"><g transform="translate(0,0)" class="label"><foreignObject width="0" height="0"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;"><span class="edgeLabel"></span></div></foreignObject></g></g><g class="edgeLabel" transform="" style="opacity: 1;"><g transform="translate(0,0)" class="label"><foreignObject width="0" height="0"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;"><span class="edgeLabel"></span></div></foreignObject></g></g><g class="edgeLabel" transform="" style="opacity: 1;"><g transform="translate(0,0)" class="label"><foreignObject width="0" height="0"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;"><span class="edgeLabel"></span></div></foreignObject></g></g><g class="edgeLabel" transform="" style="opacity: 1;"><g transform="translate(0,0)" class="label"><foreignObject width="0" height="0"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;"><span class="edgeLabel"></span></div></foreignObject></g></g><g class="edgeLabel" transform="" style="opacity: 1;"><g transform="translate(0,0)" class="label"><foreignObject width="0" height="0"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;"><span class="edgeLabel"></span></div></foreignObject></g></g><g class="edgeLabel" transform="" style="opacity: 1;"><g transform="translate(0,0)" class="label"><foreignObject width="0" height="0"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;"><span class="edgeLabel"></span></div></foreignObject></g></g><g class="edgeLabel" transform="" style="opacity: 1;"><g transform="translate(0,0)" class="label"><foreignObject width="0" height="0"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;"><span class="edgeLabel"></span></div></foreignObject></g></g><g class="edgeLabel" transform="" style="opacity: 1;"><g transform="translate(0,0)" class="label"><foreignObject width="0" height="0"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;"><span class="edgeLabel"></span></div></foreignObject></g></g><g class="edgeLabel" transform="" style="opacity: 1;"><g transform="translate(0,0)" class="label"><foreignObject width="0" height="0"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;"><span class="edgeLabel"></span></div></foreignObject></g></g><g class="edgeLabel" transform="" style="opacity: 1;"><g transform="translate(0,0)" class="label"><foreignObject width="0" height="0"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;"><span class="edgeLabel"></span></div></foreignObject></g></g><g class="edgeLabel" transform="" style="opacity: 1;"><g transform="translate(0,0)" class="label"><foreignObject width="0" height="0"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;"><span class="edgeLabel"></span></div></foreignObject></g></g><g class="edgeLabel" transform="" style="opacity: 1;"><g transform="translate(0,0)" class="label"><foreignObject width="0" height="0"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;"><span class="edgeLabel"></span></div></foreignObject></g></g><g class="edgeLabel" transform="" style="opacity: 1;"><g transform="translate(0,0)" class="label"><foreignObject width="0" height="0"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;"><span class="edgeLabel"></span></div></foreignObject></g></g><g class="edgeLabel" transform="" style="opacity: 1;"><g transform="translate(0,0)" class="label"><foreignObject width="0" height="0"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;"><span class="edgeLabel"></span></div></foreignObject></g></g><g class="edgeLabel" transform="" style="opacity: 1;"><g transform="translate(0,0)" class="label"><foreignObject width="0" height="0"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;"><span class="edgeLabel"></span></div></foreignObject></g></g><g class="edgeLabel" transform="" style="opacity: 1;"><g transform="translate(0,0)" class="label"><foreignObject width="0" height="0"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;"><span class="edgeLabel"></span></div></foreignObject></g></g><g class="edgeLabel" transform="" style="opacity: 1;"><g transform="translate(0,0)" class="label"><foreignObject width="0" height="0"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;"><span class="edgeLabel"></span></div></foreignObject></g></g><g class="edgeLabel" transform="" style="opacity: 1;"><g transform="translate(0,0)" class="label"><foreignObject width="0" height="0"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;"><span class="edgeLabel"></span></div></foreignObject></g></g></g><g class="nodes"><g class="node green" id="d" transform="translate(184,64)" style="opacity: 1;"><rect rx="5" ry="5" x="-94.5" y="-19" width="189" height="38"></rect><g class="label" transform="translate(0,0)"><g transform="translate(-84.5,-9)"><foreignObject width="169" height="18"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;">Dataflow jobs are green</div></foreignObject></g></g></g><g class="node magenta" id="k" transform="translate(184,152)" style="opacity: 1;"><rect rx="5" ry="5" x="-129" y="-19" width="258" height="38"></rect><g class="label" transform="translate(0,0)"><g transform="translate(-119,-9)"><foreignObject width="238" height="18"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;">Kubernetes services are magenta</div></foreignObject></g></g></g><g class="node orange" id="f" transform="translate(184,240)" style="opacity: 1;"><rect rx="5" ry="5" x="-87" y="-19" width="174" height="38"></rect><g class="label" transform="translate(0,0)"><g transform="translate(-77,-9)"><foreignObject width="154" height="18"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;">Producers are orange</div></foreignObject></g></g></g><g class="node cyan" id="p" transform="translate(184,328)" style="opacity: 1;"><rect rx="5" ry="5" x="-94.5" y="-19" width="189" height="38"></rect><g class="label" transform="translate(0,0)"><g transform="translate(-84.5,-9)"><foreignObject width="169" height="18"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;">PubSub topics are cyan</div></foreignObject></g></g></g><g class="node" id="g" transform="translate(184,416)" style="opacity: 1;"><rect rx="5" ry="5" x="-129" y="-19" width="258" height="38"></rect><g class="label" transform="translate(0,0)"><g transform="translate(-119,-9)"><foreignObject width="238" height="18"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;">Google Cloud services are purple</div></foreignObject></g></g></g><g class="node orange" id="f1" transform="translate(446.5,64)" style="opacity: 1;"><rect rx="5" ry="5" x="-46.5" y="-19" width="93" height="38"></rect><g class="label" transform="translate(0,0)"><g transform="translate(-36.5,-9)"><foreignObject width="73" height="18"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;">Producers</div></foreignObject></g></g></g><g class="node magenta" id="k1" transform="translate(446.5,152)" style="opacity: 1;"><rect rx="5" ry="5" x="-63.5" y="-19" width="127" height="38"></rect><g class="label" transform="translate(0,0)"><g transform="translate(-53.5,-9)"><foreignObject width="107" height="18"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;">Ingestion Edge</div></foreignObject></g></g></g><g class="node cyan" id="p1" transform="translate(446.5,240)" style="opacity: 1;"><rect rx="5" ry="5" x="-51.5" y="-19" width="103" height="38"></rect><g class="label" transform="translate(0,0)"><g transform="translate(-41.5,-9)"><foreignObject width="83" height="18"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;">Raw Topics</div></foreignObject></g></g></g><g class="node green" id="d1" transform="translate(445.5,328)" style="opacity: 1;"><rect rx="5" ry="5" x="-54" y="-19" width="108" height="38"></rect><g class="label" transform="translate(0,0)"><g transform="translate(-44,-9)"><foreignObject width="88" height="18"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;">Landfill Sink</div></foreignObject></g></g></g><g class="node" id="c1" transform="translate(445.5,416)" style="opacity: 1;"><rect rx="5" ry="5" x="-61.5" y="-19" width="123" height="38"></rect><g class="label" transform="translate(0,0)"><g transform="translate(-51.5,-9)"><foreignObject width="103" height="18"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;">Cloud Storage</div></foreignObject></g></g></g><g class="node green" id="d2" transform="translate(624.5,328)" style="opacity: 1;"><rect rx="5" ry="5" x="-40.5" y="-19" width="81" height="38"></rect><g class="label" transform="translate(0,0)"><g transform="translate(-30.5,-9)"><foreignObject width="61" height="18"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;">Decoder</div></foreignObject></g></g></g><g class="node" id="m1" transform="translate(961.5,240)" style="opacity: 1;"><rect rx="5" ry="5" x="-79.5" y="-19" width="159" height="38"></rect><g class="label" transform="translate(0,0)"><g transform="translate(-69.5,-9)"><foreignObject width="139" height="18"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;">Cloud Memorystore</div></foreignObject></g></g></g><g class="node cyan" id="p2" transform="translate(624.5,416)" style="opacity: 1;"><rect rx="5" ry="5" x="-67.5" y="-19" width="135" height="38"></rect><g class="label" transform="translate(0,0)"><g transform="translate(-57.5,-9)"><foreignObject width="115" height="18"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;">Decoded Topics</div></foreignObject></g></g></g><g class="node green" id="d3" transform="translate(413.25,529)" style="opacity: 1;"><rect rx="5" ry="5" x="-61.5" y="-19" width="123" height="38"></rect><g class="label" transform="translate(0,0)"><g transform="translate(-51.5,-9)"><foreignObject width="103" height="18"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;">BigQuery Sink</div></foreignObject></g></g></g><g class="node" id="b1" transform="translate(413.25,617)" style="opacity: 1;"><rect rx="5" ry="5" x="-43.5" y="-19" width="87" height="38"></rect><g class="label" transform="translate(0,0)"><g transform="translate(-33.5,-9)"><foreignObject width="67" height="18"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;">BigQuery</div></foreignObject></g></g></g><g class="node green" id="d4" transform="translate(580.25,529)" style="opacity: 1;"><rect rx="5" ry="5" x="-55.5" y="-19" width="111" height="38"></rect><g class="label" transform="translate(0,0)"><g transform="translate(-45.5,-9)"><foreignObject width="91" height="18"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;">Dataset Sink</div></foreignObject></g></g></g><g class="node" id="c2" transform="translate(580.25,617)" style="opacity: 1;"><rect rx="5" ry="5" x="-61.5" y="-19" width="123" height="38"></rect><g class="label" transform="translate(0,0)"><g transform="translate(-51.5,-9)"><foreignObject width="103" height="18"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;">Cloud Storage</div></foreignObject></g></g></g><g class="node green" id="d5" transform="translate(961.5,529)" style="opacity: 1;"><rect rx="5" ry="5" x="-53.5" y="-19" width="107" height="38"></rect><g class="label" transform="translate(0,0)"><g transform="translate(-43.5,-9)"><foreignObject width="87" height="18"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;">Republisher</div></foreignObject></g></g></g><g class="node cyan" id="p5" transform="translate(773.25,617)" style="opacity: 1;"><rect rx="5" ry="5" x="-81.5" y="-19" width="163" height="38"></rect><g class="label" transform="translate(0,0)"><g transform="translate(-71.5,-9)"><foreignObject width="143" height="18"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;">Per DocType Topics</div></foreignObject></g></g></g><g class="node cyan" id="p6" transform="translate(1006.75,617)" style="opacity: 1;"><rect rx="5" ry="5" x="-102" y="-19" width="204" height="38"></rect><g class="label" transform="translate(0,0)"><g transform="translate(-92,-9)"><foreignObject width="184" height="18"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;">Monitoring Sample Topics</div></foreignObject></g></g></g></g></g></g></svg>
\ No newline at end of file
+  }</style><g transform="translate(-12, -12)"><g class="output"><g class="clusters"><g class="cluster" id="subGraph0" transform="translate(184,240)" style="opacity: 1;"><rect width="328" height="440" x="-164" y="-220"></rect><g class="label"><g transform="translate(0,0)"><foreignObject width="0" height="0"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;"></div></foreignObject></g></g><text x="0" y="-206" fill="black" stroke="none" id="mermaid-1556319862364Text" style="text-anchor: middle;">Colors</text></g></g><g class="edgePaths"><g class="edgePath" style="opacity: 1;"><path class="path" d="M446.5,83L446.5,108L446.5,133" marker-end="url(#arrowhead81)" style="fill:none"></path><defs><marker id="arrowhead81" viewBox="0 0 10 10" refX="9" refY="5" markerUnits="strokeWidth" markerWidth="8" markerHeight="6" orient="auto"><path d="M 0 0 L 10 5 L 0 10 z" class="arrowheadPath" style="stroke-width: 1; stroke-dasharray: 1, 0;"></path></marker></defs></g><g class="edgePath" style="opacity: 1;"><path class="path" d="M446.5,171L446.5,196L446.5,221" marker-end="url(#arrowhead82)" style="fill:none"></path><defs><marker id="arrowhead82" viewBox="0 0 10 10" refX="9" refY="5" markerUnits="strokeWidth" markerWidth="8" markerHeight="6" orient="auto"><path d="M 0 0 L 10 5 L 0 10 z" class="arrowheadPath" style="stroke-width: 1; stroke-dasharray: 1, 0;"></path></marker></defs></g><g class="edgePath" style="opacity: 1;"><path class="path" d="M446.0681818181818,259L445.5,284L445.5,309" marker-end="url(#arrowhead83)" style="fill:none"></path><defs><marker id="arrowhead83" viewBox="0 0 10 10" refX="9" refY="5" markerUnits="strokeWidth" markerWidth="8" markerHeight="6" orient="auto"><path d="M 0 0 L 10 5 L 0 10 z" class="arrowheadPath" style="stroke-width: 1; stroke-dasharray: 1, 0;"></path></marker></defs></g><g class="edgePath" style="opacity: 1;"><path class="path" d="M445.5,347L445.5,372L445.5,397" marker-end="url(#arrowhead84)" style="fill:none"></path><defs><marker id="arrowhead84" viewBox="0 0 10 10" refX="9" refY="5" markerUnits="strokeWidth" markerWidth="8" markerHeight="6" orient="auto"><path d="M 0 0 L 10 5 L 0 10 z" class="arrowheadPath" style="stroke-width: 1; stroke-dasharray: 1, 0;"></path></marker></defs></g><g class="edgePath" style="opacity: 1;"><path class="path" d="M489.0340909090909,259L545,284L590.1704545454545,309" marker-end="url(#arrowhead85)" style="fill:none"></path><defs><marker id="arrowhead85" viewBox="0 0 10 10" refX="9" refY="5" markerUnits="strokeWidth" markerWidth="8" markerHeight="6" orient="auto"><path d="M 0 0 L 10 5 L 0 10 z" class="arrowheadPath" style="stroke-width: 1; stroke-dasharray: 1, 0;"></path></marker></defs></g><g class="edgePath" style="opacity: 1;"><path class="path" d="M882,254.54469854469855L721,284L665,309.53367875647666" marker-end="url(#arrowhead86)" style="fill:none"></path><defs><marker id="arrowhead86" viewBox="0 0 10 10" refX="9" refY="5" markerUnits="strokeWidth" markerWidth="8" markerHeight="6" orient="auto"><path d="M 0 0 L 0 0 L 0 0 z" style="fill: #333"></path></marker></defs></g><g class="edgePath" style="opacity: 1;"><path class="path" d="M624.5,347L624.5,372L624.5,397" marker-end="url(#arrowhead87)" style="fill:none"></path><defs><marker id="arrowhead87" viewBox="0 0 10 10" refX="9" refY="5" markerUnits="strokeWidth" markerWidth="8" markerHeight="6" orient="auto"><path d="M 0 0 L 10 5 L 0 10 z" class="arrowheadPath" style="stroke-width: 1; stroke-dasharray: 1, 0;"></path></marker></defs></g><g class="edgePath" style="opacity: 1;"><path class="path" d="M557,430.0591715976331L413.25,460L413.25,485L413.25,510" marker-end="url(#arrowhead88)" style="fill:none"></path><defs><marker id="arrowhead88" viewBox="0 0 10 10" refX="9" refY="5" markerUnits="strokeWidth" markerWidth="8" markerHeight="6" orient="auto"><path d="M 0 0 L 10 5 L 0 10 z" class="arrowheadPath" style="stroke-width: 1; stroke-dasharray: 1, 0;"></path></marker></defs></g><g class="edgePath" style="opacity: 1;"><path class="path" d="M413.25,548L413.25,573L413.25,598" marker-end="url(#arrowhead89)" style="fill:none"></path><defs><marker id="arrowhead89" viewBox="0 0 10 10" refX="9" refY="5" markerUnits="strokeWidth" markerWidth="8" markerHeight="6" orient="auto"><path d="M 0 0 L 10 5 L 0 10 z" class="arrowheadPath" style="stroke-width: 1; stroke-dasharray: 1, 0;"></path></marker></defs></g><g class="edgePath" style="opacity: 1;"><path class="path" d="M605.3920454545455,435L580.25,460L580.25,485L580.25,510" marker-end="url(#arrowhead90)" style="fill:none"></path><defs><marker id="arrowhead90" viewBox="0 0 10 10" refX="9" refY="5" markerUnits="strokeWidth" markerWidth="8" markerHeight="6" orient="auto"><path d="M 0 0 L 10 5 L 0 10 z" class="arrowheadPath" style="stroke-width: 1; stroke-dasharray: 1, 0;"></path></marker></defs></g><g class="edgePath" style="opacity: 1;"><path class="path" d="M580.25,548L580.25,573L580.25,598" marker-end="url(#arrowhead91)" style="fill:none"></path><defs><marker id="arrowhead91" viewBox="0 0 10 10" refX="9" refY="5" markerUnits="strokeWidth" markerWidth="8" markerHeight="6" orient="auto"><path d="M 0 0 L 10 5 L 0 10 z" class="arrowheadPath" style="stroke-width: 1; stroke-dasharray: 1, 0;"></path></marker></defs></g><g class="edgePath" style="opacity: 1;"><path class="path" d="M666.1704545454545,435L721,460L721,485L908,519.2120582120582" marker-end="url(#arrowhead92)" style="fill:none"></path><defs><marker id="arrowhead92" viewBox="0 0 10 10" refX="9" refY="5" markerUnits="strokeWidth" markerWidth="8" markerHeight="6" orient="auto"><path d="M 0 0 L 10 5 L 0 10 z" class="arrowheadPath" style="stroke-width: 1; stroke-dasharray: 1, 0;"></path></marker></defs></g><g class="edgePath" style="opacity: 1;"><path class="path" d="M961.5,259L961.5,284L961.5,328L961.5,372L961.5,416L961.5,460L961.5,485L961.5,510" marker-end="url(#arrowhead93)" style="fill:none"></path><defs><marker id="arrowhead93" viewBox="0 0 10 10" refX="9" refY="5" markerUnits="strokeWidth" markerWidth="8" markerHeight="6" orient="auto"><path d="M 0 0 L 10 5 L 0 10 z" class="arrowheadPath" style="stroke-width: 1; stroke-dasharray: 1, 0;"></path></marker></defs></g><g class="edgePath" style="opacity: 1;"><path class="path" d="M908,541.5046480743692L773.25,573L773.25,598" marker-end="url(#arrowhead94)" style="fill:none"></path><defs><marker id="arrowhead94" viewBox="0 0 10 10" refX="9" refY="5" markerUnits="strokeWidth" markerWidth="8" markerHeight="6" orient="auto"><path d="M 0 0 L 10 5 L 0 10 z" class="arrowheadPath" style="stroke-width: 1; stroke-dasharray: 1, 0;"></path></marker></defs></g><g class="edgePath" style="opacity: 1;"><path class="path" d="M981.0397727272727,548L1006.75,573L1006.75,598" marker-end="url(#arrowhead95)" style="fill:none"></path><defs><marker id="arrowhead95" viewBox="0 0 10 10" refX="9" refY="5" markerUnits="strokeWidth" markerWidth="8" markerHeight="6" orient="auto"><path d="M 0 0 L 10 5 L 0 10 z" class="arrowheadPath" style="stroke-width: 1; stroke-dasharray: 1, 0;"></path></marker></defs></g><g class="edgePath" style="opacity: 1;"><path class="path" d="M184,83L184,108L184,133" marker-end="url(#arrowhead96)" style="fill:none"></path><defs><marker id="arrowhead96" viewBox="0 0 10 10" refX="9" refY="5" markerUnits="strokeWidth" markerWidth="8" markerHeight="6" orient="auto"><path d="M 0 0 L 0 0 L 0 0 z" style="fill: #333"></path></marker></defs></g><g class="edgePath" style="opacity: 1;"><path class="path" d="M184,171L184,196L184,221" marker-end="url(#arrowhead97)" style="fill:none"></path><defs><marker id="arrowhead97" viewBox="0 0 10 10" refX="9" refY="5" markerUnits="strokeWidth" markerWidth="8" markerHeight="6" orient="auto"><path d="M 0 0 L 0 0 L 0 0 z" style="fill: #333"></path></marker></defs></g><g class="edgePath" style="opacity: 1;"><path class="path" d="M184,259L184,284L184,309" marker-end="url(#arrowhead98)" style="fill:none"></path><defs><marker id="arrowhead98" viewBox="0 0 10 10" refX="9" refY="5" markerUnits="strokeWidth" markerWidth="8" markerHeight="6" orient="auto"><path d="M 0 0 L 0 0 L 0 0 z" style="fill: #333"></path></marker></defs></g><g class="edgePath" style="opacity: 1;"><path class="path" d="M184,347L184,372L184,397" marker-end="url(#arrowhead99)" style="fill:none"></path><defs><marker id="arrowhead99" viewBox="0 0 10 10" refX="9" refY="5" markerUnits="strokeWidth" markerWidth="8" markerHeight="6" orient="auto"><path d="M 0 0 L 0 0 L 0 0 z" style="fill: #333"></path></marker></defs></g></g><g class="edgeLabels"><g class="edgeLabel" transform="" style="opacity: 1;"><g transform="translate(0,0)" class="label"><foreignObject width="0" height="0"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;"><span class="edgeLabel"></span></div></foreignObject></g></g><g class="edgeLabel" transform="" style="opacity: 1;"><g transform="translate(0,0)" class="label"><foreignObject width="0" height="0"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;"><span class="edgeLabel"></span></div></foreignObject></g></g><g class="edgeLabel" transform="" style="opacity: 1;"><g transform="translate(0,0)" class="label"><foreignObject width="0" height="0"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;"><span class="edgeLabel"></span></div></foreignObject></g></g><g class="edgeLabel" transform="" style="opacity: 1;"><g transform="translate(0,0)" class="label"><foreignObject width="0" height="0"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;"><span class="edgeLabel"></span></div></foreignObject></g></g><g class="edgeLabel" transform="" style="opacity: 1;"><g transform="translate(0,0)" class="label"><foreignObject width="0" height="0"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;"><span class="edgeLabel"></span></div></foreignObject></g></g><g class="edgeLabel" transform="" style="opacity: 1;"><g transform="translate(0,0)" class="label"><foreignObject width="0" height="0"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;"><span class="edgeLabel"></span></div></foreignObject></g></g><g class="edgeLabel" transform="" style="opacity: 1;"><g transform="translate(0,0)" class="label"><foreignObject width="0" height="0"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;"><span class="edgeLabel"></span></div></foreignObject></g></g><g class="edgeLabel" transform="" style="opacity: 1;"><g transform="translate(0,0)" class="label"><foreignObject width="0" height="0"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;"><span class="edgeLabel"></span></div></foreignObject></g></g><g class="edgeLabel" transform="" style="opacity: 1;"><g transform="translate(0,0)" class="label"><foreignObject width="0" height="0"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;"><span class="edgeLabel"></span></div></foreignObject></g></g><g class="edgeLabel" transform="" style="opacity: 1;"><g transform="translate(0,0)" class="label"><foreignObject width="0" height="0"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;"><span class="edgeLabel"></span></div></foreignObject></g></g><g class="edgeLabel" transform="" style="opacity: 1;"><g transform="translate(0,0)" class="label"><foreignObject width="0" height="0"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;"><span class="edgeLabel"></span></div></foreignObject></g></g><g class="edgeLabel" transform="" style="opacity: 1;"><g transform="translate(0,0)" class="label"><foreignObject width="0" height="0"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;"><span class="edgeLabel"></span></div></foreignObject></g></g><g class="edgeLabel" transform="" style="opacity: 1;"><g transform="translate(0,0)" class="label"><foreignObject width="0" height="0"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;"><span class="edgeLabel"></span></div></foreignObject></g></g><g class="edgeLabel" transform="" style="opacity: 1;"><g transform="translate(0,0)" class="label"><foreignObject width="0" height="0"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;"><span class="edgeLabel"></span></div></foreignObject></g></g><g class="edgeLabel" transform="" style="opacity: 1;"><g transform="translate(0,0)" class="label"><foreignObject width="0" height="0"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;"><span class="edgeLabel"></span></div></foreignObject></g></g><g class="edgeLabel" transform="" style="opacity: 1;"><g transform="translate(0,0)" class="label"><foreignObject width="0" height="0"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;"><span class="edgeLabel"></span></div></foreignObject></g></g><g class="edgeLabel" transform="" style="opacity: 1;"><g transform="translate(0,0)" class="label"><foreignObject width="0" height="0"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;"><span class="edgeLabel"></span></div></foreignObject></g></g><g class="edgeLabel" transform="" style="opacity: 1;"><g transform="translate(0,0)" class="label"><foreignObject width="0" height="0"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;"><span class="edgeLabel"></span></div></foreignObject></g></g><g class="edgeLabel" transform="" style="opacity: 1;"><g transform="translate(0,0)" class="label"><foreignObject width="0" height="0"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;"><span class="edgeLabel"></span></div></foreignObject></g></g></g><g class="nodes"><g class="node green" id="d" transform="translate(184,64)" style="opacity: 1;"><rect rx="5" ry="5" x="-94.5" y="-19" width="189" height="38"></rect><g class="label" transform="translate(0,0)"><g transform="translate(-84.5,-9)"><foreignObject width="169" height="18"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;">Dataflow jobs are green</div></foreignObject></g></g></g><g class="node magenta" id="k" transform="translate(184,152)" style="opacity: 1;"><rect rx="5" ry="5" x="-129" y="-19" width="258" height="38"></rect><g class="label" transform="translate(0,0)"><g transform="translate(-119,-9)"><foreignObject width="238" height="18"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;">Kubernetes services are magenta</div></foreignObject></g></g></g><g class="node orange" id="f" transform="translate(184,240)" style="opacity: 1;"><rect rx="5" ry="5" x="-87" y="-19" width="174" height="38"></rect><g class="label" transform="translate(0,0)"><g transform="translate(-77,-9)"><foreignObject width="154" height="18"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;">Producers are orange</div></foreignObject></g></g></g><g class="node cyan" id="p" transform="translate(184,328)" style="opacity: 1;"><rect rx="5" ry="5" x="-94.5" y="-19" width="189" height="38"></rect><g class="label" transform="translate(0,0)"><g transform="translate(-84.5,-9)"><foreignObject width="169" height="18"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;">PubSub topics are cyan</div></foreignObject></g></g></g><g class="node" id="g" transform="translate(184,416)" style="opacity: 1;"><rect rx="5" ry="5" x="-129" y="-19" width="258" height="38"></rect><g class="label" transform="translate(0,0)"><g transform="translate(-119,-9)"><foreignObject width="238" height="18"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;">Google Cloud services are purple</div></foreignObject></g></g></g><g class="node orange" id="f1" transform="translate(446.5,64)" style="opacity: 1;"><rect rx="5" ry="5" x="-46.5" y="-19" width="93" height="38"></rect><g class="label" transform="translate(0,0)"><g transform="translate(-36.5,-9)"><foreignObject width="73" height="18"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;">Producers</div></foreignObject></g></g></g><g class="node magenta" id="k1" transform="translate(446.5,152)" style="opacity: 1;"><rect rx="5" ry="5" x="-63.5" y="-19" width="127" height="38"></rect><g class="label" transform="translate(0,0)"><g transform="translate(-53.5,-9)"><foreignObject width="107" height="18"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;">Ingestion Edge</div></foreignObject></g></g></g><g class="node cyan" id="p1" transform="translate(446.5,240)" style="opacity: 1;"><rect rx="5" ry="5" x="-51.5" y="-19" width="103" height="38"></rect><g class="label" transform="translate(0,0)"><g transform="translate(-41.5,-9)"><foreignObject width="83" height="18"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;">Raw Topics</div></foreignObject></g></g></g><g class="node green" id="d1" transform="translate(445.5,328)" style="opacity: 1;"><rect rx="5" ry="5" x="-54" y="-19" width="108" height="38"></rect><g class="label" transform="translate(0,0)"><g transform="translate(-44,-9)"><foreignObject width="88" height="18"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;">Landfill Sink</div></foreignObject></g></g></g><g class="node" id="c1" transform="translate(445.5,416)" style="opacity: 1;"><rect rx="5" ry="5" x="-61.5" y="-19" width="123" height="38"></rect><g class="label" transform="translate(0,0)"><g transform="translate(-51.5,-9)"><foreignObject width="103" height="18"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;">Cloud Storage</div></foreignObject></g></g></g><g class="node green" id="d2" transform="translate(624.5,328)" style="opacity: 1;"><rect rx="5" ry="5" x="-40.5" y="-19" width="81" height="38"></rect><g class="label" transform="translate(0,0)"><g transform="translate(-30.5,-9)"><foreignObject width="61" height="18"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;">Decoder</div></foreignObject></g></g></g><g class="node" id="m1" transform="translate(961.5,240)" style="opacity: 1;"><rect rx="5" ry="5" x="-79.5" y="-19" width="159" height="38"></rect><g class="label" transform="translate(0,0)"><g transform="translate(-69.5,-9)"><foreignObject width="139" height="18"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;">Cloud Memorystore</div></foreignObject></g></g></g><g class="node cyan" id="p2" transform="translate(624.5,416)" style="opacity: 1;"><rect rx="5" ry="5" x="-67.5" y="-19" width="135" height="38"></rect><g class="label" transform="translate(0,0)"><g transform="translate(-57.5,-9)"><foreignObject width="115" height="18"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;">Decoded Topics</div></foreignObject></g></g></g><g class="node green" id="d3" transform="translate(413.25,529)" style="opacity: 1;"><rect rx="5" ry="5" x="-61.5" y="-19" width="123" height="38"></rect><g class="label" transform="translate(0,0)"><g transform="translate(-51.5,-9)"><foreignObject width="103" height="18"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;">BigQuery Sink</div></foreignObject></g></g></g><g class="node" id="b1" transform="translate(413.25,617)" style="opacity: 1;"><rect rx="5" ry="5" x="-43.5" y="-19" width="87" height="38"></rect><g class="label" transform="translate(0,0)"><g transform="translate(-33.5,-9)"><foreignObject width="67" height="18"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;">BigQuery</div></foreignObject></g></g></g><g class="node green" id="d4" transform="translate(580.25,529)" style="opacity: 1;"><rect rx="5" ry="5" x="-55.5" y="-19" width="111" height="38"></rect><g class="label" transform="translate(0,0)"><g transform="translate(-45.5,-9)"><foreignObject width="91" height="18"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;">Dataset Sink</div></foreignObject></g></g></g><g class="node" id="c2" transform="translate(580.25,617)" style="opacity: 1;"><rect rx="5" ry="5" x="-61.5" y="-19" width="123" height="38"></rect><g class="label" transform="translate(0,0)"><g transform="translate(-51.5,-9)"><foreignObject width="103" height="18"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;">Cloud Storage</div></foreignObject></g></g></g><g class="node green" id="d5" transform="translate(961.5,529)" style="opacity: 1;"><rect rx="5" ry="5" x="-53.5" y="-19" width="107" height="38"></rect><g class="label" transform="translate(0,0)"><g transform="translate(-43.5,-9)"><foreignObject width="87" height="18"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;">Republisher</div></foreignObject></g></g></g><g class="node cyan" id="p5" transform="translate(773.25,617)" style="opacity: 1;"><rect rx="5" ry="5" x="-81.5" y="-19" width="163" height="38"></rect><g class="label" transform="translate(0,0)"><g transform="translate(-71.5,-9)"><foreignObject width="143" height="18"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;">Per DocType Topics</div></foreignObject></g></g></g><g class="node cyan" id="p6" transform="translate(1006.75,617)" style="opacity: 1;"><rect rx="5" ry="5" x="-102" y="-19" width="204" height="38"></rect><g class="label" transform="translate(0,0)"><g transform="translate(-92,-9)"><foreignObject width="184" height="18"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;">Monitoring Sample Topics</div></foreignObject></g></g></g></g></g></g></svg>
\ No newline at end of file
diff --git a/docs/diagrams/workflow.mmd b/docs/diagrams/workflow.mmd
new file mode 100644
index 0000000000..1bdb522498
--- /dev/null
+++ b/docs/diagrams/workflow.mmd
@@ -0,0 +1,27 @@
+graph TD
+
+subgraph dataops/sandbox/my-project
+    dataflow
+    bigquery
+    pubsub
+    subscription
+
+    pubsub --> |gcloud pubsub subscriptions create| subscription
+    subscription --> dataflow
+    dataflow --> bigquery
+end
+
+subgraph mozilla-pipeline-schemas
+    mps[repository archive]
+end
+
+subgraph ingestion-beam
+    src[src/]
+    schemas[schemas.tar.gz]
+    bq-schemas[bq-schemas/]
+
+    src --> |mvn compile exec:java| dataflow
+    mps --> |download-schemas| schemas
+    schemas --> |generate-bq-schemas| bq-schemas
+    bq-schemas --> |update-bq-table| bigquery
+end
diff --git a/docs/diagrams/workflow.svg b/docs/diagrams/workflow.svg
new file mode 100644
index 0000000000..eb17819d96
--- /dev/null
+++ b/docs/diagrams/workflow.svg
@@ -0,0 +1,4 @@
+<svg id="mermaid-1556319859651" width="100%" xmlns="http://www.w3.org/2000/svg" style="max-width: 666.5px;" viewBox="0 0 666.5 497"><style>#mermaid-1556319859651 .label{font-family:trebuchet ms,verdana,arial;color:#333}#mermaid-1556319859651 .node circle,#mermaid-1556319859651 .node ellipse,#mermaid-1556319859651 .node polygon,#mermaid-1556319859651 .node rect{fill:#ececff;stroke:#9370db;stroke-width:1px}#mermaid-1556319859651 .node.clickable{cursor:pointer}#mermaid-1556319859651 .arrowheadPath{fill:#333}#mermaid-1556319859651 .edgePath .path{stroke:#333;stroke-width:1.5px}#mermaid-1556319859651 .edgeLabel{background-color:#e8e8e8}#mermaid-1556319859651 .cluster rect{fill:#ffffde!important;stroke:#aa3!important;stroke-width:1px!important}#mermaid-1556319859651 .cluster text{fill:#333}#mermaid-1556319859651 div.mermaidTooltip{position:absolute;text-align:center;max-width:200px;padding:2px;font-family:trebuchet ms,verdana,arial;font-size:12px;background:#ffffde;border:1px solid #aa3;border-radius:2px;pointer-events:none;z-index:100}#mermaid-1556319859651 .actor{stroke:#ccf;fill:#ececff}#mermaid-1556319859651 text.actor{fill:#000;stroke:none}#mermaid-1556319859651 .actor-line{stroke:grey}#mermaid-1556319859651 .messageLine0{marker-end:"url(#arrowhead)"}#mermaid-1556319859651 .messageLine0,#mermaid-1556319859651 .messageLine1{stroke-width:1.5;stroke-dasharray:"2 2";stroke:#333}#mermaid-1556319859651 #arrowhead{fill:#333}#mermaid-1556319859651 #crosshead path{fill:#333!important;stroke:#333!important}#mermaid-1556319859651 .messageText{fill:#333;stroke:none}#mermaid-1556319859651 .labelBox{stroke:#ccf;fill:#ececff}#mermaid-1556319859651 .labelText,#mermaid-1556319859651 .loopText{fill:#000;stroke:none}#mermaid-1556319859651 .loopLine{stroke-width:2;stroke-dasharray:"2 2";marker-end:"url(#arrowhead)";stroke:#ccf}#mermaid-1556319859651 .note{stroke:#aa3;fill:#fff5ad}#mermaid-1556319859651 .noteText{fill:#000;stroke:none;font-family:trebuchet ms,verdana,arial;font-size:14px}#mermaid-1556319859651 .section{stroke:none;opacity:.2}#mermaid-1556319859651 .section0{fill:rgba(102,102,255,.49)}#mermaid-1556319859651 .section2{fill:#fff400}#mermaid-1556319859651 .section1,#mermaid-1556319859651 .section3{fill:#fff;opacity:.2}#mermaid-1556319859651 .sectionTitle0,#mermaid-1556319859651 .sectionTitle1,#mermaid-1556319859651 .sectionTitle2,#mermaid-1556319859651 .sectionTitle3{fill:#333}#mermaid-1556319859651 .sectionTitle{text-anchor:start;font-size:11px;text-height:14px}#mermaid-1556319859651 .grid .tick{stroke:#d3d3d3;opacity:.3;shape-rendering:crispEdges}#mermaid-1556319859651 .grid path{stroke-width:0}#mermaid-1556319859651 .today{fill:none;stroke:red;stroke-width:2px}#mermaid-1556319859651 .task{stroke-width:2}#mermaid-1556319859651 .taskText{text-anchor:middle;font-size:11px}#mermaid-1556319859651 .taskTextOutsideRight{fill:#000;text-anchor:start;font-size:11px}#mermaid-1556319859651 .taskTextOutsideLeft{fill:#000;text-anchor:end;font-size:11px}#mermaid-1556319859651 .taskText0,#mermaid-1556319859651 .taskText1,#mermaid-1556319859651 .taskText2,#mermaid-1556319859651 .taskText3{fill:#fff}#mermaid-1556319859651 .task0,#mermaid-1556319859651 .task1,#mermaid-1556319859651 .task2,#mermaid-1556319859651 .task3{fill:#8a90dd;stroke:#534fbc}#mermaid-1556319859651 .taskTextOutside0,#mermaid-1556319859651 .taskTextOutside1,#mermaid-1556319859651 .taskTextOutside2,#mermaid-1556319859651 .taskTextOutside3{fill:#000}#mermaid-1556319859651 .active0,#mermaid-1556319859651 .active1,#mermaid-1556319859651 .active2,#mermaid-1556319859651 .active3{fill:#bfc7ff;stroke:#534fbc}#mermaid-1556319859651 .activeText0,#mermaid-1556319859651 .activeText1,#mermaid-1556319859651 .activeText2,#mermaid-1556319859651 .activeText3{fill:#000!important}#mermaid-1556319859651 .done0,#mermaid-1556319859651 .done1,#mermaid-1556319859651 .done2,#mermaid-1556319859651 .done3{stroke:grey;fill:#d3d3d3;stroke-width:2}#mermaid-1556319859651 .doneText0,#mermaid-1556319859651 .doneText1,#mermaid-1556319859651 .doneText2,#mermaid-1556319859651 .doneText3{fill:#000!important}#mermaid-1556319859651 .crit0,#mermaid-1556319859651 .crit1,#mermaid-1556319859651 .crit2,#mermaid-1556319859651 .crit3{stroke:#f88;fill:red;stroke-width:2}#mermaid-1556319859651 .activeCrit0,#mermaid-1556319859651 .activeCrit1,#mermaid-1556319859651 .activeCrit2,#mermaid-1556319859651 .activeCrit3{stroke:#f88;fill:#bfc7ff;stroke-width:2}#mermaid-1556319859651 .doneCrit0,#mermaid-1556319859651 .doneCrit1,#mermaid-1556319859651 .doneCrit2,#mermaid-1556319859651 .doneCrit3{stroke:#f88;fill:#d3d3d3;stroke-width:2;cursor:pointer;shape-rendering:crispEdges}#mermaid-1556319859651 .activeCritText0,#mermaid-1556319859651 .activeCritText1,#mermaid-1556319859651 .activeCritText2,#mermaid-1556319859651 .activeCritText3,#mermaid-1556319859651 .doneCritText0,#mermaid-1556319859651 .doneCritText1,#mermaid-1556319859651 .doneCritText2,#mermaid-1556319859651 .doneCritText3{fill:#000!important}#mermaid-1556319859651 .titleText{text-anchor:middle;font-size:18px;fill:#000}#mermaid-1556319859651 g.classGroup text{fill:#9370db;stroke:none;font-family:trebuchet ms,verdana,arial;font-size:10px}#mermaid-1556319859651 g.classGroup rect{fill:#ececff;stroke:#9370db}#mermaid-1556319859651 g.classGroup line{stroke:#9370db;stroke-width:1}#mermaid-1556319859651 .classLabel .box{stroke:none;stroke-width:0;fill:#ececff;opacity:.5}#mermaid-1556319859651 .classLabel .label{fill:#9370db;font-size:10px}#mermaid-1556319859651 .relation{stroke:#9370db;stroke-width:1;fill:none}#mermaid-1556319859651 #compositionEnd,#mermaid-1556319859651 #compositionStart{fill:#9370db;stroke:#9370db;stroke-width:1}#mermaid-1556319859651 #aggregationEnd,#mermaid-1556319859651 #aggregationStart{fill:#ececff;stroke:#9370db;stroke-width:1}#mermaid-1556319859651 #dependencyEnd,#mermaid-1556319859651 #dependencyStart,#mermaid-1556319859651 #extensionEnd,#mermaid-1556319859651 #extensionStart{fill:#9370db;stroke:#9370db;stroke-width:1}#mermaid-1556319859651 .branch-label,#mermaid-1556319859651 .commit-id,#mermaid-1556319859651 .commit-msg{fill:#d3d3d3;color:#d3d3d3}</style><style>#mermaid-1556319859651 {
+    color: rgb(0, 0, 0);
+    font: normal normal 400 normal 16px / normal "Times New Roman";
+  }</style><g transform="translate(-12, -12)"><g class="output"><g class="clusters"><g class="cluster" id="subGraph2" transform="translate(486,273)" style="opacity: 1;"><rect width="349" height="194" x="-174.5" y="-97"></rect><g class="label"><g transform="translate(0,0)"><foreignObject width="0" height="0"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;"></div></foreignObject></g></g><text x="0" y="-83" fill="black" stroke="none" id="mermaid-1556319859651Text" style="text-anchor: middle;">ingestion-beam</text></g><g class="cluster" id="subGraph1" transform="translate(562.5,64)" style="opacity: 1;"><rect width="216" height="88" x="-108" y="-44"></rect><g class="label"><g transform="translate(0,0)"><foreignObject width="0" height="0"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;"></div></foreignObject></g></g><text x="0" y="-30" fill="black" stroke="none" id="mermaid-1556319859651Text" style="text-anchor: middle;">mozilla-pipeline-schemas</text></g><g class="cluster" id="subGraph0" transform="translate(155.75,260.5)" style="opacity: 1;"><rect width="271.5" height="481" x="-135.75" y="-240.5"></rect><g class="label"><g transform="translate(0,0)"><foreignObject width="0" height="0"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;"></div></foreignObject></g></g><text x="0" y="-226.5" fill="black" stroke="none" id="mermaid-1556319859651Text" style="text-anchor: middle;">dataops/sandbox/my-project</text></g></g><g class="edgePaths"><g class="edgePath" style="opacity: 1;"><path class="path" d="M150,83L150,108L150,142L150,176L150,201" marker-end="url(#arrowhead88)" style="fill:none"></path><defs><marker id="arrowhead88" viewBox="0 0 10 10" refX="9" refY="5" markerUnits="strokeWidth" markerWidth="8" markerHeight="6" orient="auto"><path d="M 0 0 L 10 5 L 0 10 z" class="arrowheadPath" style="stroke-width: 1; stroke-dasharray: 1, 0;"></path></marker></defs></g><g class="edgePath" style="opacity: 1;"><path class="path" d="M150,239L150,273L150,307" marker-end="url(#arrowhead89)" style="fill:none"></path><defs><marker id="arrowhead89" viewBox="0 0 10 10" refX="9" refY="5" markerUnits="strokeWidth" markerWidth="8" markerHeight="6" orient="auto"><path d="M 0 0 L 10 5 L 0 10 z" class="arrowheadPath" style="stroke-width: 1; stroke-dasharray: 1, 0;"></path></marker></defs></g><g class="edgePath" style="opacity: 1;"><path class="path" d="M150,345L150,370L150,404L172.1320754716981,438" marker-end="url(#arrowhead90)" style="fill:none"></path><defs><marker id="arrowhead90" viewBox="0 0 10 10" refX="9" refY="5" markerUnits="strokeWidth" markerWidth="8" markerHeight="6" orient="auto"><path d="M 0 0 L 10 5 L 0 10 z" class="arrowheadPath" style="stroke-width: 1; stroke-dasharray: 1, 0;"></path></marker></defs></g><g class="edgePath" style="opacity: 1;"><path class="path" d="M404,239L404,273L190,317.65354330708664" marker-end="url(#arrowhead91)" style="fill:none"></path><defs><marker id="arrowhead91" viewBox="0 0 10 10" refX="9" refY="5" markerUnits="strokeWidth" markerWidth="8" markerHeight="6" orient="auto"><path d="M 0 0 L 10 5 L 0 10 z" class="arrowheadPath" style="stroke-width: 1; stroke-dasharray: 1, 0;"></path></marker></defs></g><g class="edgePath" style="opacity: 1;"><path class="path" d="M562.5,83L562.5,108L562.5,142L562.5,176L562.5,201" marker-end="url(#arrowhead92)" style="fill:none"></path><defs><marker id="arrowhead92" viewBox="0 0 10 10" refX="9" refY="5" markerUnits="strokeWidth" markerWidth="8" markerHeight="6" orient="auto"><path d="M 0 0 L 10 5 L 0 10 z" class="arrowheadPath" style="stroke-width: 1; stroke-dasharray: 1, 0;"></path></marker></defs></g><g class="edgePath" style="opacity: 1;"><path class="path" d="M562.5,239L562.5,273L562.5,307" marker-end="url(#arrowhead93)" style="fill:none"></path><defs><marker id="arrowhead93" viewBox="0 0 10 10" refX="9" refY="5" markerUnits="strokeWidth" markerWidth="8" markerHeight="6" orient="auto"><path d="M 0 0 L 10 5 L 0 10 z" class="arrowheadPath" style="stroke-width: 1; stroke-dasharray: 1, 0;"></path></marker></defs></g><g class="edgePath" style="opacity: 1;"><path class="path" d="M562.5,345L562.5,370L219,404L196.8679245283019,438" marker-end="url(#arrowhead94)" style="fill:none"></path><defs><marker id="arrowhead94" viewBox="0 0 10 10" refX="9" refY="5" markerUnits="strokeWidth" markerWidth="8" markerHeight="6" orient="auto"><path d="M 0 0 L 10 5 L 0 10 z" class="arrowheadPath" style="stroke-width: 1; stroke-dasharray: 1, 0;"></path></marker></defs></g></g><g class="edgeLabels"><g class="edgeLabel" transform="translate(150,142)" style="opacity: 1;"><g transform="translate(-110,-9)" class="label"><foreignObject width="220" height="18"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;"><span class="edgeLabel">gcloud pubsub subscriptions create</span></div></foreignObject></g></g><g class="edgeLabel" transform="" style="opacity: 1;"><g transform="translate(0,0)" class="label"><foreignObject width="0" height="0"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;"><span class="edgeLabel"></span></div></foreignObject></g></g><g class="edgeLabel" transform="" style="opacity: 1;"><g transform="translate(0,0)" class="label"><foreignObject width="0" height="0"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;"><span class="edgeLabel"></span></div></foreignObject></g></g><g class="edgeLabel" transform="translate(404,273)" style="opacity: 1;"><g transform="translate(-72.5,-9)" class="label"><foreignObject width="145" height="18"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;"><span class="edgeLabel">mvn compile exec:java</span></div></foreignObject></g></g><g class="edgeLabel" transform="translate(562.5,142)" style="opacity: 1;"><g transform="translate(-60.5,-9)" class="label"><foreignObject width="121" height="18"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;"><span class="edgeLabel">download-schemas</span></div></foreignObject></g></g><g class="edgeLabel" transform="translate(562.5,273)" style="opacity: 1;"><g transform="translate(-66,-9)" class="label"><foreignObject width="132" height="18"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;"><span class="edgeLabel">generate-bq-schemas</span></div></foreignObject></g></g><g class="edgeLabel" transform="translate(219,404)" style="opacity: 1;"><g transform="translate(-49,-9)" class="label"><foreignObject width="98" height="18"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;"><span class="edgeLabel">update-bq-table</span></div></foreignObject></g></g></g><g class="nodes"><g class="node" id="src" transform="translate(404,220)" style="opacity: 1;"><rect rx="0" ry="0" x="-22.5" y="-19" width="45" height="38"></rect><g class="label" transform="translate(0,0)"><g transform="translate(-12.5,-9)"><foreignObject width="25" height="18"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;">src/</div></foreignObject></g></g></g><g class="node" id="schemas" transform="translate(562.5,220)" style="opacity: 1;"><rect rx="0" ry="0" x="-63" y="-19" width="126" height="38"></rect><g class="label" transform="translate(0,0)"><g transform="translate(-53,-9)"><foreignObject width="106" height="18"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;">schemas.tar.gz</div></foreignObject></g></g></g><g class="node" id="bq-schemas" transform="translate(562.5,326)" style="opacity: 1;"><rect rx="0" ry="0" x="-55.5" y="-19" width="111" height="38"></rect><g class="label" transform="translate(0,0)"><g transform="translate(-45.5,-9)"><foreignObject width="91" height="18"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;">bq-schemas/</div></foreignObject></g></g></g><g class="node" id="dataflow" transform="translate(150,326)" style="opacity: 1;"><rect rx="0" ry="0" x="-40" y="-19" width="80" height="38"></rect><g class="label" transform="translate(0,0)"><g transform="translate(-30,-9)"><foreignObject width="60" height="18"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;">dataflow</div></foreignObject></g></g></g><g class="node" id="mps" transform="translate(562.5,64)" style="opacity: 1;"><rect rx="0" ry="0" x="-73" y="-19" width="146" height="38"></rect><g class="label" transform="translate(0,0)"><g transform="translate(-63,-9)"><foreignObject width="126" height="18"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;">repository archive</div></foreignObject></g></g></g><g class="node" id="bigquery" transform="translate(184.5,457)" style="opacity: 1;"><rect rx="0" ry="0" x="-41" y="-19" width="82" height="38"></rect><g class="label" transform="translate(0,0)"><g transform="translate(-31,-9)"><foreignObject width="62" height="18"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;">bigquery</div></foreignObject></g></g></g><g class="node" id="pubsub" transform="translate(150,64)" style="opacity: 1;"><rect rx="0" ry="0" x="-36.5" y="-19" width="73" height="38"></rect><g class="label" transform="translate(0,0)"><g transform="translate(-26.5,-9)"><foreignObject width="53" height="18"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;">pubsub</div></foreignObject></g></g></g><g class="node" id="subscription" transform="translate(150,220)" style="opacity: 1;"><rect rx="0" ry="0" x="-53" y="-19" width="106" height="38"></rect><g class="label" transform="translate(0,0)"><g transform="translate(-43,-9)"><foreignObject width="86" height="18"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; white-space: nowrap;">subscription</div></foreignObject></g></g></g></g></g></g></svg>
\ No newline at end of file
diff --git a/docs/ingestion_testing_workflow.md b/docs/ingestion_testing_workflow.md
new file mode 100644
index 0000000000..e96be09342
--- /dev/null
+++ b/docs/ingestion_testing_workflow.md
@@ -0,0 +1,85 @@
+<!-- START doctoc generated TOC please keep comment here to allow auto update -->
+<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
+
+
+- [Ingestion Testing Workflow](#ingestion-testing-workflow)
+  - [Setting up the GCS project](#setting-up-the-gcs-project)
+  - [Bootstrapping schemas from `mozilla-pipeline-schemas`](#bootstrapping-schemas-from-mozilla-pipeline-schemas)
+  - [Building the project](#building-the-project)
+
+<!-- END doctoc generated TOC please keep comment here to allow auto update -->
+
+# Ingestion Testing Workflow
+
+The ingestion-beam handles dataflow of documents from the edge into various
+sinks. You may be interested in standing up a small testing instance to validate
+the integration of the various components.
+
+![diagrams/workflow.mmd](diagrams/workflow.svg)
+__Figure__: _An overview of the various components necessary to query BigQuery
+against data from a pubsub subscription._
+
+## Setting up the GCS project
+
+Read through [`whd/gcp-quickstart`](https://github.com/whd/gcp-quickstart) for details
+about the sandboxing environment that is provided by data operations. 
+
+* Install the [Google Cloud SDK](https://cloud.google.com/sdk/)
+* Navigate to the [Google Cloud Console](https://cloud.google.com/sdk/)
+* Create a new project under `firefox.gcp.mozilla.com/dataops/sandbox`
+    - `gcloud config set project <PROJECT>`
+* Create a PubSub subscription (see `gcp-quickstart/pubsub.sh`)
+* Create a GCS bucket
+    - `gsutil mb gs://<PROJECT>`
+* Enable the [DataFlow API](https://console.cloud.google.com/marketplace/details/google/dataflow.googleapis.com)
+* Create a service account and store the key locally
+
+
+## Bootstrapping schemas from `mozilla-pipeline-schemas`
+
+* Download the latest schemas from `mozilla-pipeline-schemas` using `bin/download-schemas`. 
+    - This script may also inject testing resources into the resulting archive.
+    - A `schemas.tar.gz` will appear at the project root.
+* Generate BigQuery schemas using `bin/generate-bq-schemas`.
+    - Schemas will be written to `bq-schemas/`. 
+    ```
+    bq-schemas/
+    ├── activity-stream.impression-stats.1.bigquery.json
+    ├── coverage.coverage.1.bigquery.json
+    ├── edge-validator.error-report.1.bigquery.json
+    ├── eng-workflow.bmobugs.1.bigquery.json
+    ....
+    ```
+* Update the BigQuery table in the current project using `bin/update-bq-table`.
+    - This may take several minutes. Read the script for usage information.
+* Verify that tables have been updated by viewing the BigQuery console.
+
+
+## Building the project
+
+Follow the instructions of the project README. Here is a quick-reference for a running a job from a set of files in GCS.
+
+```bash
+export GOOGLE_APPLICATION_CREDENTIALS=keys.json
+PROJECT=$(gcloud config get-value project)
+BUCKET="gs://$PROJECT"
+
+path="$BUCKET/data/*.ndjson"
+mvn compile exec:java -Dexec.args="\
+    --runner=Dataflow \
+    --project=$PROJECT \
+    --autoscalingAlgorithm=NONE \
+    --workerMachineType=n1-standard-1 \
+    --numWorkers=1 \
+    --gcpTempLocation=$BUCKET/tmp \
+    --inputFileFormat=json \
+    --inputType=file \
+    --input=$path\
+    --outputType=bigquery \
+    --output=$PROJECT:\${document_namespace}.\${document_type}_v\${document_version} \
+    --bqWriteMethod=file_loads \
+    --tempLocation=$BUCKET/temp/bq-loads \
+    --errorOutputType=file \
+    --errorOutput=$BUCKET/error/ \
+"
+```
diff --git a/ingestion-beam/README.md b/ingestion-beam/README.md
index 2cf153b48c..8e6720bdd0 100644
--- a/ingestion-beam/README.md
+++ b/ingestion-beam/README.md
@@ -562,6 +562,9 @@ use the `bin/mvn` executable to run maven in docker:
 ./bin/mvn clean test
 ```
 
+To run the project in a sandbox against production data, see this document on
+![configuring an integration testing workflow](docs/ingestion_testing_workflow.md).
+
 # License
 
 This Source Code Form is subject to the terms of the Mozilla Public