docs: Add Logging example README (#157)

Signed-off-by: Jeff McCoy <code@jeffm.us>

docs/asciinema/Vagrantfile

@@ -26,6 +26,6 @@ Vagrant.configure("2") do |config|
   config.vm.provision "shell", inline: <<-'SCRIPT'
     # https://serverfault.com/questions/500764/dpkg-reconfigure-unable-to-re-open-stdin-no-file-or-directory
     export DEBIAN_FRONTEND=noninteractive
-    apt-get --yes install asciinema expect
+    apt-get --yes install asciinema expect jq
   SCRIPT
 end

docs/asciinema/scenarios/examples-game-logging.exp

@@ -0,0 +1,164 @@
+#!/usr/bin/expect -f
+
+# Assumptions:
+# - run as root (required to run zarf)
+# - invocation working dir is <root>/build (where zarf binary is)
+
+# Refs:
+# - https://linux.die.net/man/1/expect
+# - https://www.tcl.tk/man/tcl8.6/TclCmd/contents.html
+
+
+# variables / config
+set env(PATH) "$::env(PATH):[file normalize .]" ; # add zarf to path
+set gamedir "[file normalize ../examples/game]"
+set rcfile "/tmp/asciinema-rcfile"
+set title [
+  file tail [ file rootname [lindex $argv0] ]
+]
+set recording [lindex $argv 0]
+set log [lindex $argv 1]
+log_file $log
+set send_human {.1 .2 10 .05 1}   ; # "human-like" typing speeds on 'send -h'
+set env(PS1) {\[\033[01;32m\]root@host\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]# } ; # custom terminal input prompt
+set prompt ".*root@host.*:.*# $"
+#match_max 25000
+#set expect_out(buffer) {}
+#set down_key "\033\[B"
+
+
+# proc defs
+proc wait_input { {timeout 10} } {
+  global prompt
+  expect -timeout $timeout -re $prompt
+}
+
+proc input_wait { {seconds 1} } {
+  wait_input ; sleep $seconds
+}
+
+proc comment { {text " "} } {
+  send "# --> " ; send -h "$text \r\r" ;  input_wait 1
+}
+
+proc do { {cmd "\r"} {timeout 2} } {
+  send -h "$cmd" ; sleep 1 ; send "\r\r" ; input_wait $timeout
+}
+
+proc wait_for { {predicate "1 = 1"} {timeout 120} } {
+  set template {
+    START="$SECONDS"
+    until [ ||predicate|| ] ; do
+      sleep 2
+      if [ "$(( $SECONDS - $START ))" -gt "||timeout||" ] ; then
+        echo -e "\nwait_for::timeout!\n" > /dev/tty ; exit 1
+      fi
+    done
+  }
+  set script [ string map [ list \
+    ||predicate|| $predicate \
+    ||timeout|| $timeout \
+  ] $template ]
+  set result [ system "/bin/bash -c '$script'" ]
+}
+
+proc clean {} {
+  global gamedir
+  global rcfile
+
+  spawn bash --norc
+  send "rm -f $gamedir/*.tar.zst $rcfile\r"
+  send "zarf destroy --confirm\r"
+  send "exit\r"
+  expect eof
+}
+
+proc setup {} {
+  global rcfile
+
+  spawn bash --norc
+  send "echo 'alias ls=\"ls --color=auto\"' > $rcfile\r" ; expect *
+  send "echo 'alias grep=\"grep --color=auto\"' >> $rcfile\r" ; expect *
+  send "echo 'export PS1=\"$::env(PS1)\"' >> $rcfile\r" ; expect *
+  send "echo 'export GREP_COLOR=\"01;97;100\"' >> $rcfile\r" ; expect *
+  send "exit\r"
+  expect eof
+}
+
+
+# filesystem to ready state
+clean
+setup
+
+
+# prep cluster
+spawn bash --norc
+
+send -h "zarf init --host=localhost --components=management --confirm\r"
+expect -timeout 120 -re {.*Grafana Username[^=]*=([^\s]*)}
+set grafana_user $expect_out(1,string)
+expect -re {.*Password \(all\)[^=]*="([^"]*)"}
+set grafana_pass $expect_out(1,string)
+expect -re $prompt
+
+send -h "cd $gamedir\r" ; wait_input
+send -h "zarf package create --confirm\r" ; wait_input 120
+
+wait_for {$( curl -o /dev/null -s -w '%{http_code}' https://localhost/v2 ) == '404'}
+send -h "zarf package deploy zarf-\t --confirm\r" ; wait_input 120
+
+wait_for {$( curl -o /dev/null -s -w '%{http_code}' https://localhost ) == '200'}
+send -h "exit\r"
+expect eof
+
+
+# start recording & wait for input prompt
+spawn asciinema rec \
+  --command "/bin/bash --rcfile $rcfile" \
+  --title "$title" \
+  --idle-time-limit 3 \
+  --overwrite \
+  $recording
+expect -re {asciinema: .* done} ; wait_input
+
+comment "cluster up?"
+do "kubectl cluster-info" ; wait_input
+
+comment "game running?"
+do "kubectl get pod -l app=game"; wait_input
+
+comment "install logging component"
+send -h "zarf init --host=localhost --components=logging --confirm\r\r"
+wait_input 120
+
+comment "PLG stack up?"
+wait_for {$( curl -o /dev/null -sL -w '%{http_code}' https://localhost/monitor ) == '200'}
+system {kubectl wait -n logging pods --all --for=condition=Ready --selector=release=loki --timeout=120s}
+do "kubectl get pods -n logging" ; wait_input
+
+comment "call game to generate logs"
+do "curl -sL https://localhost | grep -e '^' -e 'main(.*$'"
+
+# wait for logs to propagate to grafana
+sleep 60
+
+comment "pod logs available to grafana (via loki)?"
+send -h "curl -sL -u '"
+send "$grafana_user:$grafana_pass"
+send -h "' \\\r"
+send {  'https://localhost/monitor/api/datasources/proxy/1/loki/api/v1/query_range?direction=BACKWARD&limit=1&query=%7Bapp%3D"game"%7D'}
+send -h "\\\r"
+send -h "  | jq '.data.result' \\\r"
+send -h {  | grep -e '^' -e '".*stdout.*$'}
+send -h "\r\r"
+wait_input 60
+
+comment "and finally, to cleanup"
+send -h "zarf destroy --confirm\r\r"
+wait_input 60
+
+comment "its gone!"
+do "kubectl cluster-info"
+
+comment "success!"
+do "exit"

examples/game/add-logging.md

@@ -0,0 +1,156 @@
+# Zarf Components - Add Logging
+
+This example demonstrates using a [Zarf component](./components.md) to inject zero-config, centralized logging into your Zarf cluster.
+
+More specifically, you'll be adding a [Promtail / Loki / Grafana (PLG)](https://github.com/grafana/loki) stack to the example game cluster by installing Zarf's "logging" component.
+
+ 
+
+
+## The Flow
+
+<a href="https://asciinema.org/a/446956?x-scenario=examples-game-logging&autoplay=1">
+<img align="right" alt="asciicast" src="https://asciinema.org/a/446956.svg?x-scenario=examples-game-logging" height="320" />
+</a>
+
+Here's what you'll do in this example:
+
+1. [Get ready](#get-ready)
+
+1. [Install the logging component](#install-the-logging-component)
+
+1. [Note the credentials](#note-the-credentials)
+
+1. [Check the logs](#check-the-logs)
+
+1. [Cleanup](#cleanup)
+
+&nbsp;
+
+&nbsp;
+
+
+## Get ready
+
+<a href="https://asciinema.org/a/446956?x-scenario=examples-game-logging&t=1">
+<img align="right" alt="asciicast" src="https://asciinema.org/a/446956.svg?x-scenario=examples-game-logging" height="256" />
+</a>
+
+This scenario builds upon the previous one, so:
+
+1. Run through the [Zarf game example](./README.md) again but _**don't** do the cleanup step_ &mdash; you're setup correctly once you can pull the game up in your browser.
+
+1. Take a deep breath&mdash;because it's good for your body&mdash;and  read on!
+
+&nbsp;
+
+&nbsp;
+
+
+## Install the logging component
+
+<a href="https://asciinema.org/a/446956?x-scenario=examples-game-logging&t=19">
+<img align="right" alt="asciicast" src="https://asciinema.org/a/446956.svg?x-scenario=examples-game-logging" height="256" />
+</a>
+
+Installing a Zarf component is _really_ easy&mdash;you just have to let `zarf init` know that you want use it.  That's it!
+
+Exactly like when you first created the game example cluster, you _move into the directory holding your init package_ and run:
+
+```sh
+cd <same dir as zarf-init.tar.zst>
+zarf init
+```
+
+You can answer the follow-on prompts in almost the exact same way as during your original install _**except** this time answer "yes" when asked whether to install the "logging" component_.
+
+Give it some time for the new logging pods to come up and you're ready to go!
+
+ > _**Note**_
+ >
+ > You can install components as part of new cluster installs too (obviously)&mdash;there's no need to update afterward if you already know you need a component.
+
+ > _**Note**_
+ >
+ > Zarf supports non-interactive installs too! See `zarf init --help` for how to make that work.
+
+&nbsp;
+
+
+## Note the credentials
+
+Go back to your terminal and review the `zarf init` command output&mdash;the very last thing printed should be a set of credentials Zarf has generated for you.
+
+Pay attention to these because you're going to need them to log into your shiny, new [Grafana](https://grafana.com/docs/) installation.
+
+The line you want will look something like this:
+
+```sh
+WARN[0026] Credentials stored in ~/.git-credentials      Gitea Username (if installed)=zarf-git-user Grafana Username=zarf-admin Password (all)="AbCDe0fGH12IJklMnOPQRSt~uVWx"
+```
+
+Pull out the `Grafana Username` and `Password (all)` values & save them for later.
+
+&nbsp;
+
+
+## Check the logs
+
+<a href="https://asciinema.org/a/446956?x-scenario=examples-game-logging&t=55">
+<img align="right" alt="asciicast" src="https://asciinema.org/a/446956.svg?x-scenario=examples-game-logging" height="256" />
+</a>
+
+We've only _just_ installed the logging utilities so we (likely) haven't had time to record anything interesting. Since log aggregation & monitoring aren't worth much without something to collect, let's get some data in there.
+
+&nbsp;
+
+
+### Generate some traffic
+
+Pull up the game in your brower&mdash;_[instructions here](./README.md#space-marine-the-demon-invasion), in case you forgot how_&mdash;and then reload the browser window a few times.
+
+Doing that sends a bunch of HTTP traffic into the cluster & should give you something worth looking at in Grafana.
+
+&nbsp;
+
+
+### Get into Grafana
+
+<a href="../../.images/get-started/plg.png">
+<img align="right" alt="dosbox" src="../../.images/get-started/plg.png" height="160" />
+</a>
+
+Now that you've got some logs worth looking at, you're ready to log into your brand new Grafana instance.
+
+Get started by navigating your browser to: `https://localhost/monitor/explore`.
+
+You'll be redirected the `/login` page where you have to sign in with the Grafana credentials you saved [in a previous step](#note-the-credentials).
+
+Once you've successfully logged in you will be redirected back to:
+
+1. the `monitor/explore` page, where
+
+1. you can select `Loki` in the dropdown, and then
+
+1. enter `{app="game"}` into the Log Browser query input field
+
+Submit that query and you'll get back a dump of all the game pod logs that Loki has collected. Neat!
+
+&nbsp;
+
+
+## Cleanup
+
+<a href="https://asciinema.org/a/446956?x-scenario=examples-game-logging&t=88">
+<img align="right" alt="asciicast" src="https://asciinema.org/a/446956.svg?x-scenario=examples-game-logging" height="256" />
+</a>
+
+Once you've had your fun it's time to clean up.
+
+In this case, since the Zarf cluster was installed specifically (and _only_) to serve this example, clean up is really easy&mdash;you just tear down the entire cluster:
+
+```sh
+zarf destroy --confirm
+```
+
+It takes just a couple moments for the _entire Zarf cluster_ to disappear&mdash;long-running system services and all&mdash;leaving your machine squeaky clean.