graph-builder: use openshift/cincinnati-graph-data instead of quay labels

Justfile

@@ -1,3 +1,21 @@
+path_prefix := "api/upgrades_info/"
+
+testdata_dir := "e2e/tests/testdata/"
+metadata_revision_file := "metadata_revision"
+metadata_reference :='reference_branch = "master"'
+
+pause_secs := "9999999"
+registry := "https://quay.io"
+repository := "openshift-release-dev/ocp-release"
+credentials_file := "${HOME}/.docker/config.json"
+
+metadata_reference_e2e:
+	printf 'reference_revision = "%s"' "$(cat {{testdata_dir}}/{{metadata_revision_file}})"
+
+metadata_reference_revision:
+	#!/usr/bin/env bash
+	read -r var <"{{testdata_dir}}"/metadata_revision; printf $var
+
 format:
 	cargo fmt --all
 
@@ -25,25 +43,77 @@ test: format
 	export RUST_BACKTRACE=1 RUST_LOG="graph-builder=trace,cincinnati=trace,dkregistry=trace"
 	cargo test --all
 
-_test component features='test-net,test-net-private' rustcargs='--ignored':
+_test component cargoargs='--features test-net,test-net-private' rustcargs='--ignored':
 	#!/usr/bin/env bash
 	set -xe
-	(pushd {{component}} && cargo test --features {{features}} -- {{rustcargs}})
+	(pushd {{component}} && cargo test {{cargoargs}} -- {{rustcargs}})
 
 
 test-net-private:
 	#!/usr/bin/env bash
 	set -e
-	just _test quay "test-net,test-net-private" ""
-	just _test cincinnati "test-net,test-net-private" ""
-	just _test graph-builder "test-net,test-net-private" ""
+	just _test quay "--features test-net,test-net-private" ""
+	just _test cincinnati "--features test-net,test-net-private" ""
+	just _test graph-builder "--features test-net,test-net-private" ""
 
 run-ci-tests:
 	#!/usr/bin/env bash
 	set -e
 	hack/run-all-tests.sh
 
-path_prefix := "api/upgrades_info/"
+# Runs the client part of the e2e test suite.
+run-e2e-test-only filter="e2e":
+	#!/usr/bin/env bash
+	set -e
+	export GRAPH_URL='http://127.0.0.1:8081/{{path_prefix}}v1/graph'
+	export E2E_METADATA_REVISION="$(just metadata_reference_revision)"
+
+	# we need to use the parent-directory here because the test runs in the graph-builder directory
+	export E2E_TESTDATA_DIR="../{{ testdata_dir }}"
+
+	just _test e2e "" "{{ filter }}"
+
+# Spawns a Cincinnati stack on localhost and runs the e2e test suite.
+run-e2e:
+	#!/usr/bin/env bash
+	set -e
+
+	just \
+		registry="{{registry}}" repository="{{repository}}" \
+		run-daemons-e2e 2>&1 &
+	DAEMON_PARENTPID=$!
+	trap "kill $DAEMON_PARENTPID" EXIT
+
+	# give the graph-builder time to scrape
+	sleep 180
+
+	for i in `seq 1 100`; do
+		just run-e2e-test-only && {
+			echo Test successful.
+			exit 0
+		} || {
+			echo Attempt failed. Trying again in 10 seconds.
+			sleep 10
+		}
+	done
+
+	echo Test failed.
+	exit 1
+
+# Capture new e2e fixtures and refresh the metadata revision file.
+e2e-fixtures-capture-only:
+	#!/usr/bin/env bash
+	set -e
+
+	for base in "stable"; do
+		for version in "4.2" "4.3"; do
+			for arch in "amd64" "s390x"; do
+				for suffix in "-production"; do
+					just get-graph-pe"${suffix}" "${base}-${version}" "${arch}" > {{testdata_dir}}/"$(just metadata_reference_revision)_${base}-${version}_${arch}${suffix}".json
+				done
+			done
+		done
+	done
 
 # Reads a graph on stdin, creates an SVG out of it and opens it with SVG-associated default viewer. Meant to be combined with one of the `get-graph-*` recipes.
 display-graph:
@@ -58,14 +128,18 @@ display-graph:
 
 	jq -cM . | {{invocation_directory()}}/hack/graph.sh | dot -Tsvg > graph.svg; xdg-open graph.svg
 
-run-graph-builder registry="https://quay.io" repository="openshift-release-dev/ocp-release" credentials_file="${HOME}/.docker/config.json":
+run-graph-builder:
 	#!/usr/bin/env bash
 	export RUST_BACKTRACE=1
-	cargo run --package graph-builder -- -c <(cat <<'EOF'
+
+	trap 'rm -rf "$TMPDIR"' EXIT
+	export TMPDIR=$(mktemp -d)
+
+	cargo run --package graph-builder -- -c <(cat <<-EOF
 		verbosity = "vvv"
 
 		[service]
-		pause_secs = 9999999
+		pause_secs = {{pause_secs}}
 		address = "127.0.0.1"
 		port = 8080
 		path_prefix = "{{path_prefix}}"
@@ -82,41 +156,65 @@ run-graph-builder registry="https://quay.io" repository="openshift-release-dev/o
 		credentials_file = "{{credentials_file}}"
 
 		[[plugin_settings]]
-		name="quay-metadata"
-		repository="{{repository}}"
+		name = "github-secondary-metadata-scrape"
+		github_org = "openshift"
+		github_repo = "cincinnati-graph-data"
+		branch = "master"
+		output_directory = "${TMPDIR}"
+		{{metadata_reference}}
 
 		[[plugin_settings]]
-		name="node-remove"
+		name = "openshift-secondary-metadata-parse"
+		data_directory = "${TMPDIR}"
 
 		[[plugin_settings]]
-		name="edge-add-remove"
+		name = "edge-add-remove"
 	EOF
 	)
 
 run-graph-builder-satellite:
-	just run-graph-builder 'sat-r220-02.lab.eng.rdu2.redhat.com' 'default_organization-custom-ocp'
+	just registry='sat-r220-02.lab.eng.rdu2.redhat.com' repository='default_organization-custom-ocp' run-graph-builder
+
+run-graph-builder-e2e:
+	just \
+		registry="{{registry}}" repository="{{repository}}" \
+		metadata_reference="$(just metadata_reference_e2e)" \
+		run-graph-builder
 
 run-policy-engine:
 	#!/usr/bin/env bash
 	export RUST_BACKTRACE=1 RUST_LOG="policy_engine=trace,cincinnati=trace,actix=trace,actix_web=trace"
-	cargo run --package policy-engine -- -vvvv --service.address 0.0.0.0 --service.path_prefix {{path_prefix}} --upstream.cincinnati.url 'http://localhost:8080/{{path_prefix}}v1/graph' --service.mandatory_client_parameters='channel'
+	cargo run --package policy-engine -- -vvvv --service.address 0.0.0.0 --service.path_prefix {{path_prefix}} --upstream.cincinnati.url 'http://127.0.0.1:8080/{{path_prefix}}v1/graph' --service.mandatory_client_parameters='channel'
 
 kill-daemons:
 	pkill graph-builder
 	pkill policy-engine
 
 run-daemons:
 	#!/usr/bin/env bash
-	just run-graph-builder #"https://quay.io" "redhat/openshift-cincinnati-test-public-manual" ~/.docker/config.json 2>&1 &
+	just run-graph-builder 2>&1 &
+	PG_PID=$!
+
+	just run-policy-engine 2>&1 &
+	PE_PID=$!
+
+	trap "kill $PG_PID $PE_PID" EXIT
+	sleep infinity
+
+run-daemons-e2e:
+	#!/usr/bin/env bash
+	just \
+		registry="{{registry}}" repository="{{repository}}" \
+		run-graph-builder-e2e 2>&1 &
 	PG_PID=$!
 
 	just run-policy-engine 2>&1 &
 	PE_PID=$!
 
 	trap "kill $PG_PID $PE_PID" EXIT
-	while true; do sleep 30; done
+	sleep infinity
 
-get-graph port channel arch host="http://localhost":
+get-graph port channel arch host="http://127.0.0.1":
 	curl --header 'Accept:application/json' {{host}}:{{port}}/{{path_prefix}}v1/graph?channel='{{channel}}'\&arch='{{arch}}' | jq .
 
 get-graph-gb:

cincinnati/Cargo.toml

@@ -44,7 +44,7 @@ twoway = "^0.2"
 walkdir = "2.3.1"
 pretty_assertions = "0.6.1"
 test-case = "1.0.0"
-prettydiff = {}
+prettydiff = "0.3.1"
 
 [build-dependencies]
 protoc-rust = "2.8"

cincinnati/src/lib.rs

@@ -450,9 +450,12 @@ impl Graph {
             .try_for_each(|mut nw| f(&mut nw))
     }
 
-    /// Get the edges expressed as version -> versions
+    /// Get the edges expressed as version -> versions; optionally include edges from/to `Release::Abstract`.
     #[cfg(any(test, feature = "test"))]
-    pub fn get_edges(&self) -> Result<MapImpl<String, SetImpl<String>>, Error> {
+    pub fn get_edges(
+        &self,
+        include_abstract: bool,
+    ) -> Result<MapImpl<String, SetImpl<String>>, Error> {
         let mut edges: MapImpl<String, SetImpl<String>> = Default::default();
 
         self.dag
@@ -467,10 +470,17 @@ impl Graph {
                         let source_version = source.version().to_string();
                         let target_version = target.version().to_string();
 
-                        edges
-                            .entry(source_version)
-                            .or_default()
-                            .insert(target_version);
+                        if include_abstract
+                            || match (source, target) {
+                                (Release::Concrete(_), Release::Concrete(_)) => true,
+                                _ => false,
+                            }
+                        {
+                            edges
+                                .entry(source_version)
+                                .or_default()
+                                .insert(target_version);
+                        }
 
                         Ok(())
                     }
@@ -937,56 +947,48 @@ pub mod testing {
         let removed_keys_left = remove_release_metadata(&mut left, unwanted_metadata_keys);
         let removed_keys_right = remove_release_metadata(&mut right, unwanted_metadata_keys);
 
-        let releases_left: std::collections::BTreeSet<Release> = (&left).into();
-        let releases_right: std::collections::BTreeSet<Release> = (&right).into();
-
-        let edges_left = right.get_edges()?;
-        let edges_right = left.get_edges()?;
+        if left == right {
+            return Ok(());
+        }
 
         let mut output: Vec<String> = vec![];
+        output.push("Graphs differ! Showing differences from left to right".into());
+        output.push("-----------------------------------------------------".into());
+
+        let edges_left = right.get_edges(false)?;
+        let edges_right = left.get_edges(false)?;
+        output.push("edges: ".into());
+        output.push(
+            prettydiff::diff_lines(
+                &serde_json::to_string_pretty(&edges_left)?,
+                &serde_json::to_string_pretty(&edges_right)?,
+            )
+            .format(),
+        );
 
-        if releases_left != releases_right {
-            output.push("Graphs differ! Showing differences from left to right".into());
-            output.push("-----------------------------------------------------".into());
-            output.push("nodes:".into());
-            output.push(
-                prettydiff::diff_lines(
-                    Box::leak(Box::new(serde_json::to_string_pretty(&releases_left)?)),
-                    Box::leak(Box::new(serde_json::to_string_pretty(&releases_right)?)),
-                )
-                .format(),
-            );
-
-            if !unwanted_metadata_keys.is_empty() {
-                output.push("removed metadata:".into());
-                output.push(
-                    prettydiff::diff_lines(
-                        &serde_json::to_string_pretty(&removed_keys_left)?,
-                        &serde_json::to_string_pretty(&removed_keys_right)?,
-                    )
-                    .format(),
-                );
-            }
-        }
+        let releases_left: std::collections::BTreeSet<Release> = (&left).into();
+        let releases_right: std::collections::BTreeSet<Release> = (&right).into();
+        output.push("nodes:".into());
+        output.push(
+            prettydiff::diff_lines(
+                Box::leak(Box::new(serde_json::to_string_pretty(&releases_left)?)),
+                Box::leak(Box::new(serde_json::to_string_pretty(&releases_right)?)),
+            )
+            .format(),
+        );
 
-        if edges_left != edges_right {
-            output.push("edges: ".into());
+        if !unwanted_metadata_keys.is_empty() {
+            output.push("removed metadata:".into());
             output.push(
                 prettydiff::diff_lines(
-                    &serde_json::to_string_pretty(&edges_left)?,
-                    &serde_json::to_string_pretty(&edges_right)?,
+                    &serde_json::to_string_pretty(&removed_keys_left)?,
+                    &serde_json::to_string_pretty(&removed_keys_right)?,
                 )
                 .format(),
             );
         }
 
-        let output = output.join("\n");
-
-        if !output.is_empty() {
-            return Err(output.into());
-        }
-
-        Ok(())
+        Err(output.join("\n").into())
     }
 
     /// Removes the metadata given by the keys and returns it.

cincinnati/src/plugins/internal/graph_builder/github_openshift_secondary_metadata_scraper/github_v3.rs

@@ -43,6 +43,14 @@ pub(crate) fn archive_entry_directory_name(org: &str, repo: &str, commit: &Commi
     format!("{}-{}-{}", &org, &repo, &commit.sha[0..7],)
 }
 
+/// Format a commit URL
+pub(crate) fn commit_url(org: &str, repo: &str, sha: &str) -> String {
+    format!(
+        "https://api.github.com/repos/{}/{}/commits/{}",
+        org, repo, sha
+    )
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;