chore(test): Integrate LDBC Tests into CI Workflow

.github/workflows/ci-dgraph-ldbc-tests.yml

@@ -0,0 +1,73 @@
+name: ci-dgraph-ldbc-tests
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+  schedule:
+    - cron: "30 * * * *"
+jobs:
+  dgraph-ldbc-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Dgraph
+        uses: actions/checkout@v3
+      - name: Checkout LDBC query repo
+        uses: actions/checkout@v3
+        with:
+          repository: dgraph-io/ldbc
+          token: ${{ secrets.DEPLOY_KEYS_GH_DGRAPHIO_LDBC }}
+          ref: master
+          path: ldbc
+      - name: Set up Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.18
+      - name: Set up Node
+        uses: actions/setup-node@v3
+        with:
+          node-version: 16
+      - name: Install protobuf-compiler
+        run: sudo apt-get install -y protobuf-compiler
+      - name: Check protobuf
+        run: |
+          cd ./protos
+          go mod tidy
+          make regenerate
+          git diff --exit-code -- .
+      - name: Make Docker Image
+        run: make image-local
+      - name: Make Linux Build
+        run: |
+          #!/bin/bash
+          # go settings
+          export GOOS=linux
+          export GOARCH=amd64
+          # make dgraph binary
+          make dgraph
+      - name: Clean Up Environment
+        run: |
+          #!/bin/bash
+          # clean cache
+          go clean -testcache
+          # build the test binary
+          cd t; go build .
+          # clean up docker containers before test execution
+          ./t -r
+      - name: Run LDBC Tests
+        run: |
+          #!/bin/bash
+          # clean cache
+          go clean -testcache
+          # go env settings
+          export GOPATH=~/go
+          # move the binary
+          cp dgraph/dgraph ~/go/bin 
+          # build the test binary
+          cd t; go build .
+          # run the ldbc tests
+          ./t --suite=ldbc
+          # clean up docker containers after test execution
+          ./t -r

systest/ldbc/alpha.yml

@@ -0,0 +1,22 @@
+version: "3.5"
+services:
+  alpha1:
+    image: dgraph/dgraph:local
+    working_dir: /data/alpha1
+    labels:
+      cluster: test
+    ports:
+      - "8080"
+      - "9080"
+    volumes:
+      - type: bind
+        source: $GOPATH/bin
+        target: /gobin
+        read_only: true
+      - type: bind
+        source: ./out/0/p
+        target: /posting
+        read_only: false
+    command: /gobin/dgraph alpha --my=alpha1:7080 --zero=zero1:5080 -p=/posting --logtostderr
+      -v=2 
+      --security "whitelist=10.0.0.0/8,172.16.0.0/12,192.168.0.0/16;"

systest/ldbc/docker-compose.yml

@@ -0,0 +1,23 @@
+version: "3.5"
+services:
+  zero1:
+    image: dgraph/dgraph:local
+    working_dir: /data/zero1
+    labels:
+      cluster: test
+    ports:
+    - "5080"
+    - "6080"
+    volumes:
+    - type: bind
+      source: $GOPATH/bin
+      target: /gobin
+      read_only: true
+    - type: volume
+      source: data
+      target: /data
+      read_only: false
+    command: /gobin/dgraph zero --raft="idx=1" --my=zero1:5080 --logtostderr
+      -v=2 --bindall
+volumes:
+  data: {}

systest/ldbc/ldbc_test.go

@@ -0,0 +1,109 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/dgraph-io/dgo/v210/protos/api"
+	"github.com/dgraph-io/dgraph/testutil"
+	"github.com/stretchr/testify/require"
+	"gopkg.in/yaml.v2"
+)
+
+type TestCases struct {
+	Tag   string `yaml:"tag"`
+	Query string `yaml:"query"`
+	Resp  string `yaml:"resp"`
+}
+
+func TestQueries(t *testing.T) {
+	dg, err := testutil.DgraphClient(testutil.ContainerAddr("alpha1", 9080))
+	if err != nil {
+		t.Fatalf("Error while getting a dgraph client: %v", err)
+	}
+
+	yfile, _ := ioutil.ReadFile("test_cases.yaml")
+
+	tc := make(map[string]TestCases)
+
+	err = yaml.Unmarshal(yfile, &tc)
+
+	if err != nil {
+		t.Fatalf("Error while greading test cases yaml: %v", err)
+	}
+
+	for _, tt := range tc {
+		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
+		resp, err := dg.NewTxn().Query(ctx, tt.Query)
+		cancel()
+
+		if ctx.Err() == context.DeadlineExceeded {
+			t.Fatal("aborting test due to query timeout")
+		}
+		require.NoError(t, err)
+		testutil.CompareJSON(t, tt.Resp, string(resp.Json))
+	}
+}
+
+func TestMain(m *testing.M) {
+	noschemaFile := filepath.Join(testutil.TestDataDirectory, "ldbcTypes.schema")
+	rdfFile := testutil.TestDataDirectory
+	if err := testutil.MakeDirEmpty([]string{"out/0", "out/1", "out/2"}); err != nil {
+		os.Exit(1)
+	}
+
+	start := time.Now()
+	fmt.Println("Bulkupload started")
+	if err := testutil.BulkLoad(testutil.BulkOpts{
+		Zero:       testutil.SockAddrZero,
+		Shards:     1,
+		RdfFile:    rdfFile,
+		SchemaFile: noschemaFile,
+	}); err != nil {
+		fmt.Println(err)
+		cleanupAndExit(1)
+	}
+
+	fmt.Printf("Took %s to bulkupload LDBC dataset\n", time.Since(start))
+
+	if err := testutil.StartAlphas("./alpha.yml"); err != nil {
+		fmt.Printf("Error while bringin up alphas. Error: %v\n", err)
+		cleanupAndExit(1)
+	}
+	schemaFile := filepath.Join(testutil.TestDataDirectory, "ldbcTypes.schema")
+	client, err := testutil.DgraphClient(testutil.ContainerAddr("alpha1", 9080))
+	if err != nil {
+		fmt.Printf("Error while creating client. Error: %v\n", err)
+		cleanupAndExit(1)
+	}
+
+	file, err := ioutil.ReadFile(schemaFile)
+	if err != nil {
+		fmt.Printf("Error while reading schema file. Error: %v\n", err)
+		cleanupAndExit(1)
+	}
+
+	if err = client.Alter(context.Background(), &api.Operation{
+		Schema: string(file),
+	}); err != nil {
+		fmt.Printf("Error while indexing. Error: %v\n", err)
+		cleanupAndExit(1)
+	}
+
+	exitCode := m.Run()
+	cleanupAndExit(exitCode)
+}
+
+func cleanupAndExit(exitCode int) {
+	if testutil.StopAlphasAndDetectRace("./alpha.yml") {
+		// if there is race fail the test
+		exitCode = 1
+	}
+	_ = os.RemoveAll("out")
+	os.Exit(exitCode)
+}