Tour of beam learning materials CI/CD refactoring and templating (#25080

) * Add support for templating in Tour of Beam learning materials * Process learning materials through the template engine * refactoring ci/cd for templated learning materials * adding template processing * refactoring learning content * fixing incorrect sample path * minor formatting comments * missed complexity added add code review comments * fix possible err overwrite Co-authored-by: Timur Sultanov <timur.sultanov@akvelon.com>
apache · Jan 19, 2023 · 3e1291c · 3e1291c
1 parent 6517731
commit 3e1291c
Show file tree

Hide file tree

Showing 108 changed files with 602 additions and 1,590 deletions.
diff --git a/learning/tour-of-beam/backend/internal/fs_content/load.go b/learning/tour-of-beam/backend/internal/fs_content/load.go
@@ -16,13 +16,14 @@
 package fs_content
 
 import (
+	"bytes"
 	"fmt"
 	"io/fs"
-	"io/ioutil"
 	"log"
 	"os"
 	"path/filepath"
 	"regexp"
+	"text/template"
 
 	tob "beam.apache.org/learning/tour-of-beam/backend/internal"
 )
@@ -38,32 +39,45 @@ const (
 )
 
 type learningPathInfo struct {
-	Sdk     string   `yaml:"sdk"`
+	Sdk     []string `yaml:"sdk"`
 	Content []string `yaml:"content"`
 }
 
 type learningModuleInfo struct {
+	Sdk        []string `yaml:"sdk"`
 	Id         string   `yaml:"id"`
 	Name       string   `yaml:"name"`
 	Complexity string   `yaml:"complexity"`
 	Content    []string `yaml:"content"`
 }
 
 type learningGroupInfo struct {
+	Sdk     []string `yaml:"sdk"`
 	Id      string   `yaml:"id"`
 	Name    string   `yaml:"name"`
 	Content []string `yaml:"content"`
 }
 
 type learningUnitInfo struct {
-	Id           string `yaml:"id"`
-	Name         string `yaml:"name"`
-	TaskName     string `yaml:"taskName"`
-	SolutionName string `yaml:"solutionName"`
+	Sdk          []string `yaml:"sdk"`
+	Id           string   `yaml:"id"`
+	Name         string   `yaml:"name"`
+	TaskName     string   `yaml:"taskName"`
+	SolutionName string   `yaml:"solutionName"`
 }
 
 func collectUnit(infopath string, ctx *sdkContext) (unit *tob.Unit, err error) {
 	info := loadLearningUnitInfo(infopath)
+
+	supported, err := isSupportedSdk(info.Sdk, ctx, infopath)
+	if err != nil {
+		return nil, err
+	}
+	if !supported {
+		log.Printf("Unit %v at %v not supported in %v\n", info.Id, infopath, ctx.sdk)
+		return nil, nil
+	}
+
 	log.Printf("Found Unit %v metadata at %v\n", info.Id, infopath)
 	ctx.idsWatcher.CheckId(info.Id)
 	builder := NewUnitBuilder(info, ctx.sdk)
@@ -77,15 +91,19 @@ func collectUnit(infopath string, ctx *sdkContext) (unit *tob.Unit, err error) {
 				return filepath.SkipDir
 
 			case d.Name() == descriptionMd:
-				content, err := ioutil.ReadFile(path)
+				templateSource, err := os.ReadFile(path)
+				if err != nil {
+					return err
+				}
+				content, err := processTemplate(templateSource, ctx.sdk)
 				if err != nil {
 					return err
 				}
 				builder.AddDescription(string(content))
 
 			// Here we rely on that WalkDir entries are lexically sorted
 			case regexp.MustCompile(hintMdRegexp).MatchString(d.Name()):
-				content, err := ioutil.ReadFile(path)
+				content, err := os.ReadFile(path)
 				if err != nil {
 					return err
 				}
@@ -97,27 +115,57 @@ func collectUnit(infopath string, ctx *sdkContext) (unit *tob.Unit, err error) {
 	return builder.Build(), err
 }
 
+func processTemplate(source []byte, sdk tob.Sdk) ([]byte, error) {
+	t := template.New("")
+	t, err := t.Parse(string(source))
+	if err != nil {
+		return nil, err
+	}
+
+	var output bytes.Buffer
+	err = t.Execute(&output, struct{ Sdk tob.Sdk }{Sdk: sdk})
+	if err != nil {
+		return nil, err
+	}
+
+	return output.Bytes(), nil
+}
+
 func collectGroup(infopath string, ctx *sdkContext) (*tob.Group, error) {
 	info := loadLearningGroupInfo(infopath)
+
+	supported, err := isSupportedSdk(info.Sdk, ctx, infopath)
+	if err != nil {
+		return nil, err
+	}
+	if !supported {
+		log.Printf("Group %v at %v not supported in %v\n", info.Id, infopath, ctx.sdk)
+		return nil, nil
+	}
+
 	log.Printf("Found Group %v metadata at %v\n", info.Name, infopath)
 	group := tob.Group{Id: info.Id, Title: info.Name}
 	for _, item := range info.Content {
 		node, err := collectNode(filepath.Join(infopath, "..", item), ctx)
 		if err != nil {
 			return &group, err
 		}
-		group.Nodes = append(group.Nodes, node)
+		if node == nil {
+			continue
+		}
+		group.Nodes = append(group.Nodes, *node)
 	}
 
 	return &group, nil
 }
 
 // Collect node which is either a unit or a group.
-func collectNode(rootpath string, ctx *sdkContext) (node tob.Node, err error) {
+func collectNode(rootpath string, ctx *sdkContext) (*tob.Node, error) {
 	files, err := os.ReadDir(rootpath)
 	if err != nil {
-		return node, err
+		return nil, err
 	}
+	node := &tob.Node{}
 	for _, f := range files {
 		switch f.Name() {
 		case unitInfoYaml:
@@ -127,46 +175,73 @@ func collectNode(rootpath string, ctx *sdkContext) (node tob.Node, err error) {
 			node.Type = tob.NODE_GROUP
 			node.Group, err = collectGroup(filepath.Join(rootpath, groupInfoYaml), ctx)
 		}
+		if err != nil {
+			return nil, err
+		}
 	}
 	if node.Type == tob.NODE_UNDEFINED {
 		return node, fmt.Errorf("node undefined at %v", rootpath)
 	}
+	if node.Group == nil && node.Unit == nil {
+		return nil, err
+	}
 	return node, err
 }
 
-func collectModule(infopath string, ctx *sdkContext) (tob.Module, error) {
+func collectModule(infopath string, ctx *sdkContext) (*tob.Module, error) {
 	info := loadLearningModuleInfo(infopath)
+
+	supported, err := isSupportedSdk(info.Sdk, ctx, infopath)
+	if err != nil {
+		return nil, err
+	}
+	if !supported {
+		log.Printf("Module %v at %v not supported in %v\n", info.Id, infopath, ctx.sdk)
+		return nil, nil
+	}
+
 	log.Printf("Found Module %v metadata at %v\n", info.Id, infopath)
 	ctx.idsWatcher.CheckId(info.Id)
 	module := tob.Module{Id: info.Id, Title: info.Name, Complexity: info.Complexity}
 	for _, item := range info.Content {
 		node, err := collectNode(filepath.Join(infopath, "..", item), ctx)
 		if err != nil {
-			return tob.Module{}, err
+			return nil, err
 		}
-		module.Nodes = append(module.Nodes, node)
+		if node == nil {
+			continue
+		}
+		module.Nodes = append(module.Nodes, *node)
 	}
 
-	return module, nil
+	return &module, nil
 }
 
-func collectSdk(infopath string) (tree tob.ContentTree, err error) {
+func collectSdk(infopath string) (trees []tob.ContentTree, err error) {
 	info := loadLearningPathInfo(infopath)
-	tree.Sdk = tob.ParseSdk(info.Sdk)
-	if tree.Sdk == tob.SDK_UNDEFINED {
-		return tree, fmt.Errorf("unknown SDK at %v", infopath)
+
+	sdks, err := getSupportedSdk(info.Sdk, infopath)
+	if err != nil {
+		return trees, err
 	}
-	log.Printf("Found Sdk %v metadata at %v\n", info.Sdk, infopath)
-	ctx := newSdkContext(tree.Sdk)
-	for _, item := range info.Content {
-		mod, err := collectModule(filepath.Join(infopath, "..", item, moduleInfoYaml), ctx)
-		if err != nil {
-			return tree, err
+	for sdk := range sdks {
+		tree := tob.ContentTree{}
+		tree.Sdk = sdk
+		log.Printf("Found Sdk %v metadata at %v\n", sdk, infopath)
+		ctx := newSdkContext(tree.Sdk)
+		for _, item := range info.Content {
+			mod, err := collectModule(filepath.Join(infopath, "..", item, moduleInfoYaml), ctx)
+			if err != nil {
+				return trees, err
+			}
+			if mod != nil {
+				tree.Modules = append(tree.Modules, *mod)
+			}
 		}
-		tree.Modules = append(tree.Modules, mod)
+		trees = append(trees, tree)
 	}
 
-	return tree, nil
+	return trees, nil
 }
 
 // Build a content tree for each SDK
@@ -179,11 +254,12 @@ func CollectLearningTree(rootpath string) (trees []tob.ContentTree, err error) {
 			return err
 		}
 		if d.Name() == contentInfoYaml {
-			tree, err := collectSdk(path)
+
+			collected, err := collectSdk(path)
 			if err != nil {
 				return err
 			}
-			trees = append(trees, tree)
+			trees = append(trees, collected...)
 			// don't walk into SDK subtree (already done by collectSdk)
 			return filepath.SkipDir
 		}
@@ -192,3 +268,25 @@ func CollectLearningTree(rootpath string) (trees []tob.ContentTree, err error) {
 
 	return trees, err
 }
+
+func isSupportedSdk(sdks []string, ctx *sdkContext, infopath string) (ok bool, err error) {
+	sdk, err := getSupportedSdk(sdks, infopath)
+	if err != nil {
+		return false, err
+	}
+	_, ok = sdk[ctx.sdk]
+	return
+}
+
+func getSupportedSdk(sdk []string, infopath string) (map[tob.Sdk]bool, error) {
+	sdks := make(map[tob.Sdk]bool)
+	for _, s := range sdk {
+		curSdk := tob.ParseSdk(s)
+		if curSdk == tob.SDK_UNDEFINED {
+			return sdks, fmt.Errorf("unknown SDK at %v", infopath)
+		}
+		sdks[curSdk] = true
+	}
+
+	return sdks, nil
+}
diff --git a/learning/tour-of-beam/backend/internal/fs_content/load_test.go b/learning/tour-of-beam/backend/internal/fs_content/load_test.go
@@ -90,3 +90,70 @@ func TestSample(t *testing.T) {
 		},
 	}, trees[1])
 }
+
+// TestTemplates test that templating engine is used correctly.
+// The test itself is intended as an example of typical template usage.
+func TestTemplateProcessing(t *testing.T) {
+	goSdkExpected := "Go SDK"
+	pythonSdkExpected := "Python SDK"
+	javaSdkExpected := "Java SDK"
+	scioSdkExpected := "SCIO SDK"
+	template := fmt.Sprintf(
+		"Using "+
+			"{{if (eq .Sdk \"go\")}}%s{{end}}"+
+			"{{if (eq .Sdk \"python\")}}%s{{end}}"+
+			"{{if (eq .Sdk \"java\")}}%s{{end}}"+
+			"{{if (eq .Sdk \"scio\")}}%s{{end}}",
+		goSdkExpected, pythonSdkExpected, javaSdkExpected, scioSdkExpected)
+
+	goOrJavaExpected := "Text for Go or Java SDK"
+	templateAboutGoOrJava := fmt.Sprintf("{{if (eq .Sdk \"go\" \"java\")}}%s{{end}}", goOrJavaExpected)
+
+	for _, s := range []struct {
+		sdk      tob.Sdk
+		template string
+		expected string
+	}{
+		{
+			sdk:      tob.SDK_GO,
+			template: template,
+			expected: fmt.Sprintf("Using %s", goSdkExpected),
+		},
+		{
+			sdk:      tob.SDK_PYTHON,
+			template: template,
+			expected: fmt.Sprintf("Using %s", pythonSdkExpected),
+		},
+		{
+			sdk:      tob.SDK_JAVA,
+			template: template,
+			expected: fmt.Sprintf("Using %s", javaSdkExpected),
+		},
+		{
+			sdk:      tob.SDK_SCIO,
+			template: template,
+			expected: fmt.Sprintf("Using %s", scioSdkExpected),
+		},
+		{
+			sdk:      tob.SDK_GO,
+			template: templateAboutGoOrJava,
+			expected: goOrJavaExpected,
+		},
+		{
+			sdk:      tob.SDK_JAVA,
+			template: templateAboutGoOrJava,
+			expected: goOrJavaExpected,
+		},
+		{
+			sdk:      tob.SDK_SCIO,
+			template: templateAboutGoOrJava,
+			expected: "",
+		},
+	} {
+		res, err := processTemplate([]byte(s.template), s.sdk)
+		if err != nil {
+			panic(err)
+		}
+		assert.Equal(t, s.expected, string(res))
+	}
+}
diff --git a/...s/learning-content/java/content-info.yaml → ...amples/learning-content/content-info.yaml b/...s/learning-content/java/content-info.yaml → ...amples/learning-content/content-info.yaml
@@ -1,4 +1,6 @@
-sdk: Java
+sdk:
+  - Java
+  - Python
 content:
   - module 1
   - module 2
diff --git a/...ent/python/module 1/group/group-info.yaml → ...ng-content/module 1/group/group-info.yaml b/...ent/python/module 1/group/group-info.yaml → ...ng-content/module 1/group/group-info.yaml
@@ -1,3 +1,5 @@
+sdk:
+  - Python
 id: group1
 name: The Group
 complexity: BASIC

diff --git a/...va/module 1/unit-challenge/description.md → ...ule 1/group/unit-challenge/description.md b/...va/module 1/unit-challenge/description.md → ...ule 1/group/unit-challenge/description.md
diff --git a/...ent/java/module 1/unit-challenge/hint1.md → ...nt/module 1/group/unit-challenge/hint1.md b/...ent/java/module 1/unit-challenge/hint1.md → ...nt/module 1/group/unit-challenge/hint1.md
diff --git a/...ent/java/module 1/unit-challenge/hint2.md → ...nt/module 1/group/unit-challenge/hint2.md b/...ent/java/module 1/unit-challenge/hint2.md → ...nt/module 1/group/unit-challenge/hint2.md
diff --git a/...va/module 1/unit-challenge/unit-info.yaml → ...ule 1/group/unit-challenge/unit-info.yaml b/...va/module 1/unit-challenge/unit-info.yaml → ...ule 1/group/unit-challenge/unit-info.yaml
@@ -1,3 +1,5 @@
+sdk:
+  - Python
 id: challenge1
 name: Challenge Name
 complexity: BASIC

diff --git a/...java/module 1/unit-example/unit-info.yaml → ...odule 1/group/unit-example/unit-info.yaml b/...java/module 1/unit-example/unit-info.yaml → ...odule 1/group/unit-example/unit-info.yaml
@@ -1,3 +1,5 @@
+sdk:
+  - Python
 id: example1
 name: Example Unit Name
 taskName: ExampleName
diff --git a/...python/module 1/intro-unit/unit-info.yaml → ...ontent/module 1/intro-unit/unit-info.yaml b/...python/module 1/intro-unit/unit-info.yaml → ...ontent/module 1/intro-unit/unit-info.yaml
@@ -1,3 +1,5 @@
+sdk:
+  - Python
 id: intro-unit 
 name: Example Unit Name
 taskName: ExampleName