[MM-53432] Refactor to support multiple job types

.github/workflows/ci.yml

@@ -12,7 +12,7 @@ jobs:
 
     strategy:
       matrix:
-        go-version: [1.19.x]
+        go-version: [1.21.x]
 
     runs-on: ubuntu-latest
 

.github/workflows/golangci-lint.yml

@@ -7,7 +7,7 @@ jobs:
     name: lint
     strategy:
       matrix:
-        go-version: [1.19.x]
+        go-version: [1.21.x]
     runs-on: ubuntu-latest
     steps:
       - name: Install Go
@@ -19,7 +19,7 @@ jobs:
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v2
         with:
-          version: v1.52.2
+          version: v1.54.2
 
           # Optional: if set to true then the action will use pre-installed Go.
           skip-go-installation: true

Makefile

@@ -41,8 +41,8 @@ DOCKER_REGISTRY_REPO    ?= mattermost/${APP_NAME}-daily
 DOCKER_USER             ?= user
 DOCKER_PASSWORD         ?= password
 ## Docker Images
-DOCKER_IMAGE_GO         += "golang:${GO_VERSION}@sha256:dd9ad81920b63c7f9f18823d888d5fdcc7e7516086fd16654d07bc437f0e2427"
-DOCKER_IMAGE_GOLINT     += "golangci/golangci-lint:v1.52.2@sha256:5fa6a92ab28ca3421c88d2b6cd794c9759d05a999aceca73053d014aad41b9d3"
+DOCKER_IMAGE_GO         += "golang:${GO_VERSION}@sha256:b17c35044f4062d83c815434615997eed97697daae8745c6dd39dc3673b87efb"
+DOCKER_IMAGE_GOLINT     += "golangci/golangci-lint:v1.54.2@sha256:abe731fe6bb335a30eab303a41dd5c2b630bb174372a4da08e3d42eab5324127"
 DOCKER_IMAGE_DOCKERLINT += "hadolint/hadolint:v2.9.2@sha256:d355bd7df747a0f124f3b5e7b21e9dafd0cb19732a276f901f0fdee243ec1f3b"
 DOCKER_IMAGE_COSIGN     += "bitnami/cosign:1.8.0@sha256:8c2c61c546258fffff18b47bb82a65af6142007306b737129a7bd5429d53629a"
 DOCKER_IMAGE_GH_CLI     += "registry.internal.mattermost.com/images/build-ci:3.16.0@sha256:f6a229a9ababef3c483f237805ee4c3dbfb63f5de4fbbf58f4c4b6ed8fcd34b6"

build/Dockerfile

@@ -1,7 +1,7 @@
 # This dockerfile is used to build Mattermost rtcd
 # A multi stage build, with golang used as a builder
 # and gcr.io/distroless/static as runner
-ARG GO_IMAGE=golang:1.19.8@sha256:dd9ad81920b63c7f9f18823d888d5fdcc7e7516086fd16654d07bc437f0e2427
+ARG GO_IMAGE=golang:1.21@sha256:b17c35044f4062d83c815434615997eed97697daae8745c6dd39dc3673b87efb
 # hadolint ignore=DL3006
 FROM ${GO_IMAGE} as builder
 

go.mod

@@ -1,6 +1,6 @@
 module github.com/mattermost/calls-offloader
 
-go 1.18
+go 1.21.1
 
 require (
 	git.mills.io/prologic/bitcask v1.0.2
@@ -9,11 +9,12 @@ require (
 	github.com/docker/docker v20.10.9+incompatible
 	github.com/gorilla/mux v1.8.0
 	github.com/kelseyhightower/envconfig v1.4.0
-	github.com/mattermost/calls-recorder v0.4.2-0.20230830151011-a4935f7c38bf
-	github.com/mattermost/mattermost/server/public v0.0.0-20230613002302-62a3ee8adcb5
+	github.com/mattermost/calls-recorder v0.5.2-0.20231005222207-35b06266dd56
+	github.com/mattermost/calls-transcriber v0.0.0-20231009225308-0909fc1dad5b
+	github.com/mattermost/mattermost/server/public v0.0.6
 	github.com/pborman/uuid v1.2.1
-	github.com/stretchr/testify v1.8.2
-	golang.org/x/crypto v0.8.0
+	github.com/stretchr/testify v1.8.4
+	golang.org/x/crypto v0.11.0
 	k8s.io/api v0.27.3
 	k8s.io/apimachinery v0.27.3
 	k8s.io/client-go v0.27.3
@@ -55,19 +56,19 @@ require (
 	github.com/plar/go-adaptive-radix-tree v1.0.4 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/sirupsen/logrus v1.9.0 // indirect
-	github.com/wiggin77/merror v1.0.4 // indirect
+	github.com/wiggin77/merror v1.0.5 // indirect
 	github.com/wiggin77/srslog v1.0.1 // indirect
 	golang.org/x/exp v0.0.0-20200908183739-ae8ad444f925 // indirect
-	golang.org/x/net v0.9.0 // indirect
-	golang.org/x/oauth2 v0.4.0 // indirect
-	golang.org/x/sys v0.7.0 // indirect
-	golang.org/x/term v0.7.0 // indirect
-	golang.org/x/text v0.9.0 // indirect
+	golang.org/x/net v0.12.0 // indirect
+	golang.org/x/oauth2 v0.7.0 // indirect
+	golang.org/x/sys v0.11.0 // indirect
+	golang.org/x/term v0.10.0 // indirect
+	golang.org/x/text v0.11.0 // indirect
 	golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect
-	google.golang.org/grpc v1.54.0 // indirect
-	google.golang.org/protobuf v1.30.0 // indirect
+	google.golang.org/grpc v1.56.1 // indirect
+	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect

public/job/job.go

@@ -9,26 +9,34 @@ import (
 	"regexp"
 
 	recorder "github.com/mattermost/calls-recorder/cmd/recorder/config"
+	transcriber "github.com/mattermost/calls-transcriber/cmd/transcriber/config"
 )
 
 type Type string
 
 const (
-	TypeRecording Type = "recording"
+	TypeRecording    Type = "recording"
+	TypeTranscribing Type = "transcribing"
 )
 
-const MinSupportedRecorderVersion = "0.4.2"
+const (
+	MinSupportedRecorderVersion    = "0.4.2"
+	MinSupportedTranscriberVersion = "0.1.0"
+)
 
-// We currently support two formats, semantic version tag or image hash (sha256).
-// TODO: Consider deprecating tag version and switch to hash only.
-var recorderRunnerREs = []*regexp.Regexp{
-	regexp.MustCompile(`^mattermost/calls-recorder@sha256:\w{64}$`),
-	regexp.MustCompile(`^mattermost/calls-recorder:v((?:0|[1-9]\d*)\.(?:0|[1-9]\d*)\.(?:0|[1-9]\d*))$`),
-	regexp.MustCompile(`^mattermost/calls-recorder-daily:v((?:0|[1-9]\d*)\.(?:0|[1-9]\d*)\.(?:0|[1-9]\d*))-dev$`),
-}
+var (
+	recorderRunnerREs = []*regexp.Regexp{
+		regexp.MustCompile(`^mattermost/calls-recorder:v((?:0|[1-9]\d*)\.(?:0|[1-9]\d*)\.(?:0|[1-9]\d*))$`),
+		regexp.MustCompile(`^mattermost/calls-recorder-daily:v((?:0|[1-9]\d*)\.(?:0|[1-9]\d*)\.(?:0|[1-9]\d*))-dev$`),
+	}
+	transcriberRunnerREs = []*regexp.Regexp{
+		regexp.MustCompile(`^mattermost/calls-transcriber:v((?:0|[1-9]\d*)\.(?:0|[1-9]\d*)\.(?:0|[1-9]\d*))$`),
+		regexp.MustCompile(`^mattermost/calls-transcriber-daily:v((?:0|[1-9]\d*)\.(?:0|[1-9]\d*)\.(?:0|[1-9]\d*))-dev$`),
+	}
+)
 
 type ServiceConfig struct {
-	Runner string
+	Runners []string
 }
 
 type Job struct {
@@ -49,49 +57,71 @@ type Config struct {
 type StopCb func(job Job, success bool) error
 
 func (c ServiceConfig) IsValid() error {
-	return RunnerIsValid(c.Runner)
+	if len(c.Runners) == 0 {
+		return fmt.Errorf("invalid empty Runners")
+	}
+
+	for _, runner := range c.Runners {
+		if err := RunnerIsValid(runner); err != nil {
+			return err
+		}
+	}
+	return nil
 }
 
 func RunnerIsValid(runner string) error {
-	if os.Getenv("DEV_MODE") == "true" {
+	if os.Getenv("DEV_MODE") == "true" || os.Getenv("TEST_MODE") == "true" {
 		return nil
 	}
 
+	if runner == "" {
+		return fmt.Errorf("should not be empty")
+	}
+
 	for _, re := range recorderRunnerREs {
 		if matches := re.FindStringSubmatch(runner); len(matches) > 1 {
 			return checkMinVersion(MinSupportedRecorderVersion, matches[1])
 		}
 	}
-	return fmt.Errorf("failed to validate runner")
+
+	for _, re := range transcriberRunnerREs {
+		if matches := re.FindStringSubmatch(runner); len(matches) > 1 {
+			return checkMinVersion(MinSupportedTranscriberVersion, matches[1])
+		}
+	}
+
+	return fmt.Errorf("failed to validate runner %q", runner)
 }
 
 func (c Config) IsValid() error {
 	if c.Type == "" {
 		return fmt.Errorf("invalid Type value: should not be empty")
 	}
 
-	if c.Runner == "" {
-		return fmt.Errorf("invalid Runner value: should not be empty")
+	if err := RunnerIsValid(c.Runner); err != nil {
+		return fmt.Errorf("invalid Runner value: %w", err)
+	}
+
+	if c.MaxDurationSec <= 0 {
+		return fmt.Errorf("invalid MaxDurationSec value: should be positive")
 	}
 
 	switch c.Type {
 	case TypeRecording:
-		if err := RunnerIsValid(c.Runner); err != nil {
-			return fmt.Errorf("invalid Runner value: %w", err)
-		}
-
 		cfg := (&recorder.RecorderConfig{}).FromMap(c.InputData)
 		cfg.SetDefaults()
 		if err := cfg.IsValid(); err != nil {
 			return fmt.Errorf("failed to validate InputData: %w", err)
 		}
+	case TypeTranscribing:
+		cfg := (&transcriber.CallTranscriberConfig{}).FromMap(c.InputData)
+		cfg.SetDefaults()
+		if err := cfg.IsValid(); err != nil {
+			return fmt.Errorf("failed to validate InputData: %w", err)
+		}
 	default:
 		return fmt.Errorf("invalid Type value: %q", c.Type)
 	}
 
-	if c.MaxDurationSec <= 0 {
-		return fmt.Errorf("invalid MaxDurationSec value: should be positive")
-	}
-
 	return nil
 }

public/job/job_test.go

@@ -44,23 +44,15 @@ func TestJobConfigIsValid(t *testing.T) {
 				Type:   TypeRecording,
 				Runner: "testrepo/calls-recorder:v0.1.0",
 			},
-			expectedError: "invalid Runner value: failed to validate runner",
+			expectedError: `invalid Runner value: failed to validate runner "testrepo/calls-recorder:v0.1.0"`,
 		},
 		{
 			name: "invalid runner",
 			cfg: Config{
 				Type:   TypeRecording,
 				Runner: "testrepo/calls-recorder@sha256:abcde",
 			},
-			expectedError: "invalid Runner value: failed to validate runner",
-		},
-		{
-			name: "invalid job type",
-			cfg: Config{
-				Type:   "invalid",
-				Runner: "mattermost/calls-recorder:v0.1.0",
-			},
-			expectedError: "invalid Type value: \"invalid\"",
+			expectedError: `invalid Runner value: failed to validate runner "testrepo/calls-recorder@sha256:abcde"`,
 		},
 		{
 			name: "invalid max duration",
@@ -72,6 +64,15 @@ func TestJobConfigIsValid(t *testing.T) {
 			},
 			expectedError: "invalid MaxDurationSec value: should be positive",
 		},
+		{
+			name: "invalid job type",
+			cfg: Config{
+				Type:           "invalid",
+				Runner:         "mattermost/calls-recorder:v" + MinSupportedRecorderVersion,
+				MaxDurationSec: 60,
+			},
+			expectedError: "invalid Type value: \"invalid\"",
+		},
 		{
 			name: "invalid version",
 			cfg: Config{
@@ -122,12 +123,15 @@ func TestServiceConfigIsValid(t *testing.T) {
 		{
 			name: "empty config",
 			cfg:  ServiceConfig{},
-			err:  "failed to validate runner",
+			err:  "invalid empty Runners",
 		},
 		{
 			name: "valid config",
 			cfg: ServiceConfig{
-				Runner: "mattermost/calls-recorder:v" + MinSupportedRecorderVersion,
+				Runners: []string{
+					"mattermost/calls-recorder:v" + MinSupportedRecorderVersion,
+					"mattermost/calls-transcriber:v" + MinSupportedTranscriberVersion,
+				},
 			},
 		},
 	}

service/client_test.go

@@ -282,7 +282,7 @@ func TestClientAutoRegister(t *testing.T) {
 
 	t.Run("unauthorized", func(t *testing.T) {
 		err = c.Init(job.ServiceConfig{
-			Runner: "",
+			Runners: nil,
 		})
 		require.EqualError(t, err, "unauthorized")
 	})
@@ -291,7 +291,7 @@ func TestClientAutoRegister(t *testing.T) {
 
 	t.Run("automatic registration", func(t *testing.T) {
 		err = c.Init(job.ServiceConfig{
-			Runner: "mattermost/calls-recorder:v" + job.MinSupportedRecorderVersion,
+			Runners: []string{"mattermost/calls-recorder:v" + job.MinSupportedRecorderVersion},
 		})
 		require.NoError(t, err)