chore(internal/postprocessor): add validation command and CI (#9575)

.github/.OwlBot.yaml

@@ -70,7 +70,7 @@ deep-remove-regex:
   - /cloudcontrolspartner/apiv1/
   - /clouddms/apiv1/
   - /cloudprofiler/apiv2/
-  - /cloudquotas/v1/
+  - /cloudquotas/apiv1/
   - /cloudtasks/apiv2/
   - /cloudtasks/apiv2beta2/
   - /cloudtasks/apiv2beta3/

.github/workflows/owlbot_validation.yml

@@ -0,0 +1,24 @@
+---
+name: owlbot_validation
+on:
+  pull_request:
+    paths:
+      - 'internal/postprocessor/config.yaml'
+      - '.github/.OwlBot.yaml'
+
+permissions:
+  contents: read
+
+jobs:
+  validate:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 1
+    - uses: actions/setup-go@v5
+      with:
+        go-version: 1.21.x
+    - run: |
+        git clone -b master --single-branch --depth=1 https://github.com/googleapis/googleapis.git 
+        go run ./internal/postprocessor validate -googleapis-dir=./googleapis

internal/postprocessor/README.md

@@ -82,3 +82,35 @@ The post-processor initializes new modules by generating the required files
 To add a new module, add the directory name of the module to `modules` in
 `google-cloud-go/internal/postprocessor/config.yaml`. Please maintain
 alphabetical ordering of the module names.
+
+## Validating your config changes
+
+The `validate` command is run as a presubmit on changes to either the
+`.github/.OwlBot.yaml` or the `internal/postprocessor/config.yaml`.
+
+If you want to run it manually, from the **repository root**, simply run the
+following:
+
+```
+go run ./internal/postprocessor validate
+```
+
+If you want to validate existence of service config yaml in the PostProcessor
+config, provide an absolute path to a local clone of `googleapis`:
+
+```
+go run ./internal/postprocessor validate -googleapis-dir=$GOOGLEAPIS
+```
+
+If you want validate a specific config file, not the repository default, then
+provide aboslute paths to either or both config files like so:
+
+```
+go run ./internal/postprocessor \
+   -owl-bot-config=$OWL_BOT_YAML \
+   -processor-config=$CONFIG_YAML
+```
+
+If you think there is an issue with the validator, just fix it in the same CL
+as the config change that triggered it. No need to update the postprocessor sha
+when the validate command is changed, it runs from HEAD of the branch.

internal/postprocessor/config.go

@@ -38,6 +38,29 @@ type config struct {
 	ManualClientInfo []*ManifestEntry
 }
 
+type serviceConfigEntry struct {
+	InputDirectory       string `yaml:"input-directory"`
+	ServiceConfig        string `yaml:"service-config"`
+	ImportPath           string `yaml:"import-path"`
+	RelPath              string `yaml:"rel-path"`
+	ReleaseLevelOverride string `yaml:"release-level-override"`
+}
+
+type postProcessorConfig struct {
+	Modules        []string              `yaml:"modules"`
+	ServiceConfigs []*serviceConfigEntry `yaml:"service-configs"`
+	ManualClients  []*ManifestEntry      `yaml:"manual-clients"`
+}
+
+type deepCopyConfig struct {
+	Source string `yaml:"source"`
+	Dest   string `yaml:"dest"`
+}
+type owlBotConfig struct {
+	DeepCopyRegex   []deepCopyConfig `yaml:"deep-copy-regex"`
+	DeepRemoveRegex []string         `yaml:"deep-remove-regex"`
+}
+
 // libraryInfo contains information about a GAPIC client.
 type libraryInfo struct {
 	// ImportPath is the Go import path for the GAPIC library.
@@ -54,54 +77,50 @@ type libraryInfo struct {
 	ReleaseLevel string
 }
 
-func (p *postProcessor) loadConfig() error {
-	var postProcessorConfig struct {
-		Modules        []string `yaml:"modules"`
-		ServiceConfigs []*struct {
-			InputDirectory       string `yaml:"input-directory"`
-			ServiceConfig        string `yaml:"service-config"`
-			ImportPath           string `yaml:"import-path"`
-			RelPath              string `yaml:"rel-path"`
-			ReleaseLevelOverride string `yaml:"release-level-override"`
-		} `yaml:"service-configs"`
-		ManualClients []*ManifestEntry `yaml:"manual-clients"`
-	}
-	b, err := os.ReadFile(filepath.Join(p.googleCloudDir, "internal", "postprocessor", "config.yaml"))
+func loadConfigs(ppcPath, obcPath string) (*postProcessorConfig, *owlBotConfig, error) {
+	var ppc postProcessorConfig
+	b, err := os.ReadFile(ppcPath)
 	if err != nil {
-		return err
+		return nil, nil, err
 	}
-	if err := yaml.Unmarshal(b, &postProcessorConfig); err != nil {
-		return err
-	}
-	var owlBotConfig struct {
-		DeepCopyRegex []struct {
-			Source string `yaml:"source"`
-			Dest   string `yaml:"dest"`
-		} `yaml:"deep-copy-regex"`
+	if err := yaml.Unmarshal(b, &ppc); err != nil {
+		return nil, nil, err
 	}
-	b2, err := os.ReadFile(filepath.Join(p.googleCloudDir, ".github", ".OwlBot.yaml"))
+	var obc owlBotConfig
+	b2, err := os.ReadFile(obcPath)
 	if err != nil {
-		return err
+		return nil, nil, err
 	}
-	if err := yaml.Unmarshal(b2, &owlBotConfig); err != nil {
+	if err := yaml.Unmarshal(b2, &obc); err != nil {
+		return nil, nil, err
+	}
+
+	return &ppc, &obc, nil
+}
+
+func (p *postProcessor) loadConfig() error {
+	ppcPath := filepath.Join(p.googleCloudDir, "internal", "postprocessor", "config.yaml")
+	obcPath := filepath.Join(p.googleCloudDir, ".github", ".OwlBot.yaml")
+	ppc, obc, err := loadConfigs(ppcPath, obcPath)
+	if err != nil {
 		return err
 	}
 
 	c := &config{
-		Modules:                postProcessorConfig.Modules,
+		Modules:                ppc.Modules,
 		ClientRelPaths:         make([]string, 0),
 		GoogleapisToImportPath: make(map[string]*libraryInfo),
-		ManualClientInfo:       postProcessorConfig.ManualClients,
+		ManualClientInfo:       ppc.ManualClients,
 	}
-	for _, v := range postProcessorConfig.ServiceConfigs {
+	for _, v := range ppc.ServiceConfigs {
 		c.GoogleapisToImportPath[v.InputDirectory] = &libraryInfo{
 			ServiceConfig: v.ServiceConfig,
 			ImportPath:    v.ImportPath,
 			RelPath:       v.RelPath,
 			ReleaseLevel:  v.ReleaseLevelOverride,
 		}
 	}
-	for _, v := range owlBotConfig.DeepCopyRegex {
+	for _, v := range obc.DeepCopyRegex {
 		i := strings.Index(v.Source, "/cloud.google.com/go")
 		li, ok := c.GoogleapisToImportPath[v.Source[1:i]]
 		if !ok {

internal/postprocessor/config.yaml

@@ -979,9 +979,6 @@ service-configs:
   - input-directory: google/devtools/cloudbuild/v2
     service-config: cloudbuild_v2.yaml
     import-path: cloud.google.com/go/cloudbuild/apiv2
-  - input-directory: google/cloud/compute/v1
-    service-config: compute_v1.yaml
-    import-path: cloud.google.com/go/compute/apiv1
   - input-directory: google/cloud/confidentialcomputing/v1
     service-config: confidentialcomputing_v1.yaml
     import-path: cloud.google.com/go/confidentialcomputing/apiv1

internal/postprocessor/main.go

@@ -70,6 +70,17 @@ func main() {
 	githubUsername := flag.String("gh-user", "googleapis", "GitHub username where repo lives.")
 	prFilepath := flag.String("pr-file", "/workspace/new_pull_request_text.txt", "Path at which to write text file if changing PR title or body.")
 
+	if len(os.Args) > 1 {
+		switch os.Args[1] {
+		case "validate":
+			log.Println("Starting config validation.")
+			if err := validate(); err != nil {
+				log.Fatal(err)
+			}
+			log.Println("Validation complete.")
+			return
+		}
+	}
 	flag.Parse()
 	ctx := context.Background()