diff --git a/go.work b/go.work new file mode 100644 index 0000000000..efb7f2a7c3 --- /dev/null +++ b/go.work @@ -0,0 +1,3 @@ +go 1.24.7 + +use ./internal/librariangen diff --git a/internal/librariangen/.gitignore b/internal/librariangen/.gitignore new file mode 100644 index 0000000000..382f954247 --- /dev/null +++ b/internal/librariangen/.gitignore @@ -0,0 +1 @@ +workspace/ \ No newline at end of file diff --git a/internal/librariangen/bazel/parser.go b/internal/librariangen/bazel/parser.go index a041903e1c..e36bd464aa 100644 --- a/internal/librariangen/bazel/parser.go +++ b/internal/librariangen/bazel/parser.go @@ -28,6 +28,7 @@ import ( // Config holds configuration extracted from a googleapis BUILD.bazel file. type Config struct { + gapicYAML string grpcServiceConfig string restNumericEnums bool serviceYAML string @@ -38,6 +39,9 @@ type Config struct { // HasGAPIC indicates whether the GAPIC generator should be run. func (c *Config) HasGAPIC() bool { return c.hasGAPIC } +// GapicYAML is the GAPIC config file in the API version directory in googleapis. +func (c *Config) GapicYAML() string { return c.gapicYAML } + // ServiceYAML is the client config file in the API version directory in googleapis. func (c *Config) ServiceYAML() string { return c.serviceYAML } @@ -81,6 +85,7 @@ func Parse(dir string) (*Config, error) { if c.restNumericEnums, err = findBool(gapicLibraryBlock, "rest_numeric_enums"); err != nil { return nil, fmt.Errorf("librariangen: failed to parse BUILD.bazel file %s: %w", fp, err) } + c.gapicYAML = strings.TrimPrefix(findString(gapicLibraryBlock, "gapic_yaml"), ":") } if err := c.Validate(); err != nil { return nil, fmt.Errorf("librariangen: invalid bazel config in %s: %w", dir, err) diff --git a/internal/librariangen/bazel/parser_test.go b/internal/librariangen/bazel/parser_test.go index 992665d76d..f0bf6f7b40 100644 --- a/internal/librariangen/bazel/parser_test.go +++ b/internal/librariangen/bazel/parser_test.go @@ -31,6 +31,7 @@ java_grpc_library( java_gapic_library( name = "asset_java_gapic", srcs = [":asset_proto_with_info"], + gapic_yaml = "cloudasset_gapic.yaml", grpc_service_config = "cloudasset_grpc_service_config.json", rest_numeric_enums = True, service_yaml = "cloudasset_v1.yaml", @@ -67,6 +68,11 @@ java_gapic_library( t.Errorf("ServiceYAML() = %q; want %q", got.ServiceYAML(), want) } }) + t.Run("GapicYAML", func(t *testing.T) { + if want := "cloudasset_gapic.yaml"; got.GapicYAML() != want { + t.Errorf("GapicYAML() = %q; want %q", got.GapicYAML(), want) + } + }) t.Run("GRPCServiceConfig", func(t *testing.T) { if want := "cloudasset_grpc_service_config.json"; got.GRPCServiceConfig() != want { t.Errorf("GRPCServiceConfig() = %q; want %q", got.GRPCServiceConfig(), want) @@ -84,7 +90,7 @@ java_gapic_library( }) } -func TestParse_serviceConfigIsTarget(t *testing.T) { +func TestParse_configIsTarget(t *testing.T) { content := ` java_grpc_library( name = "asset_java_grpc", @@ -95,6 +101,7 @@ java_grpc_library( java_gapic_library( name = "asset_java_gapic", srcs = [":asset_proto_with_info"], + gapic_yaml = ":cloudasset_gapic.yaml", grpc_service_config = "cloudasset_grpc_service_config.json", rest_numeric_enums = True, service_yaml = ":cloudasset_v1.yaml", @@ -124,6 +131,9 @@ java_gapic_library( if want := "cloudasset_v1.yaml"; got.ServiceYAML() != want { t.Errorf("ServiceYAML() = %q; want %q", got.ServiceYAML(), want) } + if want := "cloudasset_gapic.yaml"; got.GapicYAML() != want { + t.Errorf("GapicYAML() = %q; want %q", got.GapicYAML(), want) + } } func TestConfig_Validate(t *testing.T) { @@ -136,6 +146,7 @@ func TestConfig_Validate(t *testing.T) { name: "valid GAPIC", cfg: &Config{ hasGAPIC: true, + gapicYAML: "a", serviceYAML: "b", grpcServiceConfig: "c", transport: "d", @@ -149,7 +160,7 @@ func TestConfig_Validate(t *testing.T) { }, { name: "gRPC service config and transport are optional", - cfg: &Config{hasGAPIC: true, serviceYAML: "b"}, + cfg: &Config{hasGAPIC: true, serviceYAML: "b", gapicYAML: "a"}, wantErr: false, }, { diff --git a/internal/librariangen/generate/generator.go b/internal/librariangen/generate/generator.go new file mode 100644 index 0000000000..2ecf3c05a5 --- /dev/null +++ b/internal/librariangen/generate/generator.go @@ -0,0 +1,267 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package generate + +import ( + "archive/zip" + "context" + "errors" + "fmt" + "io" + "log/slog" + "os" + "path/filepath" + "strings" + + "cloud.google.com/java/internal/librariangen/bazel" + "cloud.google.com/java/internal/librariangen/execv" + "cloud.google.com/java/internal/librariangen/protoc" + "cloud.google.com/java/internal/librariangen/request" +) + +// Test substitution vars. +var ( + bazelParse = bazel.Parse + execvRun = execv.Run + requestParse = request.ParseLibrary + protocBuild = protoc.Build +) + +// Config holds the internal librariangen configuration for the generate command. +type Config struct { + // LibrarianDir is the path to the librarian-tool input directory. + // It is expected to contain the generate-request.json file. + LibrarianDir string + // InputDir is the path to the .librarian/generator-input directory from the + // language repository. + InputDir string + // OutputDir is the path to the empty directory where librariangen writes + // its output. + OutputDir string + // SourceDir is the path to a complete checkout of the googleapis repository. + SourceDir string +} + +// Validate ensures that the configuration is valid. +func (c *Config) Validate() error { + if c.LibrarianDir == "" { + return errors.New("librariangen: librarian directory must be set") + } + if c.InputDir == "" { + return errors.New("librariangen: input directory must be set") + } + if c.OutputDir == "" { + return errors.New("librariangen: output directory must be set") + } + if c.SourceDir == "" { + return errors.New("librariangen: source directory must be set") + } + return nil +} + +// Generate is the main entrypoint for the `generate` command. It orchestrates +// the entire generation process. +func Generate(ctx context.Context, cfg *Config) error { + if err := cfg.Validate(); err != nil { + return fmt.Errorf("librariangen: invalid configuration: %w", err) + } + slog.Debug("librariangen: generate command started") + defer cleanupIntermediateFiles(cfg.OutputDir) + + generateReq, err := readGenerateReq(cfg.LibrarianDir) + if err != nil { + return fmt.Errorf("librariangen: failed to read request: %w", err) + } + + if err := invokeProtoc(ctx, cfg, generateReq); err != nil { + return fmt.Errorf("librariangen: gapic generation failed: %w", err) + } + + // Unzip the generated zip file. + zipPath := filepath.Join(cfg.OutputDir, "java_gapic.zip") + if err := unzip(zipPath, cfg.OutputDir); err != nil { + return fmt.Errorf("librariangen: failed to unzip %s: %w", zipPath, err) + } + + // Unzip the inner temp-codegen.srcjar. + srcjarPath := filepath.Join(cfg.OutputDir, "temp-codegen.srcjar") + srcjarDest := filepath.Join(cfg.OutputDir, "java_gapic_srcjar") + if err := unzip(srcjarPath, srcjarDest); err != nil { + return fmt.Errorf("librariangen: failed to unzip %s: %w", srcjarPath, err) + } + + if err := restructureOutput(cfg.OutputDir, generateReq.ID); err != nil { + return fmt.Errorf("librariangen: failed to restructure output: %w", err) + } + + slog.Debug("librariangen: generate command finished") + return nil +} + +// invokeProtoc handles the protoc GAPIC generation logic for the 'generate' CLI command. +// It reads a request file, and for each API specified, it invokes protoc +// to generate the client library. It returns the module path and the path to the service YAML. +func invokeProtoc(ctx context.Context, cfg *Config, generateReq *request.Library) error { + for _, api := range generateReq.APIs { + apiServiceDir := filepath.Join(cfg.SourceDir, api.Path) + slog.Info("processing api", "service_dir", apiServiceDir) + bazelConfig, err := bazelParse(apiServiceDir) + if err != nil { + return fmt.Errorf("librariangen: failed to parse BUILD.bazel for %s: %w", apiServiceDir, err) + } + args, err := protocBuild(apiServiceDir, bazelConfig, cfg.SourceDir, cfg.OutputDir) + if err != nil { + return fmt.Errorf("librariangen: failed to build protoc command for api %q in library %q: %w", api.Path, generateReq.ID, err) + } + if err := execvRun(ctx, args, cfg.OutputDir); err != nil { + return fmt.Errorf("librariangen: protoc failed for api %q in library %q: %w", api.Path, generateReq.ID, err) + } + } + return nil +} + +// readGenerateReq reads generate-request.json from the librarian-tool input directory. +// The request file tells librariangen which library and APIs to generate. +// It is prepared by the Librarian tool and mounted at /librarian. +func readGenerateReq(librarianDir string) (*request.Library, error) { + reqPath := filepath.Join(librarianDir, "generate-request.json") + slog.Debug("librariangen: reading generate request", "path", reqPath) + + generateReq, err := requestParse(reqPath) + if err != nil { + return nil, err + } + slog.Debug("librariangen: successfully unmarshalled request", "library_id", generateReq.ID) + return generateReq, nil +} + +// moveFiles moves all files (and directories) from sourceDir to targetDir. +func moveFiles(sourceDir, targetDir string) error { + files, err := os.ReadDir(sourceDir) + if err != nil { + return fmt.Errorf("librariangen: failed to read dir %s: %w", sourceDir, err) + } + for _, f := range files { + oldPath := filepath.Join(sourceDir, f.Name()) + newPath := filepath.Join(targetDir, f.Name()) + slog.Debug("librariangen: moving file", "from", oldPath, "to", newPath) + if err := os.Rename(oldPath, newPath); err != nil { + return fmt.Errorf("librariangen: failed to move %s to %s: %w", oldPath, newPath, err) + } + } + return nil +} + +func restructureOutput(outputDir, libraryID string) error { + slog.Debug("librariangen: restructuring output directory", "dir", outputDir) + + // Define source and destination directories. + gapicSrcDir := filepath.Join(outputDir, "java_gapic_srcjar", "src", "main", "java") + gapicTestDir := filepath.Join(outputDir, "java_gapic_srcjar", "src", "test", "java") + protoSrcDir := filepath.Join(outputDir, "com") + samplesDir := filepath.Join(outputDir, "java_gapic_srcjar", "samples", "snippets") + + gapicDestDir := filepath.Join(outputDir, fmt.Sprintf("google-cloud-%s", libraryID), "src", "main", "java") + gapicTestDestDir := filepath.Join(outputDir, fmt.Sprintf("google-cloud-%s", libraryID), "src", "test", "java") + protoDestDir := filepath.Join(outputDir, fmt.Sprintf("proto-google-cloud-%s-v1", libraryID), "src", "main", "java") + samplesDestDir := filepath.Join(outputDir, "samples", "snippets") + + // Create destination directories. + destDirs := []string{gapicDestDir, gapicTestDestDir, protoDestDir, samplesDestDir} + for _, dir := range destDirs { + if err := os.MkdirAll(dir, 0755); err != nil { + return err + } + } + + // Move files. + moves := map[string]string{ + gapicSrcDir: gapicDestDir, + gapicTestDir: gapicTestDestDir, + protoSrcDir: protoDestDir, + samplesDir: samplesDestDir, + } + for src, dest := range moves { + if err := moveFiles(src, dest); err != nil { + return err + } + } + + return nil +} + +func cleanupIntermediateFiles(outputDir string) { + slog.Debug("librariangen: cleaning up intermediate files", "dir", outputDir) + filesToRemove := []string{ + "java_gapic_srcjar", + "com", + "java_gapic.zip", + "temp-codegen.srcjar", + } + for _, file := range filesToRemove { + path := filepath.Join(outputDir, file) + if err := os.RemoveAll(path); err != nil { + slog.Error("librariangen: failed to clean up intermediate file", "path", path, "error", err) + } + } +} + +func unzip(src, dest string) error { + r, err := zip.OpenReader(src) + if err != nil { + return err + } + defer r.Close() + + for _, f := range r.File { + fpath := filepath.Join(dest, f.Name) + + if !strings.HasPrefix(fpath, filepath.Clean(dest)+string(os.PathSeparator)) { + return fmt.Errorf("librariangen: illegal file path: %s", fpath) + } + + if f.FileInfo().IsDir() { + os.MkdirAll(fpath, os.ModePerm) + continue + } + + if err := os.MkdirAll(filepath.Dir(fpath), os.ModePerm); err != nil { + return err + } + + outFile, err := os.OpenFile(fpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode()) + if err != nil { + return err + } + + rc, err := f.Open() + if err != nil { + outFile.Close() + return err + } + + _, copyErr := io.Copy(outFile, rc) + rc.Close() // Error on read-only file close is less critical + closeErr := outFile.Close() + + if copyErr != nil { + return copyErr + } + if closeErr != nil { + return closeErr + } + } + return nil +} \ No newline at end of file diff --git a/internal/librariangen/generate/generator_test.go b/internal/librariangen/generate/generator_test.go new file mode 100644 index 0000000000..2e77903722 --- /dev/null +++ b/internal/librariangen/generate/generator_test.go @@ -0,0 +1,523 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package generate + +import ( + "archive/zip" + "bytes" + "context" + "errors" + "log/slog" + "os" + "path/filepath" + "strings" + "testing" +) + +// testEnv encapsulates a temporary test environment. +type testEnv struct { + tmpDir string + librarianDir string + sourceDir string + outputDir string +} + +// newTestEnv creates a new test environment. +func newTestEnv(t *testing.T) *testEnv { + t.Helper() + tmpDir, err := os.MkdirTemp("", "generator-test") + if err != nil { + t.Fatalf("failed to create temp dir: %v", err) + } + e := &testEnv{tmpDir: tmpDir} + e.librarianDir = filepath.Join(tmpDir, "librarian") + e.sourceDir = filepath.Join(tmpDir, "source") + e.outputDir = filepath.Join(tmpDir, "output") + for _, dir := range []string{e.librarianDir, e.sourceDir, e.outputDir} { + if err := os.Mkdir(dir, 0755); err != nil { + t.Fatalf("failed to create dir %s: %v", dir, err) + } + } + return e +} + +// cleanup removes the temporary directory. +func (e *testEnv) cleanup(t *testing.T) { + t.Helper() + if err := os.RemoveAll(e.tmpDir); err != nil { + t.Fatalf("failed to remove temp dir: %v", err) + } +} + +// writeRequestFile writes a generate-request.json file. +func (e *testEnv) writeRequestFile(t *testing.T, content string) { + t.Helper() + p := filepath.Join(e.librarianDir, "generate-request.json") + if err := os.WriteFile(p, []byte(content), 0644); err != nil { + t.Fatalf("failed to write request file: %v", err) + } +} + +// writeBazelFile writes a BUILD.bazel file. +func (e *testEnv) writeBazelFile(t *testing.T, apiPath, content string) { + t.Helper() + apiDir := filepath.Join(e.sourceDir, apiPath) + if err := os.MkdirAll(apiDir, 0755); err != nil { + t.Fatalf("failed to create api dir: %v", err) + } + // Create a fake .proto file, which is required by the protoc command builder. + if err := os.WriteFile(filepath.Join(apiDir, "fake.proto"), nil, 0644); err != nil { + t.Fatalf("failed to write fake proto file: %v", err) + } + p := filepath.Join(apiDir, "BUILD.bazel") + if err := os.WriteFile(p, []byte(content), 0644); err != nil { + t.Fatalf("failed to write bazel file: %v", err) + } +} + +// writeServiceYAML writes a service.yaml file. +func (e *testEnv) writeServiceYAML(t *testing.T, apiPath, title string) { + t.Helper() + apiDir := filepath.Join(e.sourceDir, apiPath) + content := "title: " + title + p := filepath.Join(apiDir, "service.yaml") + if err := os.WriteFile(p, []byte(content), 0644); err != nil { + t.Fatalf("failed to write service yaml: %v", err) + } +} + +func createFakeZip(t *testing.T, path string) { + t.Helper() + // Create a new zip archive. + newZipFile, err := os.Create(path) + if err != nil { + t.Fatalf("failed to create zip file: %v", err) + } + defer newZipFile.Close() + + zipWriter := zip.NewWriter(newZipFile) + defer zipWriter.Close() + + // Create a temporary empty zip file to be included in the main zip file. + tmpfile, err := os.CreateTemp("", "temp-zip-*.zip") + if err != nil { + t.Fatalf("failed to create temp file: %v", err) + } + defer os.Remove(tmpfile.Name()) + + tempZipWriter := zip.NewWriter(tmpfile) + // Add the src/main/java directory to the inner zip file. + _, err = tempZipWriter.Create("src/main/java/") + if err != nil { + t.Fatalf("failed to create directory in zip: %v", err) + } + _, err = tempZipWriter.Create("src/test/java/") + if err != nil { + t.Fatalf("failed to create directory in zip: %v", err) + } + tempZipWriter.Close() + + // Read the content of the temporary zip file. + zipBytes, err := os.ReadFile(tmpfile.Name()) + if err != nil { + t.Fatalf("failed to read temp zip file: %v", err) + } + + // Add the temporary zip file to the main zip file as temp-codegen.srcjar. + w, err := zipWriter.Create("temp-codegen.srcjar") + if err != nil { + t.Fatalf("failed to create empty file in zip: %v", err) + } + _, err = w.Write(zipBytes) + if err != nil { + t.Fatalf("failed to write content to zip: %v", err) + } +} + +func TestGenerate(t *testing.T) { + singleAPIRequest := `{"id": "foo", "apis": [{"path": "api/v1"}]}` + validBazel := ` +java_gapic_library( + name = "v1_gapic", + grpc_service_config = "service_config.json", + service_yaml = "service.yaml", + transport = "grpc", +) +` + invalidBazel := ` +java_gapic_library( + name = "v1_gapic", +) +` + tests := []struct { + name string + setup func(e *testEnv, t *testing.T) + protocErr error + wantErr bool + wantProtocRunCount int + }{ + { + name: "happy path", + setup: func(e *testEnv, t *testing.T) { + e.writeRequestFile(t, singleAPIRequest) + e.writeBazelFile(t, "api/v1", validBazel) + e.writeServiceYAML(t, "api/v1", "My API") + }, + wantErr: false, + wantProtocRunCount: 1, + }, + { + name: "missing request file", + setup: func(e *testEnv, t *testing.T) { + e.writeBazelFile(t, "api/v1", validBazel) + }, + wantErr: true, + }, + { + name: "missing bazel file", + setup: func(e *testEnv, t *testing.T) { + e.writeRequestFile(t, singleAPIRequest) + }, + wantErr: true, + }, + { + name: "invalid bazel config", + setup: func(e *testEnv, t *testing.T) { + e.writeRequestFile(t, singleAPIRequest) + e.writeBazelFile(t, "api/v1", invalidBazel) + }, + wantErr: true, + }, + { + name: "protoc fails", + setup: func(e *testEnv, t *testing.T) { + e.writeRequestFile(t, singleAPIRequest) + e.writeBazelFile(t, "api/v1", validBazel) + e.writeServiceYAML(t, "api/v1", "My API") + }, + protocErr: errors.New("protoc failed"), + wantErr: true, + wantProtocRunCount: 1, + }, + { + name: "unzip fails", + setup: func(e *testEnv, t *testing.T) { + e.writeRequestFile(t, singleAPIRequest) + e.writeBazelFile(t, "api/v1", validBazel) + e.writeServiceYAML(t, "api/v1", "My API") + // Create a corrupt zip file. + zipPath := filepath.Join(e.outputDir, "java_gapic.zip") + if err := os.WriteFile(zipPath, []byte("not a zip"), 0644); err != nil { + t.Fatalf("failed to write corrupt zip file: %v", err) + } + }, + wantErr: true, + wantProtocRunCount: 1, + }, + { + name: "restructureOutput fails", + setup: func(e *testEnv, t *testing.T) { + e.writeRequestFile(t, singleAPIRequest) + e.writeBazelFile(t, "api/v1", validBazel) + e.writeServiceYAML(t, "api/v1", "My API") + // Make a directory that restructureOutput needs to write to read-only. + readOnlyDir := filepath.Join(e.outputDir, "google-cloud-foo") + if err := os.Mkdir(readOnlyDir, 0400); err != nil { + t.Fatalf("failed to create read-only dir: %v", err) + } + }, + wantErr: true, + wantProtocRunCount: 1, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + e := newTestEnv(t) + defer e.cleanup(t) + tt.setup(e, t) + var protocRunCount int + execvRun = func(ctx context.Context, args []string, dir string) error { + want := "protoc" + if args[0] != want { + t.Errorf("protocRun called with %s; want %s", args[0], want) + } + if tt.protocErr == nil && tt.name != "unzip fails" { + // Simulate protoc creating the zip file. + createFakeZip(t, filepath.Join(e.outputDir, "java_gapic.zip")) + // Create the directory that is expected by restructureOutput. + if err := os.MkdirAll(filepath.Join(e.outputDir, "com"), 0755); err != nil { + t.Fatalf("failed to create directory: %v", err) + } + if err := os.MkdirAll(filepath.Join(e.outputDir, "java_gapic_srcjar", "samples", "snippets"), 0755); err != nil { + t.Fatalf("failed to create directory: %v", err) + } + } + protocRunCount++ + return tt.protocErr + } + cfg := &Config{ + LibrarianDir: e.librarianDir, + InputDir: "fake-input", + OutputDir: e.outputDir, + SourceDir: e.sourceDir, + } + if err := Generate(context.Background(), cfg); (err != nil) != tt.wantErr { + t.Errorf("Generate() error = %v, wantErr %v", err, tt.wantErr) + } + if protocRunCount != tt.wantProtocRunCount { + t.Errorf("protocRun called = %v; want %v", protocRunCount, tt.wantProtocRunCount) + } + }) + } +} + +func TestConfig_Validate(t *testing.T) { + tests := []struct { + name string + cfg *Config + wantErr bool + }{ + { + name: "valid", + cfg: &Config{ + LibrarianDir: "a", + InputDir: "b", + OutputDir: "c", + SourceDir: "d", + }, + wantErr: false, + }, + { + name: "missing librarian dir", + cfg: &Config{ + InputDir: "b", + OutputDir: "c", + SourceDir: "d", + }, + wantErr: true, + }, + { + name: "missing input dir", + cfg: &Config{ + LibrarianDir: "a", + OutputDir: "c", + SourceDir: "d", + }, + wantErr: true, + }, + { + name: "missing output dir", + cfg: &Config{ + LibrarianDir: "a", + InputDir: "b", + SourceDir: "d", + }, + wantErr: true, + }, + { + name: "missing source dir", + cfg: &Config{ + LibrarianDir: "a", + InputDir: "b", + OutputDir: "c", + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := tt.cfg.Validate(); (err != nil) != tt.wantErr { + t.Errorf("Config.Validate() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestRestructureOutput(t *testing.T) { + e := newTestEnv(t) + defer e.cleanup(t) + // Create dummy files and directories to be restructured. + if err := os.MkdirAll(filepath.Join(e.outputDir, "java_gapic_srcjar", "src", "main", "java", "com"), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(e.outputDir, "java_gapic_srcjar", "src", "main", "java", "com", "foo.java"), nil, 0644); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(e.outputDir, "java_gapic_srcjar", "src", "test", "java", "com"), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(e.outputDir, "java_gapic_srcjar", "src", "test", "java", "com", "foo_test.java"), nil, 0644); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(e.outputDir, "com"), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(e.outputDir, "com", "bar.proto"), nil, 0644); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(e.outputDir, "java_gapic_srcjar", "samples", "snippets", "com"), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(e.outputDir, "java_gapic_srcjar", "samples", "snippets", "com", "baz.java"), nil, 0644); err != nil { + t.Fatal(err) + } + + if err := restructureOutput(e.outputDir, "my-library"); err != nil { + t.Fatalf("restructureOutput() failed: %v", err) + } + + // Check that the files were moved to the correct locations. + if _, err := os.Stat(filepath.Join(e.outputDir, "google-cloud-my-library", "src", "main", "java", "com", "foo.java")); err != nil { + t.Errorf("file not moved to main: %v", err) + } + if _, err := os.Stat(filepath.Join(e.outputDir, "google-cloud-my-library", "src", "test", "java", "com", "foo_test.java")); err != nil { + t.Errorf("file not moved to test: %v", err) + } + if _, err := os.Stat(filepath.Join(e.outputDir, "proto-google-cloud-my-library-v1", "src", "main", "java", "bar.proto")); err != nil { + t.Errorf("file not moved to proto: %v", err) + } + if _, err := os.Stat(filepath.Join(e.outputDir, "samples", "snippets", "com", "baz.java")); err != nil { + t.Errorf("file not moved to samples: %v", err) + } +} + +func TestUnzip(t *testing.T) { + e := newTestEnv(t) + defer e.cleanup(t) + + t.Run("invalid zip file", func(t *testing.T) { + invalidZipPath := filepath.Join(e.outputDir, "invalid.zip") + if err := os.WriteFile(invalidZipPath, []byte("not a zip file"), 0644); err != nil { + t.Fatalf("failed to write invalid zip file: %v", err) + } + if err := unzip(invalidZipPath, e.outputDir); err == nil { + t.Error("unzip() with invalid zip file should return an error") + } + }) + + t.Run("permission denied", func(t *testing.T) { + // Create a valid zip file. + validZipPath := filepath.Join(e.outputDir, "valid.zip") + if err := os.WriteFile(validZipPath, []byte{}, 0644); err != nil { + t.Fatalf("failed to write valid zip file: %v", err) + } + // Create a zip writer to add a file to the zip. + f, err := os.OpenFile(validZipPath, os.O_RDWR, 0) + if err != nil { + t.Fatalf("failed to open zip file: %v", err) + } + defer f.Close() // Ensure file is closed + zipWriter := zip.NewWriter(f) + if _, err := zipWriter.Create("file.txt"); err != nil { + t.Fatalf("failed to create file in zip: %v", err) + } + if err := zipWriter.Close(); err != nil { // Check for errors on close + t.Fatalf("failed to close zip writer: %v", err) + } + + // Make the output directory read-only. + readOnlyDir := filepath.Join(e.tmpDir, "readonly") + if err := os.Mkdir(readOnlyDir, 0400); err != nil { + t.Fatalf("failed to create read-only dir: %v", err) + } + if err := os.Chmod(readOnlyDir, 0400); err != nil { + t.Fatalf("failed to chmod read-only dir: %v", err) + } + + if err := unzip(validZipPath, readOnlyDir); err == nil { + t.Error("unzip() with read-only destination should return an error") + } + }) + + t.Run("zip slip vulnerability", func(t *testing.T) { + // Create a zip file with a malicious file path. + maliciousZipPath := filepath.Join(e.outputDir, "malicious.zip") + f, err := os.Create(maliciousZipPath) + if err != nil { + t.Fatalf("failed to create malicious zip file: %v", err) + } + defer f.Close() + zipWriter := zip.NewWriter(f) + if _, err := zipWriter.Create("../../pwned.txt"); err != nil { + t.Fatalf("failed to create malicious file in zip: %v", err) + } + zipWriter.Close() + + destDir := filepath.Join(e.outputDir, "unzip-dest") + if err := os.Mkdir(destDir, 0755); err != nil { + t.Fatalf("failed to create unzip dest dir: %v", err) + } + + if err := unzip(maliciousZipPath, destDir); err == nil { + t.Error("unzip() with malicious zip file should return an error") + } + + // Check that the malicious file was not created. + pwnedFile := filepath.Join(e.tmpDir, "pwned.txt") + if _, err := os.Stat(pwnedFile); !os.IsNotExist(err) { + t.Errorf("malicious file was created at %s", pwnedFile) + } + }) +} + +func TestMoveFiles(t *testing.T) { + e := newTestEnv(t) + defer e.cleanup(t) + + sourceDir, err := os.MkdirTemp(e.tmpDir, "source-move-test") + if err != nil { + t.Fatalf("failed to create temp source dir: %v", err) + } + destDir := filepath.Join(e.tmpDir, "dest-move-test") + if err := os.Mkdir(destDir, 0755); err != nil { + t.Fatalf("failed to create dest dir: %v", err) + } + // Make source dir unreadable. + if err := os.Chmod(sourceDir, 0000); err != nil { + t.Fatalf("failed to chmod source dir: %v", err) + } + + if err := moveFiles(sourceDir, destDir); err == nil { + t.Error("moveFiles() with unreadable source should return an error") + } +} + +func TestCleanupIntermediateFiles(t *testing.T) { + var buf bytes.Buffer + slog.SetDefault(slog.New(slog.NewTextHandler(&buf, nil))) + + e := newTestEnv(t) + defer e.cleanup(t) + + // Create a file that cannot be deleted. + protectedDir := filepath.Join(e.outputDir, "com") + if err := os.Mkdir(protectedDir, 0755); err != nil { + t.Fatalf("failed to create protected dir: %v", err) + } + if _, err := os.Create(filepath.Join(protectedDir, "file.txt")); err != nil { + t.Fatalf("failed to create file in protected dir: %v", err) + } + // Make the directory read-only after creating the file. + if err := os.Chmod(protectedDir, 0500); err != nil { + t.Fatalf("failed to chmod protected dir: %v", err) + } + defer os.Chmod(protectedDir, 0755) // Restore permissions for cleanup. + + cleanupIntermediateFiles(e.outputDir) + + if !strings.Contains(buf.String(), "failed to clean up intermediate file") { + t.Errorf("cleanupIntermediateFiles() should log an error on failure, but did not. Log: %s", buf.String()) + } +} diff --git a/internal/librariangen/main.go b/internal/librariangen/main.go index 40e2c6b7b7..d53759d805 100644 --- a/internal/librariangen/main.go +++ b/internal/librariangen/main.go @@ -17,10 +17,13 @@ package main import ( "context" "errors" + "flag" "fmt" "log/slog" "os" "strings" + + "cloud.google.com/java/internal/librariangen/generate" ) const version = "0.1.0" @@ -30,6 +33,10 @@ func main() { os.Exit(runCLI(os.Args)) } +var ( + generateFunc = generate.Generate +) + func runCLI(args []string) int { logLevel := parseLogLevel(os.Getenv("GOOGLE_SDK_JAVA_LOGGING_LEVEL")) slog.SetDefault(slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{ @@ -77,8 +84,7 @@ func run(ctx context.Context, args []string) error { switch cmd { case "generate": - slog.Warn("librariangen: generate command is not yet implemented") - return nil + return handleGenerate(ctx, flags) case "release-init": slog.Warn("librariangen: release-init command is not yet implemented") return nil @@ -93,3 +99,17 @@ func run(ctx context.Context, args []string) error { } } + +// handleGenerate parses flags for the generate command and calls the generator. +func handleGenerate(ctx context.Context, args []string) error { + cfg := &generate.Config{} + generateFlags := flag.NewFlagSet("generate", flag.ContinueOnError) + generateFlags.StringVar(&cfg.LibrarianDir, "librarian", "/librarian", "Path to the librarian-tool input directory. Contains generate-request.json.") + generateFlags.StringVar(&cfg.InputDir, "input", "/input", "Path to the .librarian/generator-input directory from the language repository.") + generateFlags.StringVar(&cfg.OutputDir, "output", "/output", "Path to the empty directory where librariangen writes its output.") + generateFlags.StringVar(&cfg.SourceDir, "source", "/source", "Path to a complete checkout of the googleapis repository.") + if err := generateFlags.Parse(args); err != nil { + return fmt.Errorf("librariangen: failed to parse flags: %w", err) + } + return generateFunc(ctx, cfg) +} diff --git a/internal/librariangen/main_test.go b/internal/librariangen/main_test.go index 75e299f233..a48e022b08 100644 --- a/internal/librariangen/main_test.go +++ b/internal/librariangen/main_test.go @@ -18,9 +18,16 @@ import ( "context" "log/slog" "testing" + + "cloud.google.com/java/internal/librariangen/generate" ) func TestRun(t *testing.T) { + // Replace the real functions with fakes for testing. + generateFunc = func(_ context.Context, _ *generate.Config) error { + return nil + } + ctx := context.Background() tests := []struct { name string diff --git a/internal/librariangen/run-generate-library.sh b/internal/librariangen/run-generate-library.sh new file mode 100755 index 0000000000..e47d99d565 --- /dev/null +++ b/internal/librariangen/run-generate-library.sh @@ -0,0 +1,108 @@ +#!/bin/bash +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script is a development tool for the librariangen command. It +# orchestrates the end-to-end process of generating a Java client library from a +# test configuration. +# +# Key steps include: +# 1. Setting up an isolated workspace directory. +# 2. Downloading the required version of the gapic-generator-java artifact. +# 3. Compiling the librariangen Go binary. +# 4. Executing the `librariangen generate` command with pre-defined test +# inputs and capturing its output and logs. +# +# This allows for quick, reproducible test runs of the entire generation +# process. + +set -e # Exit immediately if a command exits with a non-zero status. + +cd "$(dirname "$0")" # Change to the script's directory to make relative paths work. + +# --- Configuration --- +WORKSPACE="$(pwd)/workspace" +LIBRARIANGEN_GOOGLEAPIS_DIR=../../../googleapis +LIBRARIANGEN_GOTOOLCHAIN=local +GAPIC_GENERATOR_VERSION="2.62.3" +LIBRARIANGEN_LOG="$WORKSPACE/librariangen.log" + +# --- Cleanup and Setup --- +echo "Cleaning up from last time..." +rm -rf "$WORKSPACE" +mkdir -p "$WORKSPACE" +echo "Working directory: $WORKSPACE" + +# --- Dependency Checks & Version Info --- +( + echo "--- Tool Versions ---" + echo "Go: $(GOWORK=off GOTOOLCHAIN=${LIBRARIANGEN_GOTOOLCHAIN} go version)" + echo "protoc: $(protoc --version 2>&1)" + echo "---------------------\n" +) >> "$LIBRARIANGEN_LOG" 2>&1 + +# Ensure that all required protoc dependencies are available in PATH. +if ! command -v "protoc" &> /dev/null; then + echo "Error: protoc not found in PATH. Please install it." + exit 1 +fi + +# --- Download and Prepare Tools --- +echo "Downloading gapic-generator-java version $GAPIC_GENERATOR_VERSION..." +wget -q "https://repo1.maven.org/maven2/com/google/api/gapic-generator-java/$GAPIC_GENERATOR_VERSION/gapic-generator-java-$GAPIC_GENERATOR_VERSION.jar" -O "$WORKSPACE/gapic-generator-java.jar" + +# Create wrapper script for protoc-gen-java_gapic +echo "Creating protoc-gen-java_gapic wrapper..." +cat > "$WORKSPACE/protoc-gen-java_gapic" << EOL +#!/bin/bash +set -e +exec java -cp "$WORKSPACE/gapic-generator-java.jar" com.google.api.generator.Main \$@ +EOL +chmod +x "$WORKSPACE/protoc-gen-java_gapic" + +# --- Prepare Inputs --- +LIBRARIAN_DIR="$WORKSPACE/librarian" +OUTPUT_DIR="$WORKSPACE/output" +mkdir -p "$LIBRARIAN_DIR" "$OUTPUT_DIR" + +# Use an external googleapis checkout. +if [ ! -d "$LIBRARIANGEN_GOOGLEAPIS_DIR" ]; then + echo "Error: LIBRARIANGEN_GOOGLEAPIS_DIR is not set or not a directory." + echo "Please set it to the path of your local googleapis clone." + exit 1 +fi +echo "Using googleapis source from $LIBRARIANGEN_GOOGLEAPIS_DIR" +SOURCE_DIR=$(cd "$LIBRARIANGEN_GOOGLEAPIS_DIR" && pwd) + +# Copy the generate-request.json into the librarian directory. +cp "testdata/generate/librarian/generate-request.json" "$LIBRARIAN_DIR/" + +# --- Execute --- +# Compile the librariangen binary. +BINARY_PATH="$WORKSPACE/librariangen" +echo "Compiling librariangen..." +GOWORK=off GOTOOLCHAIN=${LIBRARIANGEN_GOTOOLCHAIN} go build -o "$BINARY_PATH" . + +# Run the librariangen generate command. +echo "Running librariangen..." +PATH="$WORKSPACE:$(GOWORK=off GOTOOLCHAIN=${LIBRARIANGEN_GOTOOLCHAIN} go env GOPATH)/bin:$HOME/go/bin:$PATH" \ +"$BINARY_PATH" generate \ + --source="$SOURCE_DIR" \ + --librarian="$LIBRARIAN_DIR" \ + --output="$OUTPUT_DIR" \ + >> "$LIBRARIANGEN_LOG" 2>&1 + +echo "Library generation complete." +echo "Generated files are available in: $OUTPUT_DIR" +echo "Librariangen logs are available in: $LIBRARIANGEN_LOG" diff --git a/internal/librariangen/testdata/generate/librarian/generate-request.json b/internal/librariangen/testdata/generate/librarian/generate-request.json new file mode 100644 index 0000000000..51d386e96e --- /dev/null +++ b/internal/librariangen/testdata/generate/librarian/generate-request.json @@ -0,0 +1,22 @@ +{ + "id": "chronicle", + "version": "0.1.1", + "apis": [ + { + "path": "google/cloud/chronicle/v1", + "service_config": "chronicle_v1.yaml" + } + ], + "source_roots": [ + "chronicle", + "internal/generated/snippets/chronicle" + ], + "preserve_regex": [ + "chronicle/aliasshim/aliasshim.go", + "chronicle/CHANGES.md" + ], + "remove_regex": [ + "chronicle", + "internal/generated/snippets/chronicle" + ] +} \ No newline at end of file