From fcd19bbc3340a73e622ef26fb28bf11a1a6b8e96 Mon Sep 17 00:00:00 2001 From: Jordan Keister Date: Thu, 10 Oct 2024 09:55:38 -0500 Subject: [PATCH] change catalog-specific URL from full path to API endpoint ref solves #427 and implements phase 1 of the CatalogD expandable interface. This phase just moves from `status.contentURL` to `status.baseURL`, and will be used to provide reference to the catalog-specific API instead of the full path to JSONLines-formatted content, plus tests and documentation. Signed-off-by: Jordan Keister --- README.md | 95 +++++++++---------- api/core/v1alpha1/clustercatalog_types.go | 6 +- cmd/manager/main.go | 2 +- ....operatorframework.io_clustercatalogs.yaml | 9 +- docs/fetching-catalog-contents.md | 28 +++--- hack/scripts/demo-script.sh | 9 +- hack/scripts/gzip-demo-script.sh | 6 +- .../core/clustercatalog_controller.go | 4 +- .../core/clustercatalog_controller_test.go | 14 +-- internal/storage/localdir.go | 14 +-- internal/storage/localdir_test.go | 29 +++--- test/e2e/util.go | 9 +- test/upgrade/unpack_test.go | 2 +- 13 files changed, 114 insertions(+), 113 deletions(-) diff --git a/README.md b/README.md index 08982b9e..f4c84de2 100644 --- a/README.md +++ b/README.md @@ -38,49 +38,48 @@ Procedure steps marked with an asterisk (`*`) are likely to change with future A *Example output* ```sh - Name: operatorhubio - Namespace: - Labels: olm.operatorframework.io/metadata.name=operatorhubio - Annotations: - API Version: olm.operatorframework.io/v1alpha1 - Kind: ClusterCatalog - Metadata: - Creation Timestamp: 2024-09-12T13:37:04Z - Finalizers: - olm.operatorframework.io/delete-server-cache - Generation: 1 - Resource Version: 961 - UID: fa6bb9cf-1a36-4189-a7a0-83284c3f6f55 - Spec: - Priority: 0 - Source: - Image: - Poll Interval: 10m0s - Ref: quay.io/operatorhubio/catalog:latest - Type: Image - Status: - Conditions: - Last Transition Time: 2024-09-12T13:37:53Z - Message: Successfully unpacked and stored content from quay.io/operatorhubio/catalog:latest - Reason: Succeeded - Status: False - Type: Progressing - Last Transition Time: 2024-09-12T13:37:53Z - Message: Content from quay.io/operatorhubio/catalog:latest is being served - Reason: Available - Status: True - Type: Serving - Content URL: https://catalogd-service.olmv1-system.svc/catalogs/operatorhubio/all.json - Last Unpacked: 2024-09-12T13:37:52Z - Observed Generation: 1 - Resolved Source: - Image: - Last Poll Attempt: 2024-09-12T13:37:52Z - Last Unpacked: 2024-09-12T13:37:52Z - Ref: quay.io/operatorhubio/catalog:latest - Resolved Ref: quay.io/operatorhubio/catalog@sha256:4453a361198d39d0390fd8c1a7f07b5a5a3ae1e8dac9979ef0c4eba46299df16 - Type: Image - Events: +Name: operatorhubio +Namespace: +Labels: olm.operatorframework.io/metadata.name=operatorhubio +Annotations: +API Version: olm.operatorframework.io/v1alpha1 +Kind: ClusterCatalog +Metadata: + Creation Timestamp: 2024-10-10T19:25:01Z + Finalizers: + olm.operatorframework.io/delete-server-cache + Generation: 1 + Resource Version: 10206 + UID: 8a83a2e2-6f7f-430c-9359-258003483b12 +Spec: + Priority: 0 + Source: + Image: + Poll Interval: 10m0s + Ref: quay.io/operatorhubio/catalog:latest + Type: Image +Status: + Base URL: https://catalogd-service.olmv1-system.svc/catalogs/operatorhubio/api + Conditions: + Last Transition Time: 2024-10-10T19:25:15Z + Message: Successfully unpacked and stored content from resolved source + Observed Generation: 1 + Reason: Succeeded + Status: False + Type: Progressing + Last Transition Time: 2024-10-10T19:25:15Z + Message: Serving desired content from resolved source + Observed Generation: 1 + Reason: Available + Status: True + Type: Serving + Last Unpacked: 2024-10-10T19:25:14Z + Resolved Source: + Image: + Last Successful Poll Attempt: 2024-10-10T20:45:33Z + Ref: quay.io/operatorhubio/catalog@sha256:7fbc6c61dc36e8225d4cb392cf0ffc754c179a82b12f19b91970c09b431aa61f + Type: Image +Events: ``` 1. Port forward the `catalogd-service` service in the `olmv1-system` namespace: @@ -88,10 +87,10 @@ Procedure steps marked with an asterisk (`*`) are likely to change with future A $ kubectl -n olmv1-system port-forward svc/catalogd-service 8080:443 ``` -1. Run the following command to get a list of packages: +1. Access the `v1/all` service endpoint and filter the results to a list of packages by running the following command: ```sh - $ curl -k https://localhost:8080/catalogs/operatorhubio/all.json | jq -s '.[] | select(.schema == "olm.package") | .name' + $ curl https://localhost:8080/catalogs/operatorhubio/api/vi/all | jq -s '.[] | select(.schema == "olm.package") | .name' ``` *Example output* @@ -118,7 +117,7 @@ Procedure steps marked with an asterisk (`*`) are likely to change with future A 1. Run the following command to get a list of channels for the `ack-acm-controller` package: ```sh - $ curl -k https://localhost:8080/catalogs/operatorhubio/all.json | jq -s '.[] | select(.schema == "olm.channel") | select(.package == "ack-acm-controller") | .name' + $ curl https://localhost:8080/catalogs/operatorhubio/api/v1/all | jq -s '.[] | select(.schema == "olm.channel") | select(.package == "ack-acm-controller") | .name' ``` *Example output* @@ -132,7 +131,7 @@ Procedure steps marked with an asterisk (`*`) are likely to change with future A 1. Run the following command to get a list of bundles belonging to the `ack-acm-controller` package: ```sh - $ curl -k https://localhost:8080/catalogs/operatorhubio/all.json | jq -s '.[] | select(.schema == "olm.bundle") | select(.package == "ack-acm-controller") | .name' + $ curl https://localhost:8080/catalogs/operatorhubio/api/v1/all | jq -s '.[] | select(.schema == "olm.bundle") | select(.package == "ack-acm-controller") | .name' ``` *Example output* @@ -153,7 +152,7 @@ Thanks for your interest in contributing to `catalogd`! `catalogd` is in the very early stages of development and a more in depth contributing guide will come in the near future. -In the mean time, it is assumed you know how to make contributions to open source projects in general and this guide will only focus on how to manually test your changes (no automated testing yet). +In the meantime, it is assumed you know how to make contributions to open source projects in general and this guide will only focus on how to manually test your changes (no automated testing yet). If you have any questions, feel free to reach out to us on the Kubernetes Slack channel [#olm-dev](https://kubernetes.slack.com/archives/C0181L6JYQ2) or [create an issue](https://github.com/operator-framework/catalogd/issues/new) ### Testing Local Changes diff --git a/api/core/v1alpha1/clustercatalog_types.go b/api/core/v1alpha1/clustercatalog_types.go index ecc88403..8290633e 100644 --- a/api/core/v1alpha1/clustercatalog_types.go +++ b/api/core/v1alpha1/clustercatalog_types.go @@ -129,11 +129,9 @@ type ClusterCatalogStatus struct { // type: Image // +optional ResolvedSource *ResolvedCatalogSource `json:"resolvedSource,omitempty"` - // contentURL is a cluster-internal URL from which on-cluster components - // can read the content of a catalog + // baseURL is a cluster-internal URL from which on-cluster components can access the API endpoint for this catalog // +optional - ContentURL string `json:"contentURL,omitempty"` - + BaseURL string `json:"baseURL,omitempty"` // lastUnpacked represents the time when the // ClusterCatalog object was last unpacked successfully. // +optional diff --git a/cmd/manager/main.go b/cmd/manager/main.go index a2d0175a..b4cfee79 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -258,7 +258,7 @@ func main() { os.Exit(1) } - localStorage = storage.LocalDir{RootDir: storeDir, BaseURL: baseStorageURL} + localStorage = storage.LocalDir{RootDir: storeDir, RootURL: baseStorageURL} catalogServerConfig := serverutil.CatalogServerConfig{ ExternalAddr: externalAddr, diff --git a/config/base/crd/bases/olm.operatorframework.io_clustercatalogs.yaml b/config/base/crd/bases/olm.operatorframework.io_clustercatalogs.yaml index 67a0d774..a7a9a4a6 100644 --- a/config/base/crd/bases/olm.operatorframework.io_clustercatalogs.yaml +++ b/config/base/crd/bases/olm.operatorframework.io_clustercatalogs.yaml @@ -130,6 +130,10 @@ spec: status: description: ClusterCatalogStatus defines the observed state of ClusterCatalog properties: + baseURL: + description: baseURL is a cluster-internal URL from which on-cluster + components can access the API endpoint for this catalog + type: string conditions: description: |- conditions is a representation of the current state for this ClusterCatalog. @@ -206,11 +210,6 @@ spec: - type type: object type: array - contentURL: - description: |- - contentURL is a cluster-internal URL from which on-cluster components - can read the content of a catalog - type: string lastUnpacked: description: |- lastUnpacked represents the time when the diff --git a/docs/fetching-catalog-contents.md b/docs/fetching-catalog-contents.md index 55fb9e67..394ed087 100644 --- a/docs/fetching-catalog-contents.md +++ b/docs/fetching-catalog-contents.md @@ -1,8 +1,9 @@ # `ClusterCatalog` Interface -`catalogd` serves catalog content via an HTTP(S) endpoint +`catalogd` serves catalog content via a catalog-specific, versioned HTTP(S) endpoint. Clients access catalog information via this API endpoint and a versioned reference of the desired format. Current support includes only a complete catalog download, indicated by the path "v1/all", for example `https://catalogd-catalogservice.olmv1-system.svc/catalogs/operatorhubio/api/vi/all` would receive the complete FBC for the catalog `operatorhubio`. + ## Response Format -`catalogd` responses are encoded as a [JSON Lines](https://jsonlines.org/) stream of File-Based Catalog (FBC) [Meta](https://olm.operatorframework.io/docs/reference/file-based-catalogs/#schema) objects delimited by newlines. +`catalogd` responses retrieved via the catalog-specific v1 API are encoded as a [JSON Lines](https://jsonlines.org/) stream of File-Based Catalog (FBC) [Meta](https://olm.operatorframework.io/docs/reference/file-based-catalogs/#schema) objects delimited by newlines. ### Example For an example JSON-encoded FBC snippet @@ -79,30 +80,31 @@ For example purposes we make the following assumption: For local development, consider skipping TLS verification, such as `curl -k`, or reference external material on self-signed certificate verification. -`ClusterCatalog` CRs have a status.contentURL field whose value is the location where the content -of a catalog can be read from: +`ClusterCatalog` CRs have a status.baseURL field which identifies the catalog-specific API to access the catalog content: ```yaml status: . . - contentURL: https://catalogd-service.olmv1-system.svc/catalogs/operatorhubio/all.json + baseURL: https://catalogd-catalogservice.olmv1-system.svc/catalogs/operatorhubio/api/ resolvedSource: image: ref: quay.io/operatorhubio/catalog@sha256:e53267559addc85227c2a7901ca54b980bc900276fc24d3f4db0549cb38ecf76 type: Image ``` - ## On cluster -When making a request for the contents of the `operatorhubio` `ClusterCatalog` from within -the cluster issue a HTTP `GET` request to -`https://catalogd-service.olmv1-system.svc/catalogs/operatorhubio/all.json` +When making a request for the complete contents of the `operatorhubio` `ClusterCatalog` from within +the cluster, clients would combine `status.baseURL` with the desired API service and issue an HTTP GET request for the URL. + +For example, to receive the complete catalog data for the `operatorhubio` catalog indicated above, the client would append the service point designator `v1/all`, like: + +`https://catalogd-catalogservice.olmv1-system.svc/catalogs/operatorhubio/api/v1/all`. An example command to run a `Pod` to `curl` the catalog contents: ```sh -kubectl run fetcher --image=curlimages/curl:latest -- curl https://catalogd-service.olmv1-system.svc/catalogs/operatorhubio/all.json +kubectl run fetcher --image=curlimages/curl:latest -- curl https://catalogd-service.olmv1-system.svc/catalogs/operatorhubio/api/v1/all ``` ## Off cluster @@ -115,11 +117,11 @@ kubectl -n olmv1-system port-forward svc/catalogd-service 8080:443 ``` Once the service has been successfully forwarded to a localhost port, issue a HTTP `GET` -request to `https://localhost:8080/catalogs/operatorhubio/all.json` +request to `https://localhost:8080/catalogs/operatorhubio/api/v1/all` An example `curl` request that assumes the port-forwarding is mapped to port 8080 on the local machine: ```sh -curl http://localhost:8080/catalogs/operatorhubio/all.json +curl http://localhost:8080/catalogs/operatorhubio/api/v1/all ``` # Fetching `ClusterCatalog` contents from the `Catalogd` Service outside of the cluster @@ -190,7 +192,7 @@ This section outlines a way of exposing the `Catalogd` Service's endpoints outsi 1. Run the below example `curl` request to retrieve all of the catalog contents: ```sh - $ curl https://
/catalogs/operatorhubio/all.json + $ curl https://
/catalogs/operatorhubio/api/v1/all ``` To obtain `address` of the ingress object, you can run the below command and look for the value in the `ADDRESS` field from output: diff --git a/hack/scripts/demo-script.sh b/hack/scripts/demo-script.sh index aa17cffe..769b87b3 100755 --- a/hack/scripts/demo-script.sh +++ b/hack/scripts/demo-script.sh @@ -11,6 +11,7 @@ sleep 10 kubectl apply -f https://github.com/operator-framework/catalogd/releases/latest/download/catalogd.yaml kubectl wait --for=condition=Available -n olmv1-system deploy/catalogd-controller-manager --timeout=60s sleep 10 + # inspect crds (catalog) kubectl get crds -A @@ -19,15 +20,15 @@ kubectl apply -f config/samples/core_v1alpha1_catalog.yaml # shows catalog-sample kubectl get catalog -A # waiting for catalog to report ready status -time kubectl wait --for=condition=Unpacked catalog/operatorhubio --timeout=1m +time kubectl wait --for=condition=Serving catalog/operatorhubio --timeout=1m # port forward the catalogd-service service to interact with the HTTP server serving catalog contents (kubectl -n olmv1-system port-forward svc/catalogd-service 8080:80)& # check what 'packages' are available in this catalog -curl http://localhost:8080/catalogs/operatorhubio/all.json | jq -s '.[] | select(.schema == "olm.package") | .name' +curl http://localhost:8080/catalogs/operatorhubio/api/v1/all | jq -s '.[] | select(.schema == "olm.package") | .name' # check what channels are included in the wavefront package -curl http://localhost:8080/catalogs/operatorhubio/all.json | jq -s '.[] | select(.schema == "olm.channel") | select(.package == "wavefront") | .name' +curl http://localhost:8080/catalogs/operatorhubio/api/v1/all | jq -s '.[] | select(.schema == "olm.channel") | select(.package == "wavefront") | .name' # check what bundles are included in the wavefront package -curl http://localhost:8080/catalogs/operatorhubio/all.json | jq -s '.[] | select(.schema == "olm.bundle") | select(.package == "wavefront") | .name' +curl http://localhost:8080/catalogs/operatorhubio/api/v1/all | jq -s '.[] | select(.schema == "olm.bundle") | select(.package == "wavefront") | .name' diff --git a/hack/scripts/gzip-demo-script.sh b/hack/scripts/gzip-demo-script.sh index 49378f49..2cd1bb79 100755 --- a/hack/scripts/gzip-demo-script.sh +++ b/hack/scripts/gzip-demo-script.sh @@ -16,13 +16,13 @@ time kubectl wait --for=condition=Unpacked clustercatalog/operatorhubio --timeou sleep 5 # retrieve catalog as plaintext JSONlines -curl -k -vvv https://localhost:8080/catalogs/operatorhubio/all.json --output /tmp/cat-content.json +curl -k -vvv https://localhost:8080/catalogs/operatorhubio/api/v1/all --output /tmp/cat-content.json # advertise handling of compressed content -curl -vvv -k https://localhost:8080/catalogs/operatorhubio/all.json -H 'Accept-Encoding: gzip' --output /tmp/cat-content.gz +curl -vvv -k https://localhost:8080/catalogs/operatorhubio/api/v1/all -H 'Accept-Encoding: gzip' --output /tmp/cat-content.gz # let curl handle the compress/decompress for us -curl -vvv --compressed -k https://localhost:8080/catalogs/operatorhubio/all.json --output /tmp/cat-content-decompressed.txt +curl -vvv --compressed -k https://localhost:8080/catalogs/operatorhubio/api/v1/all --output /tmp/cat-content-decompressed.txt # show that there's no content change with changed format diff /tmp/cat-content.json /tmp/cat-content-decompressed.txt diff --git a/internal/controllers/core/clustercatalog_controller.go b/internal/controllers/core/clustercatalog_controller.go index a8ad497c..7ec49093 100644 --- a/internal/controllers/core/clustercatalog_controller.go +++ b/internal/controllers/core/clustercatalog_controller.go @@ -304,7 +304,7 @@ func updateStatusProgressing(status *v1alpha1.ClusterCatalogStatus, generation i func updateStatusServing(status *v1alpha1.ClusterCatalogStatus, result source.Result, contentURL string, generation int64) { status.ResolvedSource = result.ResolvedSource - status.ContentURL = contentURL + status.BaseURL = contentURL status.LastUnpacked = metav1.NewTime(result.UnpackTime) meta.SetStatusCondition(&status.Conditions, metav1.Condition{ Type: v1alpha1.TypeServing, @@ -317,7 +317,7 @@ func updateStatusServing(status *v1alpha1.ClusterCatalogStatus, result source.Re func updateStatusNotServing(status *v1alpha1.ClusterCatalogStatus, generation int64) { status.ResolvedSource = nil - status.ContentURL = "" + status.BaseURL = "" status.LastUnpacked = metav1.Time{} meta.SetStatusCondition(&status.Conditions, metav1.Condition{ Type: v1alpha1.TypeServing, diff --git a/internal/controllers/core/clustercatalog_controller_test.go b/internal/controllers/core/clustercatalog_controller_test.go index 347fb8c6..09fb5c20 100644 --- a/internal/controllers/core/clustercatalog_controller_test.go +++ b/internal/controllers/core/clustercatalog_controller_test.go @@ -261,7 +261,7 @@ func TestCatalogdControllerReconcile(t *testing.T) { }, }, Status: catalogdv1alpha1.ClusterCatalogStatus{ - ContentURL: "URL", + BaseURL: "URL", Conditions: []metav1.Condition{ { Type: catalogdv1alpha1.TypeServing, @@ -393,7 +393,7 @@ func TestCatalogdControllerReconcile(t *testing.T) { }, }, Status: catalogdv1alpha1.ClusterCatalogStatus{ - ContentURL: "URL", + BaseURL: "URL", LastUnpacked: metav1.Time{}, ResolvedSource: &catalogdv1alpha1.ResolvedCatalogSource{ Type: catalogdv1alpha1.SourceTypeImage, @@ -430,7 +430,7 @@ func TestCatalogdControllerReconcile(t *testing.T) { }, }, Status: catalogdv1alpha1.ClusterCatalogStatus{ - ContentURL: "", + BaseURL: "", Conditions: []metav1.Condition{ { Type: catalogdv1alpha1.TypeServing, @@ -473,7 +473,7 @@ func TestCatalogdControllerReconcile(t *testing.T) { }, }, Status: catalogdv1alpha1.ClusterCatalogStatus{ - ContentURL: "URL", + BaseURL: "URL", Conditions: []metav1.Condition{ { Type: catalogdv1alpha1.TypeProgressing, @@ -503,7 +503,7 @@ func TestCatalogdControllerReconcile(t *testing.T) { }, }, Status: catalogdv1alpha1.ClusterCatalogStatus{ - ContentURL: "URL", + BaseURL: "URL", Conditions: []metav1.Condition{ { Type: catalogdv1alpha1.TypeProgressing, @@ -544,7 +544,7 @@ func TestCatalogdControllerReconcile(t *testing.T) { }, }, Status: catalogdv1alpha1.ClusterCatalogStatus{ - ContentURL: "URL", + BaseURL: "URL", Conditions: []metav1.Condition{ { Type: catalogdv1alpha1.TypeProgressing, @@ -696,7 +696,7 @@ func TestPollingReconcilerUnpack(t *testing.T) { successfulObservedGeneration := int64(2) successfulUnpackStatus := func(mods ...func(status *catalogdv1alpha1.ClusterCatalogStatus)) catalogdv1alpha1.ClusterCatalogStatus { s := catalogdv1alpha1.ClusterCatalogStatus{ - ContentURL: "URL", + BaseURL: "URL", Conditions: []metav1.Condition{ { Type: catalogdv1alpha1.TypeProgressing, diff --git a/internal/storage/localdir.go b/internal/storage/localdir.go index 93a82b71..8d134ca3 100644 --- a/internal/storage/localdir.go +++ b/internal/storage/localdir.go @@ -21,11 +21,11 @@ import ( // atomic view of the content for a catalog. type LocalDir struct { RootDir string - BaseURL *url.URL + RootURL *url.URL } func (s LocalDir) Store(ctx context.Context, catalog string, fsys fs.FS) error { - fbcDir := filepath.Join(s.RootDir, catalog) + fbcDir := filepath.Join(s.RootDir, catalog, "api", "v1") if err := os.MkdirAll(fbcDir, 0700); err != nil { return err } @@ -43,7 +43,7 @@ func (s LocalDir) Store(ctx context.Context, catalog string, fsys fs.FS) error { }); err != nil { return fmt.Errorf("error walking FBC root: %w", err) } - fbcFile := filepath.Join(fbcDir, "all.json") + fbcFile := filepath.Join(fbcDir, "all") return os.Rename(tempFile.Name(), fbcFile) } @@ -52,25 +52,25 @@ func (s LocalDir) Delete(catalog string) error { } func (s LocalDir) ContentURL(catalog string) string { - return fmt.Sprintf("%s%s/all.json", s.BaseURL, catalog) + return fmt.Sprintf("%s%s/api", s.RootURL, catalog) } func (s LocalDir) StorageServerHandler() http.Handler { mux := http.NewServeMux() fsHandler := http.FileServer(http.FS(&filesOnlyFilesystem{os.DirFS(s.RootDir)})) - spHandler := http.StripPrefix(s.BaseURL.Path, fsHandler) + spHandler := http.StripPrefix(s.RootURL.Path, fsHandler) gzHandler := gzhttp.GzipHandler(spHandler) typeHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Add("Content-Type", "application/jsonl") gzHandler.ServeHTTP(w, r) }) - mux.Handle(s.BaseURL.Path, typeHandler) + mux.Handle(s.RootURL.Path, typeHandler) return mux } func (s LocalDir) ContentExists(catalog string) bool { - file, err := os.Stat(filepath.Join(s.RootDir, catalog, "all.json")) + file, err := os.Stat(filepath.Join(s.RootDir, catalog, "api", "v1", "all")) if err != nil { return false } diff --git a/internal/storage/localdir_test.go b/internal/storage/localdir_test.go index fd20d125..ee8cc05e 100644 --- a/internal/storage/localdir_test.go +++ b/internal/storage/localdir_test.go @@ -56,7 +56,7 @@ var _ = Describe("LocalDir Storage Test", func() { rootDir = d baseURL = &url.URL{Scheme: "http", Host: "test-addr", Path: urlPrefix} - store = LocalDir{RootDir: rootDir, BaseURL: baseURL} + store = LocalDir{RootDir: rootDir, RootURL: baseURL} unpackResultFS = &fstest.MapFS{ "bundle.yaml": &fstest.MapFile{Data: []byte(testBundle), Mode: os.ModePerm}, "package.yaml": &fstest.MapFile{Data: []byte(testPackage), Mode: os.ModePerm}, @@ -69,19 +69,20 @@ var _ = Describe("LocalDir Storage Test", func() { Expect(err).To(Not(HaveOccurred())) }) It("should store the content in the RootDir correctly", func() { - fbcFile := filepath.Join(rootDir, catalog, "all.json") + fbcDir := filepath.Join(rootDir, catalog, "api", "v1") + fbcFile := filepath.Join(fbcDir, "all") _, err := os.Stat(fbcFile) Expect(err).To(Not(HaveOccurred())) gotConfig, err := declcfg.LoadFS(ctx, unpackResultFS) Expect(err).To(Not(HaveOccurred())) - storedConfig, err := declcfg.LoadFile(os.DirFS(filepath.Join(rootDir, catalog)), "all.json") + storedConfig, err := declcfg.LoadFile(os.DirFS(fbcDir), "all") Expect(err).To(Not(HaveOccurred())) diff := cmp.Diff(gotConfig, storedConfig) Expect(diff).To(Equal("")) }) It("should form the content URL correctly", func() { - Expect(store.ContentURL(catalog)).To(Equal(fmt.Sprintf("%s%s/all.json", baseURL, catalog))) + Expect(store.ContentURL(catalog)).To(Equal(fmt.Sprintf("%s%s/api", baseURL, catalog))) }) It("should report content exists", func() { Expect(store.ContentExists(catalog)).To(BeTrue()) @@ -112,8 +113,8 @@ var _ = Describe("LocalDir Server Handler tests", func() { BeforeEach(func() { d, err := os.MkdirTemp(GinkgoT().TempDir(), "cache") Expect(err).ToNot(HaveOccurred()) - Expect(os.Mkdir(filepath.Join(d, "test-catalog"), 0700)).To(Succeed()) - store = LocalDir{RootDir: d, BaseURL: &url.URL{Path: urlPrefix}} + Expect(os.MkdirAll(filepath.Join(d, "test-catalog", "api", "v1"), 0700)).To(Succeed()) + store = LocalDir{RootDir: d, RootURL: &url.URL{Path: urlPrefix}} testServer = httptest.NewServer(store.StorageServerHandler()) }) @@ -144,15 +145,15 @@ var _ = Describe("LocalDir Server Handler tests", func() { Expect(os.WriteFile(filepath.Join(store.RootDir, "test-catalog", "foo.txt"), expectedContent, 0600)).To(Succeed()) expectFound(fmt.Sprintf("%s/%s", testServer.URL, "/catalogs/test-catalog/foo.txt"), expectedContent) }) - It("ignores accept-encoding for the path /catalogs/test-catalog/all.json with size < 1400 bytes", func() { + It("ignores accept-encoding for the path /catalogs/test-catalog/api/v1/all with size < 1400 bytes", func() { expectedContent := []byte("bar") - Expect(os.WriteFile(filepath.Join(store.RootDir, "test-catalog", "all.json"), expectedContent, 0600)).To(Succeed()) - expectFound(fmt.Sprintf("%s/%s", testServer.URL, "/catalogs/test-catalog/all.json"), expectedContent) + Expect(os.WriteFile(filepath.Join(store.RootDir, "test-catalog", "api", "v1", "all"), expectedContent, 0600)).To(Succeed()) + expectFound(fmt.Sprintf("%s/%s", testServer.URL, "/catalogs/test-catalog/api/v1/all"), expectedContent) }) - It("provides gzipped content for the path /catalogs/test-catalog/all.json with size > 1400 bytes", func() { + It("provides gzipped content for the path /catalogs/test-catalog/api/v1/all with size > 1400 bytes", func() { expectedContent := []byte(testCompressableJSON) - Expect(os.WriteFile(filepath.Join(store.RootDir, "test-catalog", "all.json"), expectedContent, 0600)).To(Succeed()) - expectFound(fmt.Sprintf("%s/%s", testServer.URL, "/catalogs/test-catalog/all.json"), expectedContent) + Expect(os.WriteFile(filepath.Join(store.RootDir, "test-catalog", "api", "v1", "all"), expectedContent, 0600)).To(Succeed()) + expectFound(fmt.Sprintf("%s/%s", testServer.URL, "/catalogs/test-catalog/api/v1/all"), expectedContent) }) It("provides json-lines format for the served JSON catalog", func() { catalog := "test-catalog" @@ -164,7 +165,7 @@ var _ = Describe("LocalDir Server Handler tests", func() { expectedContent, err := generateJSONLines([]byte(testCompressableJSON)) Expect(err).To(Not(HaveOccurred())) - expectFound(fmt.Sprintf("%s/%s", testServer.URL, "/catalogs/test-catalog/all.json"), []byte(expectedContent)) + expectFound(fmt.Sprintf("%s/%s", testServer.URL, "/catalogs/test-catalog/api/v1/all"), []byte(expectedContent)) }) It("provides json-lines format for the served YAML catalog", func() { catalog := "test-catalog" @@ -178,7 +179,7 @@ var _ = Describe("LocalDir Server Handler tests", func() { expectedContent, err := generateJSONLines(yamlData) Expect(err).To(Not(HaveOccurred())) - expectFound(fmt.Sprintf("%s/%s", testServer.URL, "/catalogs/test-catalog/all.json"), []byte(expectedContent)) + expectFound(fmt.Sprintf("%s/%s", testServer.URL, "/catalogs/test-catalog/api/v1/all"), []byte(expectedContent)) }) AfterEach(func() { testServer.Close() diff --git a/test/e2e/util.go b/test/e2e/util.go index a5a6ac1d..689737ee 100644 --- a/test/e2e/util.go +++ b/test/e2e/util.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "net/url" + "path" "strings" "k8s.io/client-go/kubernetes" @@ -17,12 +18,12 @@ func ReadTestCatalogServerContents(ctx context.Context, catalog *catalogd.Cluste if catalog == nil { return nil, fmt.Errorf("cannot read nil catalog") } - url, err := url.Parse(catalog.Status.ContentURL) + url, err := url.Parse(catalog.Status.BaseURL) if err != nil { - return nil, fmt.Errorf("error parsing clustercatalog url %s: %v", catalog.Status.ContentURL, err) + return nil, fmt.Errorf("error parsing clustercatalog url %s: %v", catalog.Status.BaseURL, err) } // url is expected to be in the format of - // http://{service_name}.{namespace}.svc/{catalog_name}/all.json + // http://{service_name}.{namespace}.svc/api/{catalog_name}/v1/all // so to get the namespace and name of the service we grab only // the hostname and split it on the '.' character ns := strings.Split(url.Hostname(), ".")[1] @@ -37,7 +38,7 @@ func ReadTestCatalogServerContents(ctx context.Context, catalog *catalogd.Cluste port = "80" } } - resp := kubeClient.CoreV1().Services(ns).ProxyGet(url.Scheme, name, port, url.Path, map[string]string{}) + resp := kubeClient.CoreV1().Services(ns).ProxyGet(url.Scheme, name, port, path.Join(url.Path, "v1", "all"), map[string]string{}) rc, err := resp.Stream(ctx) if err != nil { return nil, err diff --git a/test/upgrade/unpack_test.go b/test/upgrade/unpack_test.go index 2cc368d3..2f9a9a53 100644 --- a/test/upgrade/unpack_test.go +++ b/test/upgrade/unpack_test.go @@ -76,7 +76,7 @@ var _ = Describe("ClusterCatalog Unpacking", func() { g.Expect(cond.Reason).To(Equal(catalogd.ReasonSucceeded)) }).Should(Succeed()) - expectedFBC, err := os.ReadFile("../../testdata/catalogs/test-catalog/expected_all.json") + expectedFBC, err := os.ReadFile("../../testdata/catalogs/test-catalog/api/v1/expected_all.json") Expect(err).To(Not(HaveOccurred())) By("Making sure the catalog content is available via the http server")