From ba4dccce61bd742266a052caa6ff2251de7eb1eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferenc=20Szab=C3=B3?= Date: Fri, 15 Mar 2024 16:29:13 +0100 Subject: [PATCH] fix: Azure blob storage support in Java feature server (#2319) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add azure blob storage support in java feature server - Fix S3 integration test to work without a real AWS account - Add GCS mock to integration tests to be able to run them without a real google cloud account - Adding dependency management in maven for libraries with older incompatible versions as transitive dependencies Signed-off-by: Ferenc Szabó --- java/CONTRIBUTING.md | 4 +- java/pom.xml | 65 +++++++++-- java/serving/.gitignore | 5 +- java/serving/pom.xml | 27 +++-- .../serving/registry/AzureRegistryFile.java | 57 ++++++++++ .../service/config/ApplicationProperties.java | 9 ++ .../service/config/RegistryConfigModule.java | 23 +++- .../it/ServingRedisAzureRegistryIT.java | 105 ++++++++++++++++++ .../serving/it/ServingRedisGSRegistryIT.java | 74 +++++++----- .../serving/it/ServingRedisS3RegistryIT.java | 18 ++- 10 files changed, 335 insertions(+), 52 deletions(-) create mode 100644 java/serving/src/main/java/feast/serving/registry/AzureRegistryFile.java create mode 100644 java/serving/src/test/java/feast/serving/it/ServingRedisAzureRegistryIT.java diff --git a/java/CONTRIBUTING.md b/java/CONTRIBUTING.md index 65d43d0de5..6d53c7b5c2 100644 --- a/java/CONTRIBUTING.md +++ b/java/CONTRIBUTING.md @@ -50,7 +50,7 @@ Automatically format the code to conform the style guide by: ```sh # formats all code in the feast-java repository -mvn spotless:apply +make format-java ``` > If you're using IntelliJ, you can import these [code style settings](https://github.com/google/styleguide/blob/gh-pages/intellij-java-google-style.xml) @@ -66,7 +66,7 @@ Run all Unit tests: make test-java ``` -Run all Integration tests (note: this also runs GCS + S3 based tests which should fail): +Run all Integration tests: ``` make test-java-integration ``` diff --git a/java/pom.xml b/java/pom.xml index 59c6733784..ccb3312596 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -68,6 +68,8 @@ 0.21.0 1.6.6 30.1-jre + 3.4.34 + 4.1.101.Final ${javax.validation.version} + + com.fasterxml.jackson.core + jackson-core + ${jackson.version} + + + com.fasterxml.jackson.core + jackson-databind + ${jackson.version} + + + com.fasterxml.jackson.core + jackson-annotations + ${jackson.version} + + + + io.netty + netty-common + ${netty.version} + + + io.netty + netty-buffer + ${netty.version} + + + io.netty + netty-handler + ${netty.version} + + + io.netty + netty-transport + ${netty.version} + + + + io.projectreactor + reactor-core + ${reactor.version} + + org.junit.platform junit-platform-engine @@ -246,7 +291,7 @@ - ${license.content} + ${license.content} 1.7 @@ -264,15 +309,15 @@ - - - spotless-check - process-test-classes - - check - - - + + + spotless-check + process-test-classes + + check + + + org.apache.maven.plugins diff --git a/java/serving/.gitignore b/java/serving/.gitignore index 6c6b6d8d8f..750b7f498b 100644 --- a/java/serving/.gitignore +++ b/java/serving/.gitignore @@ -34,4 +34,7 @@ feast-serving.jar /.nb-gradle/ ## Feast Temporary Files ## -/temp/ \ No newline at end of file +/temp/ + +## Generated test data ## +**/*.parquet \ No newline at end of file diff --git a/java/serving/pom.xml b/java/serving/pom.xml index 19e54e1362..6929d65d93 100644 --- a/java/serving/pom.xml +++ b/java/serving/pom.xml @@ -16,8 +16,8 @@ ~ --> + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 @@ -121,6 +121,19 @@ 5.0.1 + + + + com.azure + azure-storage-blob + 12.25.2 + + + com.azure + azure-identity + 1.11.3 + + org.slf4j @@ -356,11 +369,11 @@ 2.7.4 test - - io.lettuce - lettuce-core - 6.0.2.RELEASE - + + io.lettuce + lettuce-core + 6.0.2.RELEASE + org.apache.commons commons-lang3 diff --git a/java/serving/src/main/java/feast/serving/registry/AzureRegistryFile.java b/java/serving/src/main/java/feast/serving/registry/AzureRegistryFile.java new file mode 100644 index 0000000000..72f6d476d5 --- /dev/null +++ b/java/serving/src/main/java/feast/serving/registry/AzureRegistryFile.java @@ -0,0 +1,57 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2021 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.serving.registry; + +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.BlobServiceClient; +import com.google.protobuf.InvalidProtocolBufferException; +import feast.proto.core.RegistryProto; +import java.util.Objects; +import java.util.Optional; + +public class AzureRegistryFile implements RegistryFile { + private final BlobClient blobClient; + private String lastKnownETag; + + public AzureRegistryFile(BlobServiceClient blobServiceClient, String url) { + String[] split = url.replace("az://", "").split("/"); + String objectPath = String.join("/", java.util.Arrays.copyOfRange(split, 1, split.length)); + this.blobClient = blobServiceClient.getBlobContainerClient(split[0]).getBlobClient(objectPath); + } + + @Override + public RegistryProto.Registry getContent() { + try { + return RegistryProto.Registry.parseFrom(blobClient.downloadContent().toBytes()); + } catch (InvalidProtocolBufferException e) { + throw new RuntimeException( + String.format( + "Couldn't read remote registry: %s. Error: %s", + blobClient.getBlobUrl(), e.getMessage())); + } + } + + @Override + public Optional getContentIfModified() { + String eTag = blobClient.getProperties().getETag(); + if (Objects.equals(eTag, this.lastKnownETag)) { + return Optional.empty(); + } else this.lastKnownETag = eTag; + + return Optional.of(getContent()); + } +} diff --git a/java/serving/src/main/java/feast/serving/service/config/ApplicationProperties.java b/java/serving/src/main/java/feast/serving/service/config/ApplicationProperties.java index 7cef10e61a..91c5440cb7 100644 --- a/java/serving/src/main/java/feast/serving/service/config/ApplicationProperties.java +++ b/java/serving/src/main/java/feast/serving/service/config/ApplicationProperties.java @@ -95,6 +95,7 @@ public static class FeastProperties { private String gcpProject; private String awsRegion; private String transformationServiceEndpoint; + private String azureStorageAccount; public String getRegistry() { return registry; @@ -205,6 +206,14 @@ public String getTransformationServiceEndpoint() { public void setTransformationServiceEndpoint(String transformationServiceEndpoint) { this.transformationServiceEndpoint = transformationServiceEndpoint; } + + public String getAzureStorageAccount() { + return azureStorageAccount; + } + + public void setAzureStorageAccount(String azureStorageAccount) { + this.azureStorageAccount = azureStorageAccount; + } } /** Store configuration class for database that this Feast Serving uses. */ diff --git a/java/serving/src/main/java/feast/serving/service/config/RegistryConfigModule.java b/java/serving/src/main/java/feast/serving/service/config/RegistryConfigModule.java index cfb4666f07..5ab951c71c 100644 --- a/java/serving/src/main/java/feast/serving/service/config/RegistryConfigModule.java +++ b/java/serving/src/main/java/feast/serving/service/config/RegistryConfigModule.java @@ -18,6 +18,9 @@ import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.azure.identity.DefaultAzureCredentialBuilder; +import com.azure.storage.blob.BlobServiceClient; +import com.azure.storage.blob.BlobServiceClientBuilder; import com.google.cloud.storage.Storage; import com.google.cloud.storage.StorageOptions; import com.google.inject.AbstractModule; @@ -43,11 +46,27 @@ public AmazonS3 awsStorage(ApplicationProperties applicationProperties) { .build(); } + @Provides + public BlobServiceClient azureStorage(ApplicationProperties applicationProperties) { + + BlobServiceClient blobServiceClient = + new BlobServiceClientBuilder() + .endpoint( + String.format( + "https://%s.blob.core.windows.net", + applicationProperties.getFeast().getAzureStorageAccount())) + .credential(new DefaultAzureCredentialBuilder().build()) + .buildClient(); + + return blobServiceClient; + } + @Provides RegistryFile registryFile( ApplicationProperties applicationProperties, Provider storageProvider, - Provider amazonS3Provider) { + Provider amazonS3Provider, + Provider azureProvider) { String registryPath = applicationProperties.getFeast().getRegistry(); Optional scheme = Optional.ofNullable(URI.create(registryPath).getScheme()); @@ -57,6 +76,8 @@ RegistryFile registryFile( return new GSRegistryFile(storageProvider.get(), registryPath); case "s3": return new S3RegistryFile(amazonS3Provider.get(), registryPath); + case "az": + return new AzureRegistryFile(azureProvider.get(), registryPath); case "": case "file": return new LocalRegistryFile(registryPath); diff --git a/java/serving/src/test/java/feast/serving/it/ServingRedisAzureRegistryIT.java b/java/serving/src/test/java/feast/serving/it/ServingRedisAzureRegistryIT.java new file mode 100644 index 0000000000..8ab658fc2a --- /dev/null +++ b/java/serving/src/test/java/feast/serving/it/ServingRedisAzureRegistryIT.java @@ -0,0 +1,105 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2021 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.serving.it; + +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.BlobServiceClient; +import com.azure.storage.blob.BlobServiceClientBuilder; +import com.azure.storage.common.StorageSharedKeyCredential; +import com.google.inject.AbstractModule; +import com.google.inject.Provides; +import feast.proto.core.RegistryProto; +import feast.serving.service.config.ApplicationProperties; +import java.io.ByteArrayInputStream; +import org.junit.jupiter.api.BeforeAll; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.junit.jupiter.Container; + +public class ServingRedisAzureRegistryIT extends ServingBaseTests { + private static final String TEST_ACCOUNT_NAME = "devstoreaccount1"; + private static final String TEST_ACCOUNT_KEY = + "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="; + private static final int BLOB_STORAGE_PORT = 10000; + private static final String TEST_CONTAINER = "test-container"; + private static final StorageSharedKeyCredential CREDENTIAL = + new StorageSharedKeyCredential(TEST_ACCOUNT_NAME, TEST_ACCOUNT_KEY); + + @Container + static final GenericContainer azureBlobMock = + new GenericContainer<>("mcr.microsoft.com/azure-storage/azurite:latest") + .waitingFor(Wait.forLogMessage("Azurite Blob service successfully listens on.*", 1)) + .withExposedPorts(BLOB_STORAGE_PORT) + .withCommand("azurite-blob", "--blobHost", "0.0.0.0"); + + private static BlobServiceClient createClient() { + return new BlobServiceClientBuilder() + .endpoint( + String.format( + "http://localhost:%d/%s", + azureBlobMock.getMappedPort(BLOB_STORAGE_PORT), TEST_ACCOUNT_NAME)) + .credential(CREDENTIAL) + .buildClient(); + } + + private static void putToStorage(RegistryProto.Registry registry) { + BlobServiceClient client = createClient(); + BlobClient blobClient = + client.getBlobContainerClient(TEST_CONTAINER).getBlobClient("registry.db"); + + blobClient.upload(new ByteArrayInputStream(registry.toByteArray())); + } + + @BeforeAll + static void setUp() { + BlobServiceClient client = createClient(); + client.createBlobContainer(TEST_CONTAINER); + + putToStorage(registryProto); + } + + @Override + ApplicationProperties.FeastProperties createFeastProperties() { + final ApplicationProperties.FeastProperties feastProperties = + TestUtils.createBasicFeastProperties( + environment.getServiceHost("redis", 6379), environment.getServicePort("redis", 6379)); + feastProperties.setRegistry(String.format("az://%s/registry.db", TEST_CONTAINER)); + + return feastProperties; + } + + @Override + void updateRegistryFile(RegistryProto.Registry registry) { + putToStorage(registry); + } + + @Override + AbstractModule registryConfig() { + return new AbstractModule() { + @Provides + public BlobServiceClient awsStorage() { + return new BlobServiceClientBuilder() + .endpoint( + String.format( + "http://localhost:%d/%s", + azureBlobMock.getMappedPort(BLOB_STORAGE_PORT), TEST_ACCOUNT_NAME)) + .credential(CREDENTIAL) + .buildClient(); + } + }; + } +} diff --git a/java/serving/src/test/java/feast/serving/it/ServingRedisGSRegistryIT.java b/java/serving/src/test/java/feast/serving/it/ServingRedisGSRegistryIT.java index 925f1887d2..96aa2077c0 100644 --- a/java/serving/src/test/java/feast/serving/it/ServingRedisGSRegistryIT.java +++ b/java/serving/src/test/java/feast/serving/it/ServingRedisGSRegistryIT.java @@ -16,47 +16,54 @@ */ package feast.serving.it; -import static org.junit.jupiter.api.Assertions.assertArrayEquals; - +import com.google.auth.oauth2.AccessToken; +import com.google.auth.oauth2.ServiceAccountCredentials; import com.google.cloud.storage.*; -import com.google.cloud.storage.testing.RemoteStorageHelper; +import com.google.inject.AbstractModule; +import com.google.inject.Provides; import feast.proto.core.RegistryProto; import feast.serving.service.config.ApplicationProperties; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; -import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.junit.jupiter.Container; public class ServingRedisGSRegistryIT extends ServingBaseTests { - static Storage storage = - RemoteStorageHelper.create() - .getOptions() - .toBuilder() - .setProjectId(System.getProperty("GCP_PROJECT", "kf-feast")) - .build() - .getService(); - static final String bucket = RemoteStorageHelper.generateBucketName(); + private static final String TEST_PROJECT = "test-project"; + private static final String TEST_BUCKET = "test-bucket"; + private static final BlobId blobId = BlobId.of(TEST_BUCKET, "registry.db");; + private static final int GCS_PORT = 4443; - static void putToStorage(BlobId blobId, RegistryProto.Registry registry) { - storage.create(BlobInfo.newBuilder(blobId).build(), registry.toByteArray()); + @Container + static final GenericContainer gcsMock = + new GenericContainer<>("fsouza/fake-gcs-server") + .withExposedPorts(GCS_PORT) + .withCreateContainerCmdModifier( + cmd -> cmd.withEntrypoint("/bin/fake-gcs-server", "-scheme", "http")); - assertArrayEquals(storage.get(blobId).getContent(), registry.toByteArray()); - } + public static final AccessToken credential = new AccessToken("test-token", null); - static BlobId blobId; + static void putToStorage(RegistryProto.Registry registry) { + Storage gcsClient = createClient(); + + gcsClient.create(BlobInfo.newBuilder(blobId).build(), registry.toByteArray()); + } @BeforeAll static void setUp() { - storage.create(BucketInfo.of(bucket)); - blobId = BlobId.of(bucket, "registry.db"); + Storage gcsClient = createClient(); + gcsClient.create(BucketInfo.of(TEST_BUCKET)); - putToStorage(blobId, registryProto); + putToStorage(registryProto); } - @AfterAll - static void tearDown() throws ExecutionException, InterruptedException { - RemoteStorageHelper.forceDelete(storage, bucket, 5, TimeUnit.SECONDS); + private static Storage createClient() { + return StorageOptions.newBuilder() + .setProjectId(TEST_PROJECT) + .setCredentials(ServiceAccountCredentials.create(credential)) + .setHost("http://localhost:" + gcsMock.getMappedPort(GCS_PORT)) + .build() + .getService(); } @Override @@ -71,6 +78,21 @@ ApplicationProperties.FeastProperties createFeastProperties() { @Override void updateRegistryFile(RegistryProto.Registry registry) { - putToStorage(blobId, registry); + putToStorage(registry); + } + + @Override + AbstractModule registryConfig() { + return new AbstractModule() { + @Provides + Storage googleStorage(ApplicationProperties applicationProperties) { + return StorageOptions.newBuilder() + .setProjectId(TEST_PROJECT) + .setCredentials(ServiceAccountCredentials.create(credential)) + .setHost("http://localhost:" + gcsMock.getMappedPort(GCS_PORT)) + .build() + .getService(); + } + }; } } diff --git a/java/serving/src/test/java/feast/serving/it/ServingRedisS3RegistryIT.java b/java/serving/src/test/java/feast/serving/it/ServingRedisS3RegistryIT.java index 12315c9e48..52e1af9065 100644 --- a/java/serving/src/test/java/feast/serving/it/ServingRedisS3RegistryIT.java +++ b/java/serving/src/test/java/feast/serving/it/ServingRedisS3RegistryIT.java @@ -17,6 +17,8 @@ package feast.serving.it; import com.adobe.testing.s3mock.testcontainers.S3MockContainer; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; import com.amazonaws.client.builder.AwsClientBuilder; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; @@ -30,13 +32,18 @@ import org.testcontainers.junit.jupiter.Container; public class ServingRedisS3RegistryIT extends ServingBaseTests { + private static final String TEST_REGION = "us-east-1"; + private static final String TEST_BUCKET = "test-bucket"; @Container static final S3MockContainer s3Mock = new S3MockContainer("2.2.3"); + private static final AWSStaticCredentialsProvider credentials = + new AWSStaticCredentialsProvider(new BasicAWSCredentials("anyAccessKey", "anySecretKey")); private static AmazonS3 createClient() { return AmazonS3ClientBuilder.standard() .withEndpointConfiguration( new AwsClientBuilder.EndpointConfiguration( - String.format("http://localhost:%d", s3Mock.getHttpServerPort()), "us-east-1")) + String.format("http://localhost:%d", s3Mock.getHttpServerPort()), TEST_REGION)) + .withCredentials(credentials) .enablePathStyleAccess() .build(); } @@ -48,13 +55,13 @@ private static void putToStorage(RegistryProto.Registry proto) { metadata.setContentType("application/protobuf"); AmazonS3 s3Client = createClient(); - s3Client.putObject("test-bucket", "registry.db", new ByteArrayInputStream(bytes), metadata); + s3Client.putObject(TEST_BUCKET, "registry.db", new ByteArrayInputStream(bytes), metadata); } @BeforeAll static void setUp() { AmazonS3 s3Client = createClient(); - s3Client.createBucket("test-bucket"); + s3Client.createBucket(TEST_BUCKET); putToStorage(registryProto); } @@ -64,7 +71,7 @@ ApplicationProperties.FeastProperties createFeastProperties() { final ApplicationProperties.FeastProperties feastProperties = TestUtils.createBasicFeastProperties( environment.getServiceHost("redis", 6379), environment.getServicePort("redis", 6379)); - feastProperties.setRegistry("s3://test-bucket/registry.db"); + feastProperties.setRegistry(String.format("s3://%s/registry.db", TEST_BUCKET)); return feastProperties; } @@ -82,7 +89,8 @@ public AmazonS3 awsStorage() { return AmazonS3ClientBuilder.standard() .withEndpointConfiguration( new AwsClientBuilder.EndpointConfiguration( - String.format("http://localhost:%d", s3Mock.getHttpServerPort()), "us-east-1")) + String.format("http://localhost:%d", s3Mock.getHttpServerPort()), TEST_REGION)) + .withCredentials(credentials) .enablePathStyleAccess() .build(); }