Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

GH-937 Update S3 client #936

Merged
merged 8 commits into from
Nov 22, 2021
Merged

GH-937 Update S3 client #936

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
c0657e6
MDP-1259: Zeppelin + reposilite PoC
Nov 18, 2021
de75f2b
reorder params
Nov 19, 2021
a5cd263
Merge pull request #1 from envomp/MDP-1259
envomp Nov 19, 2021
6e84e41
Merge branch 'dzikoysk:main' into main
envomp Nov 19, 2021
26784e9
Merge branch 'dzikoysk:main' into main
envomp Nov 22, 2021
93592f8
update documentation and tests
Nov 22, 2021
400d007
Merge remote-tracking branch 'origin/main' into main
Nov 22, 2021
9343203
Junit test fix - add missing prefixes
Nov 22, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion ...ackend/src/integration/kotlin/com/reposilite/ReposiliteRemoteIntegrationJunitExtension.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ internal class ReposiliteRemoteIntegrationJunitExtension : Extension, BeforeEach

type.getField("_extensionInitialized").set(instance, true)
type.getField("_database").set(instance, "mysql ${mariaDb.host}:${mariaDb.getMappedPort(3306)} ${mariaDb.databaseName} ${mariaDb.username} ${mariaDb.password}")
type.getField("_storageProvider").set(instance, "s3 -e ${localstack.getEndpointOverride(S3)} ${localstack.accessKey} ${localstack.secretKey} ${localstack.region} {repository}")
type.getField("_storageProvider").set(instance, "s3 -e ${localstack.getEndpointOverride(S3)} -a ${localstack.accessKey} -s ${localstack.secretKey} -r ${localstack.region} {repository}")
}
}

Expand Down
2 changes: 1 addition & 1 deletion ...egration/kotlin/com/reposilite/storage/infrastructure/S3StorageProviderIntegrationTest.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ internal class S3StorageProviderIntegrationTest : StorageProviderIntegrationTest
InMemoryLogger(),
rootDirectory.toPath(),
"test-repository",
"s3 --endpoint ${localstack.getEndpointOverride(S3).toURL()} ${localstack.accessKey} ${localstack.secretKey} ${localstack.region} test-repository"
"s3 test-repository --endpoint ${localstack.getEndpointOverride(S3).toURL()} --access-key ${localstack.accessKey} --secret-key ${localstack.secretKey} --region ${localstack.region}"
)
}

Expand Down
18 changes: 11 additions & 7 deletions reposilite-backend/src/main/kotlin/com/reposilite/settings/SharedConfiguration.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,9 +88,13 @@ class SharedConfiguration : Serializable, DeserializationHandler<SharedConfigura
@Description("# Example usage:")
@Description("# storageProvider: fs --quota 50GB")
@Description("# > S3 provider. Supported flags:")
@Description("# --endpoint = custom endpoint with which the S3 provider should communicate (optional)")
@Description("# --endpoint = overwrite the AWS endpoint (optional)")
@Description("# --access-key = overwrite AWS access-key used to authenticate (optional)")
@Description("# --secret-key = overwrite AWS secret-key used to authenticate (optional)")
@Description("# --region = overwrite AWS region (optional)")
@Description("# See software.amazon.awssdk.services.s3.S3Client for default values")
@Description("# Example usage:")
@Description("# storageProvider: s3 --endpoint custom.endpoint.com accessKey secretKey region bucket-name")
@Description("# storageProvider: s3 bucket-name --endpoint custom.endpoint.com --access-key accessKey --secret-key secretKey --region region")
@JvmField
var storageProvider = "fs --quota 100%"

Expand All @@ -104,16 +108,16 @@ class SharedConfiguration : Serializable, DeserializationHandler<SharedConfigura

@Command(name = "s3", description = ["Amazon S3 storage provider settings"])
internal class S3StorageProviderSettings : Validator() {
@Parameters(index = "0", paramLabel = "<bucket-name>")
lateinit var bucketName: String
@Option(names = ["-e", "--endpoint"], defaultValue = "")
lateinit var endpoint: String
@Parameters(index = "0", paramLabel = "<access-key>")
@Option(names = ["-a", "--access-key"], defaultValue = "")
lateinit var accessKey: String
@Parameters(index = "1", paramLabel = "<secret-key>")
@Option(names = ["-s", "--secret-key"], defaultValue = "")
lateinit var secretKey: String
@Parameters(index = "2", paramLabel = "<region>")
@Option(names = ["-r", "--region"], defaultValue = "")
lateinit var region: String
@Parameters(index = "3", paramLabel = "<bucket-name>")
lateinit var bucketName: String
}

@Description("")
Expand Down
11 changes: 9 additions & 2 deletions reposilite-backend/src/main/kotlin/com/reposilite/storage/StorageProviderFactory.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,8 +57,15 @@ object StorageProviderFactory {
val settings = loadCommandBasedConfiguration(S3StorageProviderSettings(), storageDescription).configuration

val client = S3Client.builder()
.credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create(settings.accessKey, settings.secretKey)))
.region(Region.of(settings.region))

if (settings.accessKey.isNotEmpty() && settings.secretKey.isNotEmpty()) {
client.credentialsProvider(StaticCredentialsProvider
.create(AwsBasicCredentials.create(settings.accessKey, settings.secretKey)))
}

if (settings.region.isNotEmpty()) {
client.region(Region.of(settings.region))
}

if (settings.endpoint.isNotEmpty()) {
client.endpointOverride(URI.create(settings.endpoint))
Expand Down
34 changes: 12 additions & 22 deletions ...silite-backend/src/main/kotlin/com/reposilite/storage/infrastructure/S3StorageProvider.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,7 @@ package com.reposilite.storage.infrastructure

import com.reposilite.journalist.Journalist
import com.reposilite.journalist.Logger
import com.reposilite.shared.fs.DirectoryInfo
import com.reposilite.shared.fs.DocumentInfo
import com.reposilite.shared.fs.FileDetails
import com.reposilite.shared.fs.SimpleDirectoryInfo
import com.reposilite.shared.fs.getExtension
import com.reposilite.shared.fs.getSimpleName
import com.reposilite.shared.fs.safeResolve
import com.reposilite.shared.fs.*
import com.reposilite.storage.StorageProvider
import com.reposilite.web.http.ErrorResponse
import com.reposilite.web.http.errorResponse
Expand All @@ -38,15 +32,7 @@ import panda.std.Result.ok
import panda.std.asSuccess
import software.amazon.awssdk.core.sync.RequestBody
import software.amazon.awssdk.services.s3.S3Client
import software.amazon.awssdk.services.s3.model.BucketAlreadyExistsException
import software.amazon.awssdk.services.s3.model.BucketAlreadyOwnedByYouException
import software.amazon.awssdk.services.s3.model.DeleteObjectRequest
import software.amazon.awssdk.services.s3.model.GetObjectRequest
import software.amazon.awssdk.services.s3.model.HeadObjectRequest
import software.amazon.awssdk.services.s3.model.HeadObjectResponse
import software.amazon.awssdk.services.s3.model.ListObjectsRequest
import software.amazon.awssdk.services.s3.model.NoSuchKeyException
import software.amazon.awssdk.services.s3.model.PutObjectRequest
import software.amazon.awssdk.services.s3.model.*
import java.io.File
import java.io.IOException
import java.io.InputStream
Expand All @@ -61,14 +47,18 @@ class S3StorageProvider(
) : StorageProvider, Journalist {

init {
val headBucketRequest = HeadBucketRequest.builder().bucket(bucket).build()

try {
s3.createBucket {
it.bucket(bucket)
s3.headBucket(headBucketRequest)
} catch (noSuchBucket: NoSuchBucketException) {
try {
s3.createBucket { it.bucket(bucket) }
} catch (bucketExists: BucketAlreadyExistsException) {
// ignored
} catch (buckedOwned: BucketAlreadyOwnedByYouException) {
// ignored
}
} catch (bucketExists: BucketAlreadyExistsException) {
// ignored
} catch (buckedOwned: BucketAlreadyOwnedByYouException) {
// ignored
}
}

Expand Down
Empty file added reposilite-backend/src/test/workspace/configuration.shared.cdn
View file
Open in desktop
Empty file.
16 changes: 12 additions & 4 deletions reposilite-site/docs/configuration.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,9 +76,13 @@ repositories {
# Example usage:
# storageProvider: fs --quota 50GB
# > S3 provider. Supported flags:
# --endpoint = custom endpoint with which the S3 provider should communicate (optional)
# --endpoint = overwrite the AWS endpoint (optional)
# --access-key = overwrite AWS access-key used to authenticate (optional)
# --secret-key = overwrite AWS secret-key used to authenticate (optional)
# --region = overwrite AWS region (optional)
# See software.amazon.awssdk.services.s3.S3Client for default values
# Example usage:
# storageProvider: s3 --endpoint custom.endpoint.com accessKey secretKey region bucket-name
# storageProvider: s3 bucket-name --endpoint custom.endpoint.com --access-key accessKey --secret-key secretKey --region region
storageProvider: fs --quota 100%

# List of proxied repositories associated with this repository.
Expand All @@ -105,9 +109,13 @@ repositories {
# Example usage:
# storageProvider: fs --quota 50GB
# > S3 provider. Supported flags:
# --endpoint = custom endpoint with which the S3 provider should communicate (optional)
# --endpoint = overwrite the AWS endpoint (optional)
# --access-key = overwrite AWS access-key used to authenticate (optional)
# --secret-key = overwrite AWS secret-key used to authenticate (optional)
# --region = overwrite AWS region (optional)
# See software.amazon.awssdk.services.s3.S3Client for default values
# Example usage:
# storageProvider: s3 --endpoint custom.endpoint.com accessKey secretKey region bucket-name
# storageProvider: s3 bucket-name --endpoint custom.endpoint.com --access-key accessKey --secret-key secretKey --region region
storageProvider: fs --quota 100%

# List of proxied repositories associated with this repository.
Expand Down
8 changes: 6 additions & 2 deletions reposilite-site/docs/proxy.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,13 @@ Caching may allocate thousands of artifacts, especially at the beginning - for t
# Example usage:
# storageProvider: fs --quota 50GB
# > S3 provider. Supported flags:
# --endpoint = custom endpoint with which the S3 provider should communicate (optional)
# --endpoint = overwrite the AWS endpoint (optional)
# --access-key = overwrite AWS access-key used to authenticate (optional)
# --secret-key = overwrite AWS secret-key used to authenticate (optional)
# --region = overwrite AWS region (optional)
# See software.amazon.awssdk.services.s3.S3Client for default values
# Example usage:
# storageProvider: s3 --endpoint custom.endpoint.com accessKey secretKey region bucket-name
# storageProvider: s3 bucket-name --endpoint custom.endpoint.com --access-key accessKey --secret-key secretKey --region region
storageProvider: fs --quota 85%
```

Expand Down
8 changes: 6 additions & 2 deletions reposilite-site/docs/repositories.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,9 +36,13 @@ repositories {
# Example usage:
# storageProvider: fs --quota 50GB
# > S3 provider. Supported flags:
# --endpoint = custom endpoint with which the S3 provider should communicate (optional)
# --endpoint = overwrite the AWS endpoint (optional)
# --access-key = overwrite AWS access-key used to authenticate (optional)
# --secret-key = overwrite AWS secret-key used to authenticate (optional)
# --region = overwrite AWS region (optional)
# See software.amazon.awssdk.services.s3.S3Client for default values
# Example usage:
# storageProvider: s3 --endpoint custom.endpoint.com accessKey secretKey region bucket-name
# storageProvider: s3 bucket-name --endpoint custom.endpoint.com --access-key accessKey --secret-key secretKey --region region
storageProvider: fs --quota 100%

# List of proxied repositories associated with this repository.
Expand Down