-
Notifications
You must be signed in to change notification settings - Fork 3.5k
/
test-s3-minio.R
128 lines (114 loc) · 4.15 KB
/
test-s3-minio.R
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
skip_if_not_available("s3")
skip_if_not(nzchar(Sys.which("minio")), message = "minio is not installed.")
library(dplyr)
minio_dir <- Sys.getenv("MINIO_DATA_DIR", tempfile())
minio_key <- "minioadmin"
minio_secret <- "minioadmin"
minio_port <- Sys.getenv("MINIO_PORT", "9000")
# Start minio server
dir.create(minio_dir, showWarnings = FALSE)
pid_minio <- sys::exec_background("minio", c("server", minio_dir, "--address", sprintf(":%s", minio_port)),
std_out = FALSE
)
withr::defer(tools::pskill(pid_minio))
# Helper function for minio URIs
minio_uri <- function(...) {
template <- "s3://%s:%s@%s?scheme=http&endpoint_override=localhost%s%s"
sprintf(template, minio_key, minio_secret, minio_path(...), "%3A", minio_port)
}
minio_path <- function(...) paste(now, ..., sep = "/")
# Create a "bucket" on minio for this test run, which we'll delete when done.
fs <- S3FileSystem$create(
access_key = minio_key,
secret_key = minio_secret,
scheme = "http",
endpoint_override = paste0("localhost:", minio_port),
allow_bucket_creation = TRUE,
allow_bucket_deletion = TRUE
)
limited_fs <- S3FileSystem$create(
access_key = minio_key,
secret_key = minio_secret,
scheme = "http",
endpoint_override = paste0("localhost:", minio_port),
allow_bucket_creation = FALSE,
allow_bucket_deletion = FALSE
)
now <- as.character(as.numeric(Sys.time()))
fs$CreateDir(now)
# Clean up when we're all done
withr::defer(fs$DeleteDir(now))
test_filesystem("s3", fs, minio_path, minio_uri)
test_that("CreateDir fails on bucket if allow_bucket_creation=False", {
now_tmp <- paste0(now, "-test-fail-delete")
fs$CreateDir(now_tmp)
expect_error(limited_fs$CreateDir("should-fail"))
expect_error(limited_fs$DeleteDir(now_tmp))
})
test_that("S3FileSystem input validation", {
expect_error(
S3FileSystem$create(access_key = "foo"),
"Key authentication requires both access_key and secret_key"
)
expect_error(
S3FileSystem$create(secret_key = "foo"),
"Key authentication requires both access_key and secret_key"
)
expect_error(
S3FileSystem$create(session_token = "foo"),
paste0(
"In order to initialize a session with temporary credentials, ",
"both secret_key and access_key must be provided ",
"in addition to session_token."
)
)
expect_error(
S3FileSystem$create(access_key = "foo", secret_key = "asdf", anonymous = TRUE),
'Cannot specify "access_key" and "secret_key" when anonymous = TRUE'
)
expect_error(
S3FileSystem$create(access_key = "foo", secret_key = "asdf", role_arn = "qwer"),
"Cannot provide both key authentication and role_arn"
)
expect_error(
S3FileSystem$create(access_key = "foo", secret_key = "asdf", external_id = "qwer"),
'Cannot specify "external_id" without providing a role_arn string'
)
expect_error(
S3FileSystem$create(external_id = "foo"),
'Cannot specify "external_id" without providing a role_arn string'
)
})
test_that("Confirm s3_bucket works with endpoint_override", {
bucket <- s3_bucket(
now,
access_key = minio_key,
secret_key = minio_secret,
scheme = "http",
endpoint_override = paste0("localhost:", minio_port)
)
expect_r6_class(bucket, "SubTreeFileSystem")
os <- bucket$OpenOutputStream("bucket-test.csv")
write_csv_arrow(example_data, os)
os$close()
expect_true("bucket-test.csv" %in% bucket$ls())
bucket$DeleteFile("bucket-test.csv")
})
# Cleanup
withr::deferred_run()