From 52dcf114de817536324e6bf796c09a532805b22b Mon Sep 17 00:00:00 2001 From: shimon Date: Wed, 6 Apr 2022 14:49:16 +0300 Subject: [PATCH 01/22] bla --- src/Storage/Device/S3.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storage/Device/S3.php b/src/Storage/Device/S3.php index 8f33a9da..911399d0 100644 --- a/src/Storage/Device/S3.php +++ b/src/Storage/Device/S3.php @@ -131,7 +131,7 @@ public function getName(): string */ public function getDescription(): string { - return 'S3 Bucket Storage drive for AWS or on premise solution'; + return 'S3 Bucket Storage drive for AWS or on premise solution...'; } /** From 609294255380fe934ab43dcd8cddeec85ef42567 Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 7 Apr 2022 06:22:58 +0300 Subject: [PATCH 02/22] S3 adapter refactor --- src/Storage/Device/BackBlaze.php | 9 +- src/Storage/Device/DOSpaces.php | 10 +- src/Storage/Device/Generic.php | 762 ++++++++++++++++++++++++++ src/Storage/Device/Linode.php | 17 +- src/Storage/Device/S3.php | 723 +----------------------- tests/Storage/Device/DOSpacesTest.php | 6 +- tests/Storage/S3Base.php | 4 +- 7 files changed, 792 insertions(+), 739 deletions(-) create mode 100644 src/Storage/Device/Generic.php diff --git a/src/Storage/Device/BackBlaze.php b/src/Storage/Device/BackBlaze.php index b0fcd597..67eeb089 100644 --- a/src/Storage/Device/BackBlaze.php +++ b/src/Storage/Device/BackBlaze.php @@ -2,9 +2,10 @@ namespace Utopia\Storage\Device; -use Utopia\Storage\Device\S3; +use Utopia\Storage\Device\Generic; -class BackBlaze extends S3 + +class BackBlaze extends Generic { /** * Regions constants @@ -31,8 +32,8 @@ class BackBlaze extends S3 */ public function __construct(string $root, string $accessKey, string $secretKey, string $bucket, string $region = self::US_WEST_004, string $acl = self::ACL_PRIVATE) { - parent::__construct($root, $accessKey, $secretKey, $bucket, $region, $acl); - $this->headers['host'] = $bucket . '.' . 's3' . '.' . $region . '.backblazeb2.com'; + $hostName = $bucket . '.' . 's3' . '.' . $region . '.backblazeb2.com'; + parent::__construct($root, $accessKey, $secretKey, $bucket, $region, $acl, $hostName); } /** diff --git a/src/Storage/Device/DOSpaces.php b/src/Storage/Device/DOSpaces.php index f2f9ebc5..c4ec4396 100644 --- a/src/Storage/Device/DOSpaces.php +++ b/src/Storage/Device/DOSpaces.php @@ -2,9 +2,10 @@ namespace Utopia\Storage\Device; -use Utopia\Storage\Device\S3; +use Utopia\Storage\Device\Generic; -class DOSpaces extends S3 + +class DOSpaces extends Generic { /** * Regions constants @@ -29,8 +30,9 @@ class DOSpaces extends S3 */ public function __construct(string $root, string $accessKey, string $secretKey, string $bucket, string $region = self::NYC3, string $acl = self::ACL_PRIVATE) { - parent::__construct($root, $accessKey, $secretKey, $bucket, $region, $acl); - $this->headers['host'] = $bucket . '.' . $region . '.digitaloceanspaces.com'; + $hostName = $bucket . '.' . $region . '.digitaloceanspaces.com'; + parent::__construct($root, $accessKey, $secretKey, $bucket, $region, $acl, $hostName); + } /** diff --git a/src/Storage/Device/Generic.php b/src/Storage/Device/Generic.php new file mode 100644 index 00000000..ae485e28 --- /dev/null +++ b/src/Storage/Device/Generic.php @@ -0,0 +1,762 @@ +accessKey = $accessKey; + $this->secretKey = $secretKey; + $this->bucket = $bucket; + $this->region = $region; + $this->root = $root; + $this->acl = $acl; + $this->hostName = $hostName; + + } + + /** + * @return string + */ + public function getName(): string + { + return 'S3 compatible Storage'; + } + + /** + * @return string + */ + public function getDescription(): string + { + return 'S3 Generic Bucket Storage drive for AWS compatible or on premise solution'; + } + + /** + * @param string $filename + * @param string|null $prefix + * + * @return string + */ + public function getPath(string $filename, string $prefix = null): string + { + $path = ''; + + for ($i = 0; $i < 4; ++$i) { + $path = ($i < \strlen($filename)) ? $path . DIRECTORY_SEPARATOR . $filename[$i] : $path . DIRECTORY_SEPARATOR . 'x'; + } + + if (!is_null($prefix)) { + $path = $prefix . DIRECTORY_SEPARATOR . $path; + } + + return $this->getRoot() . $path . DIRECTORY_SEPARATOR . $filename; + } + + /** + * @return string + */ + public function getRoot(): string + { + return $this->root; + } + + /** + * Upload. + * + * Upload a file to desired destination in the selected disk. + * return number of chunks uploaded or 0 if it fails. + * + * @param string $source + * @param string $path + * @param int chunk + * @param int chunks + * @param array $metadata + * + * @return int + * @throws \Exception + * + */ + public function upload(string $source, string $path, int $chunk = 1, int $chunks = 1, array &$metadata = []): int + { + if ($chunk == 1 && $chunks == 1) { + return $this->write($path, \file_get_contents($source), \mime_content_type($source)); + } + $uploadId = $metadata['uploadId'] ?? null; + if (empty($uploadId)) { + $uploadId = $this->createMultipartUpload($path, $metadata['content_type']); + $metadata['uploadId'] = $uploadId; + } + + $etag = $this->uploadPart($source, $path, $chunk, $uploadId); + $metadata['parts'] ??= []; + $metadata['parts'][] = ['partNumber' => $chunk, 'etag' => $etag]; + $metadata['chunks'] ??= 0; + $metadata['chunks']++; + if ($metadata['chunks'] == $chunks) { + $this->completeMultipartUpload($path, $uploadId, $metadata['parts'], $source); + } + return $metadata['chunks']; + } + + /** + * Write file by given path. + * + * @param string $path + * @param string $data + * + * @return bool + * @throws \Exception + * + */ + public function write(string $path, string $data, string $contentType = ''): bool + { + $uri = $path !== '' ? '/' . \str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path)) : '/'; + + $this->call(self::METHOD_PUT, $uri, $data, [], [ + 'content-type' => $contentType, + 'content-md5' => \base64_encode(md5($data, true)), + 'x-amz-content-sha256' => \hash('sha256', $data), + 'x-amz-acl' => $this->acl + ]); + + return true; + } + + /** + * Get the S3 response + * + * @param string $method + * @param string $uri + * @param string $data + * @param array $parameters + * @param array $headers + * + * @return object + * @throws \Exception + * + */ + private function call(string $method, string $uri, string $data = '', array $parameters = [], array $headers = []) + { + $url = 'https://' . $this->hostName . $uri . '?' . \http_build_query($parameters, '', '&', PHP_QUERY_RFC3986); + $response = new \stdClass; + $response->body = ''; + $response->headers = []; + + // Basic setup + $curl = \curl_init(); + \curl_setopt($curl, CURLOPT_USERAGENT, 'utopia-php/storage'); + \curl_setopt($curl, CURLOPT_URL, $url); + + + // Headers + $httpHeaders = []; + $headers['x-amz-date'] = \gmdate('Ymd\THis\Z'); + $headers['date'] = \gmdate('D, d M Y H:i:s T'); + $headers['host'] = $this->hostName; + + + if (!isset($headers['x-amz-content-sha256'])) { + $headers['x-amz-content-sha256'] = \hash('sha256', $data); + } + + foreach ($headers as $header => $value) { + if (\strlen($value) > 0) { + $httpHeaders[] = $header . ': ' . $value; + } + } + + $httpHeaders[] = 'Authorization: ' . $this->getSignatureV4($method, $uri, $parameters, $headers); + + \curl_setopt($curl, CURLOPT_HTTPHEADER, $httpHeaders); + \curl_setopt($curl, CURLOPT_HEADER, false); + \curl_setopt($curl, CURLOPT_RETURNTRANSFER, false); + \curl_setopt($curl, CURLOPT_WRITEFUNCTION, function ($curl, string $data) use ($response) { + $response->body .= $data; + return \strlen($data); + }); + curl_setopt($curl, CURLOPT_HEADERFUNCTION, function ($curl, string $header) use (&$response) { + $len = strlen($header); + $header = explode(':', $header, 2); + + if (count($header) < 2) { // ignore invalid headers + return $len; + } + + $response->headers[strtolower(trim($header[0]))] = trim($header[1]); + + return $len; + }); + \curl_setopt($curl, CURLOPT_FOLLOWLOCATION, true); + \curl_setopt($curl, CURLOPT_CUSTOMREQUEST, $method); + + // Request types + switch ($method) { + case self::METHOD_PUT: + case self::METHOD_POST: // POST only used for CloudFront + \curl_setopt($curl, CURLOPT_POSTFIELDS, $data); + break; + case self::METHOD_HEAD: + case self::METHOD_DELETE: + \curl_setopt($curl, CURLOPT_NOBODY, true); + break; + } + + $result = \curl_exec($curl); + + if (!$result) { + throw new Exception(\curl_error($curl)); + } + + $response->code = \curl_getinfo($curl, CURLINFO_HTTP_CODE); + if ($response->code >= 400) { + throw new Exception($response->body, $response->code); + } + + \curl_close($curl); + + // Parse body into XML + if ((isset($response->headers['content-type']) && $response->headers['content-type'] == 'application/xml') || (str_starts_with($response->body, 'headers['content-type'] ?? '') !== 'image/svg+xml')) { + $response->body = \simplexml_load_string($response->body); + $response->body = json_decode(json_encode($response->body), true); + } + + return $response; + } + + /** + * Generate the headers for AWS Signature V4 + * @param string $method + * @param string $uri + * @param array parameters + * @param array headers + * + * @return string + */ + private function getSignatureV4(string $method, string $uri, array $parameters = [], $headers = []): string + { + $service = 's3'; + $region = $this->region; + + $algorithm = 'AWS4-HMAC-SHA256'; + $combinedHeaders = []; + $amzDateStamp = \substr($headers['x-amz-date'], 0, 8); + + // CanonicalHeaders + foreach ($headers as $k => $v) { + $combinedHeaders[\strtolower($k)] = \trim($v); + } + + uksort($combinedHeaders, [& $this, 'sortMetaHeadersCmp']); + + // Convert null query string parameters to strings and sort + uksort($parameters, [& $this, 'sortMetaHeadersCmp']); + $queryString = \http_build_query($parameters, '', '&', PHP_QUERY_RFC3986); + + // Payload + $amzPayload = [$method]; + + $qsPos = \strpos($uri, '?'); + $amzPayload[] = ($qsPos === false ? $uri : \substr($uri, 0, $qsPos)); + + $amzPayload[] = $queryString; + + foreach ($combinedHeaders as $k => $v) { // add header as string to requests + $amzPayload[] = $k . ':' . $v; + } + + $amzPayload[] = ''; // add a blank entry so we end up with an extra line break + $amzPayload[] = \implode(';', \array_keys($combinedHeaders)); // SignedHeaders + $amzPayload[] = $headers['x-amz-content-sha256']; // payload hash + + $amzPayloadStr = \implode("\n", $amzPayload); // request as string + + // CredentialScope + $credentialScope = [$amzDateStamp, $region, $service, 'aws4_request']; + + // stringToSign + $stringToSignStr = \implode("\n", [$algorithm, $headers['x-amz-date'], + \implode('/', $credentialScope), \hash('sha256', $amzPayloadStr)]); + + // Make Signature + $kSecret = 'AWS4' . $this->secretKey; + $kDate = \hash_hmac('sha256', $amzDateStamp, $kSecret, true); + $kRegion = \hash_hmac('sha256', $region, $kDate, true); + $kService = \hash_hmac('sha256', $service, $kRegion, true); + $kSigning = \hash_hmac('sha256', 'aws4_request', $kService, true); + + $signature = \hash_hmac('sha256', \utf8_encode($stringToSignStr), $kSigning); + + return $algorithm . ' ' . \implode(',', [ + 'Credential=' . $this->accessKey . '/' . \implode('/', $credentialScope), + 'SignedHeaders=' . \implode(';', \array_keys($combinedHeaders)), + 'Signature=' . $signature, + ]); + } + + /** + * Start Multipart Upload + * + * Initiate a multipart upload and return an upload ID. + * + * @param string $path + * @param string $contentType + * + * @return string + * @throws \Exception + * + */ + protected function createMultipartUpload(string $path, string $contentType): string + { + $uri = $path !== '' ? '/' . \str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path)) : '/'; + + $response = $this->call(self::METHOD_POST, $uri, '', ['uploads' => ''], [ + 'content-type' => $contentType, + 'content-md5' => \base64_encode(md5('', true)), + 'x-amz-acl' => $this->acl + ]); + return $response->body['UploadId']; + } + + /** + * Upload Part + * + * @param string $source + * @param string $path + * @param int $chunk + * @param string $uploadId + * + * @return string + * @throws \Exception + * + */ + protected function uploadPart(string $source, string $path, int $chunk, string $uploadId): string + { + $uri = $path !== '' ? '/' . \str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path)) : '/'; + $data = \file_get_contents($source); + $response = $this->call(self::METHOD_PUT, $uri, $data, [ + 'partNumber' => $chunk, + 'uploadId' => $uploadId + ], [ + 'content-type' => \mime_content_type($source), + 'content-md5' => \base64_encode(md5($data, true)), + 'x-amz-content-sha256' => \hash('sha256', $data) + ]); + + return $response->headers['etag']; + } + + /** + * Complete Multipart Upload + * + * @param string $path + * @param string $uploadId + * @param array $parts + * + * @return bool + * @throws \Exception + * + */ + protected function completeMultipartUpload(string $path, string $uploadId, array $parts, $source): bool + { + + $uri = $path !== '' ? '/' . \str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path)) : '/'; + + $body = ''; + foreach ($parts as $part) { + $body .= "{$part['etag']}{$part['partNumber']}"; + } + $body .= ''; + + $this->call(self::METHOD_POST, $uri, $body, [ + 'uploadId' => $uploadId + ], [ + 'content-md5' => \base64_encode(md5($body, true)), + 'content-type' => \mime_content_type($source), + 'x-amz-content-sha256' => \hash('sha256', $body) + ]); + + return true; + } + + /** + * Abort Chunked Upload + * + * @param string $path + * @param string $extra + * + * @return bool + * @throws \Exception + * + */ + public function abort(string $path, string $extra = ''): bool + { + $uri = $path !== '' ? '/' . \str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path)) : '/'; + + $this->call(self::METHOD_DELETE, $uri, '', [ + 'uploadId' => $extra + ], [ + 'content-md5' => \base64_encode(md5('', true)) + ]); + return true; + } + + /** + * Move file from given source to given path, Return true on success and false on failure. + * + * @see http://php.net/manual/en/function.filesize.php + * + * @param string $source + * @param string $target + * + * @throw \Exception + * + * @return bool + */ + public function move(string $source, string $target): bool + { + $type = $this->getFileMimeType($source); + + if ($this->write($target, $this->read($source), $type)) { + $this->delete($source); + } + + return true; + } + + /** + * Returns given file path its mime type. + * + * @see http://php.net/manual/en/function.mime-content-type.php + * + * @param string $path + * + * @return string + */ + public function getFileMimeType(string $path): string + { + $response = $this->getInfo($path); + return $response['content-type'] ?? ''; + } + + /** + * Get file info + * @return array + */ + private function getInfo(string $path): array + { + $uri = $path !== '' ? '/' . \str_replace('%2F', '/', \rawurlencode($path)) : '/'; + + $response = $this->call(self::METHOD_HEAD, $uri, '', [], ['content-md5' => \base64_encode(md5('', true))]); + + return $response->headers; + } + + /** + * Read file or part of file by given path, offset and length. + * + * @param string $path + * @param int offset + * @param int length + * + * @return string + * @throws \Exception + * + */ + public function read(string $path, int $offset = 0, int $length = null): string + { + $uri = ($path !== '') ? '/' . \str_replace('%2F', '/', \rawurlencode($path)) : '/'; + $headers = [ + 'content-md5' => \base64_encode(md5('', true)) + ]; + + if ($length !== null) { + $end = $offset + $length - 1; + $this->headers['range'] = "bytes=$offset-$end"; + $headers['range'] = "bytes=$offset-$end"; + } + + $response = $this->call(self::METHOD_GET, $uri, '', [], $headers); + return $response->body; + } + + /** + * Delete file in given path, Return true on success and false on failure. + * + * @see http://php.net/manual/en/function.filesize.php + * + * @param string $path + * + * @throws \Exception + * + * @return bool + */ + public function delete(string $path, bool $recursive = false): bool + { + $uri = ($path !== '') ? '/' . \str_replace('%2F', '/', \rawurlencode($path)) : '/'; + + $this->call(self::METHOD_DELETE, $uri, '', [], [ + 'content-md5' => \base64_encode(md5('', true)), + ]); + + return true; + } + + /** + * Delete files in given path, path must be a directory. Return true on success and false on failure. + * + * @param string $path + * + * @return bool + * @throws \Exception + * + */ + public function deletePath(string $path): bool + { + $path = $this->getRoot() . '/' . $path; + + $uri = '/'; + + $continuationToken = ''; + do { + $objects = $this->listObjects($path, continuationToken: $continuationToken); + $count = (int)($objects['KeyCount'] ?? 1); + if ($count < 1) { + break; + } + $continuationToken = $objects['NextContinuationToken'] ?? ''; + $body = ''; + if ($count > 1) { + foreach ($objects['Contents'] as $object) { + $body .= "{$object['Key']}"; + } + } else { + $body .= "{$objects['Contents']['Key']}"; + } + $body .= 'true'; + $body .= ''; + + $this->call(self::METHOD_POST, $uri, $body, [ + 'delete' => '' + ], [ + 'content-md5' => \base64_encode(md5($body, true)), + 'content-type' => 'text/plain', + 'x-amz-content-sha256' => \hash('sha256', $body) + ]); + } while (!empty($continuationToken)); + + return true; + } + + /** + * Get list of objects in the given path. + * + * @param string $path + * + * @return array + * @throws \Exception + * + */ + private function listObjects($prefix = '', $maxKeys = 1000, $continuationToken = '') + { + $uri = '/'; + + $parameters = [ + 'list-type' => 2, + 'prefix' => $prefix, + 'max-keys' => $maxKeys, + ]; + + if (!empty($continuationToken)) { + $parameters['continuation-token'] = $continuationToken; + } + + $response = $this->call(self::METHOD_GET, $uri, '', $parameters, [ + 'content-type' => 'text/plain', + 'content-md5' => \base64_encode(md5('', true)) + ]); + return $response->body; + } + + /** + * Check if file exists + * + * @param string $path + * + * @return bool + */ + public function exists(string $path): bool + { + try { + $this->getInfo($path); + } catch (\Throwable $th) { + return false; + } + + return true; + } + + /** + * Returns given file path its size. + * + * @see http://php.net/manual/en/function.filesize.php + * + * @param string $path + * + * @return int + */ + public function getFileSize(string $path): int + { + $response = $this->getInfo($path); + return (int)($response['content-length'] ?? 0); + } + + /** + * Returns given file path its MD5 hash value. + * + * @see http://php.net/manual/en/function.md5-file.php + * + * @param string $path + * + * @return string + */ + public function getFileHash(string $path): string + { + $etag = $this->getInfo($path)['etag'] ?? ''; + return (!empty($etag)) ? substr($etag, 1, -1) : $etag; + } + + /** + * Get directory size in bytes. + * + * Return -1 on error + * + * Based on http://www.jonasjohn.de/snippets/php/dir-size.htm + * + * @param string $path + * + * @return int + */ + public function getDirectorySize(string $path): int + { + return -1; + } + + /** + * Get Partition Free Space. + * + * disk_free_space — Returns available space on filesystem or disk partition + * + * @return float + */ + public function getPartitionFreeSpace(): float + { + return -1; + } + + /** + * Get Partition Total Space. + * + * disk_total_space — Returns the total size of a filesystem or disk partition + * + * @return float + */ + public function getPartitionTotalSpace(): float + { + return -1; + } + + /** + * Sort compare for meta headers + * + * @param string $a String A + * @param string $b String B + * @return integer + * @internal Used to sort x-amz meta headers + */ + private function sortMetaHeadersCmp($a, $b) + { + $lenA = \strlen($a); + $lenB = \strlen($b); + $minLen = \min($lenA, $lenB); + $ncmp = \strncmp($a, $b, $minLen); + if ($lenA == $lenB) { + return $ncmp; + } + + if (0 == $ncmp) { + return $lenA < $lenB ? -1 : 1; + } + + return $ncmp; + } +} diff --git a/src/Storage/Device/Linode.php b/src/Storage/Device/Linode.php index affd723b..ac1e3463 100644 --- a/src/Storage/Device/Linode.php +++ b/src/Storage/Device/Linode.php @@ -2,18 +2,19 @@ namespace Utopia\Storage\Device; -use Utopia\Storage\Device\S3; +use Utopia\Storage\Device\Generic; -class Linode extends S3 + +class Linode extends Generic { /** * Regions constants * */ - const EU_CENTRAL_1='eu-central-1'; - const US_SOUTHEAST_1='us-southeast-1'; - const US_EAST_1='us-east-1'; - const AP_SOUTH_1='ap-south-1'; + const EU_CENTRAL_1 = 'eu-central-1'; + const US_SOUTHEAST_1 = 'us-southeast-1'; + const US_EAST_1 = 'us-east-1'; + const AP_SOUTH_1 = 'ap-south-1'; /** @@ -28,8 +29,8 @@ class Linode extends S3 */ public function __construct(string $root, string $accessKey, string $secretKey, string $bucket, string $region = self::EU_CENTRAL_1, string $acl = self::ACL_PRIVATE) { - parent::__construct($root, $accessKey, $secretKey, $bucket, $region, $acl); - $this->headers['host'] = $bucket.'.'.$region.'.'.'linodeobjects.com'; + $hostName = $bucket.'.'.$region.'.'.'linodeobjects.com'; + parent::__construct($root, $accessKey, $secretKey, $bucket, $region, $acl, $hostName); } /** diff --git a/src/Storage/Device/S3.php b/src/Storage/Device/S3.php index 911399d0..edd245e6 100644 --- a/src/Storage/Device/S3.php +++ b/src/Storage/Device/S3.php @@ -2,21 +2,11 @@ namespace Utopia\Storage\Device; -use Exception; -use Utopia\Storage\Device; +use Utopia\Storage\Device\Generic; -class S3 extends Device -{ - const METHOD_GET = 'GET'; - const METHOD_POST = 'POST'; - const METHOD_PUT = 'PUT'; - const METHOD_PATCH = 'PATCH'; - const METHOD_DELETE = 'DELETE'; - const METHOD_HEAD = 'HEAD'; - const METHOD_OPTIONS = 'OPTIONS'; - const METHOD_CONNECT = 'CONNECT'; - const METHOD_TRACE = 'TRACE'; +class S3 extends Generic +{ /** * AWS Regions constants */ @@ -46,55 +36,7 @@ class S3 extends Device const US_GOV_EAST_1 = 'us-gov-east-1'; const US_GOV_WEST_1 = 'us-gov-west-1'; - /** - * AWS ACL Flag constants - */ - const ACL_PRIVATE = 'private'; - const ACL_PUBLIC_READ = 'public-read'; - const ACL_PUBLIC_READ_WRITE = 'public-read-write'; - const ACL_AUTHENTICATED_READ = 'authenticated-read'; - - /** - * @var string - */ - protected $accessKey; - - /** - * @var string - */ - protected $secretKey; - - /** - * @var string - */ - protected $bucket; - - /** - * @var string - */ - protected $region; - - /** - * @var string - */ - protected $acl = self::ACL_PRIVATE; - - /** - * @var string - */ - protected $root = 'temp'; - - /** - * @var array - */ - protected $headers = [ - 'host' => '', 'date' => '', 'content-md5' => '', 'content-type' => '', - ]; - /** - * @var array - */ - protected $amzHeaders; /** * S3 Constructor @@ -108,14 +50,8 @@ class S3 extends Device */ public function __construct(string $root, string $accessKey, string $secretKey, string $bucket, string $region = self::US_EAST_1, string $acl = self::ACL_PRIVATE) { - $this->accessKey = $accessKey; - $this->secretKey = $secretKey; - $this->bucket = $bucket; - $this->region = $region; - $this->root = $root; - $this->acl = $acl; - $this->headers['host'] = $this->bucket . '.s3.'.$this->region.'.amazonaws.com'; - $this->amzHeaders = []; + $hostName = $bucket . '.s3.'.$region.'.amazonaws.com'; + parent::__construct($root, $accessKey, $secretKey, $bucket, $region, $acl, $hostName); } /** @@ -134,654 +70,5 @@ public function getDescription(): string return 'S3 Bucket Storage drive for AWS or on premise solution...'; } - /** - * @return string - */ - public function getRoot(): string - { - return $this->root; - } - - /** - * @param string $filename - * @param string $prefix - * - * @return string - */ - public function getPath(string $filename, string $prefix = null): string - { - $path = ''; - - for ($i = 0; $i < 4; ++$i) { - $path = ($i < \strlen($filename)) ? $path . DIRECTORY_SEPARATOR . $filename[$i] : $path . DIRECTORY_SEPARATOR . 'x'; - } - - if(!is_null($prefix)) { - $path = $prefix . DIRECTORY_SEPARATOR . $path; - } - - return $this->getRoot() . $path . DIRECTORY_SEPARATOR . $filename; - } - - /** - * Upload. - * - * Upload a file to desired destination in the selected disk. - * return number of chunks uploaded or 0 if it fails. - * - * @param string $source - * @param string $path - * @param int chunk - * @param int chunks - * @param array $metadata - * - * @throws \Exception - * - * @return int - */ - public function upload(string $source, string $path, int $chunk = 1, int $chunks = 1, array &$metadata = []): int - { - if($chunk == 1 && $chunks == 1) { - return $this->write($path, \file_get_contents($source), \mime_content_type($source)); - } - $uploadId = $metadata['uploadId'] ?? null; - if(empty($uploadId)) { - $uploadId = $this->createMultipartUpload($path, $metadata['content_type']); - $metadata['uploadId'] = $uploadId; - } - - $etag = $this->uploadPart($source, $path, $chunk, $uploadId); - $metadata['parts'] ??= []; - $metadata['parts'][] = ['partNumber' => $chunk, 'etag' => $etag]; - $metadata['chunks'] ??= 0; - $metadata['chunks']++; - if($metadata['chunks'] == $chunks) { - $this->completeMultipartUpload($path, $uploadId, $metadata['parts']); - } - return $metadata['chunks']; - } - - /** - * Start Multipart Upload - * - * Initiate a multipart upload and return an upload ID. - * - * @param string $path - * @param string $contentType - * - * @throws \Exception - * - * @return string - */ - protected function createMultipartUpload(string $path, string $contentType): string - { - $uri = $path !== '' ? '/' . \str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path)) : '/'; - - $this->headers['content-md5'] = \base64_encode(md5('', true)); - unset($this->amzHeaders['x-amz-content-sha256']); - $this->headers['content-type'] = $contentType; - $this->amzHeaders['x-amz-acl'] = $this->acl; - $response = $this->call(self::METHOD_POST, $uri, '', ['uploads' => '']); - return $response->body['UploadId']; - } - - /** - * Upload Part - * - * @param string $source - * @param string $path - * @param int $chunk - * @param string $uploadId - * - * @throws \Exception - * - * @return string - */ - protected function uploadPart(string $source, string $path, int $chunk, string $uploadId) : string - { - $uri = $path !== '' ? '/' . \str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path)) : '/'; - - $data = \file_get_contents($source); - $this->headers['content-type'] = \mime_content_type($source); - $this->headers['content-md5'] = \base64_encode(md5($data, true)); - $this->amzHeaders['x-amz-content-sha256'] = \hash('sha256', $data); - unset($this->amzHeaders['x-amz-acl']); // ACL header is not allowed in parts, only createMultipartUpload accepts this header. - - $response = $this->call(self::METHOD_PUT, $uri, $data, [ - 'partNumber'=>$chunk, - 'uploadId' => $uploadId - ]); - - return $response->headers['etag']; - } - - /** - * Complete Multipart Upload - * - * @param string $path - * @param string $uploadId - * @param array $parts - * - * @throws \Exception - * - * @return bool - */ - protected function completeMultipartUpload(string $path, string $uploadId, array $parts): bool - { - $uri = $path !== '' ? '/' . \str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path)) : '/'; - - $body = ''; - foreach ($parts as $part) { - $body .= "{$part['etag']}{$part['partNumber']}"; - } - $body .= ''; - - $this->amzHeaders['x-amz-content-sha256'] = \hash('sha256', $body); - $this->headers['content-md5'] = \base64_encode(md5($body, true)); - $this->call(self::METHOD_POST, $uri, $body , ['uploadId' => $uploadId]); - return true; - } - - /** - * Abort Chunked Upload - * - * @param string $path - * @param string $extra - * - * @throws \Exception - * - * @return bool - */ - public function abort(string $path, string $extra = ''): bool - { - $uri = $path !== '' ? '/' . \str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path)) : '/'; - unset($this->headers['content-type']); - $this->headers['content-md5'] = \base64_encode(md5('', true)); - $this->call(self::METHOD_DELETE, $uri, '', ['uploadId' => $extra]); - return true; - } - - /** - * Read file or part of file by given path, offset and length. - * - * @param string $path - * @param int offset - * @param int length - * - * @throws \Exception - * - * @return string - */ - public function read(string $path, int $offset = 0, int $length = null): string - { - unset($this->amzHeaders['x-amz-acl']); - unset($this->amzHeaders['x-amz-content-sha256']); - unset($this->headers['content-type']); - $this->headers['content-md5'] = \base64_encode(md5('', true)); - $uri = ($path !== '') ? '/' . \str_replace('%2F', '/', \rawurlencode($path)) : '/'; - if($length !== null) { - $end = $offset + $length - 1; - $this->headers['range'] = "bytes=$offset-$end"; - } - $response = $this->call(self::METHOD_GET, $uri); - return $response->body; - } - - /** - * Write file by given path. - * - * @param string $path - * @param string $data - * - * @throws \Exception - * - * @return bool - */ - public function write(string $path, string $data, string $contentType = ''): bool - { - $uri = $path !== '' ? '/' . \str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path)) : '/'; - - $this->headers['content-type'] = $contentType; - $this->headers['content-md5'] = \base64_encode(md5($data, true)); //TODO whould this work well with big file? can we skip it? - $this->amzHeaders['x-amz-content-sha256'] = \hash('sha256', $data); - $this->amzHeaders['x-amz-acl'] = $this->acl; - - $this->call(self::METHOD_PUT, $uri, $data); - - return true; - } - - /** - * Move file from given source to given path, Return true on success and false on failure. - * - * @see http://php.net/manual/en/function.filesize.php - * - * @param string $source - * @param string $target - * - * @throw \Exception - * - * @return bool - */ - public function move(string $source, string $target): bool - { - $type = $this->getFileMimeType($source); - - if ($this->write($target, $this->read($source), $type)) { - $this->delete($source); - } - - return true; - } - - /** - * Delete file in given path, Return true on success and false on failure. - * - * @see http://php.net/manual/en/function.filesize.php - * - * @param string $path - * - * @throws \Exception - * - * @return bool - */ - public function delete(string $path, bool $recursive = false): bool - { - $uri = ($path !== '') ? '/' . \str_replace('%2F', '/', \rawurlencode($path)) : '/'; - - unset($this->headers['content-type']); - unset($this->amzHeaders['x-amz-acl']); - unset($this->amzHeaders['x-amz-content-sha256']); - $this->headers['content-md5'] = \base64_encode(md5('', true)); - $this->call(self::METHOD_DELETE, $uri); - - return true; - } - - /** - * Get list of objects in the given path. - * - * @param string $path - * - * @throws \Exception - * - * @return array - */ - private function listObjects($prefix = '', $maxKeys = 1000, $continuationToken = '') - { - $uri = '/'; - $this->headers['content-type'] = 'text/plain'; - $this->headers['content-md5'] = \base64_encode(md5('', true)); - - $parameters = [ - 'list-type' => 2, - 'prefix' => $prefix, - 'max-keys' => $maxKeys, - ]; - if(!empty($continuationToken)) { - $parameters['continuation-token'] = $continuationToken; - } - $response = $this->call(self::METHOD_GET, $uri, '', $parameters); - return $response->body; - } - - /** - * Delete files in given path, path must be a directory. Return true on success and false on failure. - * - * @param string $path - * - * @throws \Exception - * - * @return bool - */ - public function deletePath(string $path): bool - { - $path = $this->getRoot() . '/' . $path; - $uri = '/'; - $continuationToken = ''; - do { - $objects = $this->listObjects($path, continuationToken: $continuationToken); - $count = (int) ($objects['KeyCount'] ?? 1); - if($count < 1) { - break; - } - $continuationToken = $objects['NextContinuationToken'] ?? ''; - $body = ''; - if($count > 1) { - foreach ($objects['Contents'] as $object) { - $body .= "{$object['Key']}"; - } - } else { - $body .= "{$objects['Contents']['Key']}"; - } - $body .= 'true'; - $body .= ''; - $this->amzHeaders['x-amz-content-sha256'] = \hash('sha256', $body); - $this->headers['content-md5'] = \base64_encode(md5($body, true)); - $this->call(self::METHOD_POST, $uri, $body, ['delete'=>'']); - } while(!empty($continuationToken)); - - return true; - } - - /** - * Check if file exists - * - * @param string $path - * - * @return bool - */ - public function exists(string $path): bool - { - try { - $this->getInfo($path); - } catch (\Throwable $th) { - return false; - } - - return true; - } - - /** - * Returns given file path its size. - * - * @see http://php.net/manual/en/function.filesize.php - * - * @param string $path - * - * @return int - */ - public function getFileSize(string $path): int - { - $response = $this->getInfo($path); - return (int)($response['content-length'] ?? 0); - } - - /** - * Returns given file path its mime type. - * - * @see http://php.net/manual/en/function.mime-content-type.php - * - * @param string $path - * - * @return string - */ - public function getFileMimeType(string $path): string - { - $response = $this->getInfo($path); - return $response['content-type'] ?? ''; - } - - /** - * Returns given file path its MD5 hash value. - * - * @see http://php.net/manual/en/function.md5-file.php - * - * @param string $path - * - * @return string - */ - public function getFileHash(string $path): string - { - $etag = $this->getInfo($path)['etag'] ?? ''; - return (!empty($etag)) ? substr($etag, 1, -1) : $etag; - } - - /** - * Get directory size in bytes. - * - * Return -1 on error - * - * Based on http://www.jonasjohn.de/snippets/php/dir-size.htm - * - * @param string $path - * - * @return int - */ - public function getDirectorySize(string $path): int - { - return -1; - } - - /** - * Get Partition Free Space. - * - * disk_free_space — Returns available space on filesystem or disk partition - * - * @return float - */ - public function getPartitionFreeSpace(): float - { - return -1; - } - - /** - * Get Partition Total Space. - * - * disk_total_space — Returns the total size of a filesystem or disk partition - * - * @return float - */ - public function getPartitionTotalSpace(): float - { - return -1; - } - - /** - * Get file info - * @return array - */ - private function getInfo(string $path): array - { - unset($this->headers['content-type']); - unset($this->amzHeaders['x-amz-acl']); - unset($this->amzHeaders['x-amz-content-sha256']); - $this->headers['content-md5'] = \base64_encode(md5('', true)); - $uri = $path !== '' ? '/' . \str_replace('%2F', '/', \rawurlencode($path)) : '/'; - $response = $this->call(self::METHOD_HEAD, $uri); - - return $response->headers; - } - - /** - * Generate the headers for AWS Signature V4 - * @param string $method - * @param string $uri - * @param array parameters - * - * @return string - */ - private function getSignatureV4(string $method, string $uri, array $parameters = []): string - { - $service = 's3'; - $region = $this->region; - - $algorithm = 'AWS4-HMAC-SHA256'; - $combinedHeaders = []; - - $amzDateStamp = \substr($this->amzHeaders['x-amz-date'], 0, 8); - - // CanonicalHeaders - foreach ($this->headers as $k => $v) { - $combinedHeaders[\strtolower($k)] = \trim($v); - } - - foreach ($this->amzHeaders as $k => $v) { - $combinedHeaders[\strtolower($k)] = \trim($v); - } - - uksort($combinedHeaders, [ & $this, 'sortMetaHeadersCmp']); - - // Convert null query string parameters to strings and sort - uksort($parameters, [ & $this, 'sortMetaHeadersCmp']); - $queryString = \http_build_query($parameters, '', '&', PHP_QUERY_RFC3986); - - // Payload - $amzPayload = [$method]; - - $qsPos = \strpos($uri, '?'); - $amzPayload[] = ($qsPos === false ? $uri : \substr($uri, 0, $qsPos)); - - $amzPayload[] = $queryString; - - foreach ($combinedHeaders as $k => $v) { // add header as string to requests - $amzPayload[] = $k . ':' . $v; - } - - $amzPayload[] = ''; // add a blank entry so we end up with an extra line break - $amzPayload[] = \implode(';', \array_keys($combinedHeaders)); // SignedHeaders - $amzPayload[] = $this->amzHeaders['x-amz-content-sha256']; // payload hash - - $amzPayloadStr = \implode("\n", $amzPayload); // request as string - - // CredentialScope - $credentialScope = [$amzDateStamp, $region, $service, 'aws4_request']; - - // stringToSign - $stringToSignStr = \implode("\n", [$algorithm, $this->amzHeaders['x-amz-date'], - \implode('/', $credentialScope), \hash('sha256', $amzPayloadStr)]); - - // Make Signature - $kSecret = 'AWS4' . $this->secretKey; - $kDate = \hash_hmac('sha256', $amzDateStamp, $kSecret, true); - $kRegion = \hash_hmac('sha256', $region, $kDate, true); - $kService = \hash_hmac('sha256', $service, $kRegion, true); - $kSigning = \hash_hmac('sha256', 'aws4_request', $kService, true); - - $signature = \hash_hmac('sha256', \utf8_encode($stringToSignStr), $kSigning); - return $algorithm . ' ' . \implode(',', [ - 'Credential=' . $this->accessKey . '/' . \implode('/', $credentialScope), - 'SignedHeaders=' . \implode(';', \array_keys($combinedHeaders)), - 'Signature=' . $signature, - ]); - } - - /** - * Get the S3 response - * - * @param string $method - * @param string $uri - * @param string $data - * @param array $parameters - * - * @throws \Exception - * - * @return object - */ - private function call(string $method, string $uri, string $data = '', array $parameters=[]) - { - $url = 'https://' . $this->headers['host'] . $uri . '?' . \http_build_query($parameters, '', '&', PHP_QUERY_RFC3986); - $response = new \stdClass; - $response->body = ''; - $response->headers = []; - - // Basic setup - $curl = \curl_init(); - \curl_setopt($curl, CURLOPT_USERAGENT, 'utopia-php/storage'); - \curl_setopt($curl, CURLOPT_URL, $url); - - // Headers - $httpHeaders = []; - $this->amzHeaders['x-amz-date'] = \gmdate('Ymd\THis\Z'); - - if (!isset($this->amzHeaders['x-amz-content-sha256'])) { - $this->amzHeaders['x-amz-content-sha256'] = \hash('sha256', $data); - } - - foreach ($this->amzHeaders as $header => $value) { - if (\strlen($value) > 0) { - $httpHeaders[] = $header . ': ' . $value; - } - } - - $this->headers['date'] = \gmdate('D, d M Y H:i:s T'); - foreach ($this->headers as $header => $value) { - if (\strlen($value) > 0) { - $httpHeaders[] = $header . ': ' . $value; - } - } - - $httpHeaders[] = 'Authorization: ' . $this->getSignatureV4($method, $uri, $parameters); - - \curl_setopt($curl, CURLOPT_HTTPHEADER, $httpHeaders); - \curl_setopt($curl, CURLOPT_HEADER, false); - \curl_setopt($curl, CURLOPT_RETURNTRANSFER, false); - \curl_setopt($curl, CURLOPT_WRITEFUNCTION, function ($curl, string $data) use ($response) { - $response->body .= $data; - return \strlen($data); - }); - curl_setopt($curl, CURLOPT_HEADERFUNCTION, function ($curl, string $header) use (&$response) { - $len = strlen($header); - $header = explode(':', $header, 2); - - if (count($header) < 2) { // ignore invalid headers - return $len; - } - - $response->headers[strtolower(trim($header[0]))] = trim($header[1]); - - return $len; - }); - \curl_setopt($curl, CURLOPT_FOLLOWLOCATION, true); - \curl_setopt($curl, CURLOPT_CUSTOMREQUEST, $method); - - // Request types - switch ($method) { - case self::METHOD_PUT: - case self::METHOD_POST: // POST only used for CloudFront - \curl_setopt($curl, CURLOPT_POSTFIELDS, $data); - break; - case self::METHOD_HEAD: - case self::METHOD_DELETE: - \curl_setopt($curl, CURLOPT_NOBODY, true); - break; - } - - $result = \curl_exec($curl); - - if (!$result) { - throw new Exception(\curl_error($curl)); - } - - $response->code = \curl_getinfo($curl, CURLINFO_HTTP_CODE); - if ($response->code >= 400) { - throw new Exception($response->body, $response->code); - } - - \curl_close($curl); - - // Parse body into XML - if ((isset($response->headers['content-type']) && $response->headers['content-type'] == 'application/xml') || (str_starts_with($response->body, 'headers['content-type'] ?? '') !== 'image/svg+xml')) { - $response->body = \simplexml_load_string($response->body); - $response->body = json_decode(json_encode($response->body), true); - } - - return $response; - } - - /** - * Sort compare for meta headers - * - * @internal Used to sort x-amz meta headers - * @param string $a String A - * @param string $b String B - * @return integer - */ - private function sortMetaHeadersCmp($a, $b) - { - $lenA = \strlen($a); - $lenB = \strlen($b); - $minLen = \min($lenA, $lenB); - $ncmp = \strncmp($a, $b, $minLen); - if ($lenA == $lenB) { - return $ncmp; - } - - if (0 == $ncmp) { - return $lenA < $lenB ? -1 : 1; - } - - return $ncmp; - } } diff --git a/tests/Storage/Device/DOSpacesTest.php b/tests/Storage/Device/DOSpacesTest.php index d7e936af..97d7840d 100644 --- a/tests/Storage/Device/DOSpacesTest.php +++ b/tests/Storage/Device/DOSpacesTest.php @@ -10,10 +10,10 @@ class DOSpacesTest extends S3Base protected function init(): void { $this->root = '/root'; - $key = $_SERVER['DO_ACCESS_KEY'] ?? ''; - $secret = $_SERVER['DO_SECRET'] ?? ''; + $key = $_SERVER['DO_ACCESS_KEY'] ?? 'OYHQUUZHZAWG7HB7746V'; + $secret = $_SERVER['DO_SECRET'] ?? 'yG2Uoq/7Kxa3zHeINzj9n5aPEMr+r9ZrqVXQkSIZH8g'; $bucket = "utopia-storage-tests"; - + $bucket = 'shimon-test-space'; $this->object = new DOSpaces($this->root, $key, $secret, $bucket, DOSpaces::NYC3, DOSpaces::ACL_PUBLIC_READ); } diff --git a/tests/Storage/S3Base.php b/tests/Storage/S3Base.php index feeb8193..01dc5836 100644 --- a/tests/Storage/S3Base.php +++ b/tests/Storage/S3Base.php @@ -133,7 +133,7 @@ public function testDeletePath() $this->assertEquals(true, $this->object->exists($path)); $this->assertEquals(true, $this->object->deletePath('bucket')); $this->assertEquals(false, $this->object->exists($path)); - + // Test Multiple Objects $path = $this->object->getPath('text-for-delete-path1.txt'); $path = str_ireplace($this->object->getRoot(), $this->object->getRoot() . DIRECTORY_SEPARATOR . 'bucket', $path); @@ -148,7 +148,7 @@ public function testDeletePath() $this->assertEquals(true, $this->object->deletePath('bucket')); $this->assertEquals(false, $this->object->exists($path)); $this->assertEquals(false, $this->object->exists($path2)); - + } From 2f25d8a31c4961d4146050a9e41c6f16a47ebe2c Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 7 Apr 2022 12:36:01 +0300 Subject: [PATCH 03/22] tidy --- src/Storage/Device/DOSpaces.php | 1 - src/Storage/Device/Generic.php | 9 +-------- tests/Storage/Device/BackBlazeTest.php | 3 +-- tests/Storage/Device/DOSpacesTest.php | 7 +++---- tests/Storage/Device/LinodeTest.php | 2 -- tests/Storage/Device/S3Test.php | 1 - 6 files changed, 5 insertions(+), 18 deletions(-) diff --git a/src/Storage/Device/DOSpaces.php b/src/Storage/Device/DOSpaces.php index c4ec4396..b3f39dad 100644 --- a/src/Storage/Device/DOSpaces.php +++ b/src/Storage/Device/DOSpaces.php @@ -32,7 +32,6 @@ public function __construct(string $root, string $accessKey, string $secretKey, { $hostName = $bucket . '.' . $region . '.digitaloceanspaces.com'; parent::__construct($root, $accessKey, $secretKey, $bucket, $region, $acl, $hostName); - } /** diff --git a/src/Storage/Device/Generic.php b/src/Storage/Device/Generic.php index ae485e28..fc959464 100644 --- a/src/Storage/Device/Generic.php +++ b/src/Storage/Device/Generic.php @@ -82,7 +82,6 @@ public function __construct(string $root, string $accessKey, string $secretKey, $this->root = $root; $this->acl = $acl; $this->hostName = $hostName; - } /** @@ -98,7 +97,7 @@ public function getName(): string */ public function getDescription(): string { - return 'S3 Generic Bucket Storage drive for AWS compatible or on premise solution'; + return 'S3 Generic Bucket Storage drive for AWS compatible solutions'; } /** @@ -217,14 +216,12 @@ private function call(string $method, string $uri, string $data = '', array $par \curl_setopt($curl, CURLOPT_USERAGENT, 'utopia-php/storage'); \curl_setopt($curl, CURLOPT_URL, $url); - // Headers $httpHeaders = []; $headers['x-amz-date'] = \gmdate('Ymd\THis\Z'); $headers['date'] = \gmdate('D, d M Y H:i:s T'); $headers['host'] = $this->hostName; - if (!isset($headers['x-amz-content-sha256'])) { $headers['x-amz-content-sha256'] = \hash('sha256', $data); } @@ -306,7 +303,6 @@ private function getSignatureV4(string $method, string $uri, array $parameters = { $service = 's3'; $region = $this->region; - $algorithm = 'AWS4-HMAC-SHA256'; $combinedHeaders = []; $amzDateStamp = \substr($headers['x-amz-date'], 0, 8); @@ -327,7 +323,6 @@ private function getSignatureV4(string $method, string $uri, array $parameters = $qsPos = \strpos($uri, '?'); $amzPayload[] = ($qsPos === false ? $uri : \substr($uri, 0, $qsPos)); - $amzPayload[] = $queryString; foreach ($combinedHeaders as $k => $v) { // add header as string to requests @@ -378,7 +373,6 @@ private function getSignatureV4(string $method, string $uri, array $parameters = protected function createMultipartUpload(string $path, string $contentType): string { $uri = $path !== '' ? '/' . \str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path)) : '/'; - $response = $this->call(self::METHOD_POST, $uri, '', ['uploads' => ''], [ 'content-type' => $contentType, 'content-md5' => \base64_encode(md5('', true)), @@ -461,7 +455,6 @@ protected function completeMultipartUpload(string $path, string $uploadId, array public function abort(string $path, string $extra = ''): bool { $uri = $path !== '' ? '/' . \str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path)) : '/'; - $this->call(self::METHOD_DELETE, $uri, '', [ 'uploadId' => $extra ], [ diff --git a/tests/Storage/Device/BackBlazeTest.php b/tests/Storage/Device/BackBlazeTest.php index b2db63eb..d85bf2ab 100644 --- a/tests/Storage/Device/BackBlazeTest.php +++ b/tests/Storage/Device/BackBlazeTest.php @@ -9,13 +9,12 @@ class BackBlazeTest extends S3Base { protected function init(): void { - $this->root = 'root'; + $key = $_SERVER['BACKBLAZE_ACCESS_KEY'] ?? ''; $secret = $_SERVER['BACKBLAZE_SECRET'] ?? ''; $bucket = "backblaze-demo"; $this->object = new BackBlaze($this->root, $key, $secret, $bucket, BackBlaze::US_WEST_004, BackBlaze::ACL_PRIVATE); - } protected function getAdapterName(): string diff --git a/tests/Storage/Device/DOSpacesTest.php b/tests/Storage/Device/DOSpacesTest.php index 97d7840d..8422d89d 100644 --- a/tests/Storage/Device/DOSpacesTest.php +++ b/tests/Storage/Device/DOSpacesTest.php @@ -9,11 +9,10 @@ class DOSpacesTest extends S3Base { protected function init(): void { - $this->root = '/root'; - $key = $_SERVER['DO_ACCESS_KEY'] ?? 'OYHQUUZHZAWG7HB7746V'; - $secret = $_SERVER['DO_SECRET'] ?? 'yG2Uoq/7Kxa3zHeINzj9n5aPEMr+r9ZrqVXQkSIZH8g'; + $key = $_SERVER['DO_ACCESS_KEY'] ?? ''; + $secret = $_SERVER['DO_SECRET'] ?? ''; $bucket = "utopia-storage-tests"; - $bucket = 'shimon-test-space'; + $this->object = new DOSpaces($this->root, $key, $secret, $bucket, DOSpaces::NYC3, DOSpaces::ACL_PUBLIC_READ); } diff --git a/tests/Storage/Device/LinodeTest.php b/tests/Storage/Device/LinodeTest.php index e5808fe5..ad5f3450 100644 --- a/tests/Storage/Device/LinodeTest.php +++ b/tests/Storage/Device/LinodeTest.php @@ -9,13 +9,11 @@ class LinodeTest extends S3Base { protected function init(): void { - $this->root = 'root'; $key = $_SERVER['LINODE_ACCESS_KEY'] ?? ''; $secret = $_SERVER['LINODE_SECRET'] ?? ''; $bucket = 'everly-test'; $this->object = new Linode($this->root, $key, $secret, $bucket, Linode::EU_CENTRAL_1, Linode::ACL_PRIVATE); - } protected function getAdapterName(): string diff --git a/tests/Storage/Device/S3Test.php b/tests/Storage/Device/S3Test.php index e9e81904..f5bd397f 100644 --- a/tests/Storage/Device/S3Test.php +++ b/tests/Storage/Device/S3Test.php @@ -10,7 +10,6 @@ class S3Test extends S3Base protected function init(): void { - $this->root = '/root'; $key = $_SERVER['S3_ACCESS_KEY'] ?? ''; $secret = $_SERVER['S3_SECRET'] ?? ''; $bucket = 'utopia-storage-tests'; From 196d8af3ca6c2de5d0f3478fa2f56cd730f026bc Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 7 Apr 2022 13:33:49 +0300 Subject: [PATCH 04/22] add generic test --- tests/Storage/Device/GenericTest.php | 30 ++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 tests/Storage/Device/GenericTest.php diff --git a/tests/Storage/Device/GenericTest.php b/tests/Storage/Device/GenericTest.php new file mode 100644 index 00000000..8a42ae80 --- /dev/null +++ b/tests/Storage/Device/GenericTest.php @@ -0,0 +1,30 @@ +object = new Generic($this->root, $key, $secret, $bucket, DOSpaces::NYC3, DOSpaces::ACL_PUBLIC_READ, $hostName); + } + + protected function getAdapterName(): string + { + return 'S3 compatible Storage'; + } + + protected function getAdapterDescription(): string + { + return 'S3 Generic Bucket Storage drive for AWS compatible solutions'; + } +} From 8a22decb874d02525f5f25c1e151317dd8b4fec7 Mon Sep 17 00:00:00 2001 From: shimon Date: Wed, 27 Apr 2022 12:33:31 +0300 Subject: [PATCH 05/22] changed blackBlase region --- tests/Storage/Device/BackBlazeTest.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/Storage/Device/BackBlazeTest.php b/tests/Storage/Device/BackBlazeTest.php index d85bf2ab..d6836ec3 100644 --- a/tests/Storage/Device/BackBlazeTest.php +++ b/tests/Storage/Device/BackBlazeTest.php @@ -12,9 +12,9 @@ protected function init(): void $key = $_SERVER['BACKBLAZE_ACCESS_KEY'] ?? ''; $secret = $_SERVER['BACKBLAZE_SECRET'] ?? ''; - $bucket = "backblaze-demo"; + $bucket = "backblaze-demo-1"; - $this->object = new BackBlaze($this->root, $key, $secret, $bucket, BackBlaze::US_WEST_004, BackBlaze::ACL_PRIVATE); + $this->object = new BackBlaze($this->root, $key, $secret, $bucket, BackBlaze::EU_CENTRAL_003, BackBlaze::ACL_PRIVATE); } protected function getAdapterName(): string From 687be2f1aa7863f1c7513c0acbd2b6f94fc9a053 Mon Sep 17 00:00:00 2001 From: shimon Date: Sun, 1 May 2022 15:34:36 +0300 Subject: [PATCH 06/22] changed s3 region --- src/Storage/Device/S3.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storage/Device/S3.php b/src/Storage/Device/S3.php index edd245e6..81208d8d 100644 --- a/src/Storage/Device/S3.php +++ b/src/Storage/Device/S3.php @@ -48,7 +48,7 @@ class S3 extends Generic * @param string $region * @param string $acl */ - public function __construct(string $root, string $accessKey, string $secretKey, string $bucket, string $region = self::US_EAST_1, string $acl = self::ACL_PRIVATE) + public function __construct(string $root, string $accessKey, string $secretKey, string $bucket, string $region = self::EU_WEST_1, string $acl = self::ACL_PRIVATE) { $hostName = $bucket . '.s3.'.$region.'.amazonaws.com'; parent::__construct($root, $accessKey, $secretKey, $bucket, $region, $acl, $hostName); From e7693809ab55ad028e7cbaa6687ab98eab65e4ee Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 2 May 2022 09:08:41 +0300 Subject: [PATCH 07/22] changed s3 region --- src/Storage/Device/Generic.php | 1 + src/Storage/Device/Linode.php | 2 +- tests/Storage/Device/BackBlazeTest.php | 4 ++-- tests/Storage/Device/LinodeTest.php | 8 ++++---- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/Storage/Device/Generic.php b/src/Storage/Device/Generic.php index fc959464..e91c840d 100644 --- a/src/Storage/Device/Generic.php +++ b/src/Storage/Device/Generic.php @@ -82,6 +82,7 @@ public function __construct(string $root, string $accessKey, string $secretKey, $this->root = $root; $this->acl = $acl; $this->hostName = $hostName; + } /** diff --git a/src/Storage/Device/Linode.php b/src/Storage/Device/Linode.php index ac1e3463..2da105e6 100644 --- a/src/Storage/Device/Linode.php +++ b/src/Storage/Device/Linode.php @@ -29,7 +29,7 @@ class Linode extends Generic */ public function __construct(string $root, string $accessKey, string $secretKey, string $bucket, string $region = self::EU_CENTRAL_1, string $acl = self::ACL_PRIVATE) { - $hostName = $bucket.'.'.$region.'.'.'linodeobjects.com'; + $hostName = 'shimon-1-test.us-east-1.linodeobjects.com'; parent::__construct($root, $accessKey, $secretKey, $bucket, $region, $acl, $hostName); } diff --git a/tests/Storage/Device/BackBlazeTest.php b/tests/Storage/Device/BackBlazeTest.php index d6836ec3..c981d4a7 100644 --- a/tests/Storage/Device/BackBlazeTest.php +++ b/tests/Storage/Device/BackBlazeTest.php @@ -10,8 +10,8 @@ class BackBlazeTest extends S3Base protected function init(): void { - $key = $_SERVER['BACKBLAZE_ACCESS_KEY'] ?? ''; - $secret = $_SERVER['BACKBLAZE_SECRET'] ?? ''; + $key = $_SERVER['BACKBLAZE_ACCESS_KEY'] ?? '0036ab2987289660000000003'; + $secret = $_SERVER['BACKBLAZE_SECRET'] ?? 'K003UXXjvol8Pq15zNxgeBU7PQAGJ3E'; $bucket = "backblaze-demo-1"; $this->object = new BackBlaze($this->root, $key, $secret, $bucket, BackBlaze::EU_CENTRAL_003, BackBlaze::ACL_PRIVATE); diff --git a/tests/Storage/Device/LinodeTest.php b/tests/Storage/Device/LinodeTest.php index ad5f3450..e14e01d2 100644 --- a/tests/Storage/Device/LinodeTest.php +++ b/tests/Storage/Device/LinodeTest.php @@ -9,11 +9,11 @@ class LinodeTest extends S3Base { protected function init(): void { - $key = $_SERVER['LINODE_ACCESS_KEY'] ?? ''; - $secret = $_SERVER['LINODE_SECRET'] ?? ''; - $bucket = 'everly-test'; + $key = $_SERVER['LINODE_ACCESS_KEY'] ?? 'IDNDUV528EG0ZV9JS5O0'; + $secret = $_SERVER['LINODE_SECRET'] ?? 'bo3tscqCRZ3KmpwBzrWNlioZm6pxl3IetTShsni0'; + $bucket = 'shimon-test'; - $this->object = new Linode($this->root, $key, $secret, $bucket, Linode::EU_CENTRAL_1, Linode::ACL_PRIVATE); + $this->object = new Linode($this->root, $key, $secret, $bucket, Linode::US_EAST_1, Linode::ACL_PRIVATE); } protected function getAdapterName(): string From 47b63558c1399b1d492fa5a78640895ee67bb45c Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 2 May 2022 10:17:55 +0300 Subject: [PATCH 08/22] test fix --- src/Storage/Device/Generic.php | 1 - src/Storage/Device/S3.php | 2 +- tests/Storage/Device/BackBlazeTest.php | 4 ++-- tests/Storage/Device/LinodeTest.php | 4 ++-- tests/Storage/Device/S3Test.php | 6 +++--- tests/Storage/S3Base.php | 2 +- 6 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/Storage/Device/Generic.php b/src/Storage/Device/Generic.php index e91c840d..bf53f162 100644 --- a/src/Storage/Device/Generic.php +++ b/src/Storage/Device/Generic.php @@ -234,7 +234,6 @@ private function call(string $method, string $uri, string $data = '', array $par } $httpHeaders[] = 'Authorization: ' . $this->getSignatureV4($method, $uri, $parameters, $headers); - \curl_setopt($curl, CURLOPT_HTTPHEADER, $httpHeaders); \curl_setopt($curl, CURLOPT_HEADER, false); \curl_setopt($curl, CURLOPT_RETURNTRANSFER, false); diff --git a/src/Storage/Device/S3.php b/src/Storage/Device/S3.php index 81208d8d..e7a46b17 100644 --- a/src/Storage/Device/S3.php +++ b/src/Storage/Device/S3.php @@ -67,7 +67,7 @@ public function getName(): string */ public function getDescription(): string { - return 'S3 Bucket Storage drive for AWS or on premise solution...'; + return 'S3 Generic Bucket Storage drive for AWS compatible solutions'; } diff --git a/tests/Storage/Device/BackBlazeTest.php b/tests/Storage/Device/BackBlazeTest.php index c981d4a7..d6836ec3 100644 --- a/tests/Storage/Device/BackBlazeTest.php +++ b/tests/Storage/Device/BackBlazeTest.php @@ -10,8 +10,8 @@ class BackBlazeTest extends S3Base protected function init(): void { - $key = $_SERVER['BACKBLAZE_ACCESS_KEY'] ?? '0036ab2987289660000000003'; - $secret = $_SERVER['BACKBLAZE_SECRET'] ?? 'K003UXXjvol8Pq15zNxgeBU7PQAGJ3E'; + $key = $_SERVER['BACKBLAZE_ACCESS_KEY'] ?? ''; + $secret = $_SERVER['BACKBLAZE_SECRET'] ?? ''; $bucket = "backblaze-demo-1"; $this->object = new BackBlaze($this->root, $key, $secret, $bucket, BackBlaze::EU_CENTRAL_003, BackBlaze::ACL_PRIVATE); diff --git a/tests/Storage/Device/LinodeTest.php b/tests/Storage/Device/LinodeTest.php index e14e01d2..1009fd6a 100644 --- a/tests/Storage/Device/LinodeTest.php +++ b/tests/Storage/Device/LinodeTest.php @@ -9,8 +9,8 @@ class LinodeTest extends S3Base { protected function init(): void { - $key = $_SERVER['LINODE_ACCESS_KEY'] ?? 'IDNDUV528EG0ZV9JS5O0'; - $secret = $_SERVER['LINODE_SECRET'] ?? 'bo3tscqCRZ3KmpwBzrWNlioZm6pxl3IetTShsni0'; + $key = $_SERVER['LINODE_ACCESS_KEY'] ?? ''; + $secret = $_SERVER['LINODE_SECRET'] ?? ''; $bucket = 'shimon-test'; $this->object = new Linode($this->root, $key, $secret, $bucket, Linode::US_EAST_1, Linode::ACL_PRIVATE); diff --git a/tests/Storage/Device/S3Test.php b/tests/Storage/Device/S3Test.php index f5bd397f..94f035b1 100644 --- a/tests/Storage/Device/S3Test.php +++ b/tests/Storage/Device/S3Test.php @@ -12,9 +12,9 @@ protected function init(): void { $key = $_SERVER['S3_ACCESS_KEY'] ?? ''; $secret = $_SERVER['S3_SECRET'] ?? ''; - $bucket = 'utopia-storage-tests'; + $bucket = 'appwrite-test-bucket'; - $this->object = new S3($this->root, $key, $secret, $bucket, S3::AP_SOUTH_1, S3::ACL_PRIVATE); + $this->object = new S3($this->root, $key, $secret, $bucket, S3::EU_WEST_1, S3::ACL_PRIVATE); } /** @@ -27,6 +27,6 @@ protected function getAdapterName() : string protected function getAdapterDescription(): string { - return 'S3 Bucket Storage drive for AWS or on premise solution'; + return 'S3 Generic Bucket Storage drive for AWS compatible solutions'; } } diff --git a/tests/Storage/S3Base.php b/tests/Storage/S3Base.php index 01dc5836..01fd5866 100644 --- a/tests/Storage/S3Base.php +++ b/tests/Storage/S3Base.php @@ -27,7 +27,7 @@ abstract protected function getAdapterDescription(): string; /** * @var string */ - protected $root = '/root'; + protected $root = 'root'; public function setUp(): void { From f30737ca52efe5d8bf66d9942deb238aa578f207 Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 2 May 2022 10:34:11 +0300 Subject: [PATCH 09/22] wasabi --- src/Storage/Device/Linode.php | 2 +- src/Storage/Device/Wasabi.php | 57 +++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 1 deletion(-) create mode 100644 src/Storage/Device/Wasabi.php diff --git a/src/Storage/Device/Linode.php b/src/Storage/Device/Linode.php index 2da105e6..ac1e3463 100644 --- a/src/Storage/Device/Linode.php +++ b/src/Storage/Device/Linode.php @@ -29,7 +29,7 @@ class Linode extends Generic */ public function __construct(string $root, string $accessKey, string $secretKey, string $bucket, string $region = self::EU_CENTRAL_1, string $acl = self::ACL_PRIVATE) { - $hostName = 'shimon-1-test.us-east-1.linodeobjects.com'; + $hostName = $bucket.'.'.$region.'.'.'linodeobjects.com'; parent::__construct($root, $accessKey, $secretKey, $bucket, $region, $acl, $hostName); } diff --git a/src/Storage/Device/Wasabi.php b/src/Storage/Device/Wasabi.php new file mode 100644 index 00000000..fc768566 --- /dev/null +++ b/src/Storage/Device/Wasabi.php @@ -0,0 +1,57 @@ + Date: Mon, 2 May 2022 10:38:48 +0300 Subject: [PATCH 10/22] wasabi test --- tests/Storage/Device/WasabiTest.php | 31 +++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 tests/Storage/Device/WasabiTest.php diff --git a/tests/Storage/Device/WasabiTest.php b/tests/Storage/Device/WasabiTest.php new file mode 100644 index 00000000..03790bc3 --- /dev/null +++ b/tests/Storage/Device/WasabiTest.php @@ -0,0 +1,31 @@ +root = 'root'; + $key = $_SERVER['WASABI_ACCESS_KEY'] ?? ''; + $secret = $_SERVER['WASABI_SECRET'] ?? ''; + $bucket = "everly-wasabi-test"; + + $this->object = new Wasabi($this->root, $key, $secret, $bucket, Wasabi::EU_CENTRAL_1, WASABI::ACL_PRIVATE); + + } + + protected function getAdapterName(): string + { + return 'Wasabi Storage'; + } + + protected function getAdapterDescription(): string + { + return 'Wasabi Storage'; + } +} From 8e74e82a3db3a471eac3a024ae0dffc832955a2a Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 2 May 2022 10:39:34 +0300 Subject: [PATCH 11/22] wasabi test --- tests/Storage/Device/WasabiTest.php | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/Storage/Device/WasabiTest.php b/tests/Storage/Device/WasabiTest.php index 03790bc3..df88829b 100644 --- a/tests/Storage/Device/WasabiTest.php +++ b/tests/Storage/Device/WasabiTest.php @@ -10,7 +10,6 @@ class WasabiTest extends S3Base { protected function init(): void { - $this->root = 'root'; $key = $_SERVER['WASABI_ACCESS_KEY'] ?? ''; $secret = $_SERVER['WASABI_SECRET'] ?? ''; $bucket = "everly-wasabi-test"; From 49c9afe3fb0a87ba3aba0e8b2eed54ee495b29be Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 2 May 2022 18:24:54 +0300 Subject: [PATCH 12/22] wasabi test --- tests/Storage/Device/WasabiTest.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/Storage/Device/WasabiTest.php b/tests/Storage/Device/WasabiTest.php index df88829b..f7efe76b 100644 --- a/tests/Storage/Device/WasabiTest.php +++ b/tests/Storage/Device/WasabiTest.php @@ -12,9 +12,9 @@ protected function init(): void { $key = $_SERVER['WASABI_ACCESS_KEY'] ?? ''; $secret = $_SERVER['WASABI_SECRET'] ?? ''; - $bucket = "everly-wasabi-test"; + $bucket = "appwrite"; - $this->object = new Wasabi($this->root, $key, $secret, $bucket, Wasabi::EU_CENTRAL_1, WASABI::ACL_PRIVATE); + $this->object = new Wasabi($this->root, $key, $secret, $bucket, WASABI::US_EAST_1, WASABI::ACL_PRIVATE); } From ab044525a39f054706c61721e20988c49de26269 Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 3 May 2022 15:47:20 +0300 Subject: [PATCH 13/22] preceding slash fix --- src/Storage/Device/Generic.php | 12 ++++++++++-- tests/Storage/S3Base.php | 2 +- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/Storage/Device/Generic.php b/src/Storage/Device/Generic.php index bf53f162..be06ea31 100644 --- a/src/Storage/Device/Generic.php +++ b/src/Storage/Device/Generic.php @@ -207,6 +207,11 @@ public function write(string $path, string $data, string $contentType = ''): boo */ private function call(string $method, string $uri, string $data = '', array $parameters = [], array $headers = []) { + + if(str_starts_with($uri,'//')){ + $uri = substr($uri, 1); + } + $url = 'https://' . $this->hostName . $uri . '?' . \http_build_query($parameters, '', '&', PHP_QUERY_RFC3986); $response = new \stdClass; $response->body = ''; @@ -575,8 +580,11 @@ public function delete(string $path, bool $recursive = false): bool */ public function deletePath(string $path): bool { - $path = $this->getRoot() . '/' . $path; - + $root = $this->getRoot(); + if(str_starts_with($root, '/')){ + $root = substr($root, 1); + } + $path = $root . '/' . $path; $uri = '/'; $continuationToken = ''; diff --git a/tests/Storage/S3Base.php b/tests/Storage/S3Base.php index 01fd5866..01dc5836 100644 --- a/tests/Storage/S3Base.php +++ b/tests/Storage/S3Base.php @@ -27,7 +27,7 @@ abstract protected function getAdapterDescription(): string; /** * @var string */ - protected $root = 'root'; + protected $root = '/root'; public function setUp(): void { From 16a82a4eb4e1b14a672c91fa998912b9417708c3 Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 3 May 2022 16:25:59 +0300 Subject: [PATCH 14/22] Wasabi --- src/Storage/Storage.php | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Storage/Storage.php b/src/Storage/Storage.php index cc27a41e..7335038a 100644 --- a/src/Storage/Storage.php +++ b/src/Storage/Storage.php @@ -15,6 +15,7 @@ class Storage const DEVICE_DO_SPACES = 'DOSpaces'; const DEVICE_BACKBLAZE = 'BackBlaze'; const DEVICE_LINODE= 'Linode'; + const DEVICE_WASABI= 'Wasabi'; /** * Devices. From d53e4b0004028644e99b7c903a75db70558e3539 Mon Sep 17 00:00:00 2001 From: shimon Date: Sun, 8 May 2022 11:02:05 +0300 Subject: [PATCH 15/22] remove leading / --- src/Storage/Device/Generic.php | 101 +++++++++++++++++++++++++++++---- 1 file changed, 89 insertions(+), 12 deletions(-) diff --git a/src/Storage/Device/Generic.php b/src/Storage/Device/Generic.php index be06ea31..ddcf4b4b 100644 --- a/src/Storage/Device/Generic.php +++ b/src/Storage/Device/Generic.php @@ -180,7 +180,17 @@ public function upload(string $source, string $path, int $chunk = 1, int $chunks */ public function write(string $path, string $data, string $contentType = ''): bool { - $uri = $path !== '' ? '/' . \str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path)) : '/'; + + $uri = '/'; + + if($path !== '') { + + $uri = \str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path)); + + if (!str_starts_with($uri, '/')) { + $uri .= '/' . $uri; + } + } $this->call(self::METHOD_PUT, $uri, $data, [], [ 'content-type' => $contentType, @@ -208,10 +218,6 @@ public function write(string $path, string $data, string $contentType = ''): boo private function call(string $method, string $uri, string $data = '', array $parameters = [], array $headers = []) { - if(str_starts_with($uri,'//')){ - $uri = substr($uri, 1); - } - $url = 'https://' . $this->hostName . $uri . '?' . \http_build_query($parameters, '', '&', PHP_QUERY_RFC3986); $response = new \stdClass; $response->body = ''; @@ -377,12 +383,23 @@ private function getSignatureV4(string $method, string $uri, array $parameters = */ protected function createMultipartUpload(string $path, string $contentType): string { - $uri = $path !== '' ? '/' . \str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path)) : '/'; + $uri = '/'; + + if($path !== '') { + + $uri = \str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path)); + + if (!str_starts_with($uri, '/')) { + $uri .= '/' . $uri; + } + } + $response = $this->call(self::METHOD_POST, $uri, '', ['uploads' => ''], [ 'content-type' => $contentType, 'content-md5' => \base64_encode(md5('', true)), 'x-amz-acl' => $this->acl ]); + return $response->body['UploadId']; } @@ -400,7 +417,17 @@ protected function createMultipartUpload(string $path, string $contentType): str */ protected function uploadPart(string $source, string $path, int $chunk, string $uploadId): string { - $uri = $path !== '' ? '/' . \str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path)) : '/'; + $uri = '/'; + + if($path !== '') { + + $uri = \str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path)); + + if (!str_starts_with($uri, '/')) { + $uri .= '/' . $uri; + } + } + $data = \file_get_contents($source); $response = $this->call(self::METHOD_PUT, $uri, $data, [ 'partNumber' => $chunk, @@ -428,7 +455,16 @@ protected function uploadPart(string $source, string $path, int $chunk, string $ protected function completeMultipartUpload(string $path, string $uploadId, array $parts, $source): bool { - $uri = $path !== '' ? '/' . \str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path)) : '/'; + $uri = '/'; + + if($path !== '') { + + $uri = \str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path)); + + if (!str_starts_with($uri, '/')) { + $uri .= '/' . $uri; + } + } $body = ''; foreach ($parts as $part) { @@ -459,12 +495,23 @@ protected function completeMultipartUpload(string $path, string $uploadId, array */ public function abort(string $path, string $extra = ''): bool { - $uri = $path !== '' ? '/' . \str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path)) : '/'; + $uri = '/'; + + if($path !== '') { + + $uri = \str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path)); + + if (!str_starts_with($uri, '/')) { + $uri .= '/' . $uri; + } + } + $this->call(self::METHOD_DELETE, $uri, '', [ 'uploadId' => $extra ], [ 'content-md5' => \base64_encode(md5('', true)) ]); + return true; } @@ -512,7 +559,16 @@ public function getFileMimeType(string $path): string */ private function getInfo(string $path): array { - $uri = $path !== '' ? '/' . \str_replace('%2F', '/', \rawurlencode($path)) : '/'; + $uri = '/'; + + if($path !== '') { + + $uri = \str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path)); + + if (!str_starts_with($uri, '/')) { + $uri .= '/' . $uri; + } + } $response = $this->call(self::METHOD_HEAD, $uri, '', [], ['content-md5' => \base64_encode(md5('', true))]); @@ -532,7 +588,17 @@ private function getInfo(string $path): array */ public function read(string $path, int $offset = 0, int $length = null): string { - $uri = ($path !== '') ? '/' . \str_replace('%2F', '/', \rawurlencode($path)) : '/'; + $uri = '/'; + + if($path !== '') { + + $uri = \str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path)); + + if (!str_starts_with($uri, '/')) { + $uri .= '/' . $uri; + } + } + $headers = [ 'content-md5' => \base64_encode(md5('', true)) ]; @@ -560,7 +626,16 @@ public function read(string $path, int $offset = 0, int $length = null): string */ public function delete(string $path, bool $recursive = false): bool { - $uri = ($path !== '') ? '/' . \str_replace('%2F', '/', \rawurlencode($path)) : '/'; + $uri = '/'; + + if($path !== '') { + + $uri = \str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path)); + + if (!str_starts_with($uri, '/')) { + $uri .= '/' . $uri; + } + } $this->call(self::METHOD_DELETE, $uri, '', [], [ 'content-md5' => \base64_encode(md5('', true)), @@ -581,9 +656,11 @@ public function delete(string $path, bool $recursive = false): bool public function deletePath(string $path): bool { $root = $this->getRoot(); + if(str_starts_with($root, '/')){ $root = substr($root, 1); } + $path = $root . '/' . $path; $uri = '/'; From 7dee35301b5ad5a5927220a0a9691bba35d347de Mon Sep 17 00:00:00 2001 From: shimon Date: Sun, 8 May 2022 13:52:03 +0300 Subject: [PATCH 16/22] run all test --- tests/Storage/Device/LinodeTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Storage/Device/LinodeTest.php b/tests/Storage/Device/LinodeTest.php index 1009fd6a..f3dbcd93 100644 --- a/tests/Storage/Device/LinodeTest.php +++ b/tests/Storage/Device/LinodeTest.php @@ -11,7 +11,7 @@ protected function init(): void { $key = $_SERVER['LINODE_ACCESS_KEY'] ?? ''; $secret = $_SERVER['LINODE_SECRET'] ?? ''; - $bucket = 'shimon-test'; + $bucket = 'appwrite-test'; $this->object = new Linode($this->root, $key, $secret, $bucket, Linode::US_EAST_1, Linode::ACL_PRIVATE); } From 3d4fbf68637e26ceddbd454e7a85f394a5134c37 Mon Sep 17 00:00:00 2001 From: Shimon Newman Date: Mon, 9 May 2022 09:38:31 +0300 Subject: [PATCH 17/22] Update src/Storage/Device/Linode.php MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matej Bačo --- src/Storage/Device/Linode.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storage/Device/Linode.php b/src/Storage/Device/Linode.php index ac1e3463..0a7058f9 100644 --- a/src/Storage/Device/Linode.php +++ b/src/Storage/Device/Linode.php @@ -29,7 +29,7 @@ class Linode extends Generic */ public function __construct(string $root, string $accessKey, string $secretKey, string $bucket, string $region = self::EU_CENTRAL_1, string $acl = self::ACL_PRIVATE) { - $hostName = $bucket.'.'.$region.'.'.'linodeobjects.com'; + $hostName = $bucket.'.'.$region.'.linodeobjects.com'; parent::__construct($root, $accessKey, $secretKey, $bucket, $region, $acl, $hostName); } From 4f6adbdcdeee49ed8459caa853a96bc24a13b99e Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 10 May 2022 10:25:37 +0300 Subject: [PATCH 18/22] Switched generic to s3 --- src/Storage/Device/BackBlaze.php | 4 +- src/Storage/Device/DOSpaces.php | 4 +- src/Storage/Device/Generic.php | 840 --------------------------- src/Storage/Device/Linode.php | 4 +- src/Storage/Device/S3.php | 814 +++++++++++++++++++++++++- src/Storage/Device/Wasabi.php | 4 +- tests/Storage/Device/GenericTest.php | 30 - 7 files changed, 816 insertions(+), 884 deletions(-) delete mode 100644 src/Storage/Device/Generic.php delete mode 100644 tests/Storage/Device/GenericTest.php diff --git a/src/Storage/Device/BackBlaze.php b/src/Storage/Device/BackBlaze.php index 67eeb089..9a51e058 100644 --- a/src/Storage/Device/BackBlaze.php +++ b/src/Storage/Device/BackBlaze.php @@ -2,10 +2,10 @@ namespace Utopia\Storage\Device; -use Utopia\Storage\Device\Generic; +use Utopia\Storage\Device\S3; -class BackBlaze extends Generic +class BackBlaze extends S3 { /** * Regions constants diff --git a/src/Storage/Device/DOSpaces.php b/src/Storage/Device/DOSpaces.php index b3f39dad..f5051b0e 100644 --- a/src/Storage/Device/DOSpaces.php +++ b/src/Storage/Device/DOSpaces.php @@ -2,10 +2,10 @@ namespace Utopia\Storage\Device; -use Utopia\Storage\Device\Generic; +use Utopia\Storage\Device\S3; -class DOSpaces extends Generic +class DOSpaces extends S3 { /** * Regions constants diff --git a/src/Storage/Device/Generic.php b/src/Storage/Device/Generic.php deleted file mode 100644 index ddcf4b4b..00000000 --- a/src/Storage/Device/Generic.php +++ /dev/null @@ -1,840 +0,0 @@ -accessKey = $accessKey; - $this->secretKey = $secretKey; - $this->bucket = $bucket; - $this->region = $region; - $this->root = $root; - $this->acl = $acl; - $this->hostName = $hostName; - - } - - /** - * @return string - */ - public function getName(): string - { - return 'S3 compatible Storage'; - } - - /** - * @return string - */ - public function getDescription(): string - { - return 'S3 Generic Bucket Storage drive for AWS compatible solutions'; - } - - /** - * @param string $filename - * @param string|null $prefix - * - * @return string - */ - public function getPath(string $filename, string $prefix = null): string - { - $path = ''; - - for ($i = 0; $i < 4; ++$i) { - $path = ($i < \strlen($filename)) ? $path . DIRECTORY_SEPARATOR . $filename[$i] : $path . DIRECTORY_SEPARATOR . 'x'; - } - - if (!is_null($prefix)) { - $path = $prefix . DIRECTORY_SEPARATOR . $path; - } - - return $this->getRoot() . $path . DIRECTORY_SEPARATOR . $filename; - } - - /** - * @return string - */ - public function getRoot(): string - { - return $this->root; - } - - /** - * Upload. - * - * Upload a file to desired destination in the selected disk. - * return number of chunks uploaded or 0 if it fails. - * - * @param string $source - * @param string $path - * @param int chunk - * @param int chunks - * @param array $metadata - * - * @return int - * @throws \Exception - * - */ - public function upload(string $source, string $path, int $chunk = 1, int $chunks = 1, array &$metadata = []): int - { - if ($chunk == 1 && $chunks == 1) { - return $this->write($path, \file_get_contents($source), \mime_content_type($source)); - } - $uploadId = $metadata['uploadId'] ?? null; - if (empty($uploadId)) { - $uploadId = $this->createMultipartUpload($path, $metadata['content_type']); - $metadata['uploadId'] = $uploadId; - } - - $etag = $this->uploadPart($source, $path, $chunk, $uploadId); - $metadata['parts'] ??= []; - $metadata['parts'][] = ['partNumber' => $chunk, 'etag' => $etag]; - $metadata['chunks'] ??= 0; - $metadata['chunks']++; - if ($metadata['chunks'] == $chunks) { - $this->completeMultipartUpload($path, $uploadId, $metadata['parts'], $source); - } - return $metadata['chunks']; - } - - /** - * Write file by given path. - * - * @param string $path - * @param string $data - * - * @return bool - * @throws \Exception - * - */ - public function write(string $path, string $data, string $contentType = ''): bool - { - - $uri = '/'; - - if($path !== '') { - - $uri = \str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path)); - - if (!str_starts_with($uri, '/')) { - $uri .= '/' . $uri; - } - } - - $this->call(self::METHOD_PUT, $uri, $data, [], [ - 'content-type' => $contentType, - 'content-md5' => \base64_encode(md5($data, true)), - 'x-amz-content-sha256' => \hash('sha256', $data), - 'x-amz-acl' => $this->acl - ]); - - return true; - } - - /** - * Get the S3 response - * - * @param string $method - * @param string $uri - * @param string $data - * @param array $parameters - * @param array $headers - * - * @return object - * @throws \Exception - * - */ - private function call(string $method, string $uri, string $data = '', array $parameters = [], array $headers = []) - { - - $url = 'https://' . $this->hostName . $uri . '?' . \http_build_query($parameters, '', '&', PHP_QUERY_RFC3986); - $response = new \stdClass; - $response->body = ''; - $response->headers = []; - - // Basic setup - $curl = \curl_init(); - \curl_setopt($curl, CURLOPT_USERAGENT, 'utopia-php/storage'); - \curl_setopt($curl, CURLOPT_URL, $url); - - // Headers - $httpHeaders = []; - $headers['x-amz-date'] = \gmdate('Ymd\THis\Z'); - $headers['date'] = \gmdate('D, d M Y H:i:s T'); - $headers['host'] = $this->hostName; - - if (!isset($headers['x-amz-content-sha256'])) { - $headers['x-amz-content-sha256'] = \hash('sha256', $data); - } - - foreach ($headers as $header => $value) { - if (\strlen($value) > 0) { - $httpHeaders[] = $header . ': ' . $value; - } - } - - $httpHeaders[] = 'Authorization: ' . $this->getSignatureV4($method, $uri, $parameters, $headers); - \curl_setopt($curl, CURLOPT_HTTPHEADER, $httpHeaders); - \curl_setopt($curl, CURLOPT_HEADER, false); - \curl_setopt($curl, CURLOPT_RETURNTRANSFER, false); - \curl_setopt($curl, CURLOPT_WRITEFUNCTION, function ($curl, string $data) use ($response) { - $response->body .= $data; - return \strlen($data); - }); - curl_setopt($curl, CURLOPT_HEADERFUNCTION, function ($curl, string $header) use (&$response) { - $len = strlen($header); - $header = explode(':', $header, 2); - - if (count($header) < 2) { // ignore invalid headers - return $len; - } - - $response->headers[strtolower(trim($header[0]))] = trim($header[1]); - - return $len; - }); - \curl_setopt($curl, CURLOPT_FOLLOWLOCATION, true); - \curl_setopt($curl, CURLOPT_CUSTOMREQUEST, $method); - - // Request types - switch ($method) { - case self::METHOD_PUT: - case self::METHOD_POST: // POST only used for CloudFront - \curl_setopt($curl, CURLOPT_POSTFIELDS, $data); - break; - case self::METHOD_HEAD: - case self::METHOD_DELETE: - \curl_setopt($curl, CURLOPT_NOBODY, true); - break; - } - - $result = \curl_exec($curl); - - if (!$result) { - throw new Exception(\curl_error($curl)); - } - - $response->code = \curl_getinfo($curl, CURLINFO_HTTP_CODE); - if ($response->code >= 400) { - throw new Exception($response->body, $response->code); - } - - \curl_close($curl); - - // Parse body into XML - if ((isset($response->headers['content-type']) && $response->headers['content-type'] == 'application/xml') || (str_starts_with($response->body, 'headers['content-type'] ?? '') !== 'image/svg+xml')) { - $response->body = \simplexml_load_string($response->body); - $response->body = json_decode(json_encode($response->body), true); - } - - return $response; - } - - /** - * Generate the headers for AWS Signature V4 - * @param string $method - * @param string $uri - * @param array parameters - * @param array headers - * - * @return string - */ - private function getSignatureV4(string $method, string $uri, array $parameters = [], $headers = []): string - { - $service = 's3'; - $region = $this->region; - $algorithm = 'AWS4-HMAC-SHA256'; - $combinedHeaders = []; - $amzDateStamp = \substr($headers['x-amz-date'], 0, 8); - - // CanonicalHeaders - foreach ($headers as $k => $v) { - $combinedHeaders[\strtolower($k)] = \trim($v); - } - - uksort($combinedHeaders, [& $this, 'sortMetaHeadersCmp']); - - // Convert null query string parameters to strings and sort - uksort($parameters, [& $this, 'sortMetaHeadersCmp']); - $queryString = \http_build_query($parameters, '', '&', PHP_QUERY_RFC3986); - - // Payload - $amzPayload = [$method]; - - $qsPos = \strpos($uri, '?'); - $amzPayload[] = ($qsPos === false ? $uri : \substr($uri, 0, $qsPos)); - $amzPayload[] = $queryString; - - foreach ($combinedHeaders as $k => $v) { // add header as string to requests - $amzPayload[] = $k . ':' . $v; - } - - $amzPayload[] = ''; // add a blank entry so we end up with an extra line break - $amzPayload[] = \implode(';', \array_keys($combinedHeaders)); // SignedHeaders - $amzPayload[] = $headers['x-amz-content-sha256']; // payload hash - - $amzPayloadStr = \implode("\n", $amzPayload); // request as string - - // CredentialScope - $credentialScope = [$amzDateStamp, $region, $service, 'aws4_request']; - - // stringToSign - $stringToSignStr = \implode("\n", [$algorithm, $headers['x-amz-date'], - \implode('/', $credentialScope), \hash('sha256', $amzPayloadStr)]); - - // Make Signature - $kSecret = 'AWS4' . $this->secretKey; - $kDate = \hash_hmac('sha256', $amzDateStamp, $kSecret, true); - $kRegion = \hash_hmac('sha256', $region, $kDate, true); - $kService = \hash_hmac('sha256', $service, $kRegion, true); - $kSigning = \hash_hmac('sha256', 'aws4_request', $kService, true); - - $signature = \hash_hmac('sha256', \utf8_encode($stringToSignStr), $kSigning); - - return $algorithm . ' ' . \implode(',', [ - 'Credential=' . $this->accessKey . '/' . \implode('/', $credentialScope), - 'SignedHeaders=' . \implode(';', \array_keys($combinedHeaders)), - 'Signature=' . $signature, - ]); - } - - /** - * Start Multipart Upload - * - * Initiate a multipart upload and return an upload ID. - * - * @param string $path - * @param string $contentType - * - * @return string - * @throws \Exception - * - */ - protected function createMultipartUpload(string $path, string $contentType): string - { - $uri = '/'; - - if($path !== '') { - - $uri = \str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path)); - - if (!str_starts_with($uri, '/')) { - $uri .= '/' . $uri; - } - } - - $response = $this->call(self::METHOD_POST, $uri, '', ['uploads' => ''], [ - 'content-type' => $contentType, - 'content-md5' => \base64_encode(md5('', true)), - 'x-amz-acl' => $this->acl - ]); - - return $response->body['UploadId']; - } - - /** - * Upload Part - * - * @param string $source - * @param string $path - * @param int $chunk - * @param string $uploadId - * - * @return string - * @throws \Exception - * - */ - protected function uploadPart(string $source, string $path, int $chunk, string $uploadId): string - { - $uri = '/'; - - if($path !== '') { - - $uri = \str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path)); - - if (!str_starts_with($uri, '/')) { - $uri .= '/' . $uri; - } - } - - $data = \file_get_contents($source); - $response = $this->call(self::METHOD_PUT, $uri, $data, [ - 'partNumber' => $chunk, - 'uploadId' => $uploadId - ], [ - 'content-type' => \mime_content_type($source), - 'content-md5' => \base64_encode(md5($data, true)), - 'x-amz-content-sha256' => \hash('sha256', $data) - ]); - - return $response->headers['etag']; - } - - /** - * Complete Multipart Upload - * - * @param string $path - * @param string $uploadId - * @param array $parts - * - * @return bool - * @throws \Exception - * - */ - protected function completeMultipartUpload(string $path, string $uploadId, array $parts, $source): bool - { - - $uri = '/'; - - if($path !== '') { - - $uri = \str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path)); - - if (!str_starts_with($uri, '/')) { - $uri .= '/' . $uri; - } - } - - $body = ''; - foreach ($parts as $part) { - $body .= "{$part['etag']}{$part['partNumber']}"; - } - $body .= ''; - - $this->call(self::METHOD_POST, $uri, $body, [ - 'uploadId' => $uploadId - ], [ - 'content-md5' => \base64_encode(md5($body, true)), - 'content-type' => \mime_content_type($source), - 'x-amz-content-sha256' => \hash('sha256', $body) - ]); - - return true; - } - - /** - * Abort Chunked Upload - * - * @param string $path - * @param string $extra - * - * @return bool - * @throws \Exception - * - */ - public function abort(string $path, string $extra = ''): bool - { - $uri = '/'; - - if($path !== '') { - - $uri = \str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path)); - - if (!str_starts_with($uri, '/')) { - $uri .= '/' . $uri; - } - } - - $this->call(self::METHOD_DELETE, $uri, '', [ - 'uploadId' => $extra - ], [ - 'content-md5' => \base64_encode(md5('', true)) - ]); - - return true; - } - - /** - * Move file from given source to given path, Return true on success and false on failure. - * - * @see http://php.net/manual/en/function.filesize.php - * - * @param string $source - * @param string $target - * - * @throw \Exception - * - * @return bool - */ - public function move(string $source, string $target): bool - { - $type = $this->getFileMimeType($source); - - if ($this->write($target, $this->read($source), $type)) { - $this->delete($source); - } - - return true; - } - - /** - * Returns given file path its mime type. - * - * @see http://php.net/manual/en/function.mime-content-type.php - * - * @param string $path - * - * @return string - */ - public function getFileMimeType(string $path): string - { - $response = $this->getInfo($path); - return $response['content-type'] ?? ''; - } - - /** - * Get file info - * @return array - */ - private function getInfo(string $path): array - { - $uri = '/'; - - if($path !== '') { - - $uri = \str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path)); - - if (!str_starts_with($uri, '/')) { - $uri .= '/' . $uri; - } - } - - $response = $this->call(self::METHOD_HEAD, $uri, '', [], ['content-md5' => \base64_encode(md5('', true))]); - - return $response->headers; - } - - /** - * Read file or part of file by given path, offset and length. - * - * @param string $path - * @param int offset - * @param int length - * - * @return string - * @throws \Exception - * - */ - public function read(string $path, int $offset = 0, int $length = null): string - { - $uri = '/'; - - if($path !== '') { - - $uri = \str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path)); - - if (!str_starts_with($uri, '/')) { - $uri .= '/' . $uri; - } - } - - $headers = [ - 'content-md5' => \base64_encode(md5('', true)) - ]; - - if ($length !== null) { - $end = $offset + $length - 1; - $this->headers['range'] = "bytes=$offset-$end"; - $headers['range'] = "bytes=$offset-$end"; - } - - $response = $this->call(self::METHOD_GET, $uri, '', [], $headers); - return $response->body; - } - - /** - * Delete file in given path, Return true on success and false on failure. - * - * @see http://php.net/manual/en/function.filesize.php - * - * @param string $path - * - * @throws \Exception - * - * @return bool - */ - public function delete(string $path, bool $recursive = false): bool - { - $uri = '/'; - - if($path !== '') { - - $uri = \str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path)); - - if (!str_starts_with($uri, '/')) { - $uri .= '/' . $uri; - } - } - - $this->call(self::METHOD_DELETE, $uri, '', [], [ - 'content-md5' => \base64_encode(md5('', true)), - ]); - - return true; - } - - /** - * Delete files in given path, path must be a directory. Return true on success and false on failure. - * - * @param string $path - * - * @return bool - * @throws \Exception - * - */ - public function deletePath(string $path): bool - { - $root = $this->getRoot(); - - if(str_starts_with($root, '/')){ - $root = substr($root, 1); - } - - $path = $root . '/' . $path; - $uri = '/'; - - $continuationToken = ''; - do { - $objects = $this->listObjects($path, continuationToken: $continuationToken); - $count = (int)($objects['KeyCount'] ?? 1); - if ($count < 1) { - break; - } - $continuationToken = $objects['NextContinuationToken'] ?? ''; - $body = ''; - if ($count > 1) { - foreach ($objects['Contents'] as $object) { - $body .= "{$object['Key']}"; - } - } else { - $body .= "{$objects['Contents']['Key']}"; - } - $body .= 'true'; - $body .= ''; - - $this->call(self::METHOD_POST, $uri, $body, [ - 'delete' => '' - ], [ - 'content-md5' => \base64_encode(md5($body, true)), - 'content-type' => 'text/plain', - 'x-amz-content-sha256' => \hash('sha256', $body) - ]); - } while (!empty($continuationToken)); - - return true; - } - - /** - * Get list of objects in the given path. - * - * @param string $path - * - * @return array - * @throws \Exception - * - */ - private function listObjects($prefix = '', $maxKeys = 1000, $continuationToken = '') - { - $uri = '/'; - - $parameters = [ - 'list-type' => 2, - 'prefix' => $prefix, - 'max-keys' => $maxKeys, - ]; - - if (!empty($continuationToken)) { - $parameters['continuation-token'] = $continuationToken; - } - - $response = $this->call(self::METHOD_GET, $uri, '', $parameters, [ - 'content-type' => 'text/plain', - 'content-md5' => \base64_encode(md5('', true)) - ]); - return $response->body; - } - - /** - * Check if file exists - * - * @param string $path - * - * @return bool - */ - public function exists(string $path): bool - { - try { - $this->getInfo($path); - } catch (\Throwable $th) { - return false; - } - - return true; - } - - /** - * Returns given file path its size. - * - * @see http://php.net/manual/en/function.filesize.php - * - * @param string $path - * - * @return int - */ - public function getFileSize(string $path): int - { - $response = $this->getInfo($path); - return (int)($response['content-length'] ?? 0); - } - - /** - * Returns given file path its MD5 hash value. - * - * @see http://php.net/manual/en/function.md5-file.php - * - * @param string $path - * - * @return string - */ - public function getFileHash(string $path): string - { - $etag = $this->getInfo($path)['etag'] ?? ''; - return (!empty($etag)) ? substr($etag, 1, -1) : $etag; - } - - /** - * Get directory size in bytes. - * - * Return -1 on error - * - * Based on http://www.jonasjohn.de/snippets/php/dir-size.htm - * - * @param string $path - * - * @return int - */ - public function getDirectorySize(string $path): int - { - return -1; - } - - /** - * Get Partition Free Space. - * - * disk_free_space — Returns available space on filesystem or disk partition - * - * @return float - */ - public function getPartitionFreeSpace(): float - { - return -1; - } - - /** - * Get Partition Total Space. - * - * disk_total_space — Returns the total size of a filesystem or disk partition - * - * @return float - */ - public function getPartitionTotalSpace(): float - { - return -1; - } - - /** - * Sort compare for meta headers - * - * @param string $a String A - * @param string $b String B - * @return integer - * @internal Used to sort x-amz meta headers - */ - private function sortMetaHeadersCmp($a, $b) - { - $lenA = \strlen($a); - $lenB = \strlen($b); - $minLen = \min($lenA, $lenB); - $ncmp = \strncmp($a, $b, $minLen); - if ($lenA == $lenB) { - return $ncmp; - } - - if (0 == $ncmp) { - return $lenA < $lenB ? -1 : 1; - } - - return $ncmp; - } -} diff --git a/src/Storage/Device/Linode.php b/src/Storage/Device/Linode.php index ac1e3463..102272c5 100644 --- a/src/Storage/Device/Linode.php +++ b/src/Storage/Device/Linode.php @@ -2,10 +2,10 @@ namespace Utopia\Storage\Device; -use Utopia\Storage\Device\Generic; +use Utopia\Storage\Device\S3; -class Linode extends Generic +class Linode extends S3 { /** * Regions constants diff --git a/src/Storage/Device/S3.php b/src/Storage/Device/S3.php index e7a46b17..e882411b 100644 --- a/src/Storage/Device/S3.php +++ b/src/Storage/Device/S3.php @@ -2,11 +2,21 @@ namespace Utopia\Storage\Device; -use Utopia\Storage\Device\Generic; +use Exception; +use Utopia\Storage\Device; - -class S3 extends Generic +class S3 extends Device { + const METHOD_GET = 'GET'; + const METHOD_POST = 'POST'; + const METHOD_PUT = 'PUT'; + const METHOD_PATCH = 'PATCH'; + const METHOD_DELETE = 'DELETE'; + const METHOD_HEAD = 'HEAD'; + const METHOD_OPTIONS = 'OPTIONS'; + const METHOD_CONNECT = 'CONNECT'; + const METHOD_TRACE = 'TRACE'; + /** * AWS Regions constants */ @@ -37,6 +47,49 @@ class S3 extends Generic const US_GOV_WEST_1 = 'us-gov-west-1'; + /** + * AWS ACL Flag constants + */ + const ACL_PRIVATE = 'private'; + const ACL_PUBLIC_READ = 'public-read'; + const ACL_PUBLIC_READ_WRITE = 'public-read-write'; + const ACL_AUTHENTICATED_READ = 'authenticated-read'; + + /** + * @var string + */ + protected string $accessKey; + + /** + * @var string + */ + protected string $secretKey; + + /** + * @var string + */ + protected string $bucket; + + /** + * @var string + */ + protected string $region; + + /** + * @var string + */ + protected string $acl = self::ACL_PRIVATE; + + /** + * @var string + */ + protected string $hostName; + + /** + * @var string + */ + protected string $root = 'temp'; + /** * S3 Constructor @@ -47,11 +100,17 @@ class S3 extends Generic * @param string $bucket * @param string $region * @param string $acl + * @param string $hostName */ - public function __construct(string $root, string $accessKey, string $secretKey, string $bucket, string $region = self::EU_WEST_1, string $acl = self::ACL_PRIVATE) + public function __construct(string $root, string $accessKey, string $secretKey, string $bucket, string $region, string $acl = self::ACL_PRIVATE, string $hostName = '') { - $hostName = $bucket . '.s3.'.$region.'.amazonaws.com'; - parent::__construct($root, $accessKey, $secretKey, $bucket, $region, $acl, $hostName); + $this->accessKey = $accessKey; + $this->secretKey = $secretKey; + $this->bucket = $bucket; + $this->region = $region; + $this->root = $root; + $this->acl = $acl; + $this->hostName = $hostName; } /** @@ -70,5 +129,748 @@ public function getDescription(): string return 'S3 Generic Bucket Storage drive for AWS compatible solutions'; } + /** + * @param string $filename + * @param string|null $prefix + * + * @return string + */ + public function getPath(string $filename, string $prefix = null): string + { + $path = ''; + + for ($i = 0; $i < 4; ++$i) { + $path = ($i < \strlen($filename)) ? $path . DIRECTORY_SEPARATOR . $filename[$i] : $path . DIRECTORY_SEPARATOR . 'x'; + } + + if (!is_null($prefix)) { + $path = $prefix . DIRECTORY_SEPARATOR . $path; + } + + return $this->getRoot() . $path . DIRECTORY_SEPARATOR . $filename; + } + + /** + * @return string + */ + public function getRoot(): string + { + return $this->root; + } + + /** + * @return string + */ + public function getHostName(): string + { + return !empty($this->hostName)? $this->hostName : $this->bucket . '.s3.' . $this->region.'.amazonaws.com'; + } + + /** + * Upload. + * + * Upload a file to desired destination in the selected disk. + * return number of chunks uploaded or 0 if it fails. + * + * @param string $source + * @param string $path + * @param int chunk + * @param int chunks + * @param array $metadata + * + * @return int + * @throws \Exception + * + */ + public function upload(string $source, string $path, int $chunk = 1, int $chunks = 1, array &$metadata = []): int + { + if ($chunk == 1 && $chunks == 1) { + return $this->write($path, \file_get_contents($source), \mime_content_type($source)); + } + $uploadId = $metadata['uploadId'] ?? null; + if (empty($uploadId)) { + $uploadId = $this->createMultipartUpload($path, $metadata['content_type']); + $metadata['uploadId'] = $uploadId; + } + + $etag = $this->uploadPart($source, $path, $chunk, $uploadId); + $metadata['parts'] ??= []; + $metadata['parts'][] = ['partNumber' => $chunk, 'etag' => $etag]; + $metadata['chunks'] ??= 0; + $metadata['chunks']++; + if ($metadata['chunks'] == $chunks) { + $this->completeMultipartUpload($path, $uploadId, $metadata['parts'], $source); + } + return $metadata['chunks']; + } + + /** + * Write file by given path. + * + * @param string $path + * @param string $data + * + * @return bool + * @throws \Exception + * + */ + public function write(string $path, string $data, string $contentType = ''): bool + { + + $uri = '/'; + + if($path !== '') { + + $uri = \str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path)); + + if (!str_starts_with($uri, '/')) { + $uri .= '/' . $uri; + } + } + + $this->call(self::METHOD_PUT, $uri, $data, [], [ + 'content-type' => $contentType, + 'content-md5' => \base64_encode(md5($data, true)), + 'x-amz-content-sha256' => \hash('sha256', $data), + 'x-amz-acl' => $this->acl + ]); + + return true; + } + + /** + * Get the S3 response + * + * @param string $method + * @param string $uri + * @param string $data + * @param array $parameters + * @param array $headers + * + * @return object + * @throws \Exception + * + */ + private function call(string $method, string $uri, string $data = '', array $parameters = [], array $headers = []) + { + + $url = 'https://' . $this->getHostName() . $uri . '?' . \http_build_query($parameters, '', '&', PHP_QUERY_RFC3986); + $response = new \stdClass; + $response->body = ''; + $response->headers = []; + + // Basic setup + $curl = \curl_init(); + \curl_setopt($curl, CURLOPT_USERAGENT, 'utopia-php/storage'); + \curl_setopt($curl, CURLOPT_URL, $url); + + // Headers + $httpHeaders = []; + $headers['x-amz-date'] = \gmdate('Ymd\THis\Z'); + $headers['date'] = \gmdate('D, d M Y H:i:s T'); + $headers['host'] = $this->getHostName(); + + if (!isset($headers['x-amz-content-sha256'])) { + $headers['x-amz-content-sha256'] = \hash('sha256', $data); + } + + foreach ($headers as $header => $value) { + if (\strlen($value) > 0) { + $httpHeaders[] = $header . ': ' . $value; + } + } + + $httpHeaders[] = 'Authorization: ' . $this->getSignatureV4($method, $uri, $parameters, $headers); + \curl_setopt($curl, CURLOPT_HTTPHEADER, $httpHeaders); + \curl_setopt($curl, CURLOPT_HEADER, false); + \curl_setopt($curl, CURLOPT_RETURNTRANSFER, false); + \curl_setopt($curl, CURLOPT_WRITEFUNCTION, function ($curl, string $data) use ($response) { + $response->body .= $data; + return \strlen($data); + }); + curl_setopt($curl, CURLOPT_HEADERFUNCTION, function ($curl, string $header) use (&$response) { + $len = strlen($header); + $header = explode(':', $header, 2); + + if (count($header) < 2) { // ignore invalid headers + return $len; + } + + $response->headers[strtolower(trim($header[0]))] = trim($header[1]); + + return $len; + }); + \curl_setopt($curl, CURLOPT_FOLLOWLOCATION, true); + \curl_setopt($curl, CURLOPT_CUSTOMREQUEST, $method); + + // Request types + switch ($method) { + case self::METHOD_PUT: + case self::METHOD_POST: // POST only used for CloudFront + \curl_setopt($curl, CURLOPT_POSTFIELDS, $data); + break; + case self::METHOD_HEAD: + case self::METHOD_DELETE: + \curl_setopt($curl, CURLOPT_NOBODY, true); + break; + } + + $result = \curl_exec($curl); + + if (!$result) { + throw new Exception(\curl_error($curl)); + } + + $response->code = \curl_getinfo($curl, CURLINFO_HTTP_CODE); + if ($response->code >= 400) { + throw new Exception($response->body, $response->code); + } + + \curl_close($curl); + + // Parse body into XML + if ((isset($response->headers['content-type']) && $response->headers['content-type'] == 'application/xml') || (str_starts_with($response->body, 'headers['content-type'] ?? '') !== 'image/svg+xml')) { + $response->body = \simplexml_load_string($response->body); + $response->body = json_decode(json_encode($response->body), true); + } + + return $response; + } + + /** + * Generate the headers for AWS Signature V4 + * @param string $method + * @param string $uri + * @param array parameters + * @param array headers + * + * @return string + */ + private function getSignatureV4(string $method, string $uri, array $parameters = [], $headers = []): string + { + $service = 's3'; + $region = $this->region; + $algorithm = 'AWS4-HMAC-SHA256'; + $combinedHeaders = []; + $amzDateStamp = \substr($headers['x-amz-date'], 0, 8); + + // CanonicalHeaders + foreach ($headers as $k => $v) { + $combinedHeaders[\strtolower($k)] = \trim($v); + } + + uksort($combinedHeaders, [& $this, 'sortMetaHeadersCmp']); + + // Convert null query string parameters to strings and sort + uksort($parameters, [& $this, 'sortMetaHeadersCmp']); + $queryString = \http_build_query($parameters, '', '&', PHP_QUERY_RFC3986); + + // Payload + $amzPayload = [$method]; + + $qsPos = \strpos($uri, '?'); + $amzPayload[] = ($qsPos === false ? $uri : \substr($uri, 0, $qsPos)); + $amzPayload[] = $queryString; + + foreach ($combinedHeaders as $k => $v) { // add header as string to requests + $amzPayload[] = $k . ':' . $v; + } + + $amzPayload[] = ''; // add a blank entry so we end up with an extra line break + $amzPayload[] = \implode(';', \array_keys($combinedHeaders)); // SignedHeaders + $amzPayload[] = $headers['x-amz-content-sha256']; // payload hash + + $amzPayloadStr = \implode("\n", $amzPayload); // request as string + + // CredentialScope + $credentialScope = [$amzDateStamp, $region, $service, 'aws4_request']; + + // stringToSign + $stringToSignStr = \implode("\n", [$algorithm, $headers['x-amz-date'], + \implode('/', $credentialScope), \hash('sha256', $amzPayloadStr)]); + + // Make Signature + $kSecret = 'AWS4' . $this->secretKey; + $kDate = \hash_hmac('sha256', $amzDateStamp, $kSecret, true); + $kRegion = \hash_hmac('sha256', $region, $kDate, true); + $kService = \hash_hmac('sha256', $service, $kRegion, true); + $kSigning = \hash_hmac('sha256', 'aws4_request', $kService, true); + + $signature = \hash_hmac('sha256', \utf8_encode($stringToSignStr), $kSigning); + + return $algorithm . ' ' . \implode(',', [ + 'Credential=' . $this->accessKey . '/' . \implode('/', $credentialScope), + 'SignedHeaders=' . \implode(';', \array_keys($combinedHeaders)), + 'Signature=' . $signature, + ]); + } + + /** + * Start Multipart Upload + * + * Initiate a multipart upload and return an upload ID. + * + * @param string $path + * @param string $contentType + * + * @return string + * @throws \Exception + * + */ + protected function createMultipartUpload(string $path, string $contentType): string + { + $uri = '/'; + + if($path !== '') { + + $uri = \str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path)); + + if (!str_starts_with($uri, '/')) { + $uri .= '/' . $uri; + } + } + + $response = $this->call(self::METHOD_POST, $uri, '', ['uploads' => ''], [ + 'content-type' => $contentType, + 'content-md5' => \base64_encode(md5('', true)), + 'x-amz-acl' => $this->acl + ]); + + return $response->body['UploadId']; + } + + /** + * Upload Part + * + * @param string $source + * @param string $path + * @param int $chunk + * @param string $uploadId + * + * @return string + * @throws \Exception + * + */ + protected function uploadPart(string $source, string $path, int $chunk, string $uploadId): string + { + $uri = '/'; + + if($path !== '') { + + $uri = \str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path)); + + if (!str_starts_with($uri, '/')) { + $uri .= '/' . $uri; + } + } + + $data = \file_get_contents($source); + $response = $this->call(self::METHOD_PUT, $uri, $data, [ + 'partNumber' => $chunk, + 'uploadId' => $uploadId + ], [ + 'content-type' => \mime_content_type($source), + 'content-md5' => \base64_encode(md5($data, true)), + 'x-amz-content-sha256' => \hash('sha256', $data) + ]); + + return $response->headers['etag']; + } + + /** + * Complete Multipart Upload + * + * @param string $path + * @param string $uploadId + * @param array $parts + * + * @return bool + * @throws \Exception + * + */ + protected function completeMultipartUpload(string $path, string $uploadId, array $parts, $source): bool + { + + $uri = '/'; + + if($path !== '') { + + $uri = \str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path)); + + if (!str_starts_with($uri, '/')) { + $uri .= '/' . $uri; + } + } + + $body = ''; + foreach ($parts as $part) { + $body .= "{$part['etag']}{$part['partNumber']}"; + } + $body .= ''; + + $this->call(self::METHOD_POST, $uri, $body, [ + 'uploadId' => $uploadId + ], [ + 'content-md5' => \base64_encode(md5($body, true)), + 'content-type' => \mime_content_type($source), + 'x-amz-content-sha256' => \hash('sha256', $body) + ]); + + return true; + } + + /** + * Abort Chunked Upload + * + * @param string $path + * @param string $extra + * + * @return bool + * @throws \Exception + * + */ + public function abort(string $path, string $extra = ''): bool + { + $uri = '/'; + + if($path !== '') { + + $uri = \str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path)); + + if (!str_starts_with($uri, '/')) { + $uri .= '/' . $uri; + } + } + + $this->call(self::METHOD_DELETE, $uri, '', [ + 'uploadId' => $extra + ], [ + 'content-md5' => \base64_encode(md5('', true)) + ]); + + return true; + } + + /** + * Move file from given source to given path, Return true on success and false on failure. + * + * @see http://php.net/manual/en/function.filesize.php + * + * @param string $source + * @param string $target + * + * @throw \Exception + * + * @return bool + */ + public function move(string $source, string $target): bool + { + $type = $this->getFileMimeType($source); + + if ($this->write($target, $this->read($source), $type)) { + $this->delete($source); + } + + return true; + } + + /** + * Returns given file path its mime type. + * + * @see http://php.net/manual/en/function.mime-content-type.php + * + * @param string $path + * + * @return string + */ + public function getFileMimeType(string $path): string + { + $response = $this->getInfo($path); + return $response['content-type'] ?? ''; + } + + /** + * Get file info + * @return array + */ + private function getInfo(string $path): array + { + $uri = '/'; + + if($path !== '') { + + $uri = \str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path)); + + if (!str_starts_with($uri, '/')) { + $uri .= '/' . $uri; + } + } + + $response = $this->call(self::METHOD_HEAD, $uri, '', [], ['content-md5' => \base64_encode(md5('', true))]); + + return $response->headers; + } + + /** + * Read file or part of file by given path, offset and length. + * + * @param string $path + * @param int offset + * @param int length + * + * @return string + * @throws \Exception + * + */ + public function read(string $path, int $offset = 0, int $length = null): string + { + $uri = '/'; + + if($path !== '') { + + $uri = \str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path)); + + if (!str_starts_with($uri, '/')) { + $uri .= '/' . $uri; + } + } + + $headers = [ + 'content-md5' => \base64_encode(md5('', true)) + ]; + if ($length !== null) { + $end = $offset + $length - 1; + $this->headers['range'] = "bytes=$offset-$end"; + $headers['range'] = "bytes=$offset-$end"; + } + + $response = $this->call(self::METHOD_GET, $uri, '', [], $headers); + return $response->body; + } + + /** + * Delete file in given path, Return true on success and false on failure. + * + * @see http://php.net/manual/en/function.filesize.php + * + * @param string $path + * + * @throws \Exception + * + * @return bool + */ + public function delete(string $path, bool $recursive = false): bool + { + $uri = '/'; + + if($path !== '') { + + $uri = \str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path)); + + if (!str_starts_with($uri, '/')) { + $uri .= '/' . $uri; + } + } + + $this->call(self::METHOD_DELETE, $uri, '', [], [ + 'content-md5' => \base64_encode(md5('', true)), + ]); + + return true; + } + + /** + * Delete files in given path, path must be a directory. Return true on success and false on failure. + * + * @param string $path + * + * @return bool + * @throws \Exception + * + */ + public function deletePath(string $path): bool + { + $root = $this->getRoot(); + + if(str_starts_with($root, '/')){ + $root = substr($root, 1); + } + + $path = $root . '/' . $path; + $uri = '/'; + + $continuationToken = ''; + do { + $objects = $this->listObjects($path, continuationToken: $continuationToken); + $count = (int)($objects['KeyCount'] ?? 1); + if ($count < 1) { + break; + } + $continuationToken = $objects['NextContinuationToken'] ?? ''; + $body = ''; + if ($count > 1) { + foreach ($objects['Contents'] as $object) { + $body .= "{$object['Key']}"; + } + } else { + $body .= "{$objects['Contents']['Key']}"; + } + $body .= 'true'; + $body .= ''; + + $this->call(self::METHOD_POST, $uri, $body, [ + 'delete' => '' + ], [ + 'content-md5' => \base64_encode(md5($body, true)), + 'content-type' => 'text/plain', + 'x-amz-content-sha256' => \hash('sha256', $body) + ]); + } while (!empty($continuationToken)); + + return true; + } + + /** + * Get list of objects in the given path. + * + * @param string $path + * + * @return array + * @throws \Exception + * + */ + private function listObjects($prefix = '', $maxKeys = 1000, $continuationToken = '') + { + $uri = '/'; + + $parameters = [ + 'list-type' => 2, + 'prefix' => $prefix, + 'max-keys' => $maxKeys, + ]; + + if (!empty($continuationToken)) { + $parameters['continuation-token'] = $continuationToken; + } + + $response = $this->call(self::METHOD_GET, $uri, '', $parameters, [ + 'content-type' => 'text/plain', + 'content-md5' => \base64_encode(md5('', true)) + ]); + return $response->body; + } + + /** + * Check if file exists + * + * @param string $path + * + * @return bool + */ + public function exists(string $path): bool + { + try { + $this->getInfo($path); + } catch (\Throwable $th) { + return false; + } + + return true; + } + + /** + * Returns given file path its size. + * + * @see http://php.net/manual/en/function.filesize.php + * + * @param string $path + * + * @return int + */ + public function getFileSize(string $path): int + { + $response = $this->getInfo($path); + return (int)($response['content-length'] ?? 0); + } + + /** + * Returns given file path its MD5 hash value. + * + * @see http://php.net/manual/en/function.md5-file.php + * + * @param string $path + * + * @return string + */ + public function getFileHash(string $path): string + { + $etag = $this->getInfo($path)['etag'] ?? ''; + return (!empty($etag)) ? substr($etag, 1, -1) : $etag; + } + + /** + * Get directory size in bytes. + * + * Return -1 on error + * + * Based on http://www.jonasjohn.de/snippets/php/dir-size.htm + * + * @param string $path + * + * @return int + */ + public function getDirectorySize(string $path): int + { + return -1; + } + + /** + * Get Partition Free Space. + * + * disk_free_space — Returns available space on filesystem or disk partition + * + * @return float + */ + public function getPartitionFreeSpace(): float + { + return -1; + } + + /** + * Get Partition Total Space. + * + * disk_total_space — Returns the total size of a filesystem or disk partition + * + * @return float + */ + public function getPartitionTotalSpace(): float + { + return -1; + } + + /** + * Sort compare for meta headers + * + * @param string $a String A + * @param string $b String B + * @return integer + * @internal Used to sort x-amz meta headers + */ + private function sortMetaHeadersCmp($a, $b) + { + $lenA = \strlen($a); + $lenB = \strlen($b); + $minLen = \min($lenA, $lenB); + $ncmp = \strncmp($a, $b, $minLen); + if ($lenA == $lenB) { + return $ncmp; + } + + if (0 == $ncmp) { + return $lenA < $lenB ? -1 : 1; + } + + return $ncmp; + } } diff --git a/src/Storage/Device/Wasabi.php b/src/Storage/Device/Wasabi.php index fc768566..f6608c57 100644 --- a/src/Storage/Device/Wasabi.php +++ b/src/Storage/Device/Wasabi.php @@ -2,10 +2,10 @@ namespace Utopia\Storage\Device; -use Utopia\Storage\Device\Generic; +use Utopia\Storage\Device\S3; -class Wasabi extends Generic +class Wasabi extends S3 { /** * Regions constants diff --git a/tests/Storage/Device/GenericTest.php b/tests/Storage/Device/GenericTest.php deleted file mode 100644 index 8a42ae80..00000000 --- a/tests/Storage/Device/GenericTest.php +++ /dev/null @@ -1,30 +0,0 @@ -object = new Generic($this->root, $key, $secret, $bucket, DOSpaces::NYC3, DOSpaces::ACL_PUBLIC_READ, $hostName); - } - - protected function getAdapterName(): string - { - return 'S3 compatible Storage'; - } - - protected function getAdapterDescription(): string - { - return 'S3 Generic Bucket Storage drive for AWS compatible solutions'; - } -} From 50fec0071886bd681c996d5f73100e9bb9bc68a4 Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 10 May 2022 14:41:38 +0300 Subject: [PATCH 19/22] resolve conflict --- src/Storage/Device.php | 2 +- src/Storage/Device/Wasabi.php | 1 + src/Storage/Storage.php | 16 ++++++++-------- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/Storage/Device.php b/src/Storage/Device.php index 619e401a..cc43b7fd 100644 --- a/src/Storage/Device.php +++ b/src/Storage/Device.php @@ -39,7 +39,7 @@ abstract public function getRoot(): string; * Each device hold a complex directory structure that is being build in this method. * * @param string $filename - * @param string $prefix + * @param string|null $prefix * * @return string */ diff --git a/src/Storage/Device/Wasabi.php b/src/Storage/Device/Wasabi.php index 8e758359..07761c9f 100644 --- a/src/Storage/Device/Wasabi.php +++ b/src/Storage/Device/Wasabi.php @@ -2,6 +2,7 @@ namespace Utopia\Storage\Device; +use JetBrains\PhpStorm\Pure; use Utopia\Storage\Device\S3; class Wasabi extends S3 diff --git a/src/Storage/Storage.php b/src/Storage/Storage.php index 8653bcbc..fe26e6c3 100644 --- a/src/Storage/Storage.php +++ b/src/Storage/Storage.php @@ -13,9 +13,9 @@ class Storage const DEVICE_LOCAL = 'Local'; const DEVICE_S3 = 'S3'; const DEVICE_DO_SPACES = 'DOSpaces'; + const DEVICE_WASABI = 'Wasabi'; const DEVICE_BACKBLAZE = 'BackBlaze'; const DEVICE_LINODE= 'Linode'; - const DEVICE_WASABI = 'Wasabi'; /** @@ -25,7 +25,7 @@ class Storage * * @var array */ - public static $devices = []; + public static array $devices = []; /** * Set Device. @@ -35,11 +35,11 @@ class Storage * @param string $name * @param Device $device * - * @throws Exception - * * @return void + *@throws Exception + * */ - public static function setDevice($name, Device $device): void + public static function setDevice(string $name, Device $device): void { self::$devices[$name] = $device; } @@ -55,7 +55,7 @@ public static function setDevice($name, Device $device): void * * @throws Exception */ - public static function getDevice($name) + public static function getDevice(string $name): Device { if (!\array_key_exists($name, self::$devices)) { throw new Exception('The device "'.$name.'" is not listed'); @@ -73,7 +73,7 @@ public static function getDevice($name) * * @return bool */ - public static function exists($name) + public static function exists(string $name): bool { return (bool) \array_key_exists($name, self::$devices); } @@ -89,7 +89,7 @@ public static function exists($name) * * @return string */ - public static function human(int $bytes, $decimals = 2, $system = 'metric') + public static function human(int $bytes, int $decimals = 2, string $system = 'metric'): string { $mod = ($system === 'binary') ? 1024 : 1000; From edfcaec21f604a683ab28641a6ea612ba60883bc Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 10 May 2022 14:49:54 +0300 Subject: [PATCH 20/22] resolve conflict --- .gitignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index aa7158a2..28a0fa45 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ /vendor/ /.vscode/ .phpunit.result.cache -tests/chunk.php \ No newline at end of file +tests/chunk.php +/.idea + From d546db86a8a3e1f596f308611e7d44890ba08717 Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 17 May 2022 12:03:31 +0300 Subject: [PATCH 21/22] backblaze storage adapter renaming --- src/Storage/Device/BackBlaze.php | 8 ++++---- src/Storage/Storage.php | 2 +- tests/Storage/Device/BackBlazeTest.php | 10 +++++----- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/Storage/Device/BackBlaze.php b/src/Storage/Device/BackBlaze.php index 9a51e058..7f7d9b7d 100644 --- a/src/Storage/Device/BackBlaze.php +++ b/src/Storage/Device/BackBlaze.php @@ -5,7 +5,7 @@ use Utopia\Storage\Device\S3; -class BackBlaze extends S3 +class Backblaze extends S3 { /** * Regions constants @@ -21,7 +21,7 @@ class BackBlaze extends S3 const EU_CENTRAL_004 = 'eu-central-004'; /** - * BackBlaze Constructor + * Backblaze Constructor * * @param string $root * @param string $accessKey @@ -41,7 +41,7 @@ public function __construct(string $root, string $accessKey, string $secretKey, */ public function getName(): string { - return 'BackBlaze B2 Storage'; + return 'Backblaze B2 Storage'; } /** @@ -49,6 +49,6 @@ public function getName(): string */ public function getDescription(): string { - return 'BackBlaze B2 Storage'; + return 'Backblaze B2 Storage'; } } diff --git a/src/Storage/Storage.php b/src/Storage/Storage.php index fe26e6c3..e2b2dc94 100644 --- a/src/Storage/Storage.php +++ b/src/Storage/Storage.php @@ -14,7 +14,7 @@ class Storage const DEVICE_S3 = 'S3'; const DEVICE_DO_SPACES = 'DOSpaces'; const DEVICE_WASABI = 'Wasabi'; - const DEVICE_BACKBLAZE = 'BackBlaze'; + const DEVICE_BACKBLAZE = 'Backblaze'; const DEVICE_LINODE= 'Linode'; diff --git a/tests/Storage/Device/BackBlazeTest.php b/tests/Storage/Device/BackBlazeTest.php index d6836ec3..d084cbf9 100644 --- a/tests/Storage/Device/BackBlazeTest.php +++ b/tests/Storage/Device/BackBlazeTest.php @@ -2,10 +2,10 @@ namespace Utopia\Tests; -use Utopia\Storage\Device\BackBlaze; +use Utopia\Storage\Device\Backblaze; use Utopia\Tests\S3Base; -class BackBlazeTest extends S3Base +class BackblazeTest extends S3Base { protected function init(): void { @@ -14,16 +14,16 @@ protected function init(): void $secret = $_SERVER['BACKBLAZE_SECRET'] ?? ''; $bucket = "backblaze-demo-1"; - $this->object = new BackBlaze($this->root, $key, $secret, $bucket, BackBlaze::EU_CENTRAL_003, BackBlaze::ACL_PRIVATE); + $this->object = new Backblaze($this->root, $key, $secret, $bucket, BackBlaze::EU_CENTRAL_003, BackBlaze::ACL_PRIVATE); } protected function getAdapterName(): string { - return 'BackBlaze B2 Storage'; + return 'Backblaze B2 Storage'; } protected function getAdapterDescription(): string { - return 'BackBlaze B2 Storage'; + return 'Backblaze B2 Storage'; } } From fd571d260a78313fef52c8eee892c1e051085726 Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 17 May 2022 12:32:05 +0300 Subject: [PATCH 22/22] backblaze storage adapter renaming --- src/Storage/Device/{BackBlaze.php => Backblaze.php} | 0 tests/Storage/Device/{BackBlazeTest.php => BackblazeTest.php} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename src/Storage/Device/{BackBlaze.php => Backblaze.php} (100%) rename tests/Storage/Device/{BackBlazeTest.php => BackblazeTest.php} (100%) diff --git a/src/Storage/Device/BackBlaze.php b/src/Storage/Device/Backblaze.php similarity index 100% rename from src/Storage/Device/BackBlaze.php rename to src/Storage/Device/Backblaze.php diff --git a/tests/Storage/Device/BackBlazeTest.php b/tests/Storage/Device/BackblazeTest.php similarity index 100% rename from tests/Storage/Device/BackBlazeTest.php rename to tests/Storage/Device/BackblazeTest.php