Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support for DataLakeStorage Gen2 File Systems #14

Merged
merged 6 commits into from
Sep 13, 2019
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
@@ -6,6 +6,7 @@ An alternative Azure Storage SDK for Go
This repository is an alternative Azure Storage SDK for Go; which supports for:

- The [Blob Storage APIs](https://docs.microsoft.com/en-us/rest/api/storageservices/blob-service-rest-api)
- The [DataLakeStorage Gen2 APIs](https://docs.microsoft.com/en-us/rest/api/storageservices/data-lake-storage-gen2)
- The [File Storage APIs](https://docs.microsoft.com/en-us/rest/api/storageservices/file-service-rest-api)
- The [Queue Storage APIs](https://docs.microsoft.com/en-us/rest/api/storageservices/queue-service-rest-api)
- The [Table Storage APIs](https://docs.microsoft.com/en-us/rest/api/storageservices/table-service-rest-api)
@@ -60,6 +61,8 @@ Depending on the API Version / API being used - different authentication mechani

Examples for all of these can be found below in [the Examples Directory](examples/).

There's a [Pull Request open adding both a SharedKey and a SharedKeyLite authorizer to Azure/go-autorest](http://github.com/Azure/go-autorest/pull/416)

## Running the Tests

Each package contains both Unit and Acceptance Tests which provision a real Storage Account on Azure, and then run tests against that.
4 changes: 4 additions & 0 deletions storage/2018-03-28/README.md
Original file line number Diff line number Diff line change
@@ -7,6 +7,10 @@ The following API's are supported by this SDK - more information about each SDK
- [Blobs API](blob/blobs)
- [Containers API](blob/containers)

## DataLakeStore Gen2

- [FileSystems API](datalakestore/filesystems)

## File Storage

- [Directories API](file/directories)
84 changes: 84 additions & 0 deletions storage/2018-03-28/datalakestore/filesystems/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
## Data Lake Storage Gen2 File Systems SDK for API version 2018-03-28

This package allows you to interact with the Data Lake Storage Gen2 File Systems API

### Supported Authorizers

* Azure Active Directory (for the Resource Endpoint `https://storage.azure.com`)

### Example Usage

```go
package main

import (
"context"
"fmt"
"os"
"time"

"github.com/Azure/go-autorest/autorest"
"github.com/Azure/go-autorest/autorest/adal"
"github.com/Azure/go-autorest/autorest/azure"
"github.com/hashicorp/go-azure-helpers/authentication"
"github.com/hashicorp/go-azure-helpers/sender"
"github.com/tombuildsstuff/giovanni/storage/2018-03-28/datalakestore/filesystems"
)

func Example() error {
accountName := "storageaccount1"
fileSystemName := "filesystem1"

builder := &authentication.Builder{
SubscriptionID: os.Getenv("ARM_SUBSCRIPTION_ID"),
ClientID: os.Getenv("ARM_CLIENT_ID"),
ClientSecret: os.Getenv("ARM_CLIENT_SECRET"),
TenantID: os.Getenv("ARM_TENANT_ID"),
Environment: os.Getenv("ARM_ENVIRONMENT"),

// Feature Toggles
SupportsClientSecretAuth: true,
}

c, err := builder.Build()
if err != nil {
return fmt.Errorf("Error building AzureRM Client: %s", err)
}

env, err := authentication.DetermineEnvironment(c.Environment)
if err != nil {
return err
}

oauthConfig, err := adal.NewOAuthConfig(env.ActiveDirectoryEndpoint, c.TenantID)
if err != nil {
return err
}

// OAuthConfigForTenant returns a pointer, which can be nil.
if oauthConfig == nil {
return fmt.Errorf("Unable to configure OAuthConfig for tenant %s", c.TenantID)
}

sender := sender.BuildSender("AzureRM")
ctx := context.Background()

storageAuth, err := config.GetAuthorizationToken(sender, oauthConfig, "https://storage.azure.com/")
if err != nil {
return fmt.Errorf("Error retrieving Authorization Token")
}


fileSystemsClient := filesystems.NewWithEnvironment(env)
fileSystemsClient.Client.Authorizer = storageAuth

input := filesystems.CreateInput{
Properties: map[string]string{},
}
if _, err = fileSystemsClient.Create(ctx, accountName, fileSystemName, input); err != nil {
return fmt.Errorf("Error creating: %s", err)
}

return nil
}
```
25 changes: 25 additions & 0 deletions storage/2018-03-28/datalakestore/filesystems/client.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
package filesystems

import (
"github.com/Azure/go-autorest/autorest"
"github.com/Azure/go-autorest/autorest/azure"
)

// Client is the base client for Data Lake Storage FileSystem
type Client struct {
autorest.Client
BaseURI string
}

// New creates an instance of the Data Lake Storage FileSystem client.
func New() Client {
return NewWithEnvironment(azure.PublicCloud)
}

// NewWithEnvironment creates an instance of the Data Lake Storage FileSystem client.
func NewWithEnvironment(environment azure.Environment) Client {
return Client{
Client: autorest.NewClientWithUserAgent(UserAgent()),
BaseURI: environment.StorageEndpointSuffix,
}
}
94 changes: 94 additions & 0 deletions storage/2018-03-28/datalakestore/filesystems/create.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
package filesystems

import (
"context"
"net/http"

"github.com/Azure/go-autorest/autorest"
"github.com/Azure/go-autorest/autorest/azure"
"github.com/Azure/go-autorest/autorest/validation"
"github.com/tombuildsstuff/giovanni/storage/internal/endpoints"
)

type CreateInput struct {
// A map of base64-encoded strings to store as user-defined properties with the File System
// Note that items may only contain ASCII characters in the ISO-8859-1 character set.
// This automatically gets converted to a comma-separated list of name and
// value pairs before sending to the API
Properties map[string]string
}

// Create creates a Data Lake Store Gen2 FileSystem within a Storage Account
func (client Client) Create(ctx context.Context, accountName string, fileSystemName string, input CreateInput) (result autorest.Response, err error) {
if accountName == "" {
return result, validation.NewError("datalakestore.Client", "Create", "`accountName` cannot be an empty string.")
}
if fileSystemName == "" {
return result, validation.NewError("datalakestore.Client", "Create", "`fileSystemName` cannot be an empty string.")
}

req, err := client.CreatePreparer(ctx, accountName, fileSystemName, input)
if err != nil {
err = autorest.NewErrorWithError(err, "datalakestore.Client", "Create", nil, "Failure preparing request")
return
}

resp, err := client.CreateSender(req)
if err != nil {
result = autorest.Response{Response: resp}
err = autorest.NewErrorWithError(err, "datalakestore.Client", "Create", resp, "Failure sending request")
return
}

result, err = client.CreateResponder(resp)
if err != nil {
err = autorest.NewErrorWithError(err, "datalakestore.Client", "Create", resp, "Failure responding to request")
}

return
}

// CreatePreparer prepares the Create request.
func (client Client) CreatePreparer(ctx context.Context, accountName string, fileSystemName string, input CreateInput) (*http.Request, error) {
pathParameters := map[string]interface{}{
"fileSystemName": autorest.Encode("path", fileSystemName),
}

queryParameters := map[string]interface{}{
"resource": autorest.Encode("query", "filesystem"),
}

headers := map[string]interface{}{
"x-ms-properties": buildProperties(input.Properties),
"x-ms-version": APIVersion,
}

preparer := autorest.CreatePreparer(
autorest.AsPut(),
autorest.WithBaseURL(endpoints.GetDataLakeStoreEndpoint(client.BaseURI, accountName)),
autorest.WithPathParameters("/{fileSystemName}", pathParameters),
autorest.WithQueryParameters(queryParameters),
autorest.WithHeaders(headers))

return preparer.Prepare((&http.Request{}).WithContext(ctx))
}

// CreateSender sends the Create request. The method will close the
// http.Response Body if it receives an error.
func (client Client) CreateSender(req *http.Request) (*http.Response, error) {
return autorest.SendWithSender(client, req,
azure.DoRetryWithRegistration(client.Client))
}

// CreateResponder handles the response to the Create request. The method always
// closes the http.Response Body.
func (client Client) CreateResponder(resp *http.Response) (result autorest.Response, err error) {
err = autorest.Respond(
resp,
client.ByInspecting(),
azure.WithErrorUnlessStatusCode(http.StatusCreated),
autorest.ByClosing())
result = autorest.Response{Response: resp}

return
}
53 changes: 53 additions & 0 deletions storage/2018-03-28/datalakestore/filesystems/create_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
package filesystems

import (
"context"
"fmt"
"testing"

"github.com/Azure/azure-sdk-for-go/profiles/latest/storage/mgmt/storage"
"github.com/tombuildsstuff/giovanni/testhelpers"
)

func TestCreateHasNoTagsByDefault(t *testing.T) {
client, err := testhelpers.Build()
if err != nil {
t.Fatal(err)
}
ctx := context.TODO()

resourceGroup := fmt.Sprintf("acctestrg-%d", testhelpers.RandomInt())
accountName := fmt.Sprintf("acctestsa%s", testhelpers.RandomString())
fileSystemName := fmt.Sprintf("acctestfs-%s", testhelpers.RandomString())

if _, err = client.BuildTestResources(ctx, resourceGroup, accountName, storage.BlobStorage); err != nil {
t.Fatal(err)
}
defer client.DestroyTestResources(ctx, resourceGroup, accountName)

fileSystemsClient := NewWithEnvironment(client.Environment)
fileSystemsClient.Client = client.PrepareWithStorageResourceManagerAuth(fileSystemsClient.Client)

t.Logf("[DEBUG] Creating an empty File System..")
input := CreateInput{
Properties: map[string]string{},
}
if _, err = fileSystemsClient.Create(ctx, accountName, fileSystemName, input); err != nil {
t.Fatal(fmt.Errorf("Error creating: %s", err))
}

t.Logf("[DEBUG] Retrieving the Properties..")
props, err := fileSystemsClient.GetProperties(ctx, accountName, fileSystemName)
if err != nil {
t.Fatal(fmt.Errorf("Error getting properties: %s", err))
}

if len(props.Properties) != 0 {
t.Fatalf("Expected 0 properties by default but got %d", len(props.Properties))
}

t.Logf("[DEBUG] Deleting File System..")
if _, err := fileSystemsClient.Delete(ctx, accountName, fileSystemName); err != nil {
t.Fatalf("Error deleting: %s", err)
}
}
85 changes: 85 additions & 0 deletions storage/2018-03-28/datalakestore/filesystems/delete.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
package filesystems

import (
"context"
"net/http"

"github.com/Azure/go-autorest/autorest"
"github.com/Azure/go-autorest/autorest/azure"
"github.com/Azure/go-autorest/autorest/validation"
"github.com/tombuildsstuff/giovanni/storage/internal/endpoints"
)

// Delete deletes a Data Lake Store Gen2 FileSystem within a Storage Account
func (client Client) Delete(ctx context.Context, accountName string, fileSystemName string) (result autorest.Response, err error) {
if accountName == "" {
return result, validation.NewError("datalakestore.Client", "Delete", "`accountName` cannot be an empty string.")
}
if fileSystemName == "" {
return result, validation.NewError("datalakestore.Client", "Delete", "`fileSystemName` cannot be an empty string.")
}

req, err := client.DeletePreparer(ctx, accountName, fileSystemName)
if err != nil {
err = autorest.NewErrorWithError(err, "datalakestore.Client", "Delete", nil, "Failure preparing request")
return
}

resp, err := client.DeleteSender(req)
if err != nil {
result = autorest.Response{Response: resp}
err = autorest.NewErrorWithError(err, "datalakestore.Client", "Delete", resp, "Failure sending request")
return
}

result, err = client.DeleteResponder(resp)
if err != nil {
err = autorest.NewErrorWithError(err, "datalakestore.Client", "Delete", resp, "Failure responding to request")
}

return
}

// DeletePreparer prepares the Delete request.
func (client Client) DeletePreparer(ctx context.Context, accountName string, fileSystemName string) (*http.Request, error) {
pathParameters := map[string]interface{}{
"fileSystemName": autorest.Encode("path", fileSystemName),
}

queryParameters := map[string]interface{}{
"resource": autorest.Encode("query", "filesystem"),
}

headers := map[string]interface{}{
"x-ms-version": APIVersion,
}

preparer := autorest.CreatePreparer(
autorest.AsDelete(),
autorest.WithBaseURL(endpoints.GetDataLakeStoreEndpoint(client.BaseURI, accountName)),
autorest.WithPathParameters("/{fileSystemName}", pathParameters),
autorest.WithQueryParameters(queryParameters),
autorest.WithHeaders(headers))

return preparer.Prepare((&http.Request{}).WithContext(ctx))
}

// DeleteSender sends the Delete request. The method will close the
// http.Response Body if it receives an error.
func (client Client) DeleteSender(req *http.Request) (*http.Response, error) {
return autorest.SendWithSender(client, req,
azure.DoRetryWithRegistration(client.Client))
}

// DeleteResponder handles the response to the Delete request. The method always
// closes the http.Response Body.
func (client Client) DeleteResponder(resp *http.Response) (result autorest.Response, err error) {
err = autorest.Respond(
resp,
client.ByInspecting(),
azure.WithErrorUnlessStatusCode(http.StatusAccepted),
autorest.ByClosing())
result = autorest.Response{Response: resp}

return
}
Loading