Skip to content

Commit

Permalink
feat: Add tables to the SDK (#2042)
Browse files Browse the repository at this point in the history 
* qs

* chantes

* changes

* changes

* stuff

* changes

* changes

* changes

* changes

* fixes

* fixes

* changes

* Fix minor stuff

* Use our assertion helper

* Fix tests

* Use our errors

* Use common schema and db

* Show by id using in syntax

* Create database using SDK

* Rename to toOpts()

* Extract getting table columns to helper

* Fix column assertions

* Refactor column assertions

* Merge two create table helper methods

* Add skip file x and x%

* Fix tests and files

* Add describe and fix integration tests

* Update comments

* Fix linter

* Fix reviewdog

* Fix test

* Fix after review part 1

* Fix after review part 2

* Fix after review part 3

* Add out of line constraint validations (WIP)

* Add type presence validation

* Validate out of line constraints

* Allow multiple out of line column constraints

* Validate out of line in alter

* Use query in create table as select

* Fix linter

* Fix after review

* Fix after review

---------

Co-authored-by: Artur Sawicki <artur.sawicki@snowflake.com>
  • Loading branch information
@sfc-gh-pbosak @sfc-gh-asawicki
sfc-gh-pbosak and sfc-gh-asawicki authored Dec 8, 2023
1 parent 66cc80a commit c1700de
Show file tree
Hide file tree
Showing 29 changed files with 6,937 additions and 330 deletions.
3 changes: 3 additions & 0 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,9 @@ test: ## run unit and integration tests
test-acceptance: ## run acceptance tests
TF_ACC=1 go test -run "^TestAcc_" -v -cover -timeout=30m ./...

test-integration: ## run SDK integration tests
go test -run "^TestInt_" -v -cover -timeout=20m ./...

test-architecture: ## check architecture constraints between packages
go test ./pkg/architests/... -v

Expand Down
6 changes: 3 additions & 3 deletions pkg/resources/file_format.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -428,7 +428,7 @@ func CreateFileFormat(d *schema.ResourceData, meta interface{}) error {
}
nullIf = append(nullIf, sdk.NullString{S: s.(string)})
}
opts.JSONNullIf = &nullIf
opts.JSONNullIf = nullIf
}
if v, ok := d.GetOk("file_extension"); ok {
opts.JSONFileExtension = sdk.String(v.(string))
Expand Down Expand Up @@ -649,7 +649,7 @@ func ReadFileFormat(d *schema.ResourceData, meta interface{}) error {
return err
}
nullIf := []string{}
for _, s := range *fileFormat.Options.JSONNullIf {
for _, s := range fileFormat.Options.JSONNullIf {
nullIf = append(nullIf, s.S)
}
if err := d.Set("null_if", nullIf); err != nil {
Expand Down Expand Up @@ -939,7 +939,7 @@ func UpdateFileFormat(d *schema.ResourceData, meta interface{}) error {
}
nullIf = append(nullIf, sdk.NullString{S: s.(string)})
}
opts.Set.JSONNullIf = &nullIf
opts.Set.JSONNullIf = nullIf
runSet = true
}
if d.HasChange("file_extension") {
Expand Down
2 changes: 2 additions & 0 deletions pkg/sdk/client.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ type Client struct {
Shares Shares
Stages Stages
Streams Streams
Tables Tables
Tags Tags
Tasks Tasks
Users Users
Expand Down Expand Up @@ -183,6 +184,7 @@ func (c *Client) initialize() {
c.Stages = &stages{client: c}
c.Streams = &streams{client: c}
c.SystemFunctions = &systemFunctions{client: c}
c.Tables = &tables{client: c}
c.Tags = &tags{client: c}
c.Tasks = &tasks{client: c}
c.Users = &users{client: c}
Expand Down
115 changes: 115 additions & 0 deletions pkg/sdk/common_table_types.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
package sdk

import "errors"

type TableRowAccessPolicy struct {
rowAccessPolicy bool `ddl:"static" sql:"ROW ACCESS POLICY"`
Name SchemaObjectIdentifier `ddl:"identifier"`
On []string `ddl:"keyword,parentheses" sql:"ON"`
}

// ColumnInlineConstraint is based on https://docs.snowflake.com/en/sql-reference/sql/create-table-constraint#inline-unique-primary-foreign-key.
type ColumnInlineConstraint struct {
Name *string `ddl:"parameter,no_equals" sql:"CONSTRAINT"`
Type ColumnConstraintType `ddl:"keyword"`
ForeignKey *InlineForeignKey `ddl:"keyword" sql:"FOREIGN KEY"`

// optional
Enforced *bool `ddl:"keyword" sql:"ENFORCED"`
NotEnforced *bool `ddl:"keyword" sql:"NOT ENFORCED"`
Deferrable *bool `ddl:"keyword" sql:"DEFERRABLE"`
NotDeferrable *bool `ddl:"keyword" sql:"NOT DEFERRABLE"`
InitiallyDeferred *bool `ddl:"keyword" sql:"INITIALLY DEFERRED"`
InitiallyImmediate *bool `ddl:"keyword" sql:"INITIALLY IMMEDIATE"`
Enable *bool `ddl:"keyword" sql:"ENABLE"`
Disable *bool `ddl:"keyword" sql:"DISABLE"`
Validate *bool `ddl:"keyword" sql:"VALIDATE"`
NoValidate *bool `ddl:"keyword" sql:"NOVALIDATE"`
Rely *bool `ddl:"keyword" sql:"RELY"`
NoRely *bool `ddl:"keyword" sql:"NORELY"`
}

func (v *ColumnInlineConstraint) validate() error {
var errs []error
switch v.Type {
case ColumnConstraintTypeForeignKey:
if !valueSet(v.ForeignKey) {
errs = append(errs, errNotSet("ColumnInlineConstraint", "ForeignKey"))
} else {
if err := v.ForeignKey.validate(); err != nil {
errs = append(errs, err)
}
}
case ColumnConstraintTypeUnique, ColumnConstraintTypePrimaryKey:
if valueSet(v.ForeignKey) {
errs = append(errs, errSet("ColumnInlineConstraint", "ForeignKey"))
}
default:
errs = append(errs, errInvalidValue("ColumnInlineConstraint", "Type", string(v.Type)))
}
if moreThanOneValueSet(v.Enforced, v.NotEnforced) {
errs = append(errs, errMoreThanOneOf("ColumnInlineConstraint", "Enforced", "NotEnforced"))
}
if moreThanOneValueSet(v.Deferrable, v.NotDeferrable) {
errs = append(errs, errMoreThanOneOf("ColumnInlineConstraint", "Deferrable", "NotDeferrable"))
}
if moreThanOneValueSet(v.InitiallyDeferred, v.InitiallyImmediate) {
errs = append(errs, errMoreThanOneOf("ColumnInlineConstraint", "InitiallyDeferred", "InitiallyImmediate"))
}
if moreThanOneValueSet(v.Enable, v.Disable) {
errs = append(errs, errMoreThanOneOf("ColumnInlineConstraint", "Enable", "Disable"))
}
if moreThanOneValueSet(v.Validate, v.NoValidate) {
errs = append(errs, errMoreThanOneOf("ColumnInlineConstraint", "Validate", "Novalidate"))
}
if moreThanOneValueSet(v.Rely, v.NoRely) {
errs = append(errs, errMoreThanOneOf("ColumnInlineConstraint", "Rely", "Norely"))
}
return errors.Join(errs...)
}

type ColumnConstraintType string

const (
ColumnConstraintTypeUnique ColumnConstraintType = "UNIQUE"
ColumnConstraintTypePrimaryKey ColumnConstraintType = "PRIMARY KEY"
ColumnConstraintTypeForeignKey ColumnConstraintType = "FOREIGN KEY"
)

type InlineForeignKey struct {
TableName string `ddl:"keyword" sql:"REFERENCES"`
ColumnName []string `ddl:"keyword,parentheses"`
Match *MatchType `ddl:"keyword" sql:"MATCH"`
On *ForeignKeyOnAction `ddl:"keyword" sql:"ON"`
}

func (v *InlineForeignKey) validate() error {
var errs []error
if !valueSet(v.TableName) {
errs = append(errs, errNotSet("InlineForeignKey", "TableName"))
}
return errors.Join(errs...)
}

type MatchType string

var (
FullMatchType MatchType = "FULL"
SimpleMatchType MatchType = "SIMPLE"
PartialMatchType MatchType = "PARTIAL"
)

type ForeignKeyOnAction struct {
OnUpdate *ForeignKeyAction `ddl:"parameter,no_equals" sql:"ON UPDATE"`
OnDelete *ForeignKeyAction `ddl:"parameter,no_equals" sql:"ON DELETE"`
}

type ForeignKeyAction string

const (
ForeignKeyCascadeAction ForeignKeyAction = "CASCADE"
ForeignKeySetNullAction ForeignKeyAction = "SET NULL"
ForeignKeySetDefaultAction ForeignKeyAction = "SET DEFAULT"
ForeignKeyRestrictAction ForeignKeyAction = "RESTRICT"
ForeignKeyNoAction ForeignKeyAction = "NO ACTION"
)
59 changes: 0 additions & 59 deletions pkg/sdk/common_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,65 +129,6 @@ func (row *propertyRow) toIntProperty() *IntProperty {
}
}

type RowAccessPolicy struct {
rowAccessPolicy bool `ddl:"static" sql:"ROW ACCESS POLICY"`
Name SchemaObjectIdentifier `ddl:"identifier"`
On []string `ddl:"keyword,parentheses" sql:"ON"`
}

type ColumnInlineConstraint struct {
NotNull *bool `ddl:"keyword" sql:"NOT NULL"`
Name *string `ddl:"parameter,no_equals" sql:"CONSTRAINT"`
Type *ColumnConstraintType `ddl:"keyword"`
ForeignKey *InlineForeignKey `ddl:"keyword" sql:"FOREIGN KEY"`

// optional
Enforced *bool `ddl:"keyword" sql:"ENFORCED"`
NotEnforced *bool `ddl:"keyword" sql:"NOT ENFORCED"`
Deferrable *bool `ddl:"keyword" sql:"DEFERRABLE"`
NotDeferrable *bool `ddl:"keyword" sql:"NOT DEFERRABLE"`
InitiallyDeferred *bool `ddl:"keyword" sql:"INITIALLY DEFERRED"`
InitiallyImmediate *bool `ddl:"keyword" sql:"INITIALLY IMMEDIATE"`
Enable *bool `ddl:"keyword" sql:"ENABLE"`
Disable *bool `ddl:"keyword" sql:"DISABLE"`
Validate *bool `ddl:"keyword" sql:"VALIDATE"`
NoValidate *bool `ddl:"keyword" sql:"NOVALIDATE"`
Rely *bool `ddl:"keyword" sql:"RELY"`
NoRely *bool `ddl:"keyword" sql:"NORELY"`
}

type ColumnConstraintType string

var (
ColumnConstraintTypeUnique ColumnConstraintType = "UNIQUE"
ColumnConstraintTypePrimaryKey ColumnConstraintType = "PRIMARY KEY"
ColumnConstraintTypeForeignKey ColumnConstraintType = "FOREIGN KEY"
)

type InlineForeignKey struct {
TableName string `ddl:"keyword" sql:"REFERENCES"`
ColumnName []string `ddl:"keyword,parentheses"`
Match *MatchType `ddl:"keyword" sql:"MATCH"`
On *ForeignKeyOnAction `ddl:"keyword" sql:"ON"`
}

func (v *InlineForeignKey) validate() error {
return nil
}

type MatchType string

var (
FullMatchType MatchType = "FULL"
SimpleMatchType MatchType = "SIMPLE"
PartialMatchType MatchType = "PARTIAL"
)

type ForeignKeyOnAction struct {
OnUpdate *bool `ddl:"parameter,no_equals" sql:"ON UPDATE"`
OnDelete *bool `ddl:"parameter,no_equals" sql:"ON DELETE"`
}

func (row *propertyRow) toBoolProperty() *BoolProperty {
var value bool
if row.Value != "" && row.Value != "null" {
Expand Down
1 change: 1 addition & 0 deletions pkg/sdk/data_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"golang.org/x/exp/slices"
)

// DataType is based on https://docs.snowflake.com/en/sql-reference/intro-summary-data-types.
type DataType string

const (
Expand Down
12 changes: 12 additions & 0 deletions pkg/sdk/errors.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,10 @@ func errNotSet(structName string, fieldNames ...string) error {
return newError(fmt.Sprintf("%v fields: %v should be set", structName, fieldNames), 2)
}

func errSet(structName string, fieldNames ...string) error {
return newError(fmt.Sprintf("%v fields: %v should not be set", structName, fieldNames), 2)
}

func errExactlyOneOf(structName string, fieldNames ...string) error {
return newError(fmt.Sprintf("exactly one of %s fields %v must be set", structName, fieldNames), 2)
}
Expand All @@ -60,6 +64,14 @@ func errAtLeastOneOf(structName string, fieldNames ...string) error {
return newError(fmt.Sprintf("at least one of %s fields %v must be set", structName, fieldNames), 2)
}

func errMoreThanOneOf(structName string, fieldNames ...string) error {
return newError(fmt.Sprintf("more than one field (%v) of %s cannot be set", fieldNames, structName), 2)
}

func errInvalidValue(structName string, fieldName string, invalidValue string) error {
return newError(fmt.Sprintf("invalid value %s of struct %s field: %s", invalidValue, structName, fieldName), 2)
}

func decodeDriverError(err error) error {
if err == nil {
return nil
Expand Down
10 changes: 6 additions & 4 deletions pkg/sdk/external_tables.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ type ExternalTables interface {
CreateWithManualPartitioning(ctx context.Context, req *CreateWithManualPartitioningExternalTableRequest) error
CreateDeltaLake(ctx context.Context, req *CreateDeltaLakeExternalTableRequest) error
CreateUsingTemplate(ctx context.Context, req *CreateExternalTableUsingTemplateRequest) error
// TODO: Add alter options from https://docs.snowflake.com/en/sql-reference/sql/alter-table#external-table-column-actions-exttablecolumnaction (for better UX)
Alter(ctx context.Context, req *AlterExternalTableRequest) error
AlterPartitions(ctx context.Context, req *AlterExternalTablePartitionRequest) error
Drop(ctx context.Context, req *DropExternalTableRequest) error
Expand Down Expand Up @@ -74,14 +75,15 @@ type CreateExternalTableOptions struct {
AwsSnsTopic *string `ddl:"parameter,single_quotes" sql:"AWS_SNS_TOPIC"`
CopyGrants *bool `ddl:"keyword" sql:"COPY GRANTS"`
Comment *string `ddl:"parameter,single_quotes" sql:"COMMENT"`
RowAccessPolicy *RowAccessPolicy `ddl:"keyword"`
RowAccessPolicy *TableRowAccessPolicy `ddl:"keyword"`
Tag []TagAssociation `ddl:"keyword,parentheses" sql:"TAG"`
}

type ExternalTableColumn struct {
Name string `ddl:"keyword"`
Type DataType `ddl:"keyword"`
AsExpression []string `ddl:"keyword,parentheses" sql:"AS"`
NotNull *bool `ddl:"keyword" sql:"NOT NULL"`
InlineConstraint *ColumnInlineConstraint
}

Expand Down Expand Up @@ -205,7 +207,7 @@ type CreateWithManualPartitioningExternalTableOptions struct {
FileFormat []ExternalTableFileFormat `ddl:"parameter,parentheses" sql:"FILE_FORMAT"`
CopyGrants *bool `ddl:"keyword" sql:"COPY GRANTS"`
Comment *string `ddl:"parameter,single_quotes" sql:"COMMENT"`
RowAccessPolicy *RowAccessPolicy `ddl:"keyword"`
RowAccessPolicy *TableRowAccessPolicy `ddl:"keyword"`
Tag []TagAssociation `ddl:"keyword,parentheses" sql:"TAG"`
}

Expand All @@ -227,7 +229,7 @@ type CreateDeltaLakeExternalTableOptions struct {
DeltaTableFormat *bool `ddl:"keyword" sql:"TABLE_FORMAT = DELTA"`
CopyGrants *bool `ddl:"keyword" sql:"COPY GRANTS"`
Comment *string `ddl:"parameter,single_quotes" sql:"COMMENT"`
RowAccessPolicy *RowAccessPolicy `ddl:"keyword"`
RowAccessPolicy *TableRowAccessPolicy `ddl:"keyword"`
Tag []TagAssociation `ddl:"keyword,parentheses" sql:"TAG"`
}

Expand All @@ -248,7 +250,7 @@ type CreateExternalTableUsingTemplateOptions struct {
FileFormat []ExternalTableFileFormat `ddl:"parameter,parentheses" sql:"FILE_FORMAT"`
AwsSnsTopic *string `ddl:"parameter,single_quotes" sql:"AWS_SNS_TOPIC"`
Comment *string `ddl:"parameter,single_quotes" sql:"COMMENT"`
RowAccessPolicy *RowAccessPolicy `ddl:"keyword"`
RowAccessPolicy *TableRowAccessPolicy `ddl:"keyword"`
Tag []TagAssociation `ddl:"keyword,parentheses" sql:"TAG"`
}

Expand Down
Loading

0 comments on commit c1700de

Please sign in to comment.