Skip to content

Commit

Permalink
fixup! [u r] Add type and domain fields to TDRSourceSpec (#6426)
Browse files Browse the repository at this point in the history
  • Loading branch information
nadove-ucsc committed Aug 1, 2024
1 parent 7fd5e03 commit c2885a5
Showing 1 changed file with 20 additions and 22 deletions.
42 changes: 20 additions & 22 deletions src/azul/terra.py
Original file line number Diff line number Diff line change
Expand Up @@ -93,20 +93,18 @@
log = logging.getLogger(__name__)


class SourceType(enum.Enum):
bigquery = 'bigquery'
parquet = 'parquet'


class SourceDomain(enum.Enum):
gcp = 'gcp'
azure = 'azure'


@attrs.frozen(kw_only=True)
class TDRSourceSpec(SourceSpec):
type: SourceType
domain: SourceDomain
class Type(enum.Enum):
bigquery = 'bigquery'
parquet = 'parquet'

class Domain(enum.Enum):
gcp = 'gcp'
azure = 'azure'

type: Type
domain: Domain
subdomain: str
name: str

Expand All @@ -120,8 +118,8 @@ def parse(cls, spec: str) -> 'TDRSourceSpec':
>>> s = TDRSourceSpec.parse('tdr:bigquery:gcp:foo:bar:/0')
>>> s # doctest: +NORMALIZE_WHITESPACE
TDRSourceSpec(prefix=Prefix(common='', partition=0),
type=<SourceType.bigquery: 'bigquery'>,
domain=<SourceDomain.gcp: 'gcp'>,
type=<Type.bigquery: 'bigquery'>,
domain=<Domain.gcp: 'gcp'>,
subdomain='foo',
name='bar')
Expand All @@ -131,12 +129,12 @@ def parse(cls, spec: str) -> 'TDRSourceSpec':
>>> TDRSourceSpec.parse('tdr:spam:gcp:foo:bar:/0')
Traceback (most recent call last):
...
ValueError: 'spam' is not a valid SourceType
ValueError: 'spam' is not a valid Type
>>> TDRSourceSpec.parse('tdr:bigquery:eggs:foo:bar:/0')
Traceback (most recent call last):
...
ValueError: 'eggs' is not a valid SourceDomain
ValueError: 'eggs' is not a valid Domain
>>> TDRSourceSpec.parse('tdr:bigquery:gcp:foo:bar:n32/0')
Traceback (most recent call last):
Expand All @@ -147,13 +145,13 @@ def parse(cls, spec: str) -> 'TDRSourceSpec':
# BigQuery (and by extension the TDR) does not allow : or / in dataset names
service, type, domain, subdomain, name = rest.split(':')
assert service == 'tdr', service
type = SourceType(type)
reject(type == SourceType.parquet, 'Parquet sources are not yet supported')
domain = SourceDomain(domain)
reject(domain == SourceDomain.azure, 'Azure sources are not yet supported')
type = cls.Type(type)
reject(type == cls.Type.parquet, 'Parquet sources are not yet supported')
domain = cls.Domain(domain)
reject(domain == cls.Domain.azure, 'Azure sources are not yet supported')
self = cls(prefix=prefix,
type=SourceType(type),
domain=SourceDomain(domain),
type=type,
domain=domain,
subdomain=subdomain,
name=name)
assert spec == str(self), spec
Expand Down

0 comments on commit c2885a5

Please sign in to comment.