Rename TDRSourceSpec.project to subdomain (#6426)

scripts/post_deploy_tdr.py

@@ -93,9 +93,9 @@ def verify_source(self,
                       ) -> None:
         source = self.tdr.lookup_source(source_spec)
         log.info('TDR client is authorized for API access to %s.', source_spec)
-        require(source.project == source_spec.project,
+        require(source.project == source_spec.subdomain,
                 'Actual Google project of TDR source differs from configured one',
-                source.project, source_spec.project)
+                source.project, source_spec.subdomain)
         # Uppercase is standard for multi-regions in the documentation but TDR
         # returns 'us' in lowercase
         require(source.location.lower() == config.tdr_source_location.lower(),

scripts/recan_bundle_tdr.py

@@ -400,7 +400,7 @@ def main(argv):
 
     tdr_source = TDRSourceRef(id=args.source_id,
                               spec=TDRSourceSpec(prefix=Prefix.of_everything,
-                                                 project='test_project',
+                                                 subdomain='test_project',
                                                  name='test_name',
                                                  is_snapshot=True))
     tdr_bundle = dss_bundle_to_tdr(dss_bundle, tdr_source)

scripts/update_subgraph_counts.py

@@ -104,13 +104,13 @@ def ideal_common_prefix(self) -> str:
 
 @attr.s(auto_attribs=True, frozen=True, kw_only=True)
 class SourceSpecArgs:
-    project: str
+    subdomain: str
     name: str
     subgraph_count: int
     explicit_prefix: Optional[str]
 
     def __str__(self) -> str:
-        params = f'{self.project!r}, {self.name!r}, {self.subgraph_count!r}'
+        params = f'{self.subdomain!r}, {self.name!r}, {self.subgraph_count!r}'
         if self.explicit_prefix is not None:
             params += f', prefix={self.explicit_prefix!r}'
         return f'mksrc({params})'
@@ -140,7 +140,7 @@ def generate_source(source: TDRSourceRef) -> SourceSpecArgs:
             prefixed_counter = SubgraphCounter.for_source(plugin, source, counter_prefix)
             if default_prefix not in prefixed_counter.partition_sizes:
                 explicit_prefix = prefixed_counter.ideal_common_prefix()
-        return SourceSpecArgs(project=source.spec.project,
+        return SourceSpecArgs(subdomain=source.spec.subdomain,
                               name=source.spec.name,
                               subgraph_count=counter.count,
                               explicit_prefix=explicit_prefix)

src/azul/terra.py

@@ -93,7 +93,7 @@
 
 @attrs.frozen(kw_only=True)
 class TDRSourceSpec(SourceSpec):
-    project: str
+    subdomain: str
     name: str
     is_snapshot: bool
 
@@ -105,24 +105,26 @@ class TDRSourceSpec(SourceSpec):
     def parse(cls, spec: str) -> 'TDRSourceSpec':
         """
         Construct an instance from its string representation, using the syntax
-        'tdr:{project}:{type}/{name}:{prefix}' ending with an optional
+        'tdr:{subdomain}:{type}/{name}:{prefix}' ending with an optional
         '/{partition_prefix_length}'.
 
         >>> s = TDRSourceSpec.parse('tdr:foo:snapshot/bar:/0')
         >>> s # doctest: +NORMALIZE_WHITESPACE
         TDRSourceSpec(prefix=Prefix(common='', partition=0),
-                      project='foo',
+                      subdomain='foo',
                       name='bar',
                       is_snapshot=True)
+
         >>> s.bq_name
         'bar'
+
         >>> str(s)
         'tdr:foo:snapshot/bar:/0'
 
         >>> d = TDRSourceSpec.parse('tdr:foo:dataset/bar:42/2')
         >>> d # doctest: +NORMALIZE_WHITESPACE
         TDRSourceSpec(prefix=Prefix(common='42', partition=2),
-                      project='foo',
+                      subdomain='foo',
                       name='bar',
                       is_snapshot=False)
         >>> d.bq_name
@@ -142,7 +144,7 @@ def parse(cls, spec: str) -> 'TDRSourceSpec':
         """
         rest, prefix = cls._parse(spec)
         # BigQuery (and by extension the TDR) does not allow : or / in dataset names
-        service, project, name = rest.split(':')
+        service, subdomain, name = rest.split(':')
         type, name = name.split('/')
         assert service == 'tdr', service
         if type == cls._type_snapshot:
@@ -152,7 +154,7 @@ def parse(cls, spec: str) -> 'TDRSourceSpec':
         else:
             assert False, type
         self = cls(prefix=prefix,
-                   project=project,
+                   subdomain=subdomain,
                    name=name,
                    is_snapshot=is_snapshot)
         assert spec == str(self), spec
@@ -181,7 +183,7 @@ def __str__(self) -> str:
         source_type = self._type_snapshot if self.is_snapshot else self._type_dataset
         return ':'.join([
             'tdr',
-            self.project,
+            self.subdomain,
             f'{source_type}/{self.name}',
             str(self.prefix)
         ])
@@ -191,7 +193,7 @@ def type_name(self):
         return self._type_snapshot if self.is_snapshot else self._type_dataset
 
     def qualify_table(self, table_name: str) -> str:
-        return '.'.join((self.project, self.bq_name, table_name))
+        return '.'.join((self.subdomain, self.bq_name, table_name))
 
     def contains(self, other: 'SourceSpec') -> bool:
         """
@@ -213,7 +215,7 @@ def contains(self, other: 'SourceSpec') -> bool:
             isinstance(other, TDRSourceSpec)
             and super().contains(other)
             and self.is_snapshot == other.is_snapshot
-            and self.project == other.project
+            and self.subdomain == other.subdomain
             and self.name == other.name
         )
 
@@ -454,11 +456,11 @@ def check_bigquery_access(self, source: TDRSourceSpec):
         """
         Verify that the client is authorized to read from TDR BigQuery tables.
         """
-        resource = f'BigQuery dataset {source.bq_name!r} in Google Cloud project {source.project!r}'
+        resource = f'BigQuery dataset {source.bq_name!r} in Google Cloud project {source.subdomain!r}'
         try:
             self.run_sql(f'''
                 SELECT *
-                FROM `{source.project}.{source.bq_name}.INFORMATION_SCHEMA.TABLES`
+                FROM `{source.subdomain}.{source.bq_name}.INFORMATION_SCHEMA.TABLES`
                 LIMIT 1
             ''')
         except Forbidden:

test/indexer/test_tdr.py

@@ -177,7 +177,7 @@ def plugin_for_source_spec(self, source_spec) -> TDR_PLUGIN:
         # noinspection PyAbstractClass
         class Plugin(MockPlugin, self._plugin_cls()):
             netloc = self.netloc
-            project_id = self.source.spec.project
+            project_id = self.source.spec.subdomain
 
         return Plugin(sources={source_spec})
 
@@ -193,7 +193,7 @@ def setUpClass(cls):
                                            command=[
                                                '--log-level=debug',
                                                '--port=9050',
-                                               '--project=' + cls.source.spec.project,
+                                               '--project=' + cls.source.spec.subdomain,
                                                '--dataset=' + cls.source.spec.bq_name
                                            ])
 
@@ -228,7 +228,7 @@ def dump_row(row: JSON) -> JSON:
             }
 
         plugin = self.plugin_for_source_spec(source)
-        bq = plugin.tdr._bigquery(source.project)
+        bq = plugin.tdr._bigquery(source.subdomain)
         table_name = plugin._full_table_name(source, table_name)
         # https://youtrack.jetbrains.com/issue/PY-50178
         # noinspection PyTypeChecker