Skip to content

Commit

Permalink
lint
Browse files Browse the repository at this point in the history
Signed-off-by: Jacob Klegar <jacob@tecton.ai>
  • Loading branch information
jklegar committed Jan 7, 2021
1 parent 4d59e79 commit 17ba056
Show file tree
Hide file tree
Showing 2 changed files with 9 additions and 5 deletions.
10 changes: 6 additions & 4 deletions sdk/python/feast/pyspark/launchers/k8s/k8s.py
Original file line number Diff line number Diff line change
Expand Up @@ -183,18 +183,20 @@ def _job_from_job_info(self, job_info: JobInfo) -> SparkJob:
def _get_staging_client(self):
uri = urlparse(self._staging_location)
return get_staging_client(uri.scheme, self._config)

def _get_azure_credentials(self):
uri = urlparse(self._staging_location)
if (uri.scheme != "wasbs"):
if uri.scheme != "wasbs":
return {}
account_name = self._config.get(opt.AZURE_BLOB_ACCOUNT_NAME)
account_key = self._config.get(opt.AZURE_BLOB_ACCOUNT_ACCESS_KEY)
if (account_name is None or account_key is None):
if account_name is None or account_key is None:
raise Exception(
f"Using Azure blob storage requires {opt.AZURE_BLOB_ACCOUNT_NAME} and {opt.AZURE_BLOB_ACCOUNT_ACCESS_KEY} to be set in config"
)
return {f"spark.hadoop.fs.azure.account.key.{account_name}.blob.core.windows.net": f"{account_key}"}
return {
f"spark.hadoop.fs.azure.account.key.{account_name}.blob.core.windows.net": f"{account_key}"
}

def historical_feature_retrieval(
self, job_params: RetrievalJobParameters
Expand Down
4 changes: 3 additions & 1 deletion sdk/python/feast/staging/storage_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -372,7 +372,9 @@ def list_files(self, uri: ParseResult) -> List[str]:
if re.match(regex, file) and file not in path
]
else:
return [f"wasbs://{bucket}@{self.account_name}.blob.core.windows.net/{path}"]
return [
f"wasbs://{bucket}@{self.account_name}.blob.core.windows.net/{path}"
]

def _uri_to_bucket_key(self, uri: ParseResult) -> Tuple[str, str]:
assert uri.hostname == f"{self.account_name}.blob.core.windows.net"
Expand Down

0 comments on commit 17ba056

Please sign in to comment.