diff --git a/automl/cloud-client/delete_dataset.py b/automl/cloud-client/delete_dataset.py new file mode 100644 index 00000000000..e6136c13b84 --- /dev/null +++ b/automl/cloud-client/delete_dataset.py @@ -0,0 +1,33 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def delete_dataset(project_id, dataset_id): + """Delete a dataset.""" + # [START automl_delete_dataset] + from google.cloud import automl + + # TODO(developer): Uncomment and set the following variables + # project_id = "YOUR_PROJECT_ID" + # dataset_id = "YOUR_DATASET_ID" + + client = automl.AutoMlClient() + # Get the full path of the dataset + dataset_full_id = client.dataset_path( + project_id, "us-central1", dataset_id + ) + response = client.delete_dataset(dataset_full_id) + + print("Dataset deleted. {}".format(response.result())) + # [END automl_delete_dataset] diff --git a/automl/cloud-client/delete_dataset_test.py b/automl/cloud-client/delete_dataset_test.py new file mode 100644 index 00000000000..8a1057a6f95 --- /dev/null +++ b/automl/cloud-client/delete_dataset_test.py @@ -0,0 +1,46 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import os + +from google.cloud import automl +import pytest + +import delete_dataset + +PROJECT_ID = os.environ["AUTOML_PROJECT_ID"] +BUCKET_ID = "{}-lcm".format(PROJECT_ID) + + +@pytest.fixture(scope="function") +def create_dataset(): + client = automl.AutoMlClient() + project_location = client.location_path(PROJECT_ID, "us-central1") + display_name = "test_" + datetime.datetime.now().strftime("%Y%m%d%H%M%S") + metadata = automl.types.TextExtractionDatasetMetadata() + dataset = automl.types.Dataset( + display_name=display_name, text_extraction_dataset_metadata=metadata + ) + response = client.create_dataset(project_location, dataset) + dataset_id = response.result().name.split("/")[-1] + + yield dataset_id + + +def test_delete_dataset(capsys, create_dataset): + # delete dataset + delete_dataset.delete_dataset(PROJECT_ID, create_dataset) + out, _ = capsys.readouterr() + assert "Dataset deleted." in out diff --git a/automl/cloud-client/export_dataset.py b/automl/cloud-client/export_dataset.py new file mode 100644 index 00000000000..45f7ee6bdd7 --- /dev/null +++ b/automl/cloud-client/export_dataset.py @@ -0,0 +1,38 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def export_dataset(project_id, dataset_id, gcs_uri): + """Export a dataset.""" + # [START automl_export_dataset] + from google.cloud import automl + + # TODO(developer): Uncomment and set the following variables + # project_id = "YOUR_PROJECT_ID" + # dataset_id = "YOUR_DATASET_ID" + # gcs_uri = "gs://YOUR_BUCKET_ID/path/to/export/" + + client = automl.AutoMlClient() + + # Get the full path of the dataset + dataset_full_id = client.dataset_path( + project_id, "us-central1", dataset_id + ) + + gcs_destination = automl.types.GcsDestination(output_uri_prefix=gcs_uri) + output_config = automl.types.OutputConfig(gcs_destination=gcs_destination) + + response = client.export_data(dataset_full_id, output_config) + print("Dataset exported. {}".format(response.result())) + # [END automl_export_dataset] diff --git a/automl/cloud-client/export_dataset_test.py b/automl/cloud-client/export_dataset_test.py new file mode 100644 index 00000000000..de8bfe5aefa --- /dev/null +++ b/automl/cloud-client/export_dataset_test.py @@ -0,0 +1,46 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import os + +import export_dataset + +PROJECT_ID = os.environ["AUTOML_PROJECT_ID"] +BUCKET_ID = "{}-lcm".format(PROJECT_ID) +PREFIX = "TEST_EXPORT_OUTPUT_" + datetime.datetime.now().strftime( + "%Y%m%d%H%M%S" +) +DATASET_ID = "TEN0000000000000000000" + + +def test_export_dataset(capsys): + # As exporting a dataset can take a long time and only one operation can be + # run on a dataset at once. Try to export a nonexistent dataset and confirm + # that the dataset was not found, but other elements of the request were\ + # valid. + try: + export_dataset.export_dataset( + PROJECT_ID, DATASET_ID, "gs://{}/{}/".format(BUCKET_ID, PREFIX) + ) + out, _ = capsys.readouterr() + assert ( + "The Dataset doesn't exist or is inaccessible for use with AutoMl." + in out + ) + except Exception as e: + assert ( + "The Dataset doesn't exist or is inaccessible for use with AutoMl." + in e.message + ) diff --git a/automl/cloud-client/get_dataset.py b/automl/cloud-client/get_dataset.py new file mode 100644 index 00000000000..a1831903c17 --- /dev/null +++ b/automl/cloud-client/get_dataset.py @@ -0,0 +1,100 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def get_dataset(project_id, dataset_id): + """Get a dataset.""" + # [START automl_language_entity_extraction_get_dataset] + # [START automl_language_sentiment_analysis_get_dataset] + # [START automl_language_text_classification_get_dataset] + # [START automl_translate_get_dataset] + # [START automl_vision_classification_get_dataset] + # [START automl_vision_object_detection_get_dataset] + from google.cloud import automl + + # TODO(developer): Uncomment and set the following variables + # project_id = "YOUR_PROJECT_ID" + # dataset_id = "YOUR_DATASET_ID" + + client = automl.AutoMlClient() + # Get the full path of the dataset + dataset_full_id = client.dataset_path( + project_id, "us-central1", dataset_id + ) + dataset = client.get_dataset(dataset_full_id) + + # Display the dataset information + print("Dataset name: {}".format(dataset.name)) + print("Dataset id: {}".format(dataset.name.split("/")[-1])) + print("Dataset display name: {}".format(dataset.display_name)) + print("Dataset create time:") + print("\tseconds: {}".format(dataset.create_time.seconds)) + print("\tnanos: {}".format(dataset.create_time.nanos)) + # [END automl_language_sentiment_analysis_get_dataset] + # [END automl_language_text_classification_get_dataset] + # [END automl_translate_get_dataset] + # [END automl_vision_classification_get_dataset] + # [END automl_vision_object_detection_get_dataset] + print( + "Text extraction dataset metadata: {}".format( + dataset.text_extraction_dataset_metadata + ) + ) + # [END automl_language_entity_extraction_get_dataset] + + # [START automl_language_sentiment_analysis_get_dataset] + print( + "Text sentiment dataset metadata: {}".format( + dataset.text_sentiment_dataset_metadata + ) + ) + # [END automl_language_sentiment_analysis_get_dataset] + + # [START automl_language_text_classification_get_dataset] + print( + "Text classification dataset metadata: {}".format( + dataset.text_classification_dataset_metadata + ) + ) + # [END automl_language_text_classification_get_dataset] + + # [START automl_translate_get_dataset] + print("Translation dataset metadata:") + print( + "\tsource_language_code: {}".format( + dataset.translation_dataset_metadata.source_language_code + ) + ) + print( + "\ttarget_language_code: {}".format( + dataset.translation_dataset_metadata.target_language_code + ) + ) + # [END automl_translate_get_dataset] + + # [START automl_vision_classification_get_dataset] + print( + "Image classification dataset metadata: {}".format( + dataset.image_classification_dataset_metadata + ) + ) + # [END automl_vision_classification_get_dataset] + + # [START automl_vision_object_detection_get_dataset] + print( + "Image object detection dataset metadata: {}".format( + dataset.image_object_detection_dataset_metadata + ) + ) + # [END automl_vision_object_detection_get_dataset] diff --git a/automl/cloud-client/get_dataset_test.py b/automl/cloud-client/get_dataset_test.py new file mode 100644 index 00000000000..3cc688eb3f9 --- /dev/null +++ b/automl/cloud-client/get_dataset_test.py @@ -0,0 +1,26 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import get_dataset + +PROJECT_ID = os.environ["AUTOML_PROJECT_ID"] +DATASET_ID = os.environ["ENTITY_EXTRACTION_DATASET_ID"] + + +def test_get_dataset(capsys): + get_dataset.get_dataset(PROJECT_ID, DATASET_ID) + out, _ = capsys.readouterr() + assert "Dataset name: " in out diff --git a/automl/cloud-client/import_dataset.py b/automl/cloud-client/import_dataset.py new file mode 100644 index 00000000000..f465bdb1e02 --- /dev/null +++ b/automl/cloud-client/import_dataset.py @@ -0,0 +1,40 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def import_dataset(project_id, dataset_id, path): + """Import a dataset.""" + # [START automl_import_data] + from google.cloud import automl + + # TODO(developer): Uncomment and set the following variables + # project_id = "YOUR_PROJECT_ID" + # dataset_id = "YOUR_DATASET_ID" + # path = "gs://YOUR_BUCKET_ID/path/to/data.csv" + + client = automl.AutoMlClient() + # Get the full path of the dataset. + dataset_full_id = client.dataset_path( + project_id, "us-central1", dataset_id + ) + # Get the multiple Google Cloud Storage URIs + input_uris = path.split(",") + gcs_source = automl.types.GcsSource(input_uris=input_uris) + input_config = automl.types.InputConfig(gcs_source=gcs_source) + # Import data from the input URI + response = client.import_data(dataset_full_id, input_config) + + print("Processing import...") + print("Data imported. {}".format(response.result())) + # [END automl_import_data] diff --git a/automl/cloud-client/import_dataset_test.py b/automl/cloud-client/import_dataset_test.py new file mode 100644 index 00000000000..2064abbcb48 --- /dev/null +++ b/automl/cloud-client/import_dataset_test.py @@ -0,0 +1,60 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import os + +from google.cloud import automl +import pytest + +import import_dataset + +PROJECT_ID = os.environ["AUTOML_PROJECT_ID"] +BUCKET_ID = "{}-lcm".format(PROJECT_ID) + + +@pytest.fixture(scope="function") +def create_dataset(): + client = automl.AutoMlClient() + project_location = client.location_path(PROJECT_ID, "us-central1") + display_name = "test_" + datetime.datetime.now().strftime("%Y%m%d%H%M%S") + metadata = automl.types.TextSentimentDatasetMetadata( + sentiment_max=4 + ) + dataset = automl.types.Dataset( + display_name=display_name, text_sentiment_dataset_metadata=metadata + ) + response = client.create_dataset(project_location, dataset) + dataset_id = response.result().name.split("/")[-1] + + yield dataset_id + + +@pytest.mark.slow +def test_import_dataset(capsys, create_dataset): + data = ( + "gs://{}/sentiment-analysis/dataset.csv".format(BUCKET_ID) + ) + dataset_id = create_dataset + import_dataset.import_dataset(PROJECT_ID, dataset_id, data) + out, _ = capsys.readouterr() + assert "Data imported." in out + + # delete created dataset + client = automl.AutoMlClient() + dataset_full_id = client.dataset_path( + PROJECT_ID, "us-central1", dataset_id + ) + response = client.delete_dataset(dataset_full_id) + response.result() diff --git a/automl/cloud-client/list_datasets.py b/automl/cloud-client/list_datasets.py new file mode 100644 index 00000000000..ae8c576e33c --- /dev/null +++ b/automl/cloud-client/list_datasets.py @@ -0,0 +1,100 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def list_datasets(project_id): + """List datasets.""" + # [START automl_language_entity_extraction_list_datasets] + # [START automl_language_sentiment_analysis_list_datasets] + # [START automl_language_text_classification_list_datasets] + # [START automl_translate_list_datasets] + # [START automl_vision_classification_list_datasets] + # [START automl_vision_object_detection_list_datasets] + from google.cloud import automl + + # TODO(developer): Uncomment and set the following variables + # project_id = "YOUR_PROJECT_ID" + + client = automl.AutoMlClient() + # A resource that represents Google Cloud Platform location. + project_location = client.location_path(project_id, "us-central1") + + # List all the datasets available in the region. + response = client.list_datasets(project_location, "") + + print("List of datasets:") + for dataset in response: + print("Dataset name: {}".format(dataset.name)) + print("Dataset id: {}".format(dataset.name.split("/")[-1])) + print("Dataset display name: {}".format(dataset.display_name)) + print("Dataset create time:") + print("\tseconds: {}".format(dataset.create_time.seconds)) + print("\tnanos: {}".format(dataset.create_time.nanos)) + # [END automl_language_sentiment_analysis_list_datasets] + # [END automl_language_text_classification_list_datasets] + # [END automl_translate_list_datasets] + # [END automl_vision_classification_list_datasets] + # [END automl_vision_object_detection_list_datasets] + print( + "Text extraction dataset metadata: {}".format( + dataset.text_extraction_dataset_metadata + ) + ) + # [END automl_language_entity_extraction_list_datasets] + + # [START automl_language_sentiment_analysis_list_datasets] + print( + "Text sentiment dataset metadata: {}".format( + dataset.text_sentiment_dataset_metadata + ) + ) + # [END automl_language_sentiment_analysis_list_datasets] + + # [START automl_language_text_classification_list_datasets] + print( + "Text classification dataset metadata: {}".format( + dataset.text_classification_dataset_metadata + ) + ) + # [END automl_language_text_classification_list_datasets] + + # [START automl_translate_list_datasets] + print("Translation dataset metadata:") + print( + "\tsource_language_code: {}".format( + dataset.translation_dataset_metadata.source_language_code + ) + ) + print( + "\ttarget_language_code: {}".format( + dataset.translation_dataset_metadata.target_language_code + ) + ) + # [END automl_translate_list_datasets] + + # [START automl_vision_classification_list_datasets] + print( + "Image classification dataset metadata: {}".format( + dataset.image_classification_dataset_metadata + ) + ) + # [END automl_vision_classification_list_datasets] + + # [START automl_vision_object_detection_list_datasets] + print( + "Image object detection dataset metadata: {}".format( + dataset.image_object_detection_dataset_metadata + ) + ) + # [END automl_vision_object_detection_list_datasets] diff --git a/automl/cloud-client/list_datasets_test.py b/automl/cloud-client/list_datasets_test.py new file mode 100644 index 00000000000..7057af815d1 --- /dev/null +++ b/automl/cloud-client/list_datasets_test.py @@ -0,0 +1,27 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import list_datasets + +PROJECT_ID = os.environ["AUTOML_PROJECT_ID"] +DATASET_ID = os.environ["ENTITY_EXTRACTION_DATASET_ID"] + + +def test_list_dataset(capsys): + # list datasets + list_datasets.list_datasets(PROJECT_ID) + out, _ = capsys.readouterr() + assert "Dataset id: {}".format(DATASET_ID) in out