From 076c4212d8e341e3b1eb232b3dda99d4c3232c08 Mon Sep 17 00:00:00 2001 From: nnegrey Date: Wed, 29 Jan 2020 09:47:49 -0700 Subject: [PATCH 1/2] automl: video beta samples move from branch to master --- automl/beta/delete_dataset.py | 33 +++++++++++++++++++++ automl/beta/delete_dataset_test.py | 46 ++++++++++++++++++++++++++++++ automl/beta/import_dataset.py | 40 ++++++++++++++++++++++++++ automl/beta/import_dataset_test.py | 41 ++++++++++++++++++++++++++ automl/beta/list_datasets.py | 45 +++++++++++++++++++++++++++++ automl/beta/list_datasets_test.py | 27 ++++++++++++++++++ 6 files changed, 232 insertions(+) create mode 100644 automl/beta/delete_dataset.py create mode 100644 automl/beta/delete_dataset_test.py create mode 100644 automl/beta/import_dataset.py create mode 100644 automl/beta/import_dataset_test.py create mode 100644 automl/beta/list_datasets.py create mode 100644 automl/beta/list_datasets_test.py diff --git a/automl/beta/delete_dataset.py b/automl/beta/delete_dataset.py new file mode 100644 index 00000000000..f1935e2b90e --- /dev/null +++ b/automl/beta/delete_dataset.py @@ -0,0 +1,33 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def delete_dataset(project_id, dataset_id): + """Delete a dataset.""" + # [START automl_delete_dataset_beta] + from google.cloud import automl_v1beta1 as automl + + # TODO(developer): Uncomment and set the following variables + # project_id = "YOUR_PROJECT_ID" + # dataset_id = "YOUR_DATASET_ID" + + client = automl.AutoMlClient() + # Get the full path of the dataset + dataset_full_id = client.dataset_path( + project_id, "us-central1", dataset_id + ) + response = client.delete_dataset(dataset_full_id) + + print("Dataset deleted. {}".format(response.result())) + # [END automl_delete_dataset_beta] diff --git a/automl/beta/delete_dataset_test.py b/automl/beta/delete_dataset_test.py new file mode 100644 index 00000000000..36630908349 --- /dev/null +++ b/automl/beta/delete_dataset_test.py @@ -0,0 +1,46 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import os + +from google.cloud import automl_v1beta1 as automl +import pytest + +import delete_dataset + +PROJECT_ID = os.environ["AUTOML_PROJECT_ID"] +BUCKET_ID = "{}-lcm".format(PROJECT_ID) + + +@pytest.fixture(scope="function") +def dataset_id(): + client = automl.AutoMlClient() + project_location = client.location_path(PROJECT_ID, "us-central1") + display_name = "test_" + datetime.datetime.now().strftime("%Y%m%d%H%M%S") + metadata = automl.types.TextExtractionDatasetMetadata() + dataset = automl.types.Dataset( + display_name=display_name, text_extraction_dataset_metadata=metadata + ) + response = client.create_dataset(project_location, dataset) + dataset_id = response.result().name.split("/")[-1] + + yield dataset_id + + +def test_delete_dataset(capsys, dataset_id): + # delete dataset + delete_dataset.delete_dataset(PROJECT_ID, dataset_id) + out, _ = capsys.readouterr() + assert "Dataset deleted." in out diff --git a/automl/beta/import_dataset.py b/automl/beta/import_dataset.py new file mode 100644 index 00000000000..c552ae6d123 --- /dev/null +++ b/automl/beta/import_dataset.py @@ -0,0 +1,40 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def import_dataset(project_id, dataset_id, path): + """Import a dataset.""" + # [START automl_import_data_beta] + from google.cloud import automl_v1beta1 as automl + + # TODO(developer): Uncomment and set the following variables + # project_id = "YOUR_PROJECT_ID" + # dataset_id = "YOUR_DATASET_ID" + # path = "gs://YOUR_BUCKET_ID/path/to/data.csv" + + client = automl.AutoMlClient() + # Get the full path of the dataset. + dataset_full_id = client.dataset_path( + project_id, "us-central1", dataset_id + ) + # Get the multiple Google Cloud Storage URIs + input_uris = path.split(",") + gcs_source = automl.types.GcsSource(input_uris=input_uris) + input_config = automl.types.InputConfig(gcs_source=gcs_source) + # Import data from the input URI + response = client.import_data(dataset_full_id, input_config) + + print("Processing import...") + print("Data imported. {}".format(response.result())) + # [END automl_import_data_beta] diff --git a/automl/beta/import_dataset_test.py b/automl/beta/import_dataset_test.py new file mode 100644 index 00000000000..35d23edc7e8 --- /dev/null +++ b/automl/beta/import_dataset_test.py @@ -0,0 +1,41 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import import_dataset + +PROJECT_ID = os.environ["AUTOML_PROJECT_ID"] +BUCKET_ID = "{}-lcm".format(PROJECT_ID) +DATASET_ID = "TEN0000000000000000000" + + +def test_import_dataset(capsys): + # As importing a dataset can take a long time and only four operations can + # be run on a dataset at once. Try to import into a nonexistent dataset and + # confirm that the dataset was not found, but other elements of the request + # were valid. + try: + data = "gs://{}/sentiment-analysis/dataset.csv".format(BUCKET_ID) + import_dataset.import_dataset(PROJECT_ID, DATASET_ID, data) + out, _ = capsys.readouterr() + assert ( + "The Dataset doesn't exist or is inaccessible for use with AutoMl." + in out + ) + except Exception as e: + assert ( + "The Dataset doesn't exist or is inaccessible for use with AutoMl." + in e.message + ) diff --git a/automl/beta/list_datasets.py b/automl/beta/list_datasets.py new file mode 100644 index 00000000000..cc2ed43c180 --- /dev/null +++ b/automl/beta/list_datasets.py @@ -0,0 +1,45 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def list_datasets(project_id): + """List datasets.""" + # [START automl_video_classification_list_datasets_beta] + from google.cloud import automl_v1beta1 as automl + + # TODO(developer): Uncomment and set the following variables + # project_id = "YOUR_PROJECT_ID" + + client = automl.AutoMlClient() + # A resource that represents Google Cloud Platform location. + project_location = client.location_path(project_id, "us-central1") + + # List all the datasets available in the region. + response = client.list_datasets(project_location, "") + + print("List of datasets:") + for dataset in response: + print("Dataset name: {}".format(dataset.name)) + print("Dataset id: {}".format(dataset.name.split("/")[-1])) + print("Dataset display name: {}".format(dataset.display_name)) + print("Dataset create time:") + print("\tseconds: {}".format(dataset.create_time.seconds)) + print("\tnanos: {}".format(dataset.create_time.nanos)) + + print( + "Video classification dataset metadata:: {}".format( + dataset.video_classification_dataset_metadata + ) + ) + # [END automl_video_classification_list_datasets_beta] diff --git a/automl/beta/list_datasets_test.py b/automl/beta/list_datasets_test.py new file mode 100644 index 00000000000..7057af815d1 --- /dev/null +++ b/automl/beta/list_datasets_test.py @@ -0,0 +1,27 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import list_datasets + +PROJECT_ID = os.environ["AUTOML_PROJECT_ID"] +DATASET_ID = os.environ["ENTITY_EXTRACTION_DATASET_ID"] + + +def test_list_dataset(capsys): + # list datasets + list_datasets.list_datasets(PROJECT_ID) + out, _ = capsys.readouterr() + assert "Dataset id: {}".format(DATASET_ID) in out From fdd590aac4546efcbe6490dfec2776d54979870c Mon Sep 17 00:00:00 2001 From: nnegrey Date: Wed, 29 Jan 2020 10:01:06 -0700 Subject: [PATCH 2/2] fix test for when create dataset doesn't return an LRO --- automl/beta/delete_dataset_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automl/beta/delete_dataset_test.py b/automl/beta/delete_dataset_test.py index 36630908349..d736aa4df0f 100644 --- a/automl/beta/delete_dataset_test.py +++ b/automl/beta/delete_dataset_test.py @@ -33,8 +33,8 @@ def dataset_id(): dataset = automl.types.Dataset( display_name=display_name, text_extraction_dataset_metadata=metadata ) - response = client.create_dataset(project_location, dataset) - dataset_id = response.result().name.split("/")[-1] + dataset = client.create_dataset(project_location, dataset) + dataset_id = dataset.name.split("/")[-1] yield dataset_id