From e91037b9dbdc01dcea2e3194ba1706b349338d71 Mon Sep 17 00:00:00 2001 From: Maciej Strzelczyk Date: Thu, 13 Oct 2022 15:35:14 +0200 Subject: [PATCH 1/5] docs(samples): Adding code samples for log reading --- samples/requirements-test.txt | 1 + samples/requirements.txt | 1 + samples/snippets/logs/read_job_logs.py | 41 ++++++++++++++++++++++++++ samples/snippets/tests/test_basics.py | 13 ++++++-- 4 files changed, 54 insertions(+), 2 deletions(-) create mode 100644 samples/snippets/logs/read_job_logs.py diff --git a/samples/requirements-test.txt b/samples/requirements-test.txt index d1808cb..6884a0a 100644 --- a/samples/requirements-test.txt +++ b/samples/requirements-test.txt @@ -1,4 +1,5 @@ pytest==7.1.3 +nox==2022.8.7 google-cloud-compute==1.6.0 google-cloud-resource-manager==1.6.2 google-cloud-storage==2.5.0 \ No newline at end of file diff --git a/samples/requirements.txt b/samples/requirements.txt index be9bb0e..4338ffe 100644 --- a/samples/requirements.txt +++ b/samples/requirements.txt @@ -1,3 +1,4 @@ isort==5.10.1 black==22.10.0 google-cloud-batch==0.3.1 +google-cloud-logging==3.2.5 diff --git a/samples/snippets/logs/read_job_logs.py b/samples/snippets/logs/read_job_logs.py new file mode 100644 index 0000000..c4cccd5 --- /dev/null +++ b/samples/snippets/logs/read_job_logs.py @@ -0,0 +1,41 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# [START batch_job_logs] +from typing import Generator, NoReturn + +from google.cloud import batch_v1 +from google.cloud import logging + + +def get_job_logs(project_id: str, job: batch_v1.Job) -> Generator[logging.TextEntry, None, None]: + log_client = logging.Client(project=project_id) + logger = log_client.logger("batch_task_logs") + + yield from logger.list_entries(filter_=f"labels.job_uid={job.uid}") + + +def print_job_logs(project_id: str, job: batch_v1.Job) -> NoReturn: + """ + Prints the log messages created by given job. + + Args: + project_id: name of the project hosting the job. + job: the job which logs you want to print. + """ + for log_entry in get_job_logs(project_id, job): + print(log_entry.payload) + +# [END batch_job_logs] diff --git a/samples/snippets/tests/test_basics.py b/samples/snippets/tests/test_basics.py index 980314d..555fda0 100644 --- a/samples/snippets/tests/test_basics.py +++ b/samples/snippets/tests/test_basics.py @@ -27,6 +27,7 @@ from ..get.get_task import get_task from ..list.list_jobs import list_jobs from ..list.list_tasks import list_tasks +from ..logs.read_job_logs import get_job_logs PROJECT = google.auth.default()[1] REGION = 'europe-north1' @@ -82,11 +83,19 @@ def _check_tasks(job_name): print('Tasks tested') +def _check_logs(job): + logs = list(get_job_logs(PROJECT, job)) + import pprint + pprint.pprint(logs) + assert len(logs) == 4 + assert all(log_msg.payload.startswith("STDOUT") for log_msg in logs) + + def test_script_job(job_name): job = create_script_job(PROJECT, REGION, job_name) - _test_body(job, additional_test=lambda: _check_tasks(job_name)) + _test_body(job, additional_test=lambda: _check_logs(job)) def test_container_job(job_name): job = create_container_job(PROJECT, REGION, job_name) - _test_body(job) + _test_body(job, additional_test=lambda: _check_tasks(job_name)) From a8f8b2ac1d6e891d2875b4c4125fc3e9db64c636 Mon Sep 17 00:00:00 2001 From: Maciej Strzelczyk Date: Fri, 14 Oct 2022 17:32:54 +0200 Subject: [PATCH 2/5] Applying suggestions --- samples/requirements-test.txt | 3 +-- samples/snippets/logs/read_job_logs.py | 13 +++++++++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/samples/requirements-test.txt b/samples/requirements-test.txt index 6884a0a..da868fd 100644 --- a/samples/requirements-test.txt +++ b/samples/requirements-test.txt @@ -1,5 +1,4 @@ pytest==7.1.3 -nox==2022.8.7 google-cloud-compute==1.6.0 google-cloud-resource-manager==1.6.2 -google-cloud-storage==2.5.0 \ No newline at end of file +google-cloud-storage==2.5.0 diff --git a/samples/snippets/logs/read_job_logs.py b/samples/snippets/logs/read_job_logs.py index c4cccd5..4f9177b 100644 --- a/samples/snippets/logs/read_job_logs.py +++ b/samples/snippets/logs/read_job_logs.py @@ -21,10 +21,15 @@ def get_job_logs(project_id: str, job: batch_v1.Job) -> Generator[logging.TextEntry, None, None]: - log_client = logging.Client(project=project_id) - logger = log_client.logger("batch_task_logs") - - yield from logger.list_entries(filter_=f"labels.job_uid={job.uid}") + # Initialize client that will be used to send requests across threads. This + # client only needs to be created once, and can be reused for multiple requests. + # After completing all of your requests, call the "__exit__()" method to safely + # clean up any remaining background resources. Alternatively, use the client as + # a context manager. + with logging.Client(project=project_id) as log_client: + logger = log_client.logger("batch_task_logs") + + yield from logger.list_entries(filter_=f"labels.job_uid={job.uid}") def print_job_logs(project_id: str, job: batch_v1.Job) -> NoReturn: From 634aff6eca44037c628db645bb920276b5d524fc Mon Sep 17 00:00:00 2001 From: Maciej Strzelczyk Date: Fri, 14 Oct 2022 18:19:57 +0200 Subject: [PATCH 3/5] Fixing the client --- samples/snippets/logs/read_job_logs.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/samples/snippets/logs/read_job_logs.py b/samples/snippets/logs/read_job_logs.py index 4f9177b..193803c 100644 --- a/samples/snippets/logs/read_job_logs.py +++ b/samples/snippets/logs/read_job_logs.py @@ -26,10 +26,10 @@ def get_job_logs(project_id: str, job: batch_v1.Job) -> Generator[logging.TextEn # After completing all of your requests, call the "__exit__()" method to safely # clean up any remaining background resources. Alternatively, use the client as # a context manager. - with logging.Client(project=project_id) as log_client: - logger = log_client.logger("batch_task_logs") + log_client = logging.Client(project=project_id) + logger = log_client.logger("batch_task_logs") - yield from logger.list_entries(filter_=f"labels.job_uid={job.uid}") + yield from logger.list_entries(filter_=f"labels.job_uid={job.uid}") def print_job_logs(project_id: str, job: batch_v1.Job) -> NoReturn: From 550ee41382f5ee0f58b00670b474153660ea6f4c Mon Sep 17 00:00:00 2001 From: Maciej Strzelczyk Date: Wed, 19 Oct 2022 15:40:54 +0200 Subject: [PATCH 4/5] Improving the snippets a bit. --- samples/snippets/logs/read_job_logs.py | 20 ++++++++++++++------ samples/snippets/tests/test_basics.py | 2 -- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/samples/snippets/logs/read_job_logs.py b/samples/snippets/logs/read_job_logs.py index 193803c..b2030c1 100644 --- a/samples/snippets/logs/read_job_logs.py +++ b/samples/snippets/logs/read_job_logs.py @@ -14,22 +14,30 @@ # [START batch_job_logs] -from typing import Generator, NoReturn +from typing import Iterable, NoReturn from google.cloud import batch_v1 from google.cloud import logging -def get_job_logs(project_id: str, job: batch_v1.Job) -> Generator[logging.TextEntry, None, None]: +def get_job_logs(project_id: str, job: batch_v1.Job) -> Iterable[logging.TextEntry]: + """ + Get logs generated by given job from Cloud Logging. + + Args: + project_id: name of the project hosting the job. + job: the job which logs you want to retrieve. + + Returns: + An iterable containing google.cloud.logging.TextEntry objects representing + retrieved logs. + """ # Initialize client that will be used to send requests across threads. This # client only needs to be created once, and can be reused for multiple requests. - # After completing all of your requests, call the "__exit__()" method to safely - # clean up any remaining background resources. Alternatively, use the client as - # a context manager. log_client = logging.Client(project=project_id) logger = log_client.logger("batch_task_logs") - yield from logger.list_entries(filter_=f"labels.job_uid={job.uid}") + return logger.list_entries(filter_=f"labels.job_uid={job.uid}") def print_job_logs(project_id: str, job: batch_v1.Job) -> NoReturn: diff --git a/samples/snippets/tests/test_basics.py b/samples/snippets/tests/test_basics.py index 555fda0..8584bd5 100644 --- a/samples/snippets/tests/test_basics.py +++ b/samples/snippets/tests/test_basics.py @@ -85,8 +85,6 @@ def _check_tasks(job_name): def _check_logs(job): logs = list(get_job_logs(PROJECT, job)) - import pprint - pprint.pprint(logs) assert len(logs) == 4 assert all(log_msg.payload.startswith("STDOUT") for log_msg in logs) From 3e4f0acbd57a424ee82b68fd3fc1fd4365fae278 Mon Sep 17 00:00:00 2001 From: Maciej Strzelczyk Date: Fri, 21 Oct 2022 13:49:58 +0200 Subject: [PATCH 5/5] Applying suggested changes --- samples/snippets/logs/read_job_logs.py | 25 +++++-------------------- samples/snippets/tests/test_basics.py | 15 ++++++++------- 2 files changed, 13 insertions(+), 27 deletions(-) diff --git a/samples/snippets/logs/read_job_logs.py b/samples/snippets/logs/read_job_logs.py index b2030c1..d9c227a 100644 --- a/samples/snippets/logs/read_job_logs.py +++ b/samples/snippets/logs/read_job_logs.py @@ -14,41 +14,26 @@ # [START batch_job_logs] -from typing import Iterable, NoReturn +from typing import NoReturn from google.cloud import batch_v1 from google.cloud import logging -def get_job_logs(project_id: str, job: batch_v1.Job) -> Iterable[logging.TextEntry]: +def print_job_logs(project_id: str, job: batch_v1.Job) -> NoReturn: """ - Get logs generated by given job from Cloud Logging. + Prints the log messages created by given job. Args: project_id: name of the project hosting the job. - job: the job which logs you want to retrieve. - - Returns: - An iterable containing google.cloud.logging.TextEntry objects representing - retrieved logs. + job: the job which logs you want to print. """ # Initialize client that will be used to send requests across threads. This # client only needs to be created once, and can be reused for multiple requests. log_client = logging.Client(project=project_id) logger = log_client.logger("batch_task_logs") - return logger.list_entries(filter_=f"labels.job_uid={job.uid}") - - -def print_job_logs(project_id: str, job: batch_v1.Job) -> NoReturn: - """ - Prints the log messages created by given job. - - Args: - project_id: name of the project hosting the job. - job: the job which logs you want to print. - """ - for log_entry in get_job_logs(project_id, job): + for log_entry in logger.list_entries(filter_=f"labels.job_uid={job.uid}"): print(log_entry.payload) # [END batch_job_logs] diff --git a/samples/snippets/tests/test_basics.py b/samples/snippets/tests/test_basics.py index 8584bd5..72a11f8 100644 --- a/samples/snippets/tests/test_basics.py +++ b/samples/snippets/tests/test_basics.py @@ -27,7 +27,7 @@ from ..get.get_task import get_task from ..list.list_jobs import list_jobs from ..list.list_tasks import list_tasks -from ..logs.read_job_logs import get_job_logs +from ..logs.read_job_logs import print_job_logs PROJECT = google.auth.default()[1] REGION = 'europe-north1' @@ -83,15 +83,16 @@ def _check_tasks(job_name): print('Tasks tested') -def _check_logs(job): - logs = list(get_job_logs(PROJECT, job)) - assert len(logs) == 4 - assert all(log_msg.payload.startswith("STDOUT") for log_msg in logs) +def _check_logs(job, capsys): + print_job_logs(PROJECT, job) + output = [line for line in capsys.readouterr().out.splitlines(keepends=False) if line != ""] + assert len(output) == 4 + assert all(log_msg.startswith("STDOUT") for log_msg in output) -def test_script_job(job_name): +def test_script_job(job_name, capsys): job = create_script_job(PROJECT, REGION, job_name) - _test_body(job, additional_test=lambda: _check_logs(job)) + _test_body(job, additional_test=lambda: _check_logs(job, capsys)) def test_container_job(job_name):