diff --git a/logging/api/README.md b/logging/api/README.md new file mode 100644 index 00000000000..6abfd13cf49 --- /dev/null +++ b/logging/api/README.md @@ -0,0 +1,64 @@ +# Cloud Logging v2 API Samples + +Sample command-line programs for retrieving Google Logging API V2 data. + +`logs_api.py` is a simple command-line program to demonstrate writing to a log, +listing its entries to view it, then deleting it via CLI operations. + +`export_logs_api.py` demonstrates how to interact with Logging sinks, which can send +logs to Google Cloud Storage, Cloud Pub/Sub, or BigQuery. In this example +we use Google Cloud Storage. It similarly exposes a CLI to run these operations. + +## Prerequisites to run locally: + + +* A Google Cloud Project + +Go to the [Google Cloud Console](https://console.cloud.google.com) to create + a project. Take note of the project ID, which is sometimes but not always + the same as your given name for a project. + +To run `export.py`, you will also need a Google Cloud Storage Bucket. + + gsutil mb gs://[YOUR_PROJECT_ID] + +You must add Cloud Logging as an owner to the bucket. To do so, add cloud-logs@google.com as +an owner to the bucket. See the [exportings logs](https://cloud.google.com/logging/docs/export/configure_export#configuring_log_sinks) +docs for complete details. + +# Set Up Your Local Dev Environment +To install, run the following commands. If you want to use [virtualenv](https://virtualenv.readthedocs.org/en/latest/) +(recommended), run the commands within a virtualenv. + +Create local credentials by running the following command and following the oauth2 flow: + + gcloud beta auth application-default login + +To run the list_logs example + + python logs_api.py --project_id= write_entry "hello world" + python logs_api.py --project_id= list_entries + python logs_api.py --project_id= delete_logger + + +The `exports_logs_api.py` samples requires a Cloud bucket that has added cloud-logs@google.com +as an owner. See: + +https://cloud.google.com/logging/docs/export/configure_export#setting_product_name_short_permissions_for_writing_exported_logs + + python export_logs_api.py --project_id=YOUR_PROJECT_ID create_sink \ + --destination_bucket=YOUR_BUCKET + + python export_logs_api.py --project_id=YOUR_PROJECT_ID update_sink\ + --destination_bucket=YOUR_BUCKET + + python export_logs_api.py --project_id=YOUR_PROJECT_ID list_sinks + + python export_logs_api.py --project_id=YOUR_PROJECT_ID delete_sink \ + --destination_bucket=YOUR_BUCKET + + +## Running on GCE, GAE, or other environments + +See our [Cloud Platform authentication guide](https://cloud.google.com/docs/authentication). + diff --git a/logging/api/export_logs_api.py b/logging/api/export_logs_api.py new file mode 100644 index 00000000000..f68efb991a9 --- /dev/null +++ b/logging/api/export_logs_api.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse + +from gcloud import logging +from oauth2client.client import GoogleCredentials + +FILTER = 'logName="projects/{}/logs/syslog" AND severity>=ERROR' +DESTINATION = 'storage.googleapis.com/{}' + + +def create_sink_if_not_exists(client, args): + # [START create] + sink = client.sink( + args.sink_name, + FILTER.format(args.project_id), + DESTINATION.format(args.destination_bucket)) + + if not sink.exists(): + sink.create() + print('Created sink {}'.format(sink.name)) + # [END create] + return sink + + +def list_sinks(client, args): + print('Listing sinks available') + + # [START list] + sinks = [] + while True: + new_sinks, token = client.list_sinks() + sinks += new_sinks + if token is None: + break + + for sink in sinks: + print('{}: {}'.format(sink.name, sink.destination)) + # [END list] + + return sinks + + +def update_sink(client, args): + """Changes the filter of a sink. + + The filter is used to determine which log statements match this sink and + will be exported to the destination. + """ + # Removes the robot in textPayload part of filter + sink = client.sink( + args.sink_name, + FILTER.format(args.project_id), + DESTINATION.format(args.destination_bucket)) + # [START update] + sink.filter = ('logName="projects/{}/logs/syslog" ' + 'AND severity>= INFO'.format(sink.project)) + print('Updated sink {}'.format(sink.name)) + sink.update() + # [END update] + + +def delete_sink(client, args): + """Deletes a sink""" + sink = client.sink( + args.sink_name, + FILTER.format(args.project_id), + DESTINATION.format(args.destination_bucket)) + # [START delete] + sink.delete() + # [END delete] + print('Deleted sink {}'.format(sink.name)) + + +def get_client(project_id): + """Builds an http client authenticated with the service account + credentials.""" + credentials = GoogleCredentials.get_application_default() + return logging.Client(project=project_id, credentials=credentials) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + '--project_id', help='Project ID you want to access.', required=True,) + + parser.add_argument( + '--sink_name', help='Output bucket to direct sink to', + default="mysink") + + subparsers = parser.add_subparsers() + + create_parser = subparsers.add_parser('create_sink') + create_parser.set_defaults(func=create_sink_if_not_exists) + create_parser.add_argument( + '--destination_bucket', help='Output bucket to direct sink to', + required=True) + + list_parser = subparsers.add_parser('list_sinks') + list_parser.set_defaults(func=list_sinks) + + update_parser = subparsers.add_parser('update_sink') + update_parser.set_defaults(func=update_sink) + update_parser.add_argument( + '--destination_bucket', help='Output bucket to direct sink to', + required=True) + + delete_parser = subparsers.add_parser('delete_sink') + delete_parser.add_argument( + '--destination_bucket', help='Output bucket to direct sink to', + required=True) + delete_parser.set_defaults(func=delete_sink) + + args = parser.parse_args() + client = get_client(args.project_id) + args.func(client, args) diff --git a/logging/api/export_logs_api_test.py b/logging/api/export_logs_api_test.py new file mode 100644 index 00000000000..85b3088c403 --- /dev/null +++ b/logging/api/export_logs_api_test.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from collections import namedtuple + +import export_logs_api + +SINK_NAME = 'test_sink' + + +def test_sinks(cloud_config): + client = export_logs_api.get_client(cloud_config.project) + + Args = namedtuple('Args', 'project_id destination_bucket sink_name') + args = Args( + destination_bucket=cloud_config.storage_bucket, + project_id=cloud_config.project, + sink_name=SINK_NAME) + + export_logs_api.create_sink_if_not_exists(client, args) + sinks = export_logs_api.list_sinks(client, args) + matched_sinks = [s for s in sinks if s.name == SINK_NAME] + assert len(matched_sinks) == 1 + export_logs_api.update_sink(client, args) + export_logs_api.delete_sink(client, args) diff --git a/logging/api/logs_api.py b/logging/api/logs_api.py new file mode 100644 index 00000000000..2ec6f6646b4 --- /dev/null +++ b/logging/api/logs_api.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse + +from gcloud import logging +from oauth2client.client import GoogleCredentials + + +def write_entry(client, args): + print('Writing log entry for logger '.format(args.logger_name)) + mylogger = client.logger(args.logger_name) + # [START write] + mylogger.log_text(args.entry) + # [END write] + + +def list_entries(client, args): + """Lists all entries for a logger""" + logger = client.logger(args.logger_name) + print('Listing all log entries for logger {}'.format(logger.name)) + # [START list] + entries = [] + while True: + new_entries, token = client.list_entries(filter_='logName="{}"'.format( + logger.full_name)) + entries += new_entries + if token is None: + break + + for entry in entries: + timestamp = entry.timestamp.isoformat() + print('{}: {}'.format + (timestamp, entry.payload)) + # [END list] + return entries + + +def delete_logger(client, args): + """Deletes a logger and all its entries. + + Note that a deletion can take several minutes to take effect. + """ + logger = client.logger(args.logger_name) + print('Deleting all logging entries for {}'.format(logger.name)) + # [START delete] + logger.delete() + # [END delete] + + +def get_client(project_id): + """Builds an http client authenticated with the service account + credentials.""" + # [START auth] + credentials = GoogleCredentials.get_application_default() + return logging.Client(project=project_id, credentials=credentials) + # [END auth] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + '--project_id', help='Project ID you want to access.', required=True) + parser.add_argument( + '--logger_name', help='Logger name', default='mylogger') + + subparsers = parser.add_subparsers() + + write_parser = subparsers.add_parser('write_entry') + write_parser.add_argument('entry') + write_parser.set_defaults(func=write_entry) + + list_parser = subparsers.add_parser('list_entries') + list_parser.set_defaults(func=list_entries) + + delete_parser = subparsers.add_parser('delete_logger') + delete_parser.set_defaults(func=delete_logger) + + args = parser.parse_args() + client = get_client(args.project_id) + args.func(client, args) diff --git a/logging/api/logs_api_test.py b/logging/api/logs_api_test.py new file mode 100644 index 00000000000..8aad5afa906 --- /dev/null +++ b/logging/api/logs_api_test.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from collections import namedtuple +import time + +from gcp.testing.flaky import flaky +import logs_api + + +@flaky +def test_logs(cloud_config): + client = logs_api.get_client(cloud_config.project) + + LOG_MESSAGE = 'hello world' + Args = namedtuple('Args', 'logger_name entry') + args = Args(logger_name='test_log', entry=LOG_MESSAGE) + + logs_api.write_entry(client, args) + time.sleep(3) + entries = logs_api.list_entries(client, args) + matched_entries = [e for e in entries if e.payload == LOG_MESSAGE] + assert len(matched_entries) > 0 + + logs_api.delete_logger(client, args) diff --git a/logging/api/requirements.txt b/logging/api/requirements.txt new file mode 100644 index 00000000000..557cfdc9fe5 --- /dev/null +++ b/logging/api/requirements.txt @@ -0,0 +1 @@ +gcloud==0.15.0