diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 000000000..e36085909 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,37 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: '' +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior: +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Logs or Screenshots** +If applicable, add logs or screenshots to help explain your problem. + +**Splunk (please complete the following information):** +- Version: [e.g. 8.0.5] +- OS: [e.g. Ubuntu 20.04.1] +- Deployment: [e.g. single-instance] + +**SDK (please complete the following information):** + - Version: [e.g. 1.6.14] + - Language Runtime Version: [e.g. Python 3.7] + - OS: [e.g. MacOS 10.15.7] + +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/custom.md b/.github/ISSUE_TEMPLATE/custom.md new file mode 100644 index 000000000..48d5f81fa --- /dev/null +++ b/.github/ISSUE_TEMPLATE/custom.md @@ -0,0 +1,10 @@ +--- +name: Custom issue template +about: Describe this issue template's purpose here. +title: '' +labels: '' +assignees: '' + +--- + + diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 000000000..bbcbbe7d6 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: '' +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. diff --git a/CHANGELOG.md b/CHANGELOG.md index 6934cee08..89c2b6450 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ # Splunk Enterprise SDK for Python Changelog +## Version 1.6.15 + +### Bug fixes +[#301](https://github.com/splunk/splunk-sdk-python/pull/301) Fix chunk synchronization +[#327](https://github.com/splunk/splunk-sdk-python/pull/327) Rename and cleanup follow-up for chunk synchronization +[#352](https://github.com/splunk/splunk-sdk-python/pull/352) Allow supplying of a key-value body when calling Context.post() + +### Minor changes +[#350](https://github.com/splunk/splunk-sdk-python/pull/350) Initial end-to-end tests for streaming, reporting, generating custom search commands +[#348](https://github.com/splunk/splunk-sdk-python/pull/348) Update copyright years to 2020 +[#346](https://github.com/splunk/splunk-sdk-python/pull/346) Readme updates to urls, terminology, and formatting +[#317](https://github.com/splunk/splunk-sdk-python/pull/317) Fix deprecation warnings + ## Version 1.6.14 ### Bug fix diff --git a/README.md b/README.md index 34c5e5fe8..fbb4f3827 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ # The Splunk Enterprise Software Development Kit for Python -#### Version 1.6.14 +#### Version 1.6.15 The Splunk Enterprise Software Development Kit (SDK) for Python contains library code and examples designed to enable developers to build applications using the Splunk platform. diff --git a/examples/searchcommands_app/setup.py b/examples/searchcommands_app/setup.py index ec2118dd0..fdb734630 100755 --- a/examples/searchcommands_app/setup.py +++ b/examples/searchcommands_app/setup.py @@ -439,7 +439,7 @@ def run(self): setup( description='Custom Search Command examples', name=os.path.basename(project_dir), - version='1.6.14', + version='1.6.15', author='Splunk, Inc.', author_email='devinfo@splunk.com', url='http://github.com/splunk/splunk-sdk-python', diff --git a/splunklib/__init__.py b/splunklib/__init__.py index a879ab9d1..668dcfe7f 100644 --- a/splunklib/__init__.py +++ b/splunklib/__init__.py @@ -16,5 +16,5 @@ from __future__ import absolute_import from splunklib.six.moves import map -__version_info__ = (1, 6, 14) +__version_info__ = (1, 6, 15) __version__ = ".".join(map(str, __version_info__)) diff --git a/splunklib/binding.py b/splunklib/binding.py index e21da6e52..a95faa480 100644 --- a/splunklib/binding.py +++ b/splunklib/binding.py @@ -1385,7 +1385,7 @@ def request(url, message, **kwargs): head = { "Content-Length": str(len(body)), "Host": host, - "User-Agent": "splunk-sdk-python/1.6.14", + "User-Agent": "splunk-sdk-python/1.6.15", "Accept": "*/*", "Connection": "Close", } # defaults diff --git a/splunklib/searchcommands/search_command.py b/splunklib/searchcommands/search_command.py index cbf8f5594..7383a5efa 100644 --- a/splunklib/searchcommands/search_command.py +++ b/splunklib/searchcommands/search_command.py @@ -856,7 +856,8 @@ def _execute(self, ifile, process): @staticmethod def _as_binary_stream(ifile): - if six.PY2: + naught = ifile.read(0) + if isinstance(naught, bytes): return ifile try: diff --git a/tests/searchcommands/chunked_data_stream.py b/tests/searchcommands/chunked_data_stream.py new file mode 100644 index 000000000..ae5363eff --- /dev/null +++ b/tests/searchcommands/chunked_data_stream.py @@ -0,0 +1,100 @@ +import collections +import csv +import io +import json + +import splunklib.searchcommands.internals +from splunklib import six + + +class Chunk(object): + def __init__(self, version, meta, data): + self.version = six.ensure_str(version) + self.meta = json.loads(meta) + dialect = splunklib.searchcommands.internals.CsvDialect + self.data = csv.DictReader(io.StringIO(data.decode("utf-8")), + dialect=dialect) + + +class ChunkedDataStreamIter(collections.Iterator): + def __init__(self, chunk_stream): + self.chunk_stream = chunk_stream + + def __next__(self): + return self.next() + + def next(self): + try: + return self.chunk_stream.read_chunk() + except EOFError: + raise StopIteration + + +class ChunkedDataStream(collections.Iterable): + def __iter__(self): + return ChunkedDataStreamIter(self) + + def __init__(self, stream): + empty = stream.read(0) + assert isinstance(empty, bytes) + self.stream = stream + + def read_chunk(self): + header = self.stream.readline() + + while len(header) > 0 and header.strip() == b'': + header = self.stream.readline() # Skip empty lines + if len(header) == 0: + raise EOFError + + version, meta, data = header.rstrip().split(b',') + metabytes = self.stream.read(int(meta)) + databytes = self.stream.read(int(data)) + return Chunk(version, metabytes, databytes) + + +def build_chunk(keyval, data=None): + metadata = six.ensure_binary(json.dumps(keyval), 'utf-8') + data_output = _build_data_csv(data) + return b"chunked 1.0,%d,%d\n%s%s" % (len(metadata), len(data_output), metadata, data_output) + + +def build_empty_searchinfo(): + return { + 'earliest_time': 0, + 'latest_time': 0, + 'search': "", + 'dispatch_dir': "", + 'sid': "", + 'args': [], + 'splunk_version': "42.3.4", + } + + +def build_getinfo_chunk(): + return build_chunk({ + 'action': 'getinfo', + 'preview': False, + 'searchinfo': build_empty_searchinfo()}) + + +def build_data_chunk(data, finished=True): + return build_chunk({'action': 'execute', 'finished': finished}, data) + + +def _build_data_csv(data): + if data is None: + return b'' + if isinstance(data, bytes): + return data + csvout = splunklib.six.StringIO() + + headers = set() + for datum in data: + headers.update(datum.keys()) + writer = csv.DictWriter(csvout, headers, + dialect=splunklib.searchcommands.internals.CsvDialect) + writer.writeheader() + for datum in data: + writer.writerow(datum) + return six.ensure_binary(csvout.getvalue()) diff --git a/tests/searchcommands/test_generator_command.py b/tests/searchcommands/test_generator_command.py new file mode 100644 index 000000000..4af61a5d2 --- /dev/null +++ b/tests/searchcommands/test_generator_command.py @@ -0,0 +1,44 @@ +import io +import time + +from . import chunked_data_stream as chunky + +from splunklib.searchcommands import Configuration, GeneratingCommand + + +def test_simple_generator(): + @Configuration() + class GeneratorTest(GeneratingCommand): + def generate(self): + for num in range(1, 10): + yield {'_time': time.time(), 'event_index': num} + generator = GeneratorTest() + in_stream = io.BytesIO() + in_stream.write(chunky.build_getinfo_chunk()) + in_stream.write(chunky.build_chunk({'action': 'execute'})) + in_stream.seek(0) + out_stream = io.BytesIO() + generator._process_protocol_v2([], in_stream, out_stream) + out_stream.seek(0) + + ds = chunky.ChunkedDataStream(out_stream) + is_first_chunk = True + finished_seen = False + expected = set(map(lambda i: str(i), range(1, 10))) + seen = set() + for chunk in ds: + if is_first_chunk: + assert chunk.meta["generating"] is True + assert chunk.meta["type"] == "stateful" + is_first_chunk = False + finished_seen = chunk.meta.get("finished", False) + for row in chunk.data: + seen.add(row["event_index"]) + print(out_stream.getvalue()) + print(expected) + print(seen) + assert expected.issubset(seen) + assert finished_seen + + + diff --git a/tests/searchcommands/test_reporting_command.py b/tests/searchcommands/test_reporting_command.py new file mode 100644 index 000000000..e5add818c --- /dev/null +++ b/tests/searchcommands/test_reporting_command.py @@ -0,0 +1,34 @@ +import io + +import splunklib.searchcommands as searchcommands +from . import chunked_data_stream as chunky + + +def test_simple_reporting_command(): + @searchcommands.Configuration() + class TestReportingCommand(searchcommands.ReportingCommand): + def reduce(self, records): + value = 0 + for record in records: + value += int(record["value"]) + yield {'sum': value} + + cmd = TestReportingCommand() + ifile = io.BytesIO() + data = list() + for i in range(0, 10): + data.append({"value": str(i)}) + ifile.write(chunky.build_getinfo_chunk()) + ifile.write(chunky.build_data_chunk(data)) + ifile.seek(0) + ofile = io.BytesIO() + cmd._process_protocol_v2([], ifile, ofile) + ofile.seek(0) + chunk_stream = chunky.ChunkedDataStream(ofile) + getinfo_response = chunk_stream.read_chunk() + assert getinfo_response.meta['type'] == 'reporting' + data_chunk = chunk_stream.read_chunk() + assert data_chunk.meta['finished'] is True # Should only be one row + data = list(data_chunk.data) + assert len(data) == 1 + assert int(data[0]['sum']) == sum(range(0, 10)) diff --git a/tests/searchcommands/test_streaming_command.py b/tests/searchcommands/test_streaming_command.py new file mode 100644 index 000000000..dcc00b53e --- /dev/null +++ b/tests/searchcommands/test_streaming_command.py @@ -0,0 +1,29 @@ +import io + +from . import chunked_data_stream as chunky +from splunklib.searchcommands import StreamingCommand, Configuration + + +def test_simple_streaming_command(): + @Configuration() + class TestStreamingCommand(StreamingCommand): + + def stream(self, records): + for record in records: + record["out_index"] = record["in_index"] + yield record + + cmd = TestStreamingCommand() + ifile = io.BytesIO() + ifile.write(chunky.build_getinfo_chunk()) + data = list() + for i in range(0, 10): + data.append({"in_index": str(i)}) + ifile.write(chunky.build_data_chunk(data, finished=True)) + ifile.seek(0) + ofile = io.BytesIO() + cmd._process_protocol_v2([], ifile, ofile) + ofile.seek(0) + output = chunky.ChunkedDataStream(ofile) + getinfo_response = output.read_chunk() + assert getinfo_response.meta["type"] == "streaming"