From 71c984c024d1001124560410a68ac509d74086c4 Mon Sep 17 00:00:00 2001 From: Alexey Petul Date: Wed, 4 Sep 2024 14:21:50 +0400 Subject: [PATCH 1/6] Fix invalid escape sequence in get_api_version func --- splunklib/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/splunklib/client.py b/splunklib/client.py index 48861880..d0aa74bd 100644 --- a/splunklib/client.py +++ b/splunklib/client.py @@ -769,7 +769,7 @@ def get_api_version(self, path): # For example, "/services/search/jobs" is using API v1 api_version = 1 - versionSearch = re.search('(?:servicesNS\/[^/]+\/[^/]+|services)\/[^/]+\/v(\d+)\/', path) + versionSearch = re.search(r'(?:servicesNS\/[^/]+\/[^/]+|services)\/[^/]+\/v(\d+)\/', path) if versionSearch: api_version = int(versionSearch.group(1)) From 62acf2c9aa0098ea9d42fb308d39521312d15391 Mon Sep 17 00:00:00 2001 From: Szymon Date: Wed, 9 Apr 2025 14:46:47 +0200 Subject: [PATCH 2/6] Reapply "Fix pipeline" This reverts commit 3b6d974dca26fd4bf29b5bf316f56b5f87b6bd90. --- .github/workflows/test.yml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index dc8bd208..329c686c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,7 +1,7 @@ name: Python CI on: - [ push, pull_request, workflow_dispatch ] + [ push, workflow_dispatch ] jobs: build: @@ -11,11 +11,22 @@ jobs: matrix: os: - ubuntu-latest - python: [ 3.7, 3.9, 3.13] + python: [ 3.9, 3.13 ] splunk-version: - "8.1" - "8.2" - "latest" + include: + - os: ubuntu-22.04 + python: 3.7 + splunk-version: "8.1" + - os: ubuntu-22.04 + python: 3.7 + splunk-version: "8.2" + - os: ubuntu-22.04 + python: 3.7 + splunk-version: "latest" + fail-fast: false steps: From 1f3fa3544781e0ddf31d5d540bf99878c1971566 Mon Sep 17 00:00:00 2001 From: Szymon Date: Wed, 9 Apr 2025 14:49:14 +0200 Subject: [PATCH 3/6] Reapply "Use relative imports for splunk-sdk" This reverts commit de9f0f8685c9106d6145efc7bfae3147b0c18441. --- splunklib/binding.py | 4 ++-- splunklib/client.py | 8 ++++---- splunklib/modularinput/event.py | 2 +- splunklib/modularinput/event_writer.py | 2 +- splunklib/searchcommands/search_command.py | 4 +--- 5 files changed, 9 insertions(+), 11 deletions(-) diff --git a/splunklib/binding.py b/splunklib/binding.py index 25a09948..c5f361b8 100644 --- a/splunklib/binding.py +++ b/splunklib/binding.py @@ -39,8 +39,8 @@ from http import client from http.cookies import SimpleCookie from xml.etree.ElementTree import XML, ParseError -from splunklib.data import record -from splunklib import __version__ +from .data import record +from . import __version__ logger = logging.getLogger(__name__) diff --git a/splunklib/client.py b/splunklib/client.py index ee390c9e..c78ea88d 100644 --- a/splunklib/client.py +++ b/splunklib/client.py @@ -68,9 +68,9 @@ from time import sleep from urllib import parse -from splunklib import data -from splunklib.data import record -from splunklib.binding import (AuthenticationError, Context, HTTPError, UrlEncoded, +from . import data +from .data import record +from .binding import (AuthenticationError, Context, HTTPError, UrlEncoded, _encode, _make_cookie_header, _NoAuthenticationToken, namespace) @@ -3999,4 +3999,4 @@ def batch_save(self, *documents): data = json.dumps(documents) return json.loads( - self._post('batch_save', headers=KVStoreCollectionData.JSON_HEADER, body=data).body.read().decode('utf-8')) \ No newline at end of file + self._post('batch_save', headers=KVStoreCollectionData.JSON_HEADER, body=data).body.read().decode('utf-8')) diff --git a/splunklib/modularinput/event.py b/splunklib/modularinput/event.py index 7ee7266a..bebd61e4 100644 --- a/splunklib/modularinput/event.py +++ b/splunklib/modularinput/event.py @@ -15,7 +15,7 @@ from io import TextIOBase import xml.etree.ElementTree as ET -from splunklib.utils import ensure_str +from ..utils import ensure_str class Event: diff --git a/splunklib/modularinput/event_writer.py b/splunklib/modularinput/event_writer.py index 7be3845a..7ea37ca8 100644 --- a/splunklib/modularinput/event_writer.py +++ b/splunklib/modularinput/event_writer.py @@ -15,7 +15,7 @@ import sys import traceback -from splunklib.utils import ensure_str +from ..utils import ensure_str from .event import ET diff --git a/splunklib/searchcommands/search_command.py b/splunklib/searchcommands/search_command.py index 7e8f771e..e66f70c3 100644 --- a/splunklib/searchcommands/search_command.py +++ b/splunklib/searchcommands/search_command.py @@ -34,11 +34,8 @@ from urllib.parse import urlsplit from warnings import warn from xml.etree import ElementTree -from splunklib.utils import ensure_str - # Relative imports -import splunklib from . import Boolean, Option, environment from .internals import ( CommandLineParser, @@ -53,6 +50,7 @@ RecordWriterV2, json_encode_string) from ..client import Service +from ..utils import ensure_str # ---------------------------------------------------------------------------------------------------------------------- From 6bd47f265c916a26a766f1168ec52665dc96d6b5 Mon Sep 17 00:00:00 2001 From: maszyk99 Date: Fri, 27 Jun 2025 20:30:25 +0200 Subject: [PATCH 4/6] SDK-23: check if developer added custom map method --- splunklib/searchcommands/internals.py | 2 +- splunklib/searchcommands/reporting_command.py | 21 ++++++++++++------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/splunklib/searchcommands/internals.py b/splunklib/searchcommands/internals.py index abceac30..5f20c3fa 100644 --- a/splunklib/searchcommands/internals.py +++ b/splunklib/searchcommands/internals.py @@ -554,7 +554,7 @@ def write_record(self, record): def write_records(self, records): self._ensure_validity() - records = list(records) + records = [] if records is NotImplemented else list(records) write_record = self._write_record for record in records: write_record(record) diff --git a/splunklib/searchcommands/reporting_command.py b/splunklib/searchcommands/reporting_command.py index 5df3dc7e..e455a159 100644 --- a/splunklib/searchcommands/reporting_command.py +++ b/splunklib/searchcommands/reporting_command.py @@ -77,21 +77,26 @@ def map(self, records): """ return NotImplemented - def prepare(self): - - phase = self.phase + def _has_custom_method(self, method_name): + method = getattr(self.__class__, method_name, None) + base_method = getattr(ReportingCommand, method_name, None) + return callable(method) and (method is not base_method) - if phase == 'map': - # noinspection PyUnresolvedReferences - self._configuration = self.map.ConfigurationSettings(self) + def prepare(self): + if self.phase == 'map': + if self._has_custom_method('map'): + phase_method = getattr(self.__class__, 'map') + self._configuration = phase_method.ConfigurationSettings(self) + else: + self._configuration = self.ConfigurationSettings(self) return - if phase == 'reduce': + if self.phase == 'reduce': streaming_preop = chain((self.name, 'phase="map"', str(self._options)), self.fieldnames) self._configuration.streaming_preop = ' '.join(streaming_preop) return - raise RuntimeError(f'Unrecognized reporting command phase: {json_encode_string(str(phase))}') + raise RuntimeError(f'Unrecognized reporting command phase: {json_encode_string(str(self.phase))}') def reduce(self, records): """ Override this method to produce a reporting data structure. From 42bf5e145aa8ccfd2d9257026d243b34f9c912f4 Mon Sep 17 00:00:00 2001 From: maszyk99 Date: Thu, 3 Jul 2025 13:05:22 +0200 Subject: [PATCH 5/6] SDK-23: add unit test with map --- .../searchcommands/test_reporting_command.py | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/tests/searchcommands/test_reporting_command.py b/tests/searchcommands/test_reporting_command.py index 2111447d..dbda9cd8 100644 --- a/tests/searchcommands/test_reporting_command.py +++ b/tests/searchcommands/test_reporting_command.py @@ -32,3 +32,42 @@ def reduce(self, records): data = list(data_chunk.data) assert len(data) == 1 assert int(data[0]['sum']) == sum(range(0, 10)) + + +def test_simple_reporting_command_with_map(): + @searchcommands.Configuration() + class MapAndReduceReportingCommand(searchcommands.ReportingCommand): + def map(self, records): + for record in records: + record["value"] = str(int(record["value"]) * 2) + yield record + + def reduce(self, records): + total = 0 + for record in records: + total += int(record["value"]) + yield {"sum": total} + + cmd = MapAndReduceReportingCommand() + ifile = io.BytesIO() + + input_data = [{"value": str(i)} for i in range(5)] + + mapped_data = list(cmd.map(input_data)) + + ifile.write(chunky.build_getinfo_chunk()) + ifile.write(chunky.build_data_chunk(mapped_data)) + ifile.seek(0) + + ofile = io.BytesIO() + cmd._process_protocol_v2([], ifile, ofile) + + ofile.seek(0) + chunk_stream = chunky.ChunkedDataStream(ofile) + chunk_stream.read_chunk() + data_chunk = chunk_stream.read_chunk() + assert data_chunk.meta['finished'] is True + + result = list(data_chunk.data) + expected_sum = sum(i * 2 for i in range(5)) + assert int(result[0]["sum"]) == expected_sum From c6c269cf3144ee582001f540749453542ce6c7f5 Mon Sep 17 00:00:00 2001 From: Szymon Date: Mon, 7 Jul 2025 13:21:50 +0200 Subject: [PATCH 6/6] Add loggers in Service, add test for service creation --- splunklib/searchcommands/search_command.py | 7 ++- tests/searchcommands/test_search_command.py | 49 +++++++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/splunklib/searchcommands/search_command.py b/splunklib/searchcommands/search_command.py index e66f70c3..ba2a4dc0 100644 --- a/splunklib/searchcommands/search_command.py +++ b/splunklib/searchcommands/search_command.py @@ -340,24 +340,29 @@ def service(self): of :code:`None` is returned. """ + if self._service is not None: return self._service metadata = self._metadata if metadata is None: + self.logger.warning("Missing metadata for service creation.") return None try: searchinfo = self._metadata.searchinfo except AttributeError: + self.logger.warning("Missing searchinfo in metadata for service creation.") return None splunkd_uri = searchinfo.splunkd_uri - if splunkd_uri is None: + if splunkd_uri is None or splunkd_uri == "" or splunkd_uri == " ": + self.logger.warning(f"Incorrect value for Splunkd URI: {splunkd_uri!r} in metadata") return None + uri = urlsplit(splunkd_uri, allow_fragments=False) self._service = Service( diff --git a/tests/searchcommands/test_search_command.py b/tests/searchcommands/test_search_command.py index c8fe7d80..849a8888 100755 --- a/tests/searchcommands/test_search_command.py +++ b/tests/searchcommands/test_search_command.py @@ -22,14 +22,17 @@ import codecs import os import re +import logging from io import TextIOWrapper +from unittest.mock import MagicMock, patch import pytest import splunklib from splunklib.searchcommands import Configuration, StreamingCommand from splunklib.searchcommands.decorators import ConfigurationSetting, Option +from splunklib.searchcommands.internals import ObjectView from splunklib.searchcommands.search_command import SearchCommand from splunklib.client import Service from splunklib.utils import ensure_binary @@ -265,6 +268,52 @@ def test_process_scpv2(self): _package_directory = os.path.dirname(os.path.abspath(__file__)) +class TestSearchCommandService(TestCase): + def setUp(self): + TestCase.setUp(self) + self.command = SearchCommand() + console_handler = logging.StreamHandler() + console_handler.setLevel(logging.WARNING) + self.command.logger.addHandler(console_handler) + + def test_service_exists(self): + self.command._service = Service() + self.assertIsNotNone(self.command.service) + + def test_service_not_exists(self): + self.assertIsNone(self.command.service) + + def test_missing_metadata(self): + with self.assertLogs(self.command.logger, level='WARNING') as log: + service = self.command.service + self.assertIsNone(service) + self.assertTrue(any("Missing metadata for service creation." in message for message in log.output)) + + def test_missing_searchinfo(self): + with self.assertLogs(self.command.logger, level='WARNING') as log: + self.command._metadata = ObjectView({}) + self.assertIsNone(self.command.service) + self.assertTrue(any("Missing searchinfo in metadata for service creation." in message for message in log.output)) + + + def test_missing_splunkd_uri(self): + with self.assertLogs(self.command.logger, level='WARNING') as log: + metadata = ObjectView({"searchinfo": ObjectView({"splunkd_uri": ""})}) + self.command._metadata = metadata + self.assertIsNone(self.command.service) + self.assertTrue(any("Incorrect value for Splunkd URI: '' in metadata" in message for message in log.output)) + + + + def test_service_returns_valid_service_object(self): + metadata = ObjectView({"searchinfo":ObjectView({"splunkd_uri":"https://127.0.0.1:8089", + "session_key":"mock_session_key", + "app":"search", + })}) + self.command._metadata = metadata + self.assertIsInstance(self.command.service, Service) + + if __name__ == "__main__": main()