diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index e01d12ce..af2f96fb 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,3 +1,7 @@ +**7.2.1**: + +- Check the release note and download the package/source from [Here](https://github.com/splunk/eventgen/releases/tag/7.2.1) + **7.2.0**: - Check the release note and download the package/source from [Here](https://github.com/splunk/eventgen/releases/tag/7.2.0) diff --git a/docs/Gemfile.lock b/docs/Gemfile.lock index df772d69..3ee15976 100644 --- a/docs/Gemfile.lock +++ b/docs/Gemfile.lock @@ -33,7 +33,7 @@ GEM github-pages (201) activesupport (= 4.2.11.1) github-pages-health-check (= 1.16.1) - jekyll (= 3.8.5) + jekyll (= 3.9.0) jekyll-avatar (= 0.6.0) jekyll-coffeescript (= 1.1.1) jekyll-commonmark-ghpages (= 0.1.6) @@ -67,7 +67,7 @@ GEM jekyll-theme-time-machine (= 0.1.1) jekyll-titles-from-headings (= 0.5.1) jemoji (= 0.10.2) - kramdown (= 1.17.0) + kramdown (>= 2.3.0) liquid (= 4.0.0) listen (= 3.1.5) mercenary (~> 0.3) @@ -87,14 +87,14 @@ GEM http_parser.rb (0.6.0) i18n (0.9.5) concurrent-ruby (~> 1.0) - jekyll (3.8.5) + jekyll (3.9.0) addressable (~> 2.4) colorator (~> 1.0) em-websocket (~> 0.5) i18n (~> 0.7) jekyll-sass-converter (~> 1.0) jekyll-watch (~> 2.0) - kramdown (~> 1.14) + kramdown (>= 1.17, < 3) liquid (~> 4.0) mercenary (~> 0.3.3) pathutil (~> 0.9) @@ -192,30 +192,34 @@ GEM gemoji (~> 3.0) html-pipeline (~> 2.2) jekyll (~> 3.0) - kramdown (1.17.0) + kramdown (2.3.0) + rexml liquid (4.0.0) listen (3.1.5) rb-fsevent (~> 0.9, >= 0.9.4) rb-inotify (~> 0.9, >= 0.9.7) ruby_dep (~> 1.2) mercenary (0.3.6) - mini_portile2 (2.4.0) + mini_portile2 (2.5.0) minima (2.5.0) jekyll (~> 3.5) jekyll-feed (~> 0.9) jekyll-seo-tag (~> 2.1) minitest (5.12.2) multipart-post (2.1.1) - nokogiri (1.10.8) - mini_portile2 (~> 2.4.0) + nokogiri (1.11.0) + mini_portile2 (~> 2.5.0) + racc (~> 1.4) octokit (4.14.0) sawyer (~> 0.8.0, >= 0.5.3) pathutil (0.16.2) forwardable-extended (~> 2.6) public_suffix (3.1.1) + racc (1.5.2) rb-fsevent (0.10.3) rb-inotify (0.10.0) ffi (~> 1.0) + rexml (3.2.4) rouge (3.11.0) ruby-enum (0.7.2) i18n diff --git a/pyproject.toml b/pyproject.toml index ad62596c..fd788846 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.masonry.api" [tool.poetry] name = "splunk_eventgen" -version = "7.2.0" +version = "7.2.1" description = "Splunk Event Generator to produce real-time, representative data" authors = [ "Brian Bingham ", "Tony Lee ", "Jack Meixensperger ",] license = "Apache-2.0" diff --git a/splunk_eventgen/__main__.py b/splunk_eventgen/__main__.py index 7d6d5186..869ab63d 100644 --- a/splunk_eventgen/__main__.py +++ b/splunk_eventgen/__main__.py @@ -274,6 +274,8 @@ def build_splunk_app(dest, source=os.getcwd(), remove=True): target_file = os.path.join(dest, "sa_eventgen_{}.spl".format(EVENTGEN_VERSION)) splunk_app = os.path.join(FILE_LOCATION, "splunk_app") splunk_app_samples = os.path.join(splunk_app, "samples") + if os.path.exists(splunk_app_samples): + shutil.rmtree(splunk_app_samples) shutil.copytree(os.path.join(FILE_LOCATION, "samples"), splunk_app_samples) try: shutil.copytree(splunk_app, directory) @@ -291,7 +293,7 @@ def build_splunk_app(dest, source=os.getcwd(), remove=True): shutil.copyfile(eventgen_conf, directory_default_dir) # install 3rd lib dependencies - install_target = os.path.join(directory, "lib") + install_target = os.path.join(directory, "bin") install_cmd = ( "pip install --requirement splunk_eventgen/lib/requirements.txt --upgrade --no-compile " + "--no-binary :all: --target " diff --git a/splunk_eventgen/eventgen_core.py b/splunk_eventgen/eventgen_core.py index 02378963..b2a59f17 100644 --- a/splunk_eventgen/eventgen_core.py +++ b/splunk_eventgen/eventgen_core.py @@ -266,9 +266,7 @@ def _create_generator_pool(self, workercount=20): worker = Thread( target=self._generator_do_work, args=(self.workerQueue, self.loggingQueue), - kwargs={ - "output_counter": self.output_counters[i], - }, + kwargs={"output_counter": self.output_counters[i]}, ) worker.setDaemon(True) worker.start() @@ -277,9 +275,7 @@ def _create_generator_pool(self, workercount=20): worker = Thread( target=self._generator_do_work, args=(self.workerQueue, self.loggingQueue), - kwargs={ - "output_counter": None, - }, + kwargs={"output_counter": None}, ) worker.setDaemon(True) worker.start() diff --git a/splunk_eventgen/lib/eventgentimer.py b/splunk_eventgen/lib/eventgentimer.py index 181eb730..38c3e991 100644 --- a/splunk_eventgen/lib/eventgentimer.py +++ b/splunk_eventgen/lib/eventgentimer.py @@ -182,7 +182,6 @@ def real_run(self): raweventsize=raw_event_size, ) self.perdayrater.rate() - self.perdayrater.queue_it(count) self.rater.queue_it(count) self.countdown = self.interval self.executions += 1 diff --git a/splunk_eventgen/lib/generatorplugin.py b/splunk_eventgen/lib/generatorplugin.py index 2f72fa51..ee0520d9 100644 --- a/splunk_eventgen/lib/generatorplugin.py +++ b/splunk_eventgen/lib/generatorplugin.py @@ -33,16 +33,8 @@ def __str__(self): def __repr__(self): return self.__str__() - def build_events( - self, eventsDict, startTime, earliest, latest, ignore_tokens=False - ): + def send_events(self, send_objects, startTime): """Ready events for output by replacing tokens and updating the output queue""" - # Replace tokens first so that perDayVolume evaluates the correct event length - send_objects = self.replace_tokens( - eventsDict, earliest, latest, ignore_tokens=ignore_tokens - ) - # after replace_tokens() is called, we don't need eventsDict - del eventsDict try: self._out.bulksend(send_objects) self._sample.timestamp = None diff --git a/splunk_eventgen/lib/logging_config/__init__.py b/splunk_eventgen/lib/logging_config/__init__.py index c58ad603..fda8519d 100644 --- a/splunk_eventgen/lib/logging_config/__init__.py +++ b/splunk_eventgen/lib/logging_config/__init__.py @@ -21,10 +21,7 @@ }, "filters": {}, "handlers": { - "console": { - "class": "logging.StreamHandler", - "formatter": "default", - }, + "console": {"class": "logging.StreamHandler", "formatter": "default"}, "eventgen_main": { "class": "logging.handlers.RotatingFileHandler", "formatter": "default", diff --git a/splunk_eventgen/lib/plugins/generator/default.py b/splunk_eventgen/lib/plugins/generator/default.py index 60109580..d059dcb5 100644 --- a/splunk_eventgen/lib/plugins/generator/default.py +++ b/splunk_eventgen/lib/plugins/generator/default.py @@ -73,7 +73,8 @@ def gen(self, count, earliest, latest, samplename=None): % (self._sample.name, self._sample.app, len(eventsDict)) ) - GeneratorPlugin.build_events(self, eventsDict, startTime, earliest, latest) + send_objects = self.replace_tokens(eventsDict, earliest, latest) + self.send_events(send_objects, startTime) def load(): diff --git a/splunk_eventgen/lib/plugins/generator/perdayvolumegenerator.py b/splunk_eventgen/lib/plugins/generator/perdayvolumegenerator.py index 40b80849..d51103a4 100644 --- a/splunk_eventgen/lib/plugins/generator/perdayvolumegenerator.py +++ b/splunk_eventgen/lib/plugins/generator/perdayvolumegenerator.py @@ -33,6 +33,21 @@ def gen(self, count, earliest, latest, samplename=None): ) startTime = datetime.datetime.now() + # Pre-generate each event in the sample once, so we can replace tokens and calculate any volume differences + preReplacementSize = sum( + [len(event["_raw"]) for event in self._sample.sampleDict] + ) + allEvents = self.replace_tokens(self._sample.sampleDict, earliest, latest) + postReplacementSize = sum([len(event["_raw"]) for event in allEvents]) + replacementVolumeRatio = preReplacementSize / postReplacementSize + size = size * replacementVolumeRatio + logger.debug( + "Token replacement size factor: {}, new interval target size: {}".format( + replacementVolumeRatio, size + ) + ) + del allEvents + # Create a counter for the current byte size of the read in samples currentSize = 0 @@ -94,8 +109,10 @@ def gen(self, count, earliest, latest, samplename=None): % (self._sample.name, self._sample.app, len(eventsDict)) ) - # build the events and replace tokens - self.build_events(eventsDict, startTime, earliest, latest) + # replace tokens and send events + send_objects = self.replace_tokens(eventsDict, earliest, latest) + del eventsDict + self.send_events(send_objects, startTime) def load(): diff --git a/splunk_eventgen/lib/plugins/generator/weblog.py b/splunk_eventgen/lib/plugins/generator/weblog.py old mode 100755 new mode 100644 diff --git a/splunk_eventgen/lib/plugins/output/counter.py b/splunk_eventgen/lib/plugins/output/counter.py old mode 100755 new mode 100644 diff --git a/splunk_eventgen/lib/plugins/output/devnull.py b/splunk_eventgen/lib/plugins/output/devnull.py old mode 100755 new mode 100644 diff --git a/splunk_eventgen/splunk_app/README.md b/splunk_eventgen/splunk_app/README.md new file mode 100644 index 00000000..b73ca0bc --- /dev/null +++ b/splunk_eventgen/splunk_app/README.md @@ -0,0 +1,13 @@ +### Introduction + +SA-Eventgen (Splunk App Eventgen) allows users to generate and index custom event data on their Splunk instance. +This app is currently built and maintained through the [Eventgen repository](https://github.com/splunk/eventgen). + +### Documentation + +The general Eventgen documentation is hosted on [Github](http://splunk.github.io/eventgen). +For installation and configuration instructions specific to the Splunk App, see the +[app install](http://splunk.github.io/eventgen/SETUP.html#splunk-app-installation) page of the documentation. + +Finally, see to the [Eventgen Reference Guide](http://splunk.github.io/eventgen/REFERENCE.html#eventgenconfspec) +for information on Eventgen configuration options. diff --git a/splunk_eventgen/splunk_app/README/eventgen.conf.spec b/splunk_eventgen/splunk_app/README/eventgen.conf.spec index dc63d07e..f197afcd 100644 --- a/splunk_eventgen/splunk_app/README/eventgen.conf.spec +++ b/splunk_eventgen/splunk_app/README/eventgen.conf.spec @@ -17,7 +17,6 @@ ## IMPORTANT! Do not specify any settings under a default stanza ## The layering system will not behave appropriately ## Use [global] instead -[default] [global] disabled = false diff --git a/splunk_eventgen/splunk_app/README/inputs.conf.spec b/splunk_eventgen/splunk_app/README/inputs.conf.spec index 14f970be..e408f372 100644 --- a/splunk_eventgen/splunk_app/README/inputs.conf.spec +++ b/splunk_eventgen/splunk_app/README/inputs.conf.spec @@ -1,2 +1,3 @@ [modinput_eventgen://] verbosity = +python.version = python3 diff --git a/splunk_eventgen/splunk_app/default/app.conf b/splunk_eventgen/splunk_app/default/app.conf index 8943aba1..c33b5fc5 100644 --- a/splunk_eventgen/splunk_app/default/app.conf +++ b/splunk_eventgen/splunk_app/default/app.conf @@ -8,13 +8,13 @@ ## Splunk app configuration file [install] -is_configured = true +is_configured = false state = enabled build = 1 [launcher] author = Splunk Inc. -version = 7.2.0 +version = 7.2.1 description = SA-Eventgen app for dynamic data generation [package] @@ -23,3 +23,7 @@ id = SA-Eventgen [ui] is_visible = true label = SA-Eventgen + +[triggers] +reload.eventgen = simple +reload.inputs.eventgen = access_endpoints /data/inputs/eventgen diff --git a/splunk_eventgen/splunk_app/metadata/local.meta b/splunk_eventgen/splunk_app/metadata/local.meta deleted file mode 100644 index 09069ad0..00000000 --- a/splunk_eventgen/splunk_app/metadata/local.meta +++ /dev/null @@ -1,23 +0,0 @@ -[app/install/state] -version = 4.3 -modtime = 1331844441.589340000 - -[inputs/script%3A%2F%2F%24SPLUNK_HOME%2Fetc%2Fapps%2FSA-Eventgen%2Fbin%2Feventgen.py] -version = 4.3.2 -modtime = 1340395082.546770000 - -[app/install/is_configured] -version = 4.3.2 -modtime = 1340395082.652048000 - -[views/log_viewer] -version = 6.1.2 -modtime = 1405368344.241276000 - -[views/perf] -version = 6.1.2 -modtime = 1402343428.848001000 - -[inputs/eventgen_modinput%3A%2F%2Fmain] -version = 6.1.2 -modtime = 1405542782.814751000 diff --git a/splunk_eventgen/splunk_app/static/appIcon.png b/splunk_eventgen/splunk_app/static/appIcon.png new file mode 100644 index 00000000..a4b9444e Binary files /dev/null and b/splunk_eventgen/splunk_app/static/appIcon.png differ diff --git a/splunk_eventgen/splunk_app/static/appIcon_2x.png b/splunk_eventgen/splunk_app/static/appIcon_2x.png new file mode 100644 index 00000000..11e6d68b Binary files /dev/null and b/splunk_eventgen/splunk_app/static/appIcon_2x.png differ diff --git a/tests/large/conf/eventgen_perdayvolume.conf b/tests/large/conf/eventgen_perdayvolume.conf new file mode 100755 index 00000000..f177db74 --- /dev/null +++ b/tests/large/conf/eventgen_perdayvolume.conf @@ -0,0 +1,6 @@ +[sample] +sampleDir = ../sample +sampletype = raw +outputMode = file +fileName = tests/large/results/eventgen_perdayvolume.result +perDayVolume = 1 diff --git a/tests/large/conf/eventgen_perdayvolume_large_token.conf b/tests/large/conf/eventgen_perdayvolume_large_token.conf new file mode 100755 index 00000000..9e72f818 --- /dev/null +++ b/tests/large/conf/eventgen_perdayvolume_large_token.conf @@ -0,0 +1,10 @@ +[sample] +sampleDir = ../sample +sampletype = raw +outputMode = file +fileName = tests/large/results/eventgen_perdayvolume.result +perDayVolume = 1 + +token.0.token = @@integer +token.0.replacementType = static +token.0.replacement = "supercalifragilisticexpialidocious" diff --git a/tests/large/conf/eventgen_perdayvolume_small_token.conf b/tests/large/conf/eventgen_perdayvolume_small_token.conf new file mode 100755 index 00000000..13964e10 --- /dev/null +++ b/tests/large/conf/eventgen_perdayvolume_small_token.conf @@ -0,0 +1,10 @@ +[sample] +sampleDir = ../sample +sampletype = raw +outputMode = file +fileName = tests/large/results/eventgen_perdayvolume.result +perDayVolume = 1 + +token.0.token = @@integer +token.0.replacementType = random +token.0.replacement = integer[0:10] diff --git a/tests/large/test_perdayvolume.py b/tests/large/test_perdayvolume.py new file mode 100644 index 00000000..0c626005 --- /dev/null +++ b/tests/large/test_perdayvolume.py @@ -0,0 +1,30 @@ +from pytest import mark + + +def calculate_perdayvolume(events, runtime, interval=60): + # Calculate expected data volume output (GB) if run for 24 hours + # Get the integer # of intervals, data is only generated on completion of an interval + num_intervals = runtime // interval + event_volume = sum([len(event) for event in events]) + total_volume = event_volume / 1024 / 1024 / 1024 * 60 * 24 + perdayvolume = total_volume / num_intervals + return perdayvolume + + +@mark.parametrize( + ("conf_filename", "execution_timeout", "perdayvolume"), + [ + ("eventgen_perdayvolume.conf", 300, 1), + ("eventgen_perdayvolume_small_token.conf", 300, 1), + ("eventgen_perdayvolume_large_token.conf", 300, 1), + ], +) +def test_perdayvolume( + eventgen_test_helper, conf_filename, execution_timeout, perdayvolume +): + # Test accuracy of small volume target with no token replacements + # TODO: using outputMode=file for now, test helper unable to collect all generated events w/ outputMode=stdout + events = eventgen_test_helper(conf_filename, execution_timeout).get_events() + assert ( + 0.98 < (calculate_perdayvolume(events, execution_timeout) / perdayvolume) < 1.02 + )