diff --git a/README.md b/README.md index 0257c85..932dbfe 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ Parses XLSX/XLS/CSV files into ElasticSearch using column titles from a specifie - support for importing to TLS enabled elasticsearch servers. - supports concurrent parsing of excel files and importing for better performance. - parses the whole file before starting the import - to make sure your index is not left in an undesired state. +- specify elasticsearch ingest pipeline for a given spreadsheet. ## Prerequisites The application requires ElasticSearch as its output. @@ -21,7 +22,7 @@ The application requires ElasticSearch as its output. 2. Grab the latest .jar file from [releases](https://github.com/codingchili/parser-excel-elasticsearch/releases). -Tested with ElasticSearch 5.6.2, 6.4.2 and 7.0.0-alpha1 +Tested with ElasticSearch 5.6.2, 6.4.2, 7.0.0-alpha1 and 7.4.0. ## Running with docker ```console @@ -35,7 +36,7 @@ connect to it from another machine when binding to all interfaces. Running the application, filename and index is required, to import from the terminal run: ```console -java -Xmx2g -jar excelastic.jar --mapping mappingName --clear +java -Xmx2g -jar excelastic.jar --mapping mappingName --pipeline pipelineName --clear ``` If running with --clear, then the existing index will be cleared before the import starts. diff --git a/excelastic.png b/excelastic.png index c3c4b51..af0cd0e 100644 Binary files a/excelastic.png and b/excelastic.png differ diff --git a/pom.xml b/pom.xml index d117be8..3f33e63 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.codingchili excelastic - 1.3.6 + 1.3.7 diff --git a/src/main/java/com/codingchili/ApplicationLauncher.java b/src/main/java/com/codingchili/ApplicationLauncher.java index adaed71..b476155 100644 --- a/src/main/java/com/codingchili/ApplicationLauncher.java +++ b/src/main/java/com/codingchili/ApplicationLauncher.java @@ -21,7 +21,7 @@ */ public class ApplicationLauncher { private final ApplicationLogger logger = new ApplicationLogger(getClass()); - public static String VERSION = "1.3.6"; + public static String VERSION = "1.3.7"; private Vertx vertx; public static void main(String[] args) { diff --git a/src/main/java/com/codingchili/Model/ElasticWriter.java b/src/main/java/com/codingchili/Model/ElasticWriter.java index b05d42a..3662785 100644 --- a/src/main/java/com/codingchili/Model/ElasticWriter.java +++ b/src/main/java/com/codingchili/Model/ElasticWriter.java @@ -158,10 +158,18 @@ private void updateStatus(HttpClientResponse response, ImportEvent event, int re } private String createImportHeader(ImportEvent event) { + JsonObject indexBody = new JsonObject(); + indexBody.put("_index", event.getIndex()) + .put("_type", event.getMapping()); + + event.getPipeline() + .filter(value -> !value.isEmpty()) + .ifPresent((pipeline) -> indexBody.put("pipeline", pipeline)); + return new JsonObject() - .put("index", new JsonObject() - .put("_index", event.getIndex()) - .put("_type", event.getMapping())).encode() + "\n"; + .put("index", indexBody) + .encode() + "\n"; + } /** diff --git a/src/main/java/com/codingchili/Model/ImportEvent.java b/src/main/java/com/codingchili/Model/ImportEvent.java index b952f09..4d98cb0 100644 --- a/src/main/java/com/codingchili/Model/ImportEvent.java +++ b/src/main/java/com/codingchili/Model/ImportEvent.java @@ -11,19 +11,22 @@ /** * @author Robin Duda *

- * Contains infromation about an import request. + * Contains information about an import request. */ public class ImportEvent { private static final String ARG_CLEAR = "--clear"; private static final String ARG_OFFSET = "--offset"; private static final String ARG_MAPPING = "--mapping"; + private static final String ARG_PIPELINE = "--pipeline"; private static final String OFFSET = "offset"; private static final String MAPPING = "mapping"; + private static final String PIPELINE = "pipeline"; private static final String OPTIONS = "options"; private static final String CLEAR = "clear"; private FileParser parser; private Boolean clearExisting; private String mapping; + private Optional pipeline; private String index; private String uploadId; private int offset; @@ -38,6 +41,7 @@ public static ImportEvent fromParams(MultiMap params) { return new ImportEvent() .setIndex(params.get(INDEX)) .setMapping(getMappingByParams(params)) + .setPipeline(params.get(PIPELINE)) .setClearExisting(params.get(OPTIONS).equals(CLEAR)) .setUploadId(params.get(UPLOAD_ID)) .setOffset(Integer.parseInt(params.get(OFFSET))); @@ -53,15 +57,17 @@ public static ImportEvent fromCommandLineArgs(String[] args) { return new ImportEvent() .setIndex(args[1]) .setOffset(getArgParamValue(args, ARG_OFFSET).map(Integer::parseInt).orElse(1)) - .setClearExisting(Arrays.asList(args).contains(ARG_CLEAR)) - .setMapping(getArgParamValue(args, ARG_MAPPING).orElse("default")); + .setMapping(getArgParamValue(args, ARG_MAPPING).orElse("default")) + .setPipeline(getArgParamValue(args, ARG_PIPELINE).orElse("")) + .setClearExisting(Arrays.asList(args).contains(ARG_CLEAR)); + } private static Optional getArgParamValue(String[] args, String argName) { for (int i = 0; i < args.length; i++) { if (args[i].equals(argName)) { if (i + 1 < args.length) { - return Optional.of(args[i + i]); + return Optional.of(args[i + 1]); } } } @@ -132,4 +138,14 @@ public ImportEvent setIndex(String index) { this.index = index; return this; } + + public Optional getPipeline() { + return pipeline; + } + + public ImportEvent setPipeline(String pipeline) { + this.pipeline = Optional.ofNullable(pipeline); + return this; + } + } diff --git a/src/main/resources/templates/index.jade b/src/main/resources/templates/index.jade index 3285f2c..c3c29be 100644 --- a/src/main/resources/templates/index.jade +++ b/src/main/resources/templates/index.jade @@ -32,6 +32,10 @@ html(lang='en') label.col-lg-3.control-label(for='mapping') Mapping .col-lg-9 input#index.form-control(type='text', name='mapping', placeholder='default') + .form-group + label.col-lg-3.control-label(for='pipeline') Pipeline + .col-lg-9 + input#index.form-control(type='text', name='pipeline', placeholder='none if empty') a.text-center.clickable#excel-options-show show excel options .form-group(hidden)#excel-options