failures = new ArrayList<>();
+
+ String currentFieldName = null;
+ while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+ if (token == XContentParser.Token.FIELD_NAME) {
+ currentFieldName = parser.currentName();
+ } else if (token.isValue()) {
+ if (SearchResponse.Cluster.INDICES_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
+ indexExpression = parser.text();
+ } else if (SearchResponse.Cluster.STATUS_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
+ status = parser.text();
+ } else if (SearchResponse.TIMED_OUT.match(currentFieldName, parser.getDeprecationHandler())) {
+ timedOut = parser.booleanValue();
+ } else if (SearchResponse.TOOK.match(currentFieldName, parser.getDeprecationHandler())) {
+ took = parser.longValue();
+ } else {
+ parser.skipChildren();
+ }
+ } else if (RestActions._SHARDS_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
+ while ((token = parser.nextToken()) != Token.END_OBJECT) {
+ if (token == Token.FIELD_NAME) {
+ currentFieldName = parser.currentName();
+ } else if (token.isValue()) {
+ if (RestActions.FAILED_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
+ failedShards = parser.intValue();
+ } else if (RestActions.SUCCESSFUL_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
+ successfulShards = parser.intValue();
+ } else if (RestActions.TOTAL_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
+ totalShards = parser.intValue();
+ } else if (RestActions.SKIPPED_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
+ skippedShards = parser.intValue();
+ } else {
+ parser.skipChildren();
+ }
+ } else {
+ parser.skipChildren();
+ }
+ }
+ } else if (token == Token.START_ARRAY) {
+ if (RestActions.FAILURES_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
+ while (parser.nextToken() != Token.END_ARRAY) {
+ failures.add(ParsedShardSearchFailure.fromXContent(parser));
+ }
+ } else {
+ parser.skipChildren();
+ }
+ } else {
+ parser.skipChildren();
+ }
+ }
+
+ Integer totalShardsFinal = totalShards == -1 ? null : totalShards;
+ Integer successfulShardsFinal = successfulShards == -1 ? null : successfulShards;
+ Integer skippedShardsFinal = skippedShards == -1 ? null : skippedShards;
+ Integer failedShardsFinal = failedShards == -1 ? null : failedShards;
+ TimeValue tookTimeValue = took == -1L ? null : new TimeValue(took);
+ boolean skipUnavailable = SearchResponse.Cluster.SKIP_UNAVAILABLE_DEFAULT; // skipUnavailable is not exposed to XContent, so just use default
+
+ return new SearchResponse.Cluster(
+ clusterName,
+ indexExpression,
+ skipUnavailable,
+ SearchResponse.Cluster.Status.valueOf(status.toUpperCase(Locale.ROOT)),
+ totalShardsFinal,
+ successfulShardsFinal,
+ skippedShardsFinal,
+ failedShardsFinal,
+ failures,
+ tookTimeValue,
+ timedOut
+ );
+ }
+ }
+}
diff --git a/src/main/java/org/elasticsearch/action/search/ParsedShardSearchFailure.java b/src/main/java/org/elasticsearch/action/search/ParsedShardSearchFailure.java
new file mode 100644
index 000000000..ea3bb74a9
--- /dev/null
+++ b/src/main/java/org/elasticsearch/action/search/ParsedShardSearchFailure.java
@@ -0,0 +1,73 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.action.search;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.search.SearchShardTarget;
+import org.elasticsearch.transport.RemoteClusterAware;
+import org.elasticsearch.xcontent.XContentParser;
+
+import java.io.IOException;
+
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+
+/**
+ * Represents a failure to search on a specific shard.
+ */
+public class ParsedShardSearchFailure {
+
+ public static ShardSearchFailure fromXContent(XContentParser parser) throws IOException {
+ XContentParser.Token token;
+ ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser);
+ String currentFieldName = null;
+ int shardId = -1;
+ String indexName = null;
+ String clusterAlias = null;
+ String nodeId = null;
+ ElasticsearchException exception = null;
+ while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+ if (token == XContentParser.Token.FIELD_NAME) {
+ currentFieldName = parser.currentName();
+ } else if (token.isValue()) {
+ if (ShardSearchFailure.SHARD_FIELD.equals(currentFieldName)) {
+ shardId = parser.intValue();
+ } else if (ShardSearchFailure.INDEX_FIELD.equals(currentFieldName)) {
+ String[] split = RemoteClusterAware.splitIndexName(parser.text());
+ clusterAlias = split[0];
+ indexName = split[1];
+ } else if (ShardSearchFailure.NODE_FIELD.equals(currentFieldName)) {
+ nodeId = parser.text();
+ } else {
+ parser.skipChildren();
+ }
+ } else if (token == XContentParser.Token.START_OBJECT) {
+ if (ShardSearchFailure.REASON_FIELD.equals(currentFieldName)) {
+ exception = ElasticsearchException.fromXContent(parser);
+ } else {
+ parser.skipChildren();
+ }
+ } else {
+ parser.skipChildren();
+ }
+ }
+ SearchShardTarget searchShardTarget = null;
+ if (nodeId != null) {
+ searchShardTarget = new SearchShardTarget(
+ nodeId,
+ new ShardId(new Index(indexName, IndexMetadata.INDEX_UUID_NA_VALUE), shardId),
+ clusterAlias
+ );
+ }
+ return new ShardSearchFailure(exception, searchShardTarget);
+ }
+}
diff --git a/src/main/java/org/elasticsearch/action/support/master/ParsedAcknowledgedResponse.java b/src/main/java/org/elasticsearch/action/support/master/ParsedAcknowledgedResponse.java
new file mode 100644
index 000000000..1b724e464
--- /dev/null
+++ b/src/main/java/org/elasticsearch/action/support/master/ParsedAcknowledgedResponse.java
@@ -0,0 +1,64 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+package org.elasticsearch.action.support.master;
+
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.xcontent.ConstructingObjectParser;
+import org.elasticsearch.xcontent.ObjectParser;
+import org.elasticsearch.xcontent.ParseField;
+import org.elasticsearch.xcontent.XContentParser;
+
+import java.io.IOException;
+
+import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
+
+/**
+ * A response to an action which updated the cluster state, but needs to report whether any relevant nodes failed to apply the update. For
+ * instance, a {@link org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest} may update a mapping in the index metadata, but
+ * one or more data nodes may fail to acknowledge the new mapping within the ack timeout. If this happens then clients must accept that
+ * subsequent requests that rely on the mapping update may return errors from the lagging data nodes.
+ *
+ * Actions which return a payload-free acknowledgement of success should generally prefer to use {@link ActionResponse.Empty} instead of
+ * {@link AcknowledgedResponse}, and other listeners should generally prefer {@link Void}.
+ */
+public class ParsedAcknowledgedResponse {
+
+ private static final ParseField ACKNOWLEDGED = new ParseField(AcknowledgedResponse.ACKNOWLEDGED_KEY);
+
+ public static void declareAcknowledgedField(ConstructingObjectParser objectParser) {
+ objectParser.declareField(
+ constructorArg(),
+ (parser, context) -> parser.booleanValue(),
+ ACKNOWLEDGED,
+ ObjectParser.ValueType.BOOLEAN
+ );
+ }
+
+ /**
+ * A generic parser that simply parses the acknowledged flag
+ */
+ private static final ConstructingObjectParser ACKNOWLEDGED_FLAG_PARSER = new ConstructingObjectParser<>(
+ "acknowledged_flag",
+ true,
+ args -> (Boolean) args[0]
+ );
+
+ static {
+ ACKNOWLEDGED_FLAG_PARSER.declareField(
+ constructorArg(),
+ (parser, context) -> parser.booleanValue(),
+ ACKNOWLEDGED,
+ ObjectParser.ValueType.BOOLEAN
+ );
+ }
+
+ public static AcknowledgedResponse fromXContent(XContentParser parser) throws IOException {
+ return AcknowledgedResponse.of(ACKNOWLEDGED_FLAG_PARSER.apply(parser, null));
+ }
+}
diff --git a/src/main/java/org/elasticsearch/action/support/master/ParsedShardsAcknowledgedResponse.java b/src/main/java/org/elasticsearch/action/support/master/ParsedShardsAcknowledgedResponse.java
new file mode 100644
index 000000000..6c58f3a16
--- /dev/null
+++ b/src/main/java/org/elasticsearch/action/support/master/ParsedShardsAcknowledgedResponse.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.action.support.master;
+
+import org.elasticsearch.xcontent.ConstructingObjectParser;
+import org.elasticsearch.xcontent.ObjectParser;
+
+import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
+
+public class ParsedShardsAcknowledgedResponse {
+
+ public static void declareAcknowledgedAndShardsAcknowledgedFields(
+ ConstructingObjectParser objectParser
+ ) {
+ ParsedAcknowledgedResponse.declareAcknowledgedField(objectParser);
+ objectParser.declareField(
+ constructorArg(),
+ (parser, context) -> parser.booleanValue(),
+ ShardsAcknowledgedResponse.SHARDS_ACKNOWLEDGED,
+ ObjectParser.ValueType.BOOLEAN
+ );
+ }
+}
diff --git a/src/main/java/org/elasticsearch/action/update/ParsedUpdateResponse.java b/src/main/java/org/elasticsearch/action/update/ParsedUpdateResponse.java
new file mode 100644
index 000000000..ca1272cee
--- /dev/null
+++ b/src/main/java/org/elasticsearch/action/update/ParsedUpdateResponse.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.action.update;
+
+import org.elasticsearch.action.ParsedDocWriteResponse;
+import org.elasticsearch.index.get.GetResult;
+import org.elasticsearch.xcontent.XContentParser;
+
+import java.io.IOException;
+
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+
+public class ParsedUpdateResponse {
+
+ public static UpdateResponse fromXContent(XContentParser parser) throws IOException {
+ ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser);
+
+ UpdateResponse.Builder context = new UpdateResponse.Builder();
+ while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
+ parseXContentFields(parser, context);
+ }
+ return context.build();
+ }
+
+ /**
+ * Parse the current token and update the parsing context appropriately.
+ */
+ public static void parseXContentFields(XContentParser parser, UpdateResponse.Builder context) throws IOException {
+ XContentParser.Token token = parser.currentToken();
+ String currentFieldName = parser.currentName();
+
+ if (UpdateResponse.GET.equals(currentFieldName)) {
+ if (token == XContentParser.Token.START_OBJECT) {
+ context.setGetResult(GetResult.fromXContentEmbedded(parser));
+ }
+ } else {
+ ParsedDocWriteResponse.parseInnerToXContent(parser, context);
+ }
+ }
+}
diff --git a/src/main/java/org/elasticsearch/aggregations/pipeline/BucketSelectorPipelineAggregationBuilder.java b/src/main/java/org/elasticsearch/aggregations/pipeline/BucketSelectorPipelineAggregationBuilder.java
new file mode 100644
index 000000000..9f4029313
--- /dev/null
+++ b/src/main/java/org/elasticsearch/aggregations/pipeline/BucketSelectorPipelineAggregationBuilder.java
@@ -0,0 +1,221 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.aggregations.pipeline;
+
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.TransportVersions;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.script.Script;
+import org.elasticsearch.search.aggregations.pipeline.AbstractPipelineAggregationBuilder;
+import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.TreeMap;
+
+import static org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.Parser.BUCKETS_PATH;
+import static org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.Parser.GAP_POLICY;
+
+public class BucketSelectorPipelineAggregationBuilder extends AbstractPipelineAggregationBuilder {
+ public static final String NAME = "bucket_selector";
+
+ private final Map bucketsPathsMap;
+ private final Script script;
+ private GapPolicy gapPolicy = GapPolicy.SKIP;
+
+ public BucketSelectorPipelineAggregationBuilder(String name, Map bucketsPathsMap, Script script) {
+ super(name, NAME, new TreeMap<>(bucketsPathsMap).values().toArray(new String[bucketsPathsMap.size()]));
+ this.bucketsPathsMap = bucketsPathsMap;
+ this.script = script;
+ }
+
+ /**
+ * Read from a stream.
+ */
+ public BucketSelectorPipelineAggregationBuilder(StreamInput in) throws IOException {
+ super(in, NAME);
+ bucketsPathsMap = in.readMap(StreamInput::readString, StreamInput::readString);
+ script = new Script(in);
+ gapPolicy = GapPolicy.readFrom(in);
+ }
+
+ @Override
+ protected void doWriteTo(StreamOutput out) throws IOException {
+ out.writeMap(bucketsPathsMap, StreamOutput::writeString, StreamOutput::writeString);
+ script.writeTo(out);
+ gapPolicy.writeTo(out);
+ }
+
+ /**
+ * Sets the gap policy to use for this aggregation.
+ */
+ public BucketSelectorPipelineAggregationBuilder gapPolicy(GapPolicy gapPolicy) {
+ if (gapPolicy == null) {
+ throw new IllegalArgumentException("[gapPolicy] must not be null: [" + name + "]");
+ }
+ this.gapPolicy = gapPolicy;
+ return this;
+ }
+
+ /**
+ * Gets the gap policy to use for this aggregation.
+ */
+ public GapPolicy gapPolicy() {
+ return gapPolicy;
+ }
+
+ @Override
+ protected PipelineAggregator createInternal(Map metadata) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ protected XContentBuilder internalXContent(XContentBuilder builder, Params params) throws IOException {
+ builder.field(BUCKETS_PATH.getPreferredName(), bucketsPathsMap);
+ builder.field(Script.SCRIPT_PARSE_FIELD.getPreferredName(), script);
+ builder.field(GAP_POLICY.getPreferredName(), gapPolicy.getName());
+ return builder;
+ }
+
+ public static BucketSelectorPipelineAggregationBuilder parse(String reducerName, XContentParser parser) throws IOException {
+ XContentParser.Token token;
+ Script script = null;
+ String currentFieldName = null;
+ Map bucketsPathsMap = null;
+ GapPolicy gapPolicy = null;
+
+ while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+ if (token == XContentParser.Token.FIELD_NAME) {
+ currentFieldName = parser.currentName();
+ } else if (token == XContentParser.Token.VALUE_STRING) {
+ if (BUCKETS_PATH.match(currentFieldName, parser.getDeprecationHandler())) {
+ bucketsPathsMap = new HashMap<>();
+ bucketsPathsMap.put("_value", parser.text());
+ } else if (GAP_POLICY.match(currentFieldName, parser.getDeprecationHandler())) {
+ gapPolicy = GapPolicy.parse(parser.text(), parser.getTokenLocation());
+ } else if (Script.SCRIPT_PARSE_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
+ script = Script.parse(parser);
+ } else {
+ throw new ParsingException(
+ parser.getTokenLocation(),
+ "Unknown key for a " + token + " in [" + reducerName + "]: [" + currentFieldName + "]."
+ );
+ }
+ } else if (token == XContentParser.Token.START_ARRAY) {
+ if (BUCKETS_PATH.match(currentFieldName, parser.getDeprecationHandler())) {
+ List paths = new ArrayList<>();
+ while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+ String path = parser.text();
+ paths.add(path);
+ }
+ bucketsPathsMap = new HashMap<>();
+ for (int i = 0; i < paths.size(); i++) {
+ bucketsPathsMap.put("_value" + i, paths.get(i));
+ }
+ } else {
+ throw new ParsingException(
+ parser.getTokenLocation(),
+ "Unknown key for a " + token + " in [" + reducerName + "]: [" + currentFieldName + "]."
+ );
+ }
+ } else if (token == XContentParser.Token.START_OBJECT) {
+ if (Script.SCRIPT_PARSE_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
+ script = Script.parse(parser);
+ } else if (BUCKETS_PATH.match(currentFieldName, parser.getDeprecationHandler())) {
+ Map map = parser.map();
+ bucketsPathsMap = new HashMap<>();
+ for (Map.Entry entry : map.entrySet()) {
+ bucketsPathsMap.put(entry.getKey(), String.valueOf(entry.getValue()));
+ }
+ } else {
+ throw new ParsingException(
+ parser.getTokenLocation(),
+ "Unknown key for a " + token + " in [" + reducerName + "]: [" + currentFieldName + "]."
+ );
+ }
+ } else {
+ throw new ParsingException(parser.getTokenLocation(), "Unexpected token " + token + " in [" + reducerName + "].");
+ }
+ }
+
+ if (bucketsPathsMap == null) {
+ throw new ParsingException(
+ parser.getTokenLocation(),
+ "Missing required field [" + BUCKETS_PATH.getPreferredName() + "] for bucket_selector aggregation [" + reducerName + "]"
+ );
+ }
+
+ if (script == null) {
+ throw new ParsingException(
+ parser.getTokenLocation(),
+ "Missing required field ["
+ + Script.SCRIPT_PARSE_FIELD.getPreferredName()
+ + "] for bucket_selector aggregation ["
+ + reducerName
+ + "]"
+ );
+ }
+
+ BucketSelectorPipelineAggregationBuilder factory = new BucketSelectorPipelineAggregationBuilder(
+ reducerName,
+ bucketsPathsMap,
+ script
+ );
+ if (gapPolicy != null) {
+ factory.gapPolicy(gapPolicy);
+ }
+ return factory;
+ }
+
+ @Override
+ protected void validate(ValidationContext context) {
+ context.validateHasParent(NAME, name);
+ }
+
+ @Override
+ protected boolean overrideBucketsPath() {
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(super.hashCode(), bucketsPathsMap, script, gapPolicy);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) return true;
+ if (obj == null || getClass() != obj.getClass()) return false;
+ if (super.equals(obj) == false) return false;
+
+ BucketSelectorPipelineAggregationBuilder other = (BucketSelectorPipelineAggregationBuilder) obj;
+ return Objects.equals(bucketsPathsMap, other.bucketsPathsMap)
+ && Objects.equals(script, other.script)
+ && Objects.equals(gapPolicy, other.gapPolicy);
+ }
+
+ @Override
+ public String getWriteableName() {
+ return NAME;
+ }
+
+ @Override
+ public TransportVersion getMinimalSupportedVersion() {
+ return TransportVersions.ZERO;
+ }
+}
diff --git a/src/main/java/org/elasticsearch/aggregations/pipeline/MovFnPipelineAggregationBuilder.java b/src/main/java/org/elasticsearch/aggregations/pipeline/MovFnPipelineAggregationBuilder.java
new file mode 100644
index 000000000..71a8ea494
--- /dev/null
+++ b/src/main/java/org/elasticsearch/aggregations/pipeline/MovFnPipelineAggregationBuilder.java
@@ -0,0 +1,246 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.aggregations.pipeline;
+
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.TransportVersions;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.script.Script;
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.pipeline.AbstractPipelineAggregationBuilder;
+import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.xcontent.ConstructingObjectParser;
+import org.elasticsearch.xcontent.ObjectParser;
+import org.elasticsearch.xcontent.ParseField;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.Parser.BUCKETS_PATH;
+import static org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.Parser.FORMAT;
+import static org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.Parser.GAP_POLICY;
+import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
+
+public class MovFnPipelineAggregationBuilder extends AbstractPipelineAggregationBuilder {
+ public static final String NAME = "moving_fn";
+ private static final ParseField WINDOW = new ParseField("window");
+ private static final ParseField SHIFT = new ParseField("shift");
+
+ private final Script script;
+ private final String bucketsPathString;
+ private String format = null;
+ private GapPolicy gapPolicy = GapPolicy.SKIP;
+ private int window;
+ private int shift;
+
+ public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>(
+ NAME,
+ false,
+ (args, name) -> new MovFnPipelineAggregationBuilder(name, (String) args[0], (Script) args[1], (int) args[2])
+ );
+ static {
+ PARSER.declareString(constructorArg(), BUCKETS_PATH_FIELD);
+ PARSER.declareField(
+ constructorArg(),
+ (p, c) -> Script.parse(p),
+ Script.SCRIPT_PARSE_FIELD,
+ ObjectParser.ValueType.OBJECT_OR_STRING
+ );
+ PARSER.declareInt(constructorArg(), WINDOW);
+
+ PARSER.declareInt(MovFnPipelineAggregationBuilder::setShift, SHIFT);
+ PARSER.declareString(MovFnPipelineAggregationBuilder::format, FORMAT);
+ PARSER.declareField(MovFnPipelineAggregationBuilder::gapPolicy, p -> {
+ if (p.currentToken() == XContentParser.Token.VALUE_STRING) {
+ return GapPolicy.parse(p.text().toLowerCase(Locale.ROOT), p.getTokenLocation());
+ }
+ throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]");
+ }, GAP_POLICY, ObjectParser.ValueType.STRING);
+ };
+
+ public MovFnPipelineAggregationBuilder(String name, String bucketsPath, Script script, int window) {
+ super(name, NAME, new String[] { bucketsPath });
+ this.bucketsPathString = bucketsPath;
+ this.script = script;
+ if (window <= 0) {
+ throw new IllegalArgumentException("[" + WINDOW.getPreferredName() + "] must be a positive, non-zero integer.");
+ }
+ this.window = window;
+ }
+
+ public MovFnPipelineAggregationBuilder(StreamInput in) throws IOException {
+ super(in, NAME);
+ bucketsPathString = in.readString();
+ script = new Script(in);
+ format = in.readOptionalString();
+ gapPolicy = GapPolicy.readFrom(in);
+ window = in.readInt();
+ shift = in.readInt();
+ }
+
+ @Override
+ protected void doWriteTo(StreamOutput out) throws IOException {
+ out.writeString(bucketsPathString);
+ script.writeTo(out);
+ out.writeOptionalString(format);
+ gapPolicy.writeTo(out);
+ out.writeInt(window);
+ out.writeInt(shift);
+ }
+
+ /**
+ * Sets the format to use on the output of this aggregation.
+ */
+ public MovFnPipelineAggregationBuilder format(String format) {
+ if (Strings.isNullOrEmpty(format)) {
+ throw new IllegalArgumentException("[" + FORMAT.getPreferredName() + "] must not be null or an empty string.");
+ }
+ this.format = format;
+ return this;
+ }
+
+ /**
+ * Gets the format to use on the output of this aggregation.
+ */
+ public String format() {
+ return format;
+ }
+
+ protected DocValueFormat formatter() {
+ if (format != null) {
+ return new DocValueFormat.Decimal(format);
+ }
+ return DocValueFormat.RAW;
+ }
+
+ /**
+ * Sets the gap policy to use for this aggregation.
+ */
+ public MovFnPipelineAggregationBuilder gapPolicy(GapPolicy gapPolicy) {
+ if (gapPolicy == null) {
+ throw new IllegalArgumentException("[" + GAP_POLICY.getPreferredName() + "] must not be null.");
+ }
+ this.gapPolicy = gapPolicy;
+ return this;
+ }
+
+ /**
+ * Gets the gap policy to use for this aggregation.
+ */
+ public GapPolicy gapPolicy() {
+ return gapPolicy;
+ }
+
+ /**
+ * Returns the window size for this aggregation
+ */
+ public int getWindow() {
+ return window;
+ }
+
+ /**
+ * Sets the window size for this aggregation
+ */
+ public void setWindow(int window) {
+ if (window <= 0) {
+ throw new IllegalArgumentException("[" + WINDOW.getPreferredName() + "] must be a positive, non-zero integer.");
+ }
+ this.window = window;
+ }
+
+ public void setShift(int shift) {
+ this.shift = shift;
+ }
+
+ @Override
+ protected void validate(ValidationContext context) {
+ if (window <= 0) {
+ context.addValidationError("[" + WINDOW.getPreferredName() + "] must be a positive, non-zero integer.");
+ }
+ context.validateParentAggSequentiallyOrderedWithoutSkips(NAME, name);
+ }
+
+ @Override
+ protected PipelineAggregator createInternal(Map metadata) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ protected XContentBuilder internalXContent(XContentBuilder builder, Params params) throws IOException {
+ builder.field(BUCKETS_PATH.getPreferredName(), bucketsPathString);
+ builder.field(Script.SCRIPT_PARSE_FIELD.getPreferredName(), script);
+ if (format != null) {
+ builder.field(FORMAT.getPreferredName(), format);
+ }
+ builder.field(GAP_POLICY.getPreferredName(), gapPolicy.getName());
+ builder.field(WINDOW.getPreferredName(), window);
+ builder.field(SHIFT.getPreferredName(), shift);
+ return builder;
+ }
+
+ /**
+ * Used for serialization testing, since pipeline aggs serialize themselves as a named object but are parsed
+ * as a regular object with the name passed in.
+ */
+ static MovFnPipelineAggregationBuilder parse(XContentParser parser) throws IOException {
+ parser.nextToken();
+ if (parser.currentToken().equals(XContentParser.Token.START_OBJECT)) {
+ parser.nextToken();
+ if (parser.currentToken().equals(XContentParser.Token.FIELD_NAME)) {
+ String aggName = parser.currentName();
+ parser.nextToken(); // "moving_fn"
+ parser.nextToken(); // start_object
+ return PARSER.apply(parser, aggName);
+ }
+ }
+
+ throw new IllegalStateException("Expected aggregation name but none found");
+ }
+
+ @Override
+ protected boolean overrideBucketsPath() {
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(super.hashCode(), bucketsPathString, script, format, gapPolicy, window, shift);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) return true;
+ if (obj == null || getClass() != obj.getClass()) return false;
+ if (super.equals(obj) == false) return false;
+ MovFnPipelineAggregationBuilder other = (MovFnPipelineAggregationBuilder) obj;
+ return Objects.equals(bucketsPathString, other.bucketsPathString)
+ && Objects.equals(script, other.script)
+ && Objects.equals(format, other.format)
+ && Objects.equals(gapPolicy, other.gapPolicy)
+ && Objects.equals(window, other.window)
+ && Objects.equals(shift, other.shift);
+ }
+
+ @Override
+ public String getWriteableName() {
+ return NAME;
+ }
+
+ @Override
+ public TransportVersion getMinimalSupportedVersion() {
+ return TransportVersions.ZERO;
+ }
+}
diff --git a/src/main/java/org/elasticsearch/index/reindex/ParsedBulkByScrollResponse.java b/src/main/java/org/elasticsearch/index/reindex/ParsedBulkByScrollResponse.java
new file mode 100644
index 000000000..6e575e004
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/reindex/ParsedBulkByScrollResponse.java
@@ -0,0 +1,96 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.index.reindex;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.action.bulk.BulkItemResponse.Failure;
+import org.elasticsearch.index.reindex.ScrollableHitSource.SearchFailure;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.xcontent.ObjectParser;
+import org.elasticsearch.xcontent.ParseField;
+import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xcontent.XContentParser.Token;
+
+import java.io.IOException;
+
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+
+/**
+ * Response used for actions that index many documents using a scroll request.
+ */
+public class ParsedBulkByScrollResponse {
+
+ @SuppressWarnings("unchecked")
+ private static final ObjectParser PARSER = new ObjectParser<>(
+ "bulk_by_scroll_response",
+ true,
+ BulkByScrollResponseBuilder::new
+ );
+ static {
+ PARSER.declareLong(BulkByScrollResponseBuilder::setTook, new ParseField(BulkByScrollResponse.TOOK_FIELD));
+ PARSER.declareBoolean(BulkByScrollResponseBuilder::setTimedOut, new ParseField(BulkByScrollResponse.TIMED_OUT_FIELD));
+ PARSER.declareObjectArray(BulkByScrollResponseBuilder::setFailures, (p, c) -> parseFailure(p), new ParseField(BulkByScrollResponse.FAILURES_FIELD));
+ // since the result of BulkByScrollResponse.Status are mixed we also parse that in this
+ ParsedBulkByScrollTask.ParsedStatus.declareFields(PARSER);
+ }
+
+ public static BulkByScrollResponse fromXContent(XContentParser parser) {
+ return PARSER.apply(parser, null).buildResponse();
+ }
+
+ private static Object parseFailure(XContentParser parser) throws IOException {
+ ensureExpectedToken(Token.START_OBJECT, parser.currentToken(), parser);
+ Token token;
+ String index = null;
+ String id = null;
+ Integer status = null;
+ Integer shardId = null;
+ String nodeId = null;
+ ElasticsearchException bulkExc = null;
+ ElasticsearchException searchExc = null;
+ while ((token = parser.nextToken()) != Token.END_OBJECT) {
+ ensureExpectedToken(Token.FIELD_NAME, token, parser);
+ String name = parser.currentName();
+ token = parser.nextToken();
+ if (token == Token.START_ARRAY) {
+ parser.skipChildren();
+ } else if (token == Token.START_OBJECT) {
+ switch (name) {
+ case SearchFailure.REASON_FIELD -> searchExc = ElasticsearchException.fromXContent(parser);
+ case Failure.CAUSE_FIELD -> bulkExc = ElasticsearchException.fromXContent(parser);
+ default -> parser.skipChildren();
+ }
+ } else if (token == Token.VALUE_STRING) {
+ switch (name) {
+ // This field is the same as SearchFailure.index
+ case Failure.INDEX_FIELD -> index = parser.text();
+ case Failure.ID_FIELD -> id = parser.text();
+ case SearchFailure.NODE_FIELD -> nodeId = parser.text();
+ }
+ } else if (token == Token.VALUE_NUMBER) {
+ switch (name) {
+ case Failure.STATUS_FIELD -> status = parser.intValue();
+ case SearchFailure.SHARD_FIELD -> shardId = parser.intValue();
+ }
+ }
+ }
+ if (bulkExc != null) {
+ return new Failure(index, id, bulkExc, RestStatus.fromCode(status));
+ } else if (searchExc != null) {
+ if (status == null) {
+ return new SearchFailure(searchExc, index, shardId, nodeId);
+ } else {
+ return new SearchFailure(searchExc, index, shardId, nodeId, RestStatus.fromCode(status));
+ }
+ } else {
+ throw new ElasticsearchParseException("failed to parse failures array. At least one of {reason,cause} must be present");
+ }
+ }
+}
diff --git a/src/main/java/org/elasticsearch/index/reindex/ParsedBulkByScrollTask.java b/src/main/java/org/elasticsearch/index/reindex/ParsedBulkByScrollTask.java
new file mode 100644
index 000000000..1f00506f3
--- /dev/null
+++ b/src/main/java/org/elasticsearch/index/reindex/ParsedBulkByScrollTask.java
@@ -0,0 +1,184 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.index.reindex;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.core.Tuple;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.xcontent.ConstructingObjectParser;
+import org.elasticsearch.xcontent.ObjectParser;
+import org.elasticsearch.xcontent.ParseField;
+import org.elasticsearch.xcontent.XContentParseException;
+import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xcontent.XContentParser.Token;
+
+import java.io.IOException;
+
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
+
+/**
+ * Task storing information about a currently running BulkByScroll request.
+ *
+ * When the request is not sliced, this task is the only task created, and starts an action to perform search requests.
+ *
+ * When the request is sliced, this task can either represent a coordinating task (using
+ * {@link BulkByScrollTask#setWorkerCount(int)}) or a worker task that performs search queries (using
+ * {@link BulkByScrollTask#setWorker(float, Integer)}).
+ *
+ * We don't always know if this task will be a leader or worker task when it's created, because if slices is set to "auto" it may
+ * be either depending on the number of shards in the source indices. We figure that out when the request is handled and set it on this
+ * class with {@link #setWorkerCount(int)} or {@link #setWorker(float, Integer)}.
+ */
+public class ParsedBulkByScrollTask {
+
+ /**
+ * Status of the reindex, update by query, or delete by query. While in
+ * general we allow {@linkplain Task.Status} implementations to make
+ * backwards incompatible changes to their {@link Task.Status#toXContent}
+ * implementations, this one has become defacto standardized because Kibana
+ * parses it. As such, we should be very careful about removing things from
+ * this.
+ */
+ public static class ParsedStatus {
+
+ static final ConstructingObjectParser, Void> RETRIES_PARSER = new ConstructingObjectParser<>(
+ "bulk_by_scroll_task_status_retries",
+ true,
+ a -> new Tuple<>(((Long) a[0]), (Long) a[1])
+ );
+ static {
+ RETRIES_PARSER.declareLong(constructorArg(), new ParseField(BulkByScrollTask.Status.RETRIES_BULK_FIELD));
+ RETRIES_PARSER.declareLong(constructorArg(), new ParseField(BulkByScrollTask.Status.RETRIES_SEARCH_FIELD));
+ }
+
+ public static void declareFields(ObjectParser parser) {
+ parser.declareInt(BulkByScrollTask.StatusBuilder::setSliceId, new ParseField(BulkByScrollTask.Status.SLICE_ID_FIELD));
+ parser.declareLong(BulkByScrollTask.StatusBuilder::setTotal, new ParseField(BulkByScrollTask.Status.TOTAL_FIELD));
+ parser.declareLong(BulkByScrollTask.StatusBuilder::setUpdated, new ParseField(BulkByScrollTask.Status.UPDATED_FIELD));
+ parser.declareLong(BulkByScrollTask.StatusBuilder::setCreated, new ParseField(BulkByScrollTask.Status.CREATED_FIELD));
+ parser.declareLong(BulkByScrollTask.StatusBuilder::setDeleted, new ParseField(BulkByScrollTask.Status.DELETED_FIELD));
+ parser.declareInt(BulkByScrollTask.StatusBuilder::setBatches, new ParseField(BulkByScrollTask.Status.BATCHES_FIELD));
+ parser.declareLong(BulkByScrollTask.StatusBuilder::setVersionConflicts, new ParseField(BulkByScrollTask.Status.VERSION_CONFLICTS_FIELD));
+ parser.declareLong(BulkByScrollTask.StatusBuilder::setNoops, new ParseField(BulkByScrollTask.Status.NOOPS_FIELD));
+ parser.declareObject(BulkByScrollTask.StatusBuilder::setRetries, RETRIES_PARSER, new ParseField(BulkByScrollTask.Status.RETRIES_FIELD));
+ parser.declareLong(BulkByScrollTask.StatusBuilder::setThrottled, new ParseField(BulkByScrollTask.Status.THROTTLED_RAW_FIELD));
+ parser.declareFloat(BulkByScrollTask.StatusBuilder::setRequestsPerSecond, new ParseField(BulkByScrollTask.Status.REQUESTS_PER_SEC_FIELD));
+ parser.declareString(BulkByScrollTask.StatusBuilder::setReasonCancelled, new ParseField(BulkByScrollTask.Status.CANCELED_FIELD));
+ parser.declareLong(BulkByScrollTask.StatusBuilder::setThrottledUntil, new ParseField(BulkByScrollTask.Status.THROTTLED_UNTIL_RAW_FIELD));
+ parser.declareObjectArray(
+ BulkByScrollTask.StatusBuilder::setSliceStatuses,
+ (p, c) -> ParsedStatusOrException.fromXContent(p),
+ new ParseField(BulkByScrollTask.Status.SLICES_FIELD)
+ );
+ }
+
+ public static BulkByScrollTask.Status fromXContent(XContentParser parser) throws IOException {
+ XContentParser.Token token;
+ if (parser.currentToken() == Token.START_OBJECT) {
+ token = parser.nextToken();
+ } else {
+ token = parser.nextToken();
+ }
+ ensureExpectedToken(Token.START_OBJECT, token, parser);
+ token = parser.nextToken();
+ ensureExpectedToken(Token.FIELD_NAME, token, parser);
+ return innerFromXContent(parser);
+ }
+
+ public static BulkByScrollTask.Status innerFromXContent(XContentParser parser) throws IOException {
+ Token token = parser.currentToken();
+ String fieldName = parser.currentName();
+ ensureExpectedToken(XContentParser.Token.FIELD_NAME, token, parser);
+ BulkByScrollTask.StatusBuilder builder = new BulkByScrollTask.StatusBuilder();
+ while ((token = parser.nextToken()) != Token.END_OBJECT) {
+ if (token == Token.FIELD_NAME) {
+ fieldName = parser.currentName();
+ } else if (token == Token.START_OBJECT) {
+ if (fieldName.equals(BulkByScrollTask.Status.RETRIES_FIELD)) {
+ builder.setRetries(ParsedStatus.RETRIES_PARSER.parse(parser, null));
+ } else {
+ parser.skipChildren();
+ }
+ } else if (token == Token.START_ARRAY) {
+ if (fieldName.equals(BulkByScrollTask.Status.SLICES_FIELD)) {
+ while ((token = parser.nextToken()) != Token.END_ARRAY) {
+ builder.addToSliceStatuses(ParsedStatusOrException.fromXContent(parser));
+ }
+ } else {
+ parser.skipChildren();
+ }
+ } else { // else if it is a value
+ switch (fieldName) {
+ case BulkByScrollTask.Status.SLICE_ID_FIELD -> builder.setSliceId(parser.intValue());
+ case BulkByScrollTask.Status.TOTAL_FIELD -> builder.setTotal(parser.longValue());
+ case BulkByScrollTask.Status.UPDATED_FIELD -> builder.setUpdated(parser.longValue());
+ case BulkByScrollTask.Status.CREATED_FIELD -> builder.setCreated(parser.longValue());
+ case BulkByScrollTask.Status.DELETED_FIELD -> builder.setDeleted(parser.longValue());
+ case BulkByScrollTask.Status.BATCHES_FIELD -> builder.setBatches(parser.intValue());
+ case BulkByScrollTask.Status.VERSION_CONFLICTS_FIELD -> builder.setVersionConflicts(parser.longValue());
+ case BulkByScrollTask.Status.NOOPS_FIELD -> builder.setNoops(parser.longValue());
+ case BulkByScrollTask.Status.THROTTLED_RAW_FIELD -> builder.setThrottled(parser.longValue());
+ case BulkByScrollTask.Status.REQUESTS_PER_SEC_FIELD -> builder.setRequestsPerSecond(parser.floatValue());
+ case BulkByScrollTask.Status.CANCELED_FIELD -> builder.setReasonCancelled(parser.text());
+ case BulkByScrollTask.Status.THROTTLED_UNTIL_RAW_FIELD -> builder.setThrottledUntil(parser.longValue());
+ }
+ }
+ }
+ return builder.buildStatus();
+ }
+ }
+
+ /**
+ * The status of a slice of the request. Successful requests store the {@link StatusOrException#status} while failing requests store a
+ * {@link StatusOrException#exception}.
+ */
+ public static class ParsedStatusOrException {
+
+ /**
+ * Since {@link StatusOrException} can contain either an {@link Exception} or a {@link Status} we need to peek
+ * at a field first before deciding what needs to be parsed since the same object could contains either.
+ * The {@link #EXPECTED_EXCEPTION_FIELDS} contains the fields that are expected when the serialised object
+ * was an instance of exception and the {@link Status#FIELDS_SET} is the set of fields expected when the
+ * serialized object was an instance of Status.
+ */
+ public static BulkByScrollTask.StatusOrException fromXContent(XContentParser parser) throws IOException {
+ XContentParser.Token token = parser.currentToken();
+ if (token == null) {
+ token = parser.nextToken();
+ }
+ if (token == Token.VALUE_NULL) {
+ return null;
+ } else {
+ ensureExpectedToken(XContentParser.Token.START_OBJECT, token, parser);
+ token = parser.nextToken();
+ // This loop is present only to ignore unknown tokens. It breaks as soon as we find a field
+ // that is allowed.
+ while (token != Token.END_OBJECT) {
+ ensureExpectedToken(Token.FIELD_NAME, token, parser);
+ String fieldName = parser.currentName();
+ // weird way to ignore unknown tokens
+ if (BulkByScrollTask.Status.FIELDS_SET.contains(fieldName)) {
+ return new BulkByScrollTask.StatusOrException(ParsedStatus.innerFromXContent(parser));
+ } else if (BulkByScrollTask.StatusOrException.EXPECTED_EXCEPTION_FIELDS.contains(fieldName)) {
+ return new BulkByScrollTask.StatusOrException(ElasticsearchException.innerFromXContent(parser, false));
+ } else {
+ // Ignore unknown tokens
+ token = parser.nextToken();
+ if (token == Token.START_OBJECT || token == Token.START_ARRAY) {
+ parser.skipChildren();
+ }
+ token = parser.nextToken();
+ }
+ }
+ throw new XContentParseException("Unable to parse StatusFromException. Expected fields not found.");
+ }
+ }
+ }
+}
diff --git a/src/main/java/org/elasticsearch/join/aggregations/ChildrenAggregationBuilder.java b/src/main/java/org/elasticsearch/join/aggregations/ChildrenAggregationBuilder.java
new file mode 100644
index 000000000..3af59349c
--- /dev/null
+++ b/src/main/java/org/elasticsearch/join/aggregations/ChildrenAggregationBuilder.java
@@ -0,0 +1,168 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.join.aggregations;
+
+import org.apache.lucene.search.Query;
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.TransportVersions;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
+import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
+import org.elasticsearch.search.aggregations.AggregatorFactory;
+import org.elasticsearch.search.aggregations.support.AggregationContext;
+import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
+import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
+import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
+import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.aggregations.support.ValuesSourceType;
+import org.elasticsearch.xcontent.ParseField;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Objects;
+
+public class ChildrenAggregationBuilder extends ValuesSourceAggregationBuilder {
+
+ public static final String NAME = "children";
+ private static final ParseField TYPE_FIELD = new ParseField("type");
+
+ private final String childType;
+ private Query parentFilter;
+ private Query childFilter;
+
+ /**
+ * @param name
+ * the name of this aggregation
+ * @param childType
+ * the type of children documents
+ */
+ public ChildrenAggregationBuilder(String name, String childType) {
+ super(name);
+ if (childType == null) {
+ throw new IllegalArgumentException("[childType] must not be null: [" + name + "]");
+ }
+ this.childType = childType;
+ }
+
+ protected ChildrenAggregationBuilder(ChildrenAggregationBuilder clone, Builder factoriesBuilder, Map metadata) {
+ super(clone, factoriesBuilder, metadata);
+ this.childType = clone.childType;
+ this.childFilter = clone.childFilter;
+ this.parentFilter = clone.parentFilter;
+ }
+
+ @Override
+ protected ValuesSourceType defaultValueSourceType() {
+ return CoreValuesSourceType.KEYWORD;
+ }
+
+ @Override
+ protected AggregationBuilder shallowCopy(Builder factoriesBuilder, Map metadata) {
+ return new ChildrenAggregationBuilder(this, factoriesBuilder, metadata);
+ }
+
+ /**
+ * Read from a stream.
+ */
+ public ChildrenAggregationBuilder(StreamInput in) throws IOException {
+ super(in);
+ childType = in.readString();
+ }
+
+ @Override
+ protected void innerWriteTo(StreamOutput out) throws IOException {
+ out.writeString(childType);
+ }
+
+ @Override
+ public BucketCardinality bucketCardinality() {
+ return BucketCardinality.ONE;
+ }
+
+ @Override
+ protected ValuesSourceAggregatorFactory innerBuild(
+ AggregationContext context,
+ ValuesSourceConfig config,
+ AggregatorFactory parent,
+ Builder subFactoriesBuilder
+ ) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ protected ValuesSourceConfig resolveConfig(AggregationContext context) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
+ builder.field(TYPE_FIELD.getPreferredName(), childType);
+ return builder;
+ }
+
+ public static ChildrenAggregationBuilder parse(String aggregationName, XContentParser parser) throws IOException {
+ String childType = null;
+
+ XContentParser.Token token;
+ String currentFieldName = null;
+ while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+ if (token == XContentParser.Token.FIELD_NAME) {
+ currentFieldName = parser.currentName();
+ } else if (token == XContentParser.Token.VALUE_STRING) {
+ if ("type".equals(currentFieldName)) {
+ childType = parser.text();
+ } else {
+ throw new ParsingException(
+ parser.getTokenLocation(),
+ "Unknown key for a " + token + " in [" + aggregationName + "]: [" + currentFieldName + "]."
+ );
+ }
+ } else {
+ throw new ParsingException(parser.getTokenLocation(), "Unexpected token " + token + " in [" + aggregationName + "].");
+ }
+ }
+
+ if (childType == null) {
+ throw new ParsingException(
+ parser.getTokenLocation(),
+ "Missing [child_type] field for children aggregation [" + aggregationName + "]"
+ );
+ }
+
+ return new ChildrenAggregationBuilder(aggregationName, childType);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(super.hashCode(), childType);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) return true;
+ if (obj == null || getClass() != obj.getClass()) return false;
+ if (super.equals(obj) == false) return false;
+ ChildrenAggregationBuilder other = (ChildrenAggregationBuilder) obj;
+ return Objects.equals(childType, other.childType);
+ }
+
+ @Override
+ public String getType() {
+ return NAME;
+ }
+
+ @Override
+ public TransportVersion getMinimalSupportedVersion() {
+ return TransportVersions.ZERO;
+ }
+}
diff --git a/src/main/java/org/elasticsearch/join/query/HasChildQueryBuilder.java b/src/main/java/org/elasticsearch/join/query/HasChildQueryBuilder.java
new file mode 100644
index 000000000..0f7b7c4a8
--- /dev/null
+++ b/src/main/java/org/elasticsearch/join/query/HasChildQueryBuilder.java
@@ -0,0 +1,338 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+package org.elasticsearch.join.query;
+
+import org.apache.lucene.search.MatchNoDocsQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.join.ScoreMode;
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.TransportVersions;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.index.query.AbstractQueryBuilder;
+import org.elasticsearch.index.query.InnerHitBuilder;
+import org.elasticsearch.index.query.InnerHitContextBuilder;
+import org.elasticsearch.index.query.NestedQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryRewriteContext;
+import org.elasticsearch.index.query.SearchExecutionContext;
+import org.elasticsearch.xcontent.ParseField;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * A query builder for {@code has_child} query.
+ */
+public class HasChildQueryBuilder extends AbstractQueryBuilder {
+ public static final String NAME = "has_child";
+
+ /**
+ * The default maximum number of children that are required to match for the parent to be considered a match.
+ */
+ public static final int DEFAULT_MAX_CHILDREN = Integer.MAX_VALUE;
+ /**
+ * The default minimum number of children that are required to match for the parent to be considered a match.
+ */
+ public static final int DEFAULT_MIN_CHILDREN = 1;
+ private static final ScoreMode DEFAULT_SCORE_MODE = ScoreMode.None;
+
+ /**
+ * The default value for ignore_unmapped.
+ */
+ public static final boolean DEFAULT_IGNORE_UNMAPPED = false;
+
+ private static final ParseField QUERY_FIELD = new ParseField("query");
+ private static final ParseField TYPE_FIELD = new ParseField("type");
+ private static final ParseField MAX_CHILDREN_FIELD = new ParseField("max_children");
+ private static final ParseField MIN_CHILDREN_FIELD = new ParseField("min_children");
+ private static final ParseField SCORE_MODE_FIELD = new ParseField("score_mode");
+ private static final ParseField INNER_HITS_FIELD = new ParseField("inner_hits");
+ private static final ParseField IGNORE_UNMAPPED_FIELD = new ParseField("ignore_unmapped");
+
+ private final QueryBuilder query;
+ private final String type;
+ private final ScoreMode scoreMode;
+ private InnerHitBuilder innerHitBuilder;
+ private int minChildren = DEFAULT_MIN_CHILDREN;
+ private int maxChildren = DEFAULT_MAX_CHILDREN;
+ private boolean ignoreUnmapped = DEFAULT_IGNORE_UNMAPPED;
+
+ public HasChildQueryBuilder(String type, QueryBuilder query, ScoreMode scoreMode) {
+ this(type, query, DEFAULT_MIN_CHILDREN, DEFAULT_MAX_CHILDREN, scoreMode, null);
+ }
+
+ private HasChildQueryBuilder(
+ String type,
+ QueryBuilder query,
+ int minChildren,
+ int maxChildren,
+ ScoreMode scoreMode,
+ InnerHitBuilder innerHitBuilder
+ ) {
+ this.type = requireValue(type, "[" + NAME + "] requires 'type' field");
+ this.query = requireValue(query, "[" + NAME + "] requires 'query' field");
+ this.scoreMode = requireValue(scoreMode, "[" + NAME + "] requires 'score_mode' field");
+ this.innerHitBuilder = innerHitBuilder;
+ this.minChildren = minChildren;
+ this.maxChildren = maxChildren;
+ }
+
+ /**
+ * Read from a stream.
+ */
+ public HasChildQueryBuilder(StreamInput in) throws IOException {
+ super(in);
+ type = in.readString();
+ minChildren = in.readInt();
+ maxChildren = in.readInt();
+ scoreMode = ScoreMode.values()[in.readVInt()];
+ query = in.readNamedWriteable(QueryBuilder.class);
+ innerHitBuilder = in.readOptionalWriteable(InnerHitBuilder::new);
+ ignoreUnmapped = in.readBoolean();
+ }
+
+ @Override
+ protected void doWriteTo(StreamOutput out) throws IOException {
+ out.writeString(type);
+ out.writeInt(minChildren);
+ out.writeInt(maxChildren);
+ out.writeVInt(scoreMode.ordinal());
+ out.writeNamedWriteable(query);
+ out.writeOptionalWriteable(innerHitBuilder);
+ out.writeBoolean(ignoreUnmapped);
+ }
+
+ /**
+ * Defines the minimum number of children that are required to match for the parent to be considered a match and
+ * the maximum number of children that are required to match for the parent to be considered a match.
+ */
+ public HasChildQueryBuilder minMaxChildren(int min, int max) {
+ if (min <= 0) {
+ throw new IllegalArgumentException("[" + NAME + "] requires positive 'min_children' field");
+ }
+ if (max <= 0) {
+ throw new IllegalArgumentException("[" + NAME + "] requires positive 'max_children' field");
+ }
+ if (max < min) {
+ throw new IllegalArgumentException("[" + NAME + "] 'max_children' is less than 'min_children'");
+ }
+ this.minChildren = min;
+ this.maxChildren = max;
+ return this;
+ }
+
+ /**
+ * Returns inner hit definition in the scope of this query and reusing the defined type and query.
+ */
+ public InnerHitBuilder innerHit() {
+ return innerHitBuilder;
+ }
+
+ public HasChildQueryBuilder innerHit(InnerHitBuilder innerHit) {
+ this.innerHitBuilder = innerHit;
+ innerHitBuilder.setIgnoreUnmapped(ignoreUnmapped);
+ return this;
+ }
+
+ /**
+ * Returns the children query to execute.
+ */
+ public QueryBuilder query() {
+ return query;
+ }
+
+ /**
+ * Returns the child type
+ */
+ public String childType() {
+ return type;
+ }
+
+ /**
+ * Returns how the scores from the matching child documents are mapped into the parent document.
+ */
+ public ScoreMode scoreMode() {
+ return scoreMode;
+ }
+
+ /**
+ * Returns the minimum number of children that are required to match for the parent to be considered a match.
+ * The default is {@value #DEFAULT_MIN_CHILDREN}
+ */
+ public int minChildren() {
+ return minChildren;
+ }
+
+ /**
+ * Returns the maximum number of children that are required to match for the parent to be considered a match.
+ * The default is {@value #DEFAULT_MAX_CHILDREN}
+ */
+ public int maxChildren() {
+ return maxChildren;
+ }
+
+ /**
+ * Sets whether the query builder should ignore unmapped types (and run a
+ * {@link MatchNoDocsQuery} in place of this query) or throw an exception if
+ * the type is unmapped.
+ */
+ public HasChildQueryBuilder ignoreUnmapped(boolean ignoreUnmapped) {
+ this.ignoreUnmapped = ignoreUnmapped;
+ if (innerHitBuilder != null) {
+ innerHitBuilder.setIgnoreUnmapped(ignoreUnmapped);
+ }
+ return this;
+ }
+
+ /**
+ * Gets whether the query builder will ignore unmapped types (and run a
+ * {@link MatchNoDocsQuery} in place of this query) or throw an exception if
+ * the type is unmapped.
+ */
+ public boolean ignoreUnmapped() {
+ return ignoreUnmapped;
+ }
+
+ @Override
+ protected void doXContent(XContentBuilder builder, Params params) throws IOException {
+ builder.startObject(NAME);
+ builder.field(QUERY_FIELD.getPreferredName());
+ query.toXContent(builder, params);
+ builder.field(TYPE_FIELD.getPreferredName(), type);
+ if (false == scoreMode.equals(DEFAULT_SCORE_MODE)) {
+ builder.field(SCORE_MODE_FIELD.getPreferredName(), NestedQueryBuilder.scoreModeAsString(scoreMode));
+ }
+ if (minChildren != DEFAULT_MIN_CHILDREN) {
+ builder.field(MIN_CHILDREN_FIELD.getPreferredName(), minChildren);
+ }
+ if (maxChildren != DEFAULT_MAX_CHILDREN) {
+ builder.field(MAX_CHILDREN_FIELD.getPreferredName(), maxChildren);
+ }
+ if (ignoreUnmapped != DEFAULT_IGNORE_UNMAPPED) {
+ builder.field(IGNORE_UNMAPPED_FIELD.getPreferredName(), ignoreUnmapped);
+ }
+ boostAndQueryNameToXContent(builder);
+ if (innerHitBuilder != null) {
+ builder.field(INNER_HITS_FIELD.getPreferredName(), innerHitBuilder, params);
+ }
+ builder.endObject();
+ }
+
+ public static HasChildQueryBuilder fromXContent(XContentParser parser) throws IOException {
+ float boost = AbstractQueryBuilder.DEFAULT_BOOST;
+ String childType = null;
+ ScoreMode scoreMode = DEFAULT_SCORE_MODE;
+ int minChildren = HasChildQueryBuilder.DEFAULT_MIN_CHILDREN;
+ int maxChildren = HasChildQueryBuilder.DEFAULT_MAX_CHILDREN;
+ boolean ignoreUnmapped = DEFAULT_IGNORE_UNMAPPED;
+ String queryName = null;
+ InnerHitBuilder innerHitBuilder = null;
+ String currentFieldName = null;
+ XContentParser.Token token;
+ QueryBuilder iqb = null;
+ while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+ if (token == XContentParser.Token.FIELD_NAME) {
+ currentFieldName = parser.currentName();
+ } else if (token == XContentParser.Token.START_OBJECT) {
+ if (QUERY_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
+ iqb = parseInnerQueryBuilder(parser);
+ } else if (INNER_HITS_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
+ innerHitBuilder = InnerHitBuilder.fromXContent(parser);
+ } else {
+ throw new ParsingException(parser.getTokenLocation(), "[has_child] query does not support [" + currentFieldName + "]");
+ }
+ } else if (token.isValue()) {
+ if (TYPE_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
+ childType = parser.text();
+ } else if (SCORE_MODE_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
+ scoreMode = NestedQueryBuilder.parseScoreMode(parser.text());
+ } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
+ boost = parser.floatValue();
+ } else if (MIN_CHILDREN_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
+ minChildren = parser.intValue(true);
+ } else if (MAX_CHILDREN_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
+ maxChildren = parser.intValue(true);
+ } else if (IGNORE_UNMAPPED_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
+ ignoreUnmapped = parser.booleanValue();
+ } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
+ queryName = parser.text();
+ } else {
+ throw new ParsingException(parser.getTokenLocation(), "[has_child] query does not support [" + currentFieldName + "]");
+ }
+ }
+ }
+ HasChildQueryBuilder hasChildQueryBuilder = new HasChildQueryBuilder(childType, iqb, scoreMode);
+ hasChildQueryBuilder.minMaxChildren(minChildren, maxChildren);
+ hasChildQueryBuilder.queryName(queryName);
+ hasChildQueryBuilder.boost(boost);
+ hasChildQueryBuilder.ignoreUnmapped(ignoreUnmapped);
+ if (innerHitBuilder != null) {
+ hasChildQueryBuilder.innerHit(innerHitBuilder);
+ }
+ return hasChildQueryBuilder;
+ }
+
+ @Override
+ public String getWriteableName() {
+ return NAME;
+ }
+
+ @Override
+ protected Query doToQuery(SearchExecutionContext context) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ protected boolean doEquals(HasChildQueryBuilder that) {
+ return Objects.equals(query, that.query)
+ && Objects.equals(type, that.type)
+ && Objects.equals(scoreMode, that.scoreMode)
+ && Objects.equals(minChildren, that.minChildren)
+ && Objects.equals(maxChildren, that.maxChildren)
+ && Objects.equals(innerHitBuilder, that.innerHitBuilder)
+ && Objects.equals(ignoreUnmapped, that.ignoreUnmapped);
+ }
+
+ @Override
+ protected int doHashCode() {
+ return Objects.hash(query, type, scoreMode, minChildren, maxChildren, innerHitBuilder, ignoreUnmapped);
+ }
+
+ @Override
+ protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws IOException {
+ QueryBuilder rewrittenQuery = query.rewrite(queryRewriteContext);
+ if (rewrittenQuery != query) {
+ HasChildQueryBuilder hasChildQueryBuilder = new HasChildQueryBuilder(
+ type,
+ rewrittenQuery,
+ minChildren,
+ maxChildren,
+ scoreMode,
+ innerHitBuilder
+ );
+ hasChildQueryBuilder.ignoreUnmapped(ignoreUnmapped);
+ return hasChildQueryBuilder;
+ }
+ return this;
+ }
+
+ @Override
+ protected void extractInnerHitBuilders(Map innerHits) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public TransportVersion getMinimalSupportedVersion() {
+ return TransportVersions.ZERO;
+ }
+}
diff --git a/src/main/java/org/elasticsearch/plugin/nlpcn/ActionRequestRestExecuter.java b/src/main/java/org/elasticsearch/plugin/nlpcn/ActionRequestRestExecuter.java
new file mode 100644
index 000000000..f73841fd5
--- /dev/null
+++ b/src/main/java/org/elasticsearch/plugin/nlpcn/ActionRequestRestExecuter.java
@@ -0,0 +1,62 @@
+package org.elasticsearch.plugin.nlpcn;
+
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.rest.RestChannel;
+import org.elasticsearch.rest.action.RestRefCountedChunkedToXContentListener;
+import org.nlpcn.es4sql.exception.SqlParseException;
+import org.nlpcn.es4sql.query.SqlElasticDeleteByQueryRequestBuilder;
+import org.nlpcn.es4sql.query.SqlElasticRequestBuilder;
+import org.nlpcn.es4sql.query.join.JoinRequestBuilder;
+
+import java.io.IOException;
+
+
+public class ActionRequestRestExecuter {
+
+ private RestChannel channel;
+ private Client client;
+ private SqlElasticRequestBuilder requestBuilder;
+
+ public ActionRequestRestExecuter(SqlElasticRequestBuilder requestBuilder, RestChannel channel, final Client client) {
+ this.requestBuilder = requestBuilder;
+ this.channel = channel;
+ this.client = client;
+ }
+
+
+
+ /**
+ * Execute the ActionRequest and returns the REST response using the channel.
+ */
+ public void execute() throws Exception {
+ ActionRequest request = requestBuilder.request();
+
+ //todo: maby change to instanceof multi?
+ if(requestBuilder instanceof JoinRequestBuilder){
+ executeJoinRequestAndSendResponse();
+ }
+ else if (request instanceof SearchRequest) {
+ client.search((SearchRequest) request, new RestRefCountedChunkedToXContentListener<>(channel));
+ } else if (requestBuilder instanceof SqlElasticDeleteByQueryRequestBuilder) {
+ throw new UnsupportedOperationException("currently not support delete on elastic 2.0.0");
+ }
+ else if(request instanceof GetIndexRequest) {
+ this.requestBuilder.getBuilder().execute( new GetIndexRequestRestListener(channel, (GetIndexRequest) request));
+ }
+
+
+ else {
+ throw new Exception(String.format("Unsupported ActionRequest provided: %s", request.getClass().getName()));
+ }
+ }
+
+ private void executeJoinRequestAndSendResponse() throws IOException, SqlParseException {
+ ElasticJoinExecutor executor = ElasticJoinExecutor.createJoinExecutor(client,requestBuilder);
+ executor.run();
+ executor.sendResponse(channel);
+ }
+
+}
diff --git a/src/main/java/org/elasticsearch/plugin/nlpcn/ComperableHitResult.java b/src/main/java/org/elasticsearch/plugin/nlpcn/ComperableHitResult.java
new file mode 100644
index 000000000..aa3b3da94
--- /dev/null
+++ b/src/main/java/org/elasticsearch/plugin/nlpcn/ComperableHitResult.java
@@ -0,0 +1,75 @@
+package org.elasticsearch.plugin.nlpcn;
+
+import com.google.common.base.Joiner;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.lookup.Source;
+import org.nlpcn.es4sql.Util;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Created by Eliran on 9/9/2016.
+ */
+public class ComperableHitResult {
+ private SearchHit hit;
+ private String comperator;
+ private boolean isAllNull;
+ private Map flattenMap;
+ public ComperableHitResult(SearchHit hit , String[] fieldsOrder ,String seperator) {
+ this.hit = hit;
+ Map hitAsMap = Source.fromBytes(hit.getSourceRef()).source();
+ this.flattenMap = new HashMap<>();
+ List results = new ArrayList<>();
+ this.isAllNull = true;
+
+ for(int i = 0 ; i< fieldsOrder.length ;i++){
+ String field = fieldsOrder[i];
+ Object result = Util.deepSearchInMap(hitAsMap,field);
+ if(result == null){
+ results.add("");
+ }
+ else {
+ this.isAllNull = false;
+ results.add(result.toString());
+ this.flattenMap.put(field,result);
+ }
+ }
+ this.comperator = Joiner.on(seperator).join(results);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ ComperableHitResult that = (ComperableHitResult) o;
+
+ if (!comperator.equals(that.comperator)) return false;
+
+ return true;
+ }
+
+ public boolean isAllNull() {
+ return isAllNull;
+ }
+
+ @Override
+ public int hashCode() {
+ return comperator.hashCode();
+ }
+
+ public String getComperator() {
+ return comperator;
+ }
+
+ public Map getFlattenMap() {
+ return flattenMap;
+ }
+
+ public SearchHit getOriginalHit(){
+ return hit;
+ }
+}
diff --git a/src/main/java/org/elasticsearch/plugin/nlpcn/DeleteByQueryRestListener.java b/src/main/java/org/elasticsearch/plugin/nlpcn/DeleteByQueryRestListener.java
deleted file mode 100644
index d5dcae133..000000000
--- a/src/main/java/org/elasticsearch/plugin/nlpcn/DeleteByQueryRestListener.java
+++ /dev/null
@@ -1,65 +0,0 @@
-package org.elasticsearch.plugin.nlpcn;
-
-import org.elasticsearch.action.ShardOperationFailedException;
-import org.elasticsearch.action.deletebyquery.DeleteByQueryResponse;
-import org.elasticsearch.action.deletebyquery.IndexDeleteByQueryResponse;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentBuilderString;
-import org.elasticsearch.rest.BytesRestResponse;
-import org.elasticsearch.rest.RestChannel;
-import org.elasticsearch.rest.RestResponse;
-import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.rest.action.support.RestBuilderListener;
-
-
-public class DeleteByQueryRestListener extends RestBuilderListener {
-
- public DeleteByQueryRestListener(RestChannel channel) {
- super(channel);
- }
-
- @Override
- public RestResponse buildResponse(DeleteByQueryResponse result, XContentBuilder builder) throws Exception {
- RestStatus restStatus = result.status();
- builder.startObject();
- builder.startObject(Fields._INDICES);
- for (IndexDeleteByQueryResponse indexDeleteByQueryResponse : result.getIndices().values()) {
- builder.startObject(indexDeleteByQueryResponse.getIndex(), XContentBuilder.FieldCaseConversion.NONE);
-
- builder.startObject(Fields._SHARDS);
- builder.field(Fields.TOTAL, indexDeleteByQueryResponse.getTotalShards());
- builder.field(Fields.SUCCESSFUL, indexDeleteByQueryResponse.getSuccessfulShards());
- builder.field(Fields.FAILED, indexDeleteByQueryResponse.getFailedShards());
- ShardOperationFailedException[] failures = indexDeleteByQueryResponse.getFailures();
- if (failures != null && failures.length > 0) {
- builder.startArray(Fields.FAILURES);
- for (ShardOperationFailedException shardFailure : failures) {
- builder.startObject();
- builder.field(Fields.INDEX, shardFailure.index());
- builder.field(Fields.SHARD, shardFailure.shardId());
- builder.field(Fields.REASON, shardFailure.reason());
- builder.endObject();
- }
- builder.endArray();
- }
- builder.endObject();
-
- builder.endObject();
- }
- builder.endObject();
- builder.endObject();
- return new BytesRestResponse(restStatus, builder);
- }
-
- static final class Fields {
- static final XContentBuilderString _INDICES = new XContentBuilderString("_indices");
- static final XContentBuilderString _SHARDS = new XContentBuilderString("_shards");
- static final XContentBuilderString TOTAL = new XContentBuilderString("total");
- static final XContentBuilderString SUCCESSFUL = new XContentBuilderString("successful");
- static final XContentBuilderString FAILED = new XContentBuilderString("failed");
- static final XContentBuilderString FAILURES = new XContentBuilderString("failures");
- static final XContentBuilderString INDEX = new XContentBuilderString("index");
- static final XContentBuilderString SHARD = new XContentBuilderString("shard");
- static final XContentBuilderString REASON = new XContentBuilderString("reason");
- }
-}
diff --git a/src/main/java/org/elasticsearch/plugin/nlpcn/ElasticHitsExecutor.java b/src/main/java/org/elasticsearch/plugin/nlpcn/ElasticHitsExecutor.java
new file mode 100644
index 000000000..75de36e3d
--- /dev/null
+++ b/src/main/java/org/elasticsearch/plugin/nlpcn/ElasticHitsExecutor.java
@@ -0,0 +1,14 @@
+package org.elasticsearch.plugin.nlpcn;
+
+import org.elasticsearch.search.SearchHits;
+import org.nlpcn.es4sql.exception.SqlParseException;
+
+import java.io.IOException;
+
+/**
+ * Created by Eliran on 21/8/2016.
+ */
+public interface ElasticHitsExecutor {
+ void run() throws IOException, SqlParseException ;
+ SearchHits getHits();
+}
diff --git a/src/main/java/org/elasticsearch/plugin/nlpcn/ElasticJoinExecutor.java b/src/main/java/org/elasticsearch/plugin/nlpcn/ElasticJoinExecutor.java
index 270542c38..87dc7880f 100644
--- a/src/main/java/org/elasticsearch/plugin/nlpcn/ElasticJoinExecutor.java
+++ b/src/main/java/org/elasticsearch/plugin/nlpcn/ElasticJoinExecutor.java
@@ -1,23 +1,19 @@
package org.elasticsearch.plugin.nlpcn;
-import com.google.common.collect.ImmutableMap;
+import org.apache.lucene.search.TotalHits;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.action.search.SearchType;
-import org.elasticsearch.client.Client;
-import org.elasticsearch.common.text.StringText;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentType;
-
-import org.elasticsearch.rest.BytesRestResponse;
+import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.rest.RestResponse;
+import org.elasticsearch.search.lookup.Source;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.core.TimeValue;
import org.elasticsearch.rest.RestChannel;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
-import org.elasticsearch.search.internal.InternalSearchHit;
-import org.elasticsearch.search.internal.InternalSearchHits;
+import org.elasticsearch.search.sort.FieldSortBuilder;
+import org.elasticsearch.search.sort.SortOrder;
import org.nlpcn.es4sql.domain.Field;
import org.nlpcn.es4sql.exception.SqlParseException;
import org.nlpcn.es4sql.query.SqlElasticRequestBuilder;
@@ -27,12 +23,18 @@
import org.nlpcn.es4sql.query.join.TableInJoinRequestBuilder;
import java.io.IOException;
-import java.util.*;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
/**
* Created by Eliran on 15/9/2015.
*/
-public abstract class ElasticJoinExecutor {
+public abstract class ElasticJoinExecutor implements ElasticHitsExecutor {
protected SearchHits results ;
protected MetaSearchResult metaResults;
protected final int MAX_RESULTS_ON_ONE_FETCH = 10000;
@@ -48,60 +50,27 @@ protected ElasticJoinExecutor(JoinRequestBuilder requestBuilder) {
&& (secondTableReturnedField == null || secondTableReturnedField.size() == 0);
}
- public void sendResponse(RestChannel channel){
- try {
- String json = resultAsString();
- BytesRestResponse bytesRestResponse = new BytesRestResponse(RestStatus.OK, json);
- channel.sendResponse(bytesRestResponse);
- } catch (IOException e) {
- e.printStackTrace();
- }
+ public void sendResponse(RestChannel channel) throws IOException {
+ XContentBuilder builder = ElasticUtils.hitsAsXContentBuilder(results, metaResults);
+ RestResponse bytesRestResponse = new RestResponse(RestStatus.OK, builder);
+ channel.sendResponse(bytesRestResponse);
}
+ @Override
public void run() throws IOException, SqlParseException {
long timeBefore = System.currentTimeMillis();
- List combinedSearchHits = innerRun();
+ List combinedSearchHits = innerRun();
int resultsSize = combinedSearchHits.size();
- InternalSearchHit[] hits = combinedSearchHits.toArray(new InternalSearchHit[resultsSize]);
- this.results = new InternalSearchHits(hits, resultsSize,1.0f);
+ SearchHit[] hits = combinedSearchHits.toArray(new SearchHit[resultsSize]);
+ this.results = SearchHits.unpooled(hits, new TotalHits(resultsSize, TotalHits.Relation.EQUAL_TO), 1.0f);
long joinTimeInMilli = System.currentTimeMillis() - timeBefore;
this.metaResults.setTookImMilli(joinTimeInMilli);
}
- //use our deserializer instead of results toXcontent because the source field is differnet from sourceAsMap.
- public String resultAsString() throws IOException {
- if(this.results == null) return null;
- Object[] searchHits;
- searchHits = new Object[(int) this.results.totalHits()];
- int i = 0;
- for(SearchHit hit : this.results) {
- HashMap value = new HashMap<>();
- value.put("_id",hit.getId());
- value.put("_type", hit.getType());
- value.put("_score", hit.score());
- value.put("_source", hit.sourceAsMap());
- searchHits[i] = value;
- i++;
- }
- HashMap hits = new HashMap<>();
- hits.put("total",this.results.totalHits());
- hits.put("max_score",this.results.maxScore());
- hits.put("hits",searchHits);
- XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON).prettyPrint();
- builder.startObject();
- builder.field("took", metaResults.getTookImMilli());
- builder.field("timed_out",metaResults.isTimedOut());
- builder.field("_shards", ImmutableMap.of("total", metaResults.getTotalNumOfShards(),
- "successful", metaResults.getSuccessfulShards()
- , "failed", metaResults.getFailedShards()));
- builder.field("hits",hits) ;
- builder.endObject();
-
- return builder.string();
- }
- protected abstract List innerRun() throws IOException, SqlParseException ;
+ protected abstract List innerRun() throws IOException, SqlParseException ;
+ @Override
public SearchHits getHits(){
return results;
}
@@ -120,11 +89,11 @@ else if (requestBuilder instanceof NestedLoopsElasticRequestBuilder){
}
}
- protected void mergeSourceAndAddAliases(Map secondTableHitSource, InternalSearchHit searchHit,String t1Alias,String t2Alias) {
- Map results = mapWithAliases(searchHit.getSource(), t1Alias);
+ protected void mergeSourceAndAddAliases(Map secondTableHitSource, Map hitSource, String t1Alias, String t2Alias) {
+ Map results = mapWithAliases(hitSource, t1Alias);
results.putAll(mapWithAliases(secondTableHitSource, t2Alias));
- searchHit.getSource().clear();
- searchHit.getSource().putAll(results);
+ hitSource.clear();
+ hitSource.putAll(results);
}
protected Map mapWithAliases(Map source, String alias) {
@@ -175,14 +144,14 @@ protected Object deepSearchInMap(Map fieldsMap, String name) {
}
- protected void addUnmatchedResults(List combinedResults, Collection firstTableSearchHits, List secondTableReturnedFields,int currentNumOfIds, int totalLimit,String t1Alias,String t2Alias) {
+ protected void addUnmatchedResults(List combinedResults, Collection firstTableSearchHits, List secondTableReturnedFields,int currentNumOfIds, int totalLimit,String t1Alias,String t2Alias) {
boolean limitReached = false;
for(SearchHitsResult hitsResult : firstTableSearchHits){
if(!hitsResult.isMatchedWithOtherTable())
- for (InternalSearchHit hit : hitsResult.getSearchHits()) {
+ for (SearchHit hit : hitsResult.getSearchHits()) {
//todo: decide which id to put or type. or maby its ok this way. just need to doc.
- InternalSearchHit unmachedResult = createUnmachedResult(secondTableReturnedFields, hit.docId(), t1Alias, t2Alias, hit);
+ SearchHit unmachedResult = createUnmachedResult(secondTableReturnedFields, hit.docId(), t1Alias, t2Alias, hit);
combinedResults.add(unmachedResult);
currentNumOfIds++;
if (currentNumOfIds >= totalLimit) {
@@ -195,18 +164,20 @@ protected void addUnmatchedResults(List combinedResults, Coll
}
}
- protected InternalSearchHit createUnmachedResult( List secondTableReturnedFields, int docId, String t1Alias, String t2Alias, SearchHit hit) {
- String unmatchedId = hit.id() + "|0";
- StringText unamatchedType = new StringText(hit.getType() + "|null");
-
- InternalSearchHit searchHit = new InternalSearchHit(docId, unmatchedId, unamatchedType, hit.getFields());
+ protected SearchHit createUnmachedResult( List secondTableReturnedFields, int docId, String t1Alias, String t2Alias, SearchHit hit) {
+ String unmatchedId = hit.getId() + "|0";
+ SearchHit searchHit = SearchHit.unpooled(docId, unmatchedId);
+ searchHit.addDocumentFields(hit.getDocumentFields(), Collections.emptyMap());
searchHit.sourceRef(hit.getSourceRef());
- searchHit.sourceAsMap().clear();
- searchHit.sourceAsMap().putAll(hit.sourceAsMap());
+ Source source = Source.fromBytes(searchHit.getSourceRef());
+ Map hitSource = source.source();
+ hitSource.clear();
+ hitSource.putAll(Source.fromBytes(hit.getSourceRef()).source());
Map emptySecondTableHitSource = createNullsSource(secondTableReturnedFields);
- mergeSourceAndAddAliases(emptySecondTableHitSource, searchHit,t1Alias,t2Alias);
+ mergeSourceAndAddAliases(emptySecondTableHitSource, hitSource, t1Alias,t2Alias);
+ searchHit.sourceRef(Source.fromMap(hitSource, source.sourceContentType()).internalSourceRef());
return searchHit;
}
@@ -233,11 +204,12 @@ protected SearchResponse scrollOneTimeWithMax(Client client,TableInJoinRequestBu
.setScroll(new TimeValue(60000))
.setSize(MAX_RESULTS_ON_ONE_FETCH);
boolean ordered = tableRequest.getOriginalSelect().isOrderdSelect();
- if(!ordered) scrollRequest.setSearchType(SearchType.SCAN);
+ if(!ordered) scrollRequest.addSort(FieldSortBuilder.DOC_FIELD_NAME, SortOrder.ASC);
responseWithHits = scrollRequest.get();
//on ordered select - not using SCAN , elastic returns hits on first scroll
- if(!ordered)
- responseWithHits = client.prepareSearchScroll(responseWithHits.getScrollId()).setScroll(new TimeValue(600000)).get();
+ //es5.0 elastic always return docs on scan
+// if(!ordered)
+// responseWithHits = client.prepareSearchScroll(responseWithHits.getScrollId()).setScroll(new TimeValue(600000)).get();
return responseWithHits;
}
diff --git a/src/main/java/org/elasticsearch/plugin/nlpcn/ElasticResultHandler.java b/src/main/java/org/elasticsearch/plugin/nlpcn/ElasticResultHandler.java
index 18b48158b..33ea7a54f 100644
--- a/src/main/java/org/elasticsearch/plugin/nlpcn/ElasticResultHandler.java
+++ b/src/main/java/org/elasticsearch/plugin/nlpcn/ElasticResultHandler.java
@@ -1,13 +1,8 @@
package org.elasticsearch.plugin.nlpcn;
import org.elasticsearch.search.SearchHit;
-import org.elasticsearch.search.aggregations.Aggregation;
-import org.elasticsearch.search.aggregations.Aggregations;
-import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
+import org.elasticsearch.search.lookup.Source;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
import java.util.Map;
/**
@@ -15,7 +10,7 @@
*/
public class ElasticResultHandler {
public static Object getFieldValue(SearchHit hit,String field){
- return deepSearchInMap(hit.sourceAsMap(),field);
+ return deepSearchInMap(Source.fromBytes(hit.getSourceRef()).source(),field);
}
private static Object deepSearchInMap(Map fieldsMap, String name) {
diff --git a/src/main/java/org/elasticsearch/plugin/nlpcn/ElasticUtils.java b/src/main/java/org/elasticsearch/plugin/nlpcn/ElasticUtils.java
new file mode 100644
index 000000000..5546a15c0
--- /dev/null
+++ b/src/main/java/org/elasticsearch/plugin/nlpcn/ElasticUtils.java
@@ -0,0 +1,74 @@
+package org.elasticsearch.plugin.nlpcn;
+
+import com.google.common.collect.ImmutableMap;
+import org.apache.lucene.search.TotalHits;
+import org.elasticsearch.action.search.SearchRequestBuilder;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.search.lookup.Source;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentFactory;
+import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.SearchHits;
+import org.elasticsearch.search.sort.FieldSortBuilder;
+import org.elasticsearch.search.sort.SortOrder;
+import org.nlpcn.es4sql.domain.Select;
+
+import java.io.IOException;
+import java.util.HashMap;
+
+/**
+ * Created by Eliran on 2/9/2016.
+ */
+public class ElasticUtils {
+
+ public static SearchResponse scrollOneTimeWithHits(Client client, SearchRequestBuilder requestBuilder, Select originalSelect, int resultSize) {
+ SearchResponse responseWithHits;SearchRequestBuilder scrollRequest = requestBuilder
+ .setScroll(new TimeValue(60000))
+ .setSize(resultSize);
+ boolean ordered = originalSelect.isOrderdSelect();
+ if(!ordered) scrollRequest.addSort(FieldSortBuilder.DOC_FIELD_NAME, SortOrder.ASC);
+ responseWithHits = scrollRequest.get();
+ //on ordered select - not using SCAN , elastic returns hits on first scroll
+ //es5.0 elastic always return docs on scan
+// if(!ordered) {
+// responseWithHits = client.prepareSearchScroll(responseWithHits.getScrollId()).setScroll(new TimeValue(600000)).get();
+// }
+ return responseWithHits;
+ }
+
+
+ //use our deserializer instead of results toXcontent because the source field is differnet from sourceAsMap.
+ public static XContentBuilder hitsAsXContentBuilder(SearchHits results, MetaSearchResult metaResults) throws IOException {
+ if(results == null) return null;
+ Object[] searchHits;
+ searchHits = new Object[(int) results.getTotalHits().value()];
+ int i = 0;
+ for(SearchHit hit : results) {
+ HashMap value = new HashMap<>();
+ value.put("_id",hit.getId());
+ value.put("_score", hit.getScore());
+ value.put("_source", Source.fromBytes(hit.getSourceRef()).source());
+ searchHits[i] = value;
+ i++;
+ }
+ HashMap hits = new HashMap<>();
+ TotalHits totalHits = results.getTotalHits();
+ hits.put("total", ImmutableMap.of("value", totalHits.value(),
+ "relation", totalHits.relation() == TotalHits.Relation.EQUAL_TO ? "eq" : "gte"));
+ hits.put("max_score",results.getMaxScore());
+ hits.put("hits",searchHits);
+ XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON).prettyPrint();
+ builder.startObject();
+ builder.field("took", metaResults.getTookImMilli());
+ builder.field("timed_out",metaResults.isTimedOut());
+ builder.field("_shards", ImmutableMap.of("total", metaResults.getTotalNumOfShards(),
+ "successful", metaResults.getSuccessfulShards()
+ , "failed", metaResults.getFailedShards()));
+ builder.field("hits",hits) ;
+ builder.endObject();
+ return builder;
+ }
+}
diff --git a/src/main/java/org/elasticsearch/plugin/nlpcn/GetIndexRequestRestListener.java b/src/main/java/org/elasticsearch/plugin/nlpcn/GetIndexRequestRestListener.java
index a08b18869..d0cbec317 100644
--- a/src/main/java/org/elasticsearch/plugin/nlpcn/GetIndexRequestRestListener.java
+++ b/src/main/java/org/elasticsearch/plugin/nlpcn/GetIndexRequestRestListener.java
@@ -2,24 +2,19 @@
import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
-import org.elasticsearch.action.deletebyquery.DeleteByQueryResponse;
-import org.elasticsearch.cluster.metadata.AliasMetaData;
-import org.elasticsearch.cluster.metadata.MappingMetaData;
-import org.elasticsearch.common.collect.ImmutableOpenMap;
-import org.elasticsearch.common.hppc.cursors.ObjectObjectCursor;
+import org.elasticsearch.cluster.metadata.AliasMetadata;
+
import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentBuilderString;
-import org.elasticsearch.rest.BytesRestResponse;
+import org.elasticsearch.xcontent.ToXContent;
+import org.elasticsearch.xcontent.XContentBuilder;
import org.elasticsearch.rest.RestChannel;
import org.elasticsearch.rest.RestResponse;
import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.rest.action.support.RestBuilderListener;
-import org.elasticsearch.search.warmer.IndexWarmersMetaData;
+import org.elasticsearch.rest.action.RestBuilderListener;
import java.io.IOException;
import java.util.List;
+import java.util.Map;
/**
* Created by Eliran on 6/10/2015.
@@ -35,7 +30,7 @@ public GetIndexRequestRestListener(RestChannel channel,GetIndexRequest getIndexR
@Override
public RestResponse buildResponse(GetIndexResponse getIndexResponse, XContentBuilder builder) throws Exception {
- GetIndexRequest.Feature[] features = getIndexRequest.featuresAsEnums();
+ GetIndexRequest.Feature[] features = getIndexRequest.features();
String[] indices = getIndexResponse.indices();
builder.startObject();
@@ -47,14 +42,11 @@ public RestResponse buildResponse(GetIndexResponse getIndexResponse, XContentBui
writeAliases(getIndexResponse.aliases().get(index), builder, channel.request());
break;
case MAPPINGS:
- writeMappings(getIndexResponse.mappings().get(index), builder, channel.request());
+ writeMappings((Map) getIndexResponse.mappings().get(index).rawSourceAsMap(), builder, channel.request());
break;
case SETTINGS:
writeSettings(getIndexResponse.settings().get(index), builder, channel.request());
break;
- case WARMERS:
- writeWarmers(getIndexResponse.warmers().get(index), builder, channel.request());
- break;
default:
throw new IllegalStateException("feature [" + feature + "] is not valid");
}
@@ -64,24 +56,24 @@ public RestResponse buildResponse(GetIndexResponse getIndexResponse, XContentBui
}
builder.endObject();
- return new BytesRestResponse(RestStatus.OK, builder);
+ return new RestResponse(RestStatus.OK, builder);
}
- private void writeAliases(List aliases, XContentBuilder builder, ToXContent.Params params) throws IOException {
+ private void writeAliases(List aliases, XContentBuilder builder, ToXContent.Params params) throws IOException {
builder.startObject(Fields.ALIASES);
if (aliases != null) {
- for (AliasMetaData alias : aliases) {
- AliasMetaData.Builder.toXContent(alias, builder, params);
+ for (AliasMetadata alias : aliases) {
+ AliasMetadata.Builder.toXContent(alias, builder, params);
}
}
builder.endObject();
}
- private void writeMappings(ImmutableOpenMap mappings, XContentBuilder builder, ToXContent.Params params) throws IOException {
+ private void writeMappings(Map> mappings, XContentBuilder builder, ToXContent.Params params) throws IOException {
builder.startObject(Fields.MAPPINGS);
if (mappings != null) {
- for (ObjectObjectCursor typeEntry : mappings) {
- builder.field(typeEntry.key);
- builder.map(typeEntry.value.sourceAsMap());
+ for (Map.Entry> typeEntry : mappings.entrySet()) {
+ builder.field(typeEntry.getKey());
+ builder.map(typeEntry.getValue());
}
}
builder.endObject();
@@ -93,20 +85,11 @@ private void writeSettings(Settings settings, XContentBuilder builder, ToXConten
builder.endObject();
}
- private void writeWarmers(List warmers, XContentBuilder builder, ToXContent.Params params) throws IOException {
- builder.startObject(Fields.WARMERS);
- if (warmers != null) {
- for (IndexWarmersMetaData.Entry warmer : warmers) {
- IndexWarmersMetaData.FACTORY.toXContent(warmer, builder, params);
- }
- }
- builder.endObject();
- }
static class Fields {
- static final XContentBuilderString ALIASES = new XContentBuilderString("aliases");
- static final XContentBuilderString MAPPINGS = new XContentBuilderString("mappings");
- static final XContentBuilderString SETTINGS = new XContentBuilderString("settings");
- static final XContentBuilderString WARMERS = new XContentBuilderString("warmers");
+ static final String ALIASES = "aliases";
+ static final String MAPPINGS = "mappings";
+ static final String SETTINGS = "settings";
+ static final String WARMERS = "warmers";
}
}
\ No newline at end of file
diff --git a/src/main/java/org/elasticsearch/plugin/nlpcn/HashJoinComparisonStructure.java b/src/main/java/org/elasticsearch/plugin/nlpcn/HashJoinComparisonStructure.java
index 239e27374..98edfface 100644
--- a/src/main/java/org/elasticsearch/plugin/nlpcn/HashJoinComparisonStructure.java
+++ b/src/main/java/org/elasticsearch/plugin/nlpcn/HashJoinComparisonStructure.java
@@ -1,9 +1,13 @@
package org.elasticsearch.plugin.nlpcn;
-import org.elasticsearch.search.internal.InternalSearchHit;
+import org.elasticsearch.search.SearchHit;
import org.nlpcn.es4sql.domain.Field;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
/**
* Created by Eliran on 2/11/2015.
@@ -32,11 +36,11 @@ public HashMap>> getComparisons() {
return comparisonIDtoComparisonFields;
}
- public void insertIntoComparisonHash(String comparisonID,String comparisonKey,InternalSearchHit hit){
+ public void insertIntoComparisonHash(String comparisonID,String comparisonKey,SearchHit hit){
HashMap comparisonHash = this.comparisonIDtoComparisonHash.get(comparisonID);
SearchHitsResult currentSearchHitsResult = comparisonHash.get(comparisonKey);
if(currentSearchHitsResult == null) {
- currentSearchHitsResult = new SearchHitsResult(new ArrayList(),false);
+ currentSearchHitsResult = new SearchHitsResult(new ArrayList(),false);
comparisonHash.put(comparisonKey, currentSearchHitsResult);
}
currentSearchHitsResult.getSearchHits().add(hit);
diff --git a/src/main/java/org/elasticsearch/plugin/nlpcn/HashJoinElasticExecutor.java b/src/main/java/org/elasticsearch/plugin/nlpcn/HashJoinElasticExecutor.java
index dd4f52840..32b704112 100644
--- a/src/main/java/org/elasticsearch/plugin/nlpcn/HashJoinElasticExecutor.java
+++ b/src/main/java/org/elasticsearch/plugin/nlpcn/HashJoinElasticExecutor.java
@@ -3,27 +3,30 @@
import com.alibaba.druid.sql.ast.statement.SQLJoinTableSource;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.action.search.SearchType;
-import org.elasticsearch.client.Client;
-import org.elasticsearch.common.text.StringText;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.index.query.BoolFilterBuilder;
+import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.core.TimeValue;
import org.elasticsearch.index.query.BoolQueryBuilder;
-import org.elasticsearch.index.query.FilterBuilders;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit;
-import org.elasticsearch.search.internal.InternalSearchHit;
+import org.elasticsearch.search.lookup.Source;
import org.nlpcn.es4sql.domain.Field;
import org.nlpcn.es4sql.domain.Select;
import org.nlpcn.es4sql.domain.Where;
import org.nlpcn.es4sql.exception.SqlParseException;
import org.nlpcn.es4sql.query.join.HashJoinElasticRequestBuilder;
import org.nlpcn.es4sql.query.join.TableInJoinRequestBuilder;
-import org.nlpcn.es4sql.query.maker.FilterMaker;
import org.nlpcn.es4sql.query.maker.QueryMaker;
import java.io.IOException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
/**
* Created by Eliran on 22/8/2015.
@@ -47,7 +50,8 @@ public HashJoinElasticExecutor(Client client, HashJoinElasticRequestBuilder requ
this.alreadyMatched = new HashSet<>();
}
- public List innerRun() throws IOException, SqlParseException {
+ @Override
+ public List innerRun() throws IOException, SqlParseException {
Map>> optimizationTermsFilterStructure =
initOptimizationStructure();
@@ -61,7 +65,7 @@ public List innerRun() throws IOException, SqlParseException
updateRequestWithTermsFilter(optimizationTermsFilterStructure, secondTableRequest);
}
- List combinedResult = createCombinedResults(secondTableRequest);
+ List combinedResult = createCombinedResults(secondTableRequest);
int currentNumOfResults = combinedResult.size();
int totalLimit = requestBuilder.getTotalLimit();
@@ -76,9 +80,9 @@ public List innerRun() throws IOException, SqlParseException
t2Alias);
}
if(firstTableRequest.getOriginalSelect().isOrderdSelect()){
- Collections.sort(combinedResult,new Comparator() {
+ Collections.sort(combinedResult,new Comparator() {
@Override
- public int compare(InternalSearchHit o1, InternalSearchHit o2) {
+ public int compare(SearchHit o1, SearchHit o2) {
return o1.docId() - o2.docId();
}
});
@@ -105,8 +109,8 @@ private void updateFirstTableLimitIfNeeded() {
}
}
- private List createCombinedResults( TableInJoinRequestBuilder secondTableRequest) {
- List combinedResult = new ArrayList<>();
+ private List createCombinedResults( TableInJoinRequestBuilder secondTableRequest) {
+ List combinedResult = new ArrayList<>();
int resultIds = 0;
int totalLimit = this.requestBuilder.getTotalLimit();
Integer hintLimit = secondTableRequest.getHintLimit();
@@ -117,10 +121,10 @@ private List createCombinedResults( TableInJoinRequestBuilder
finishedScrolling = true;
} else {
searchResponse = secondTableRequest.getRequestBuilder()
- .setSearchType(SearchType.SCAN)
.setScroll(new TimeValue(60000))
.setSize(MAX_RESULTS_ON_ONE_FETCH).get();
- searchResponse = client.prepareSearchScroll(searchResponse.getScrollId()).setScroll(new TimeValue(600000)).get();
+ //es5.0 no need to scroll again!
+// searchResponse = client.prepareSearchScroll(searchResponse.getScrollId()).setScroll(new TimeValue(600000)).get();
finishedScrolling = false;
}
updateMetaSearchResults(searchResponse);
@@ -134,19 +138,19 @@ private List createCombinedResults( TableInJoinRequestBuilder
if (limitReached) break;
//todo: need to run on comparisons. for each comparison check if exists and add.
HashMap>> comparisons = this.hashJoinComparisonStructure.getComparisons();
-
+ Map secondTableHitSource = Source.fromBytes(secondTableHit.getSourceRef()).source();
for (Map.Entry>> comparison : comparisons.entrySet()) {
String comparisonID = comparison.getKey();
List> t1ToT2FieldsComparison = comparison.getValue();
- String key = getComparisonKey(t1ToT2FieldsComparison, secondTableHit, false, null);
+ String key = getComparisonKey(t1ToT2FieldsComparison, secondTableHitSource, false, null);
SearchHitsResult searchHitsResult = this.hashJoinComparisonStructure.searchForMatchingSearchHits(comparisonID, key);
if (searchHitsResult != null && searchHitsResult.getSearchHits().size() > 0) {
searchHitsResult.setMatchedWithOtherTable(true);
- List searchHits = searchHitsResult.getSearchHits();
- for (InternalSearchHit matchingHit : searchHits) {
- String combinedId = matchingHit.id() + "|" + secondTableHit.getId();
+ List searchHits = searchHitsResult.getSearchHits();
+ for (SearchHit matchingHit : searchHits) {
+ String combinedId = matchingHit.getId() + "|" + secondTableHit.getId();
//in order to prevent same matching when using OR on hashJoins.
if(this.alreadyMatched.contains(combinedId)){
continue;
@@ -156,18 +160,22 @@ private List createCombinedResults( TableInJoinRequestBuilder
}
Map copiedSource = new HashMap();
- copyMaps(copiedSource,secondTableHit.sourceAsMap());
+ copyMaps(copiedSource, secondTableHitSource);
onlyReturnedFields(copiedSource, secondTableRequest.getReturnedFields(),secondTableRequest.getOriginalSelect().isSelectAll());
- InternalSearchHit searchHit = new InternalSearchHit(matchingHit.docId(), combinedId, new StringText(matchingHit.getType() + "|" + secondTableHit.getType()), matchingHit.getFields());
+ SearchHit searchHit = SearchHit.unpooled(matchingHit.docId(), combinedId);
+ searchHit.addDocumentFields(matchingHit.getDocumentFields(), Collections.emptyMap());
searchHit.sourceRef(matchingHit.getSourceRef());
- searchHit.sourceAsMap().clear();
- searchHit.sourceAsMap().putAll(matchingHit.sourceAsMap());
+ Source source = Source.fromBytes(searchHit.getSourceRef());
+ Map hitSource = source.source();
+ hitSource.clear();
+ hitSource.putAll(Source.fromBytes(matchingHit.getSourceRef()).source());
String t1Alias = requestBuilder.getFirstTable().getAlias();
String t2Alias = requestBuilder.getSecondTable().getAlias();
- mergeSourceAndAddAliases(copiedSource, searchHit, t1Alias, t2Alias);
+ mergeSourceAndAddAliases(copiedSource, hitSource, t1Alias, t2Alias);
+ searchHit.sourceRef(Source.fromMap(hitSource, source.sourceContentType()).internalSourceRef());
combinedResult.add(searchHit);
resultIds++;
@@ -201,17 +209,22 @@ private void createKeyToResultsAndFillOptimizationStructure(Map>> comparisons = this.hashJoinComparisonStructure.getComparisons();
+ Map hitSource = Source.fromBytes(hit.getSourceRef()).source();
for (Map.Entry>> comparison : comparisons.entrySet()) {
String comparisonID = comparison.getKey();
List> t1ToT2FieldsComparison = comparison.getValue();
- String key = getComparisonKey(t1ToT2FieldsComparison, hit, true, optimizationTermsFilterStructure.get(comparisonID));
+ String key = getComparisonKey(t1ToT2FieldsComparison, hitSource, true, optimizationTermsFilterStructure.get(comparisonID));
//int docid , id
- InternalSearchHit searchHit = new InternalSearchHit(resultIds, hit.id(), new StringText(hit.getType()), hit.getFields());
+ SearchHit searchHit = SearchHit.unpooled(resultIds, hit.getId());
+ searchHit.addDocumentFields(hit.getDocumentFields(), Collections.emptyMap());
searchHit.sourceRef(hit.getSourceRef());
- onlyReturnedFields(searchHit.sourceAsMap(), firstTableRequest.getReturnedFields(),firstTableRequest.getOriginalSelect().isSelectAll());
+ Source source = Source.fromBytes(searchHit.getSourceRef());
+ Map searchHitSource = source.source();
+ onlyReturnedFields(searchHitSource, firstTableRequest.getReturnedFields(),firstTableRequest.getOriginalSelect().isSelectAll());
+ searchHit.sourceRef(Source.fromMap(searchHitSource, source.sourceContentType()).internalSourceRef());
resultIds++;
this.hashJoinComparisonStructure.insertIntoComparisonHash(comparisonID, key, searchHit);
}
@@ -236,7 +249,7 @@ private List scrollTillLimit(TableInJoinRequestBuilder tableInJoinReq
updateMetaSearchResults(scrollResp);
List hitsWithScan = new ArrayList<>();
int curentNumOfResults = 0;
- SearchHit[] hits = scrollResp.getHits().hits();
+ SearchHit[] hits = scrollResp.getHits().getHits();
if (hintLimit == null) hintLimit = MAX_RESULTS_FOR_FIRST_TABLE;
@@ -269,42 +282,30 @@ private boolean needToOptimize(Map>> optimizatio
private void updateRequestWithTermsFilter(Map>> optimizationTermsFilterStructure, TableInJoinRequestBuilder secondTableRequest) throws SqlParseException {
Select select = secondTableRequest.getOriginalSelect();
- BoolFilterBuilder orFilter = FilterBuilders.boolFilter();
+
BoolQueryBuilder orQuery = QueryBuilders.boolQuery();
for(Map> optimization : optimizationTermsFilterStructure.values()) {
- BoolFilterBuilder andFilter = FilterBuilders.boolFilter();
BoolQueryBuilder andQuery = QueryBuilders.boolQuery();
for (Map.Entry> keyToValues : optimization.entrySet()) {
String fieldName = keyToValues.getKey();
List