diff --git a/changelog/unreleased/solr-18083-fix-read-only-behavior.yml b/changelog/unreleased/solr-18083-fix-read-only-behavior.yml new file mode 100644 index 00000000000..2aae02026c5 --- /dev/null +++ b/changelog/unreleased/solr-18083-fix-read-only-behavior.yml @@ -0,0 +1,9 @@ +# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc +title: Fix operational issues with readOnly collections, such as restarting SolrNodes and replicating from the leader. +type: fixed # added, changed, fixed, deprecated, removed, dependency_update, security, other +authors: + - name: Houston Putman + nick: HoustonPutman +links: + - name: SOLR-18083 + url: https://issues.apache.org/jira/browse/SOLR-18083 diff --git a/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java b/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java index 0b8c2a744da..bf023d3cb09 100644 --- a/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java +++ b/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java @@ -303,6 +303,8 @@ private void commitOnLeader(String leaderBaseUrl, String coreName) // ureq.getParams().set(UpdateParams.OPEN_SEARCHER, onlyLeaderIndexes); // Why do we need to open searcher if "onlyLeaderIndexes"? ureq.getParams().set(UpdateParams.OPEN_SEARCHER, false); + // If the leader is readOnly, do not fail since the core is already committed. + ureq.getParams().set(UpdateParams.FAIL_ON_READ_ONLY, false); ureq.setAction(AbstractUpdateRequest.ACTION.COMMIT, false, true).process(client); } } @@ -657,15 +659,17 @@ public final void doSyncOrReplicateRecovery(SolrCore core) throws Exception { break; } - // we wait a bit so that any updates on the leader - // that started before they saw recovering state - // are sure to have finished (see SOLR-7141 for - // discussion around current value) - // TODO since SOLR-11216, we probably won't need this - try { - Thread.sleep(waitForUpdatesWithStaleStatePauseMilliSeconds); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); + if (!core.readOnly) { + // we wait a bit so that any updates on the leader + // that started before they saw recovering state + // are sure to have finished (see SOLR-7141 for + // discussion around current value) + // TODO since SOLR-11216, we probably won't need this + try { + Thread.sleep(waitForUpdatesWithStaleStatePauseMilliSeconds); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } } // first thing we just try to sync diff --git a/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContext.java b/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContext.java index 16a29f89a58..96401345503 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContext.java +++ b/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContext.java @@ -195,7 +195,7 @@ void runLeaderProcess(boolean weAreReplacement) throws KeeperException, Interrup // first cancel any current recovery core.getUpdateHandler().getSolrCoreState().cancelRecovery(); - if (weAreReplacement) { + if (weAreReplacement && !core.readOnly) { // wait a moment for any floating updates to finish try { Thread.sleep(2500); diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java index 24c7dc83b0c..4e73bd530f3 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrCore.java +++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java @@ -2476,7 +2476,7 @@ public RefCounted openNewSearcher( true, directoryFactory); } else { - RefCounted writer = getSolrCoreState().getIndexWriter(this); + RefCounted writer = getSolrCoreState().getIndexWriter(this, false); DirectoryReader newReader = null; try { newReader = indexReaderFactory.newReader(writer.get(), this); diff --git a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java index 70480602807..c9878048dbb 100644 --- a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java +++ b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java @@ -530,14 +530,14 @@ IndexFetchResult fetchLatestIndex(boolean forceReplication, boolean forceCoreRel // we just clear ours and commit log.info("New index in Leader. Deleting mine..."); RefCounted iw = - solrCore.getUpdateHandler().getSolrCoreState().getIndexWriter(solrCore); + solrCore.getUpdateHandler().getSolrCoreState().getIndexWriter(solrCore, false); try { iw.get().deleteAll(); } finally { iw.decref(); } assert TestInjection.injectDelayBeforeFollowerCommitRefresh(); - if (skipCommitOnLeaderVersionZero) { + if (skipCommitOnLeaderVersionZero || solrCore.readOnly) { openNewSearcherAndUpdateCommitPoint(); } else { SolrQueryRequest req = new LocalSolrQueryRequest(solrCore, new ModifiableSolrParams()); @@ -624,7 +624,7 @@ IndexFetchResult fetchLatestIndex(boolean forceReplication, boolean forceCoreRel // are successfully deleted solrCore.getUpdateHandler().newIndexWriter(true); RefCounted writer = - solrCore.getUpdateHandler().getSolrCoreState().getIndexWriter(null); + solrCore.getUpdateHandler().getSolrCoreState().getIndexWriter(null, false); try { IndexWriter indexWriter = writer.get(); int c = 0; diff --git a/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java b/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java index cdc5de9313b..e803bdd1d6b 100644 --- a/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java @@ -1378,7 +1378,7 @@ public void inform(SolrCore core) { // ensure the writer is initialized so that we have a list of commit points RefCounted iw = - core.getUpdateHandler().getSolrCoreState().getIndexWriter(core); + core.getUpdateHandler().getSolrCoreState().getIndexWriter(core, false); iw.decref(); } catch (IOException e) { diff --git a/solr/core/src/java/org/apache/solr/handler/RequestHandlerUtils.java b/solr/core/src/java/org/apache/solr/handler/RequestHandlerUtils.java index 7b5d791ae42..7a04edad33e 100644 --- a/solr/core/src/java/org/apache/solr/handler/RequestHandlerUtils.java +++ b/solr/core/src/java/org/apache/solr/handler/RequestHandlerUtils.java @@ -99,6 +99,7 @@ public static void updateCommit(CommitUpdateCommand cmd, SolrParams params) { cmd.maxOptimizeSegments = params.getInt(UpdateParams.MAX_OPTIMIZE_SEGMENTS, cmd.maxOptimizeSegments); cmd.prepareCommit = params.getBool(UpdateParams.PREPARE_COMMIT, cmd.prepareCommit); + cmd.failOnReadOnly = params.getBool(UpdateParams.FAIL_ON_READ_ONLY, cmd.failOnReadOnly); } /** diff --git a/solr/core/src/java/org/apache/solr/update/CommitUpdateCommand.java b/solr/core/src/java/org/apache/solr/update/CommitUpdateCommand.java index 6a749af2ac9..2e4d0be5fb2 100644 --- a/solr/core/src/java/org/apache/solr/update/CommitUpdateCommand.java +++ b/solr/core/src/java/org/apache/solr/update/CommitUpdateCommand.java @@ -39,6 +39,7 @@ public class CommitUpdateCommand extends UpdateCommand { public boolean expungeDeletes = false; public boolean softCommit = false; public boolean prepareCommit = false; + public boolean failOnReadOnly = true; // fail the commit if the core or collection is readOnly /** * User provided commit data. Can be let to null if there is none. It is possible to commit this @@ -98,6 +99,9 @@ public String toString() { .append(softCommit) .append(",prepareCommit=") .append(prepareCommit); + if (!failOnReadOnly) { + sb.append(",failOnReadOnly=").append(failOnReadOnly); + } if (commitData != null) { sb.append(",commitData=").append(commitData); } diff --git a/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java b/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java index 767c1ad0b77..e7c9474f508 100644 --- a/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java +++ b/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java @@ -106,8 +106,9 @@ private void closeIndexWriter(IndexWriterCloser closer) { } @Override - public RefCounted getIndexWriter(SolrCore core) throws IOException { - if (core != null && (!core.indexEnabled || core.readOnly)) { + public RefCounted getIndexWriter(SolrCore core, boolean failOnReadOnly) + throws IOException { + if (core != null && (!core.indexEnabled || (core.readOnly && failOnReadOnly))) { throw new SolrException( SolrException.ErrorCode.SERVICE_UNAVAILABLE, "Indexing is temporarily disabled"); } diff --git a/solr/core/src/java/org/apache/solr/update/SolrCoreState.java b/solr/core/src/java/org/apache/solr/update/SolrCoreState.java index 4d6f02d229f..23adf63bd97 100644 --- a/solr/core/src/java/org/apache/solr/update/SolrCoreState.java +++ b/solr/core/src/java/org/apache/solr/update/SolrCoreState.java @@ -178,7 +178,19 @@ public void deregisterInFlightUpdate() { * * @throws IOException If there is a low-level I/O error. */ - public abstract RefCounted getIndexWriter(SolrCore core) throws IOException; + public RefCounted getIndexWriter(SolrCore core) throws IOException { + return getIndexWriter(core, true); + } + + /** + * Get the current IndexWriter. If a new IndexWriter must be created, use the settings from the + * given {@link SolrCore}. Be very careful using the {@code failOnReadOnly=false} flag, by default + * it should be true if the returned indexWriter will be used for writing. + * + * @throws IOException If there is a low-level I/O error. + */ + public abstract RefCounted getIndexWriter(SolrCore core, boolean failOnReadOnly) + throws IOException; /** * Rollback the current IndexWriter. When creating the new IndexWriter use the settings from the diff --git a/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java index 82424a5f06a..fe6f44501e7 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java +++ b/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java @@ -127,7 +127,7 @@ public static DistribPhase parseParam(final String param) { protected final SolrQueryResponse rsp; private final AtomicUpdateDocumentMerger docMerger; - private final UpdateLog ulog; + protected final UpdateLog ulog; private final VersionInfo vinfo; private final boolean versionsStored; private boolean returnVersions; diff --git a/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java index 34532421bf5..12c2695a29d 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java +++ b/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java @@ -161,7 +161,15 @@ public void processCommit(CommitUpdateCommand cmd) throws IOException { assert TestInjection.injectFailUpdateRequests(); if (isReadOnly()) { - throw new SolrException(ErrorCode.FORBIDDEN, "Collection " + collection + " is read-only."); + if (cmd.failOnReadOnly) { + throw new SolrException(ErrorCode.FORBIDDEN, "Collection " + collection + " is read-only."); + } else { + // Committing on a readOnly core/collection is a no-op, since the core was committed when + // becoming read-only and hasn't had any updates since. + assert ulog == null || !ulog.hasUncommittedChanges() + : "Uncommitted changes found when trying to commit on a read-only collection"; + return; + } } List nodes = null; diff --git a/solr/solrj/src/java/org/apache/solr/common/params/UpdateParams.java b/solr/solrj/src/java/org/apache/solr/common/params/UpdateParams.java index f14252fb970..df6b11f1f3a 100644 --- a/solr/solrj/src/java/org/apache/solr/common/params/UpdateParams.java +++ b/solr/solrj/src/java/org/apache/solr/common/params/UpdateParams.java @@ -46,6 +46,9 @@ public interface UpdateParams { /** expert: calls IndexWriter.prepareCommit */ public static String PREPARE_COMMIT = "prepareCommit"; + /** Fail a commit when the core or collection is in read-only mode */ + public static String FAIL_ON_READ_ONLY = "failOnReadOnly"; + /** Rollback update commands */ public static String ROLLBACK = "rollback";