From 4eb27f8b305cd3d7dac146b2034876b28a5957ed Mon Sep 17 00:00:00 2001 From: Nick Shanin Date: Thu, 1 Oct 2026 21:28:11 -0400 Subject: [PATCH 1/8] SOLR-7550: tolerate version failures during peer sync --- .../java/org/apache/solr/update/PeerSync.java | 15 ++++++- .../org/apache/solr/update/PeerSyncTest.java | 39 +++++++++++++++++++ 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/solr/core/src/java/org/apache/solr/update/PeerSync.java b/solr/core/src/java/org/apache/solr/update/PeerSync.java index 8476dc84a979..bf0bb450c97d 100644 --- a/solr/core/src/java/org/apache/solr/update/PeerSync.java +++ b/solr/core/src/java/org/apache/solr/update/PeerSync.java @@ -364,7 +364,8 @@ private void requestVersions(String replica) { shardHandler.submit(sreq, replica, sreq.params); } - private boolean handleResponse(ShardResponse srsp) { + @VisibleForTesting + boolean handleResponse(ShardResponse srsp) { ShardRequest sreq = srsp.getShardRequest(); if (srsp.getException() != null) { @@ -419,6 +420,18 @@ private boolean handleResponse(ShardResponse srsp) { return true; } + if (cantReachIsSuccess + && sreq.purpose == SHARD_REQUEST_PURPOSE_GET_VERSIONS + && srsp.getException() instanceof SolrException + && ((SolrException) srsp.getException()).code() == 500) { + log.warn( + "{} got a 500 from {}, counting as success. {} Replica may not have loaded the core.", + msg(), + srsp.getShardAddress(), + srsp.getException()); + return true; + } + // TODO: we should return the above information so that when we can request a recovery through // zookeeper, we do that for these nodes diff --git a/solr/core/src/test/org/apache/solr/update/PeerSyncTest.java b/solr/core/src/test/org/apache/solr/update/PeerSyncTest.java index 9e733abc19ec..fe8bd00fd97f 100644 --- a/solr/core/src/test/org/apache/solr/update/PeerSyncTest.java +++ b/solr/core/src/test/org/apache/solr/update/PeerSyncTest.java @@ -20,6 +20,7 @@ import static org.hamcrest.core.StringContains.containsString; import java.io.IOException; +import java.lang.reflect.Field; import java.util.ArrayList; import java.util.Arrays; import java.util.LinkedHashSet; @@ -37,6 +38,8 @@ import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.StrUtils; +import org.apache.solr.handler.component.ShardRequest; +import org.apache.solr.handler.component.ShardResponse; import org.apache.solr.schema.IndexSchema; import org.apache.solr.update.PeerSync.MissedUpdatesRequest; import org.apache.solr.update.processor.DistributedUpdateProcessor; @@ -680,4 +683,40 @@ private static void testHandleVersionsWithRangesMissingTwoRanges() { } } } + + @Test + public void testPeerSyncIgnores500FromVersionRequestWhenCantReachIsSuccess() throws Exception { + PeerSync peerSync = + new PeerSync(h.getCore(), List.of("http://example.com/solr/core"), 10, true); + + ShardRequest request = new ShardRequest(); + request.purpose = PeerSync.SHARD_REQUEST_PURPOSE_GET_VERSIONS; + + ShardResponse response = new ShardResponse(); + response.setShardRequest(request); + setException(response, new SolrException(SolrException.ErrorCode.SERVER_ERROR, "boom")); + + assertTrue(peerSync.handleResponse(response)); + } + + @Test + public void testPeerSyncStillFails500FromUpdateRequest() throws Exception { + PeerSync peerSync = + new PeerSync(h.getCore(), List.of("http://example.com/solr/core"), 10, true); + + ShardRequest request = new ShardRequest(); + request.purpose = PeerSync.SHARD_REQUEST_PURPOSE_GET_UPDATES; + + ShardResponse response = new ShardResponse(); + response.setShardRequest(request); + setException(response, new SolrException(SolrException.ErrorCode.SERVER_ERROR, "boom")); + + assertFalse(peerSync.handleResponse(response)); + } + + private static void setException(ShardResponse response, Throwable exception) throws Exception { + Field field = ShardResponse.class.getDeclaredField("exception"); + field.setAccessible(true); + field.set(response, exception); + } } From 701bdef81e9df79740cd3697f41479edb6dfce9b Mon Sep 17 00:00:00 2001 From: Nick Shanin Date: Sat, 3 Oct 2026 16:41:09 -0400 Subject: [PATCH 2/8] SOLR-7550: mock ShardResponse, close PeerSync in tests, neutral log text, add changelog --- .../unreleased/SOLR-7550-peersync-500.yml | 7 +++ .../java/org/apache/solr/update/PeerSync.java | 2 +- .../org/apache/solr/update/PeerSyncTest.java | 56 ++++++++++--------- 3 files changed, 38 insertions(+), 27 deletions(-) create mode 100644 changelog/unreleased/SOLR-7550-peersync-500.yml diff --git a/changelog/unreleased/SOLR-7550-peersync-500.yml b/changelog/unreleased/SOLR-7550-peersync-500.yml new file mode 100644 index 000000000000..2d5eb304ae7f --- /dev/null +++ b/changelog/unreleased/SOLR-7550-peersync-500.yml @@ -0,0 +1,7 @@ +title: PeerSync during leader election now treats a 500 response to the versions request from a replica as unreachable, like a 503, so a replica whose core failed to load no longer blocks the election +type: fixed +authors: + - name: Nick Shanin +links: + - name: SOLR-7550 + url: https://issues.apache.org/jira/browse/SOLR-7550 diff --git a/solr/core/src/java/org/apache/solr/update/PeerSync.java b/solr/core/src/java/org/apache/solr/update/PeerSync.java index bf0bb450c97d..0bb339b58ec5 100644 --- a/solr/core/src/java/org/apache/solr/update/PeerSync.java +++ b/solr/core/src/java/org/apache/solr/update/PeerSync.java @@ -425,7 +425,7 @@ boolean handleResponse(ShardResponse srsp) { && srsp.getException() instanceof SolrException && ((SolrException) srsp.getException()).code() == 500) { log.warn( - "{} got a 500 from {}, counting as success. {} Replica may not have loaded the core.", + "{} got a 500 from {}, counting as success ", msg(), srsp.getShardAddress(), srsp.getException()); diff --git a/solr/core/src/test/org/apache/solr/update/PeerSyncTest.java b/solr/core/src/test/org/apache/solr/update/PeerSyncTest.java index fe8bd00fd97f..1e086043f5ba 100644 --- a/solr/core/src/test/org/apache/solr/update/PeerSyncTest.java +++ b/solr/core/src/test/org/apache/solr/update/PeerSyncTest.java @@ -18,9 +18,10 @@ import static org.apache.solr.update.processor.DistributingUpdateProcessorFactory.DISTRIB_UPDATE_PARAM; import static org.hamcrest.core.StringContains.containsString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import java.io.IOException; -import java.lang.reflect.Field; import java.util.ArrayList; import java.util.Arrays; import java.util.LinkedHashSet; @@ -686,37 +687,40 @@ private static void testHandleVersionsWithRangesMissingTwoRanges() { @Test public void testPeerSyncIgnores500FromVersionRequestWhenCantReachIsSuccess() throws Exception { - PeerSync peerSync = - new PeerSync(h.getCore(), List.of("http://example.com/solr/core"), 10, true); - - ShardRequest request = new ShardRequest(); - request.purpose = PeerSync.SHARD_REQUEST_PURPOSE_GET_VERSIONS; - - ShardResponse response = new ShardResponse(); - response.setShardRequest(request); - setException(response, new SolrException(SolrException.ErrorCode.SERVER_ERROR, "boom")); - - assertTrue(peerSync.handleResponse(response)); + assumeWorkingMockito(); + try (PeerSync peerSync = + new PeerSync(h.getCore(), List.of("http://example.com/solr/core"), 10, true)) { + ShardResponse response = + failedResponse( + PeerSync.SHARD_REQUEST_PURPOSE_GET_VERSIONS, + new SolrException(SolrException.ErrorCode.SERVER_ERROR, "boom")); + + assertTrue(peerSync.handleResponse(response)); + } } @Test public void testPeerSyncStillFails500FromUpdateRequest() throws Exception { - PeerSync peerSync = - new PeerSync(h.getCore(), List.of("http://example.com/solr/core"), 10, true); + assumeWorkingMockito(); + try (PeerSync peerSync = + new PeerSync(h.getCore(), List.of("http://example.com/solr/core"), 10, true)) { + ShardResponse response = + failedResponse( + PeerSync.SHARD_REQUEST_PURPOSE_GET_UPDATES, + new SolrException(SolrException.ErrorCode.SERVER_ERROR, "boom")); + + assertFalse(peerSync.handleResponse(response)); + } + } + private static ShardResponse failedResponse(int purpose, Throwable exception) { ShardRequest request = new ShardRequest(); - request.purpose = PeerSync.SHARD_REQUEST_PURPOSE_GET_UPDATES; - - ShardResponse response = new ShardResponse(); - response.setShardRequest(request); - setException(response, new SolrException(SolrException.ErrorCode.SERVER_ERROR, "boom")); - - assertFalse(peerSync.handleResponse(response)); - } + request.purpose = purpose; - private static void setException(ShardResponse response, Throwable exception) throws Exception { - Field field = ShardResponse.class.getDeclaredField("exception"); - field.setAccessible(true); - field.set(response, exception); + ShardResponse response = mock(ShardResponse.class); + when(response.getShardRequest()).thenReturn(request); + when(response.getException()).thenReturn(exception); + when(response.getShardAddress()).thenReturn("http://example.com/solr/core"); + return response; } } From 7bde698fc30d36528aecb7f885a391c7ab60499d Mon Sep 17 00:00:00 2001 From: Nick Shanin Date: Sun, 4 Oct 2026 01:43:07 +0000 Subject: [PATCH 3/8] SOLR-7550: build real ShardResponse in tests instead of mocking it ShardResponse is final, so Mockito cannot mock it and the new PeerSyncTest cases could never run as written. Build a real ShardResponse instead. That needs setException and setShardAddress callable from the update package, so widen those two setters to public; setShardRequest and setSolrResponse on the same class are already public. The Mockito imports and guards added with the tests are no longer needed and are dropped. --- .../solr/handler/component/ShardResponse.java | 4 ++-- .../test/org/apache/solr/update/PeerSyncTest.java | 13 +++++-------- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/handler/component/ShardResponse.java b/solr/core/src/java/org/apache/solr/handler/component/ShardResponse.java index 289a6e941829..75bc447138c8 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/ShardResponse.java +++ b/solr/core/src/java/org/apache/solr/handler/component/ShardResponse.java @@ -78,7 +78,7 @@ void setShard(String shard) { this.shard = shard; } - void setException(Throwable exception) { + public void setException(Throwable exception) { this.exception = exception; } @@ -97,7 +97,7 @@ public String getShardAddress() { return this.shardAddress; } - void setShardAddress(String addr) { + public void setShardAddress(String addr) { this.shardAddress = addr; } } diff --git a/solr/core/src/test/org/apache/solr/update/PeerSyncTest.java b/solr/core/src/test/org/apache/solr/update/PeerSyncTest.java index 1e086043f5ba..bd0348dcf758 100644 --- a/solr/core/src/test/org/apache/solr/update/PeerSyncTest.java +++ b/solr/core/src/test/org/apache/solr/update/PeerSyncTest.java @@ -18,8 +18,6 @@ import static org.apache.solr.update.processor.DistributingUpdateProcessorFactory.DISTRIB_UPDATE_PARAM; import static org.hamcrest.core.StringContains.containsString; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; import java.io.IOException; import java.util.ArrayList; @@ -687,7 +685,6 @@ private static void testHandleVersionsWithRangesMissingTwoRanges() { @Test public void testPeerSyncIgnores500FromVersionRequestWhenCantReachIsSuccess() throws Exception { - assumeWorkingMockito(); try (PeerSync peerSync = new PeerSync(h.getCore(), List.of("http://example.com/solr/core"), 10, true)) { ShardResponse response = @@ -701,7 +698,6 @@ public void testPeerSyncIgnores500FromVersionRequestWhenCantReachIsSuccess() thr @Test public void testPeerSyncStillFails500FromUpdateRequest() throws Exception { - assumeWorkingMockito(); try (PeerSync peerSync = new PeerSync(h.getCore(), List.of("http://example.com/solr/core"), 10, true)) { ShardResponse response = @@ -717,10 +713,11 @@ private static ShardResponse failedResponse(int purpose, Throwable exception) { ShardRequest request = new ShardRequest(); request.purpose = purpose; - ShardResponse response = mock(ShardResponse.class); - when(response.getShardRequest()).thenReturn(request); - when(response.getException()).thenReturn(exception); - when(response.getShardAddress()).thenReturn("http://example.com/solr/core"); + // ShardResponse is final, so it cannot be mocked; build a real one. + ShardResponse response = new ShardResponse(); + response.setShardRequest(request); + response.setException(exception); + response.setShardAddress("http://example.com/solr/core"); return response; } } From b1ef1caddd19da509c5ec967f5ad24e1f094606e Mon Sep 17 00:00:00 2001 From: Nick Shanin Date: Sun, 4 Oct 2026 02:47:57 +0000 Subject: [PATCH 4/8] SOLR-7550: move handleResponse tests to their own test class PeerSyncTest calls initCore from its constructor, and the test runner constructs the class once per test method, so every method added to that suite leaks a whole harness core and the suite-end ObjectReleaseTracker check fails with the leaked core graph. The two handleResponse tests therefore live in a new PeerSyncHandleResponseTest that extends SolrTestCase and calls initCore once from @BeforeClass. PeerSyncTest is back to its main-branch contents. --- .../update/PeerSyncHandleResponseTest.java | 79 +++++++++++++++++++ .../org/apache/solr/update/PeerSyncTest.java | 40 ---------- 2 files changed, 79 insertions(+), 40 deletions(-) create mode 100644 solr/core/src/test/org/apache/solr/update/PeerSyncHandleResponseTest.java diff --git a/solr/core/src/test/org/apache/solr/update/PeerSyncHandleResponseTest.java b/solr/core/src/test/org/apache/solr/update/PeerSyncHandleResponseTest.java new file mode 100644 index 000000000000..780f99ef6229 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/update/PeerSyncHandleResponseTest.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.update; + +import java.util.List; +import org.apache.solr.SolrTestCase; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.common.SolrException; +import org.apache.solr.handler.component.ShardRequest; +import org.apache.solr.handler.component.ShardResponse; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * Unit tests for {@link PeerSync#handleResponse}. These live outside {@link PeerSyncTest} because + * that suite calls initCore from its constructor, so every additional test method in it leaks a + * harness core. + */ +public class PeerSyncHandleResponseTest extends SolrTestCase { + + @BeforeClass + public static void beforeClass() throws Exception { + SolrTestCaseJ4.newRandomConfig(); + initCore("solrconfig-tlog.xml", "schema.xml"); + } + + @Test + public void testPeerSyncIgnores500FromVersionRequestWhenCantReachIsSuccess() throws Exception { + try (PeerSync peerSync = + new PeerSync(h.getCore(), List.of("http://example.com/solr/core"), 10, true)) { + ShardResponse response = + failedResponse( + PeerSync.SHARD_REQUEST_PURPOSE_GET_VERSIONS, + new SolrException(SolrException.ErrorCode.SERVER_ERROR, "boom")); + + assertTrue(peerSync.handleResponse(response)); + } + } + + @Test + public void testPeerSyncStillFails500FromUpdateRequest() throws Exception { + try (PeerSync peerSync = + new PeerSync(h.getCore(), List.of("http://example.com/solr/core"), 10, true)) { + ShardResponse response = + failedResponse( + PeerSync.SHARD_REQUEST_PURPOSE_GET_UPDATES, + new SolrException(SolrException.ErrorCode.SERVER_ERROR, "boom")); + + assertFalse(peerSync.handleResponse(response)); + } + } + + private static ShardResponse failedResponse(int purpose, Throwable exception) { + ShardRequest request = new ShardRequest(); + request.purpose = purpose; + + // ShardResponse is final, so it cannot be mocked; build a real one. + ShardResponse response = new ShardResponse(); + response.setShardRequest(request); + response.setException(exception); + response.setShardAddress("http://example.com/solr/core"); + return response; + } +} diff --git a/solr/core/src/test/org/apache/solr/update/PeerSyncTest.java b/solr/core/src/test/org/apache/solr/update/PeerSyncTest.java index bd0348dcf758..9e733abc19ec 100644 --- a/solr/core/src/test/org/apache/solr/update/PeerSyncTest.java +++ b/solr/core/src/test/org/apache/solr/update/PeerSyncTest.java @@ -37,8 +37,6 @@ import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.StrUtils; -import org.apache.solr.handler.component.ShardRequest; -import org.apache.solr.handler.component.ShardResponse; import org.apache.solr.schema.IndexSchema; import org.apache.solr.update.PeerSync.MissedUpdatesRequest; import org.apache.solr.update.processor.DistributedUpdateProcessor; @@ -682,42 +680,4 @@ private static void testHandleVersionsWithRangesMissingTwoRanges() { } } } - - @Test - public void testPeerSyncIgnores500FromVersionRequestWhenCantReachIsSuccess() throws Exception { - try (PeerSync peerSync = - new PeerSync(h.getCore(), List.of("http://example.com/solr/core"), 10, true)) { - ShardResponse response = - failedResponse( - PeerSync.SHARD_REQUEST_PURPOSE_GET_VERSIONS, - new SolrException(SolrException.ErrorCode.SERVER_ERROR, "boom")); - - assertTrue(peerSync.handleResponse(response)); - } - } - - @Test - public void testPeerSyncStillFails500FromUpdateRequest() throws Exception { - try (PeerSync peerSync = - new PeerSync(h.getCore(), List.of("http://example.com/solr/core"), 10, true)) { - ShardResponse response = - failedResponse( - PeerSync.SHARD_REQUEST_PURPOSE_GET_UPDATES, - new SolrException(SolrException.ErrorCode.SERVER_ERROR, "boom")); - - assertFalse(peerSync.handleResponse(response)); - } - } - - private static ShardResponse failedResponse(int purpose, Throwable exception) { - ShardRequest request = new ShardRequest(); - request.purpose = purpose; - - // ShardResponse is final, so it cannot be mocked; build a real one. - ShardResponse response = new ShardResponse(); - response.setShardRequest(request); - response.setException(exception); - response.setShardAddress("http://example.com/solr/core"); - return response; - } } From adf27bf2ad093aaf8bbe42d17d5660d7db28b913 Mon Sep 17 00:00:00 2001 From: Nick Shanin Date: Sun, 4 Oct 2026 02:51:21 +0000 Subject: [PATCH 5/8] SOLR-7550: boot the handleResponse tests with EmbeddedSolrServerTestRule SolrTestCase does not carry the SolrTestCaseJ4 harness (no initCore, no static h), so the new class follows the RootFieldTest pattern: start the embedded server rule, create the standard test collection, and take the core from its container with its reference counted. --- .../update/PeerSyncHandleResponseTest.java | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/update/PeerSyncHandleResponseTest.java b/solr/core/src/test/org/apache/solr/update/PeerSyncHandleResponseTest.java index 780f99ef6229..569e17ebcf61 100644 --- a/solr/core/src/test/org/apache/solr/update/PeerSyncHandleResponseTest.java +++ b/solr/core/src/test/org/apache/solr/update/PeerSyncHandleResponseTest.java @@ -21,28 +21,37 @@ import org.apache.solr.SolrTestCase; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.common.SolrException; +import org.apache.solr.core.SolrCore; import org.apache.solr.handler.component.ShardRequest; import org.apache.solr.handler.component.ShardResponse; +import org.apache.solr.util.EmbeddedSolrServerTestRule; import org.junit.BeforeClass; +import org.junit.ClassRule; import org.junit.Test; /** * Unit tests for {@link PeerSync#handleResponse}. These live outside {@link PeerSyncTest} because * that suite calls initCore from its constructor, so every additional test method in it leaks a - * harness core. + * harness core. The core here only exists so a PeerSync can be constructed; handleResponse itself + * reads nothing from it beyond the log prefix. */ public class PeerSyncHandleResponseTest extends SolrTestCase { + @ClassRule + public static final EmbeddedSolrServerTestRule solrTestRule = new EmbeddedSolrServerTestRule(); + @BeforeClass public static void beforeClass() throws Exception { + solrTestRule.startSolr(SolrTestCaseJ4.TEST_HOME()); SolrTestCaseJ4.newRandomConfig(); - initCore("solrconfig-tlog.xml", "schema.xml"); + solrTestRule.newCollection().withConfigSet(SolrTestCaseJ4.TEST_COLL1_CONF()).create(); } @Test public void testPeerSyncIgnores500FromVersionRequestWhenCantReachIsSuccess() throws Exception { - try (PeerSync peerSync = - new PeerSync(h.getCore(), List.of("http://example.com/solr/core"), 10, true)) { + try (SolrCore core = solrTestRule.getCoreContainer().getCore("collection1"); + PeerSync peerSync = + new PeerSync(core, List.of("http://example.com/solr/core"), 10, true)) { ShardResponse response = failedResponse( PeerSync.SHARD_REQUEST_PURPOSE_GET_VERSIONS, @@ -54,8 +63,9 @@ public void testPeerSyncIgnores500FromVersionRequestWhenCantReachIsSuccess() thr @Test public void testPeerSyncStillFails500FromUpdateRequest() throws Exception { - try (PeerSync peerSync = - new PeerSync(h.getCore(), List.of("http://example.com/solr/core"), 10, true)) { + try (SolrCore core = solrTestRule.getCoreContainer().getCore("collection1"); + PeerSync peerSync = + new PeerSync(core, List.of("http://example.com/solr/core"), 10, true)) { ShardResponse response = failedResponse( PeerSync.SHARD_REQUEST_PURPOSE_GET_UPDATES, From 687165651f9210e333a03c566a27f59f40527174 Mon Sep 17 00:00:00 2001 From: Nick Shanin Date: Sun, 4 Oct 2026 02:55:15 +0000 Subject: [PATCH 6/8] SOLR-7550: tidy formatting for PeerSyncHandleResponseTest --- .../org/apache/solr/update/PeerSyncHandleResponseTest.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/update/PeerSyncHandleResponseTest.java b/solr/core/src/test/org/apache/solr/update/PeerSyncHandleResponseTest.java index 569e17ebcf61..69ba48355c0a 100644 --- a/solr/core/src/test/org/apache/solr/update/PeerSyncHandleResponseTest.java +++ b/solr/core/src/test/org/apache/solr/update/PeerSyncHandleResponseTest.java @@ -50,8 +50,7 @@ public static void beforeClass() throws Exception { @Test public void testPeerSyncIgnores500FromVersionRequestWhenCantReachIsSuccess() throws Exception { try (SolrCore core = solrTestRule.getCoreContainer().getCore("collection1"); - PeerSync peerSync = - new PeerSync(core, List.of("http://example.com/solr/core"), 10, true)) { + PeerSync peerSync = new PeerSync(core, List.of("http://example.com/solr/core"), 10, true)) { ShardResponse response = failedResponse( PeerSync.SHARD_REQUEST_PURPOSE_GET_VERSIONS, @@ -64,8 +63,7 @@ public void testPeerSyncIgnores500FromVersionRequestWhenCantReachIsSuccess() thr @Test public void testPeerSyncStillFails500FromUpdateRequest() throws Exception { try (SolrCore core = solrTestRule.getCoreContainer().getCore("collection1"); - PeerSync peerSync = - new PeerSync(core, List.of("http://example.com/solr/core"), 10, true)) { + PeerSync peerSync = new PeerSync(core, List.of("http://example.com/solr/core"), 10, true)) { ShardResponse response = failedResponse( PeerSync.SHARD_REQUEST_PURPOSE_GET_UPDATES, From 5f4570b20ad4d830a4affdfd2e45527753093729 Mon Sep 17 00:00:00 2001 From: Nick Shanin Date: Thu, 15 Oct 2026 03:35:23 -0400 Subject: [PATCH 7/8] SOLR-7550: count only a core init failure, not any 500, as success in peer sync --- .../unreleased/SOLR-7550-peersync-500.yml | 2 +- .../java/org/apache/solr/update/PeerSync.java | 6 ++- .../update/PeerSyncHandleResponseTest.java | 44 +++++++++++++++++-- 3 files changed, 45 insertions(+), 7 deletions(-) diff --git a/changelog/unreleased/SOLR-7550-peersync-500.yml b/changelog/unreleased/SOLR-7550-peersync-500.yml index 2d5eb304ae7f..75762a12a8b6 100644 --- a/changelog/unreleased/SOLR-7550-peersync-500.yml +++ b/changelog/unreleased/SOLR-7550-peersync-500.yml @@ -1,4 +1,4 @@ -title: PeerSync during leader election now treats a 500 response to the versions request from a replica as unreachable, like a 503, so a replica whose core failed to load no longer blocks the election +title: PeerSync counts a core initialization failure on the versions request as success when cantReachIsSuccess is set, so a core that will not load no longer blocks leader election or a sync request type: fixed authors: - name: Nick Shanin diff --git a/solr/core/src/java/org/apache/solr/update/PeerSync.java b/solr/core/src/java/org/apache/solr/update/PeerSync.java index 0bb339b58ec5..23f14a3d13fe 100644 --- a/solr/core/src/java/org/apache/solr/update/PeerSync.java +++ b/solr/core/src/java/org/apache/solr/update/PeerSync.java @@ -42,6 +42,7 @@ import org.apache.solr.common.util.EnvUtils; import org.apache.solr.common.util.IOUtils; import org.apache.solr.core.SolrCore; +import org.apache.solr.core.SolrCoreInitializationException; import org.apache.solr.core.SolrInfoBean; import org.apache.solr.handler.component.ShardHandler; import org.apache.solr.handler.component.ShardHandlerFactory; @@ -423,9 +424,10 @@ boolean handleResponse(ShardResponse srsp) { if (cantReachIsSuccess && sreq.purpose == SHARD_REQUEST_PURPOSE_GET_VERSIONS && srsp.getException() instanceof SolrException - && ((SolrException) srsp.getException()).code() == 500) { + && SolrCoreInitializationException.class.getName() + .equals(((SolrException) srsp.getException()).getThrowable())) { log.warn( - "{} got a 500 from {}, counting as success ", + "{} got a core init failure from {}, counting as success ", msg(), srsp.getShardAddress(), srsp.getException()); diff --git a/solr/core/src/test/org/apache/solr/update/PeerSyncHandleResponseTest.java b/solr/core/src/test/org/apache/solr/update/PeerSyncHandleResponseTest.java index 69ba48355c0a..75e5949ba8a7 100644 --- a/solr/core/src/test/org/apache/solr/update/PeerSyncHandleResponseTest.java +++ b/solr/core/src/test/org/apache/solr/update/PeerSyncHandleResponseTest.java @@ -17,17 +17,25 @@ package org.apache.solr.update; +import java.io.IOException; +import java.lang.invoke.MethodHandles; import java.util.List; import org.apache.solr.SolrTestCase; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.common.SolrException; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.core.SolrCore; +import org.apache.solr.core.SolrCoreInitializationException; import org.apache.solr.handler.component.ShardRequest; import org.apache.solr.handler.component.ShardResponse; +import org.apache.solr.servlet.ResponseUtils; import org.apache.solr.util.EmbeddedSolrServerTestRule; import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Unit tests for {@link PeerSync#handleResponse}. These live outside {@link PeerSyncTest} because @@ -37,6 +45,8 @@ */ public class PeerSyncHandleResponseTest extends SolrTestCase { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + @ClassRule public static final EmbeddedSolrServerTestRule solrTestRule = new EmbeddedSolrServerTestRule(); @@ -48,26 +58,41 @@ public static void beforeClass() throws Exception { } @Test - public void testPeerSyncIgnores500FromVersionRequestWhenCantReachIsSuccess() throws Exception { + public void testPeerSyncCountsCoreInitFailureFromVersionRequestAsSuccess() throws Exception { try (SolrCore core = solrTestRule.getCoreContainer().getCore("collection1"); PeerSync peerSync = new PeerSync(core, List.of("http://example.com/solr/core"), 10, true)) { ShardResponse response = failedResponse( PeerSync.SHARD_REQUEST_PURPOSE_GET_VERSIONS, - new SolrException(SolrException.ErrorCode.SERVER_ERROR, "boom")); + serverError( + new SolrCoreInitializationException("collection1", new IOException("boom")))); assertTrue(peerSync.handleResponse(response)); } } @Test - public void testPeerSyncStillFails500FromUpdateRequest() throws Exception { + public void testPeerSyncStillFailsOtherServerErrorFromVersionRequest() throws Exception { + try (SolrCore core = solrTestRule.getCoreContainer().getCore("collection1"); + PeerSync peerSync = new PeerSync(core, List.of("http://example.com/solr/core"), 10, true)) { + ShardResponse response = + failedResponse( + PeerSync.SHARD_REQUEST_PURPOSE_GET_VERSIONS, + serverError(new SolrException(SolrException.ErrorCode.SERVER_ERROR, "boom"))); + + assertFalse(peerSync.handleResponse(response)); + } + } + + @Test + public void testPeerSyncStillFailsCoreInitFailureFromUpdateRequest() throws Exception { try (SolrCore core = solrTestRule.getCoreContainer().getCore("collection1"); PeerSync peerSync = new PeerSync(core, List.of("http://example.com/solr/core"), 10, true)) { ShardResponse response = failedResponse( PeerSync.SHARD_REQUEST_PURPOSE_GET_UPDATES, - new SolrException(SolrException.ErrorCode.SERVER_ERROR, "boom")); + serverError( + new SolrCoreInitializationException("collection1", new IOException("boom")))); assertFalse(peerSync.handleResponse(response)); } @@ -84,4 +109,15 @@ private static ShardResponse failedResponse(int purpose, Throwable exception) { response.setShardAddress("http://example.com/solr/core"); return response; } + + // The server writes the exception class into the response metadata, so build the error the + // same way the response writer does. + @SuppressWarnings("unchecked") + private static SolrException serverError(Throwable cause) { + NamedList info = new SimpleOrderedMap<>(); + ResponseUtils.getErrorInfo(cause, info, log); + SolrException exception = new SolrException(SolrException.ErrorCode.SERVER_ERROR, "boom"); + exception.setMetadata((NamedList) info.get("metadata")); + return exception; + } } From fcc8a874abbb8fbd465a835da6903f64de7009b2 Mon Sep 17 00:00:00 2001 From: Nick Shanin Date: Thu, 15 Oct 2026 18:09:38 +0000 Subject: [PATCH 8/8] SOLR-7550: tidy line wrap in PeerSync.java --- solr/core/src/java/org/apache/solr/update/PeerSync.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/solr/core/src/java/org/apache/solr/update/PeerSync.java b/solr/core/src/java/org/apache/solr/update/PeerSync.java index 23f14a3d13fe..7add2ada2170 100644 --- a/solr/core/src/java/org/apache/solr/update/PeerSync.java +++ b/solr/core/src/java/org/apache/solr/update/PeerSync.java @@ -424,7 +424,8 @@ boolean handleResponse(ShardResponse srsp) { if (cantReachIsSuccess && sreq.purpose == SHARD_REQUEST_PURPOSE_GET_VERSIONS && srsp.getException() instanceof SolrException - && SolrCoreInitializationException.class.getName() + && SolrCoreInitializationException.class + .getName() .equals(((SolrException) srsp.getException()).getThrowable())) { log.warn( "{} got a core init failure from {}, counting as success ",