From 7be21fdb29a1d8a18e14e1a90d21b9d5215bf511 Mon Sep 17 00:00:00 2001
From: Valery Kharseko <vharseko@3a-systems.ru>
Date: Wed, 18 Nov 2026 06:37:46 +0000
Subject: [PATCH] [#1303] Wait for both sides of replication initialization (#1304)

---
 opendj-server-legacy/src/test/java/org/opends/server/replication/service/ReplicationDomainTest.java |  369 ++++++++++++++++++++++++++++++++++++++++++++-------
 1 files changed, 314 insertions(+), 55 deletions(-)

diff --git a/opendj-server-legacy/src/test/java/org/opends/server/replication/service/ReplicationDomainTest.java b/opendj-server-legacy/src/test/java/org/opends/server/replication/service/ReplicationDomainTest.java
index f460e63..08b86dc 100644
--- a/opendj-server-legacy/src/test/java/org/opends/server/replication/service/ReplicationDomainTest.java
+++ b/opendj-server-legacy/src/test/java/org/opends/server/replication/service/ReplicationDomainTest.java
@@ -20,12 +20,12 @@
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
 import static java.util.concurrent.TimeUnit.SECONDS;
 import static org.opends.messages.ReplicationMessages.*;
-import static org.opends.server.TestCaseUtils.generateThreadDump;
 import static org.opends.server.util.CollectionUtils.*;
 import static org.testng.Assert.*;
 
 import java.net.SocketTimeoutException;
 import java.io.InputStream;
+import java.io.OutputStream;
 import java.util.concurrent.FutureTask;
 import java.util.concurrent.ExecutionException;
 import java.util.Collections;
@@ -418,6 +418,202 @@
     }
   }
 
+  /** A receiver can have all data while the exporter is still accounting for it (issue #1303). */
+  @Test
+  public void exportChecksWaitForExporterAfterDataArrives() throws Exception
+  {
+    final int entryCount = 5000;
+    DN baseDN = DN.valueOf("o=test");
+    ReplicationServer rs1 = null;
+    ReplicationServer rs2 = null;
+    RecordingExportDomain exporter = null;
+    FakeReplicationDomain importer = null;
+    ReplicationBroker originalBroker = null;
+    CountDownLatch published = new CountDownLatch(1);
+    CountDownLatch resume = new CountDownLatch(1);
+    CountDownLatch brokerReady = new CountDownLatch(1);
+    CountDownLatch exportReturned = new CountDownLatch(1);
+    CountDownLatch finishExport = new CountDownLatch(1);
+    FutureTask<Void> check = null;
+    Thread checkThread = null;
+    try
+    {
+      int[] ports = TestCaseUtils.findFreePorts(2);
+      rs1 = createReplicationServer(11, ports[0], "exportCounterRace1", 100);
+      rs2 = createReplicationServer(12, ports[1], "exportCounterRace2", 100, "localhost:" + ports[0]);
+      String exportedData = buildExportedData(entryCount);
+      StringBuffer importedData = new StringBuffer();
+      exporter = new RecordingExportDomain(baseDN, 1, newTreeSet("localhost:" + ports[0]),
+          0, exportedData, entryCount)
+      {
+        @Override
+        protected void initializeRemote(int target, int taskServer, Task task, int window) throws DirectoryException
+        {
+          try
+          {
+            assertTrue(brokerReady.await(90, SECONDS), "the test broker was not installed");
+          }
+          catch (InterruptedException e)
+          {
+            Thread.currentThread().interrupt();
+            throw new IllegalStateException(e);
+          }
+          super.initializeRemote(target, taskServer, task, window);
+        }
+
+        @Override
+        void beforeCompletion() throws InterruptedException
+        {
+          exportReturned.countDown();
+          assertTrue(finishExport.await(90, SECONDS), "the export completion was not released");
+        }
+      };
+      importer = new FakeReplicationDomain(baseDN, 2, newTreeSet("localhost:" + ports[1]),
+          0, null, importedData, 0);
+      waitTopologyKnowsReplica(importer, 1);
+      waitTopologyKnowsReplica(exporter, 2);
+
+      originalBroker = exporter.broker;
+      ReplicationBroker delegate = originalBroker;
+      exporter.broker = org.mockito.Mockito.mock(ReplicationBroker.class, (org.mockito.stubbing.Answer<Object>) invocation -> {
+        Object result;
+        try
+        {
+          result = invocation.getMethod().invoke(delegate, invocation.getArguments());
+        }
+        catch (java.lang.reflect.InvocationTargetException e)
+        {
+          throw e.getCause();
+        }
+        if (invocation.getMethod().getName().equals("publish")
+            && invocation.getArguments().length == 2
+            && invocation.getArguments()[0] instanceof org.opends.server.replication.protocol.EntryMsg)
+        {
+          published.countDown();
+          if (!resume.await(30, SECONDS))
+          {
+            throw new AssertionError("exporter was not resumed");
+          }
+        }
+        return result;
+      });
+      brokerReady.countDown();
+      RecordingInitializeTask task = new RecordingInitializeTask();
+      importer.initializeFromRemote(1, task);
+      assertTrue(published.await(30, SECONDS), "the export did not publish its data");
+      newTimer().repeatUntilSuccess(() -> assertEquals(importedData.length(), exportedData.length()));
+      ImportExportContext exportContext = exporter.getImportExportContext();
+      assertNotNull(exportContext, "the export must still own its context");
+      assertEquals(exportContext.getLeftEntryCount(), (long) entryCount,
+          "the exporter must still be paused before accounting");
+
+      RecordingExportDomain finalExporter = exporter;
+      FakeReplicationDomain finalImporter = importer;
+      CountDownLatch checkStarted = new CountDownLatch(1);
+      check = new FutureTask<>(() -> {
+        checkStarted.countDown();
+        assertExportSuccessful(finalExporter, finalImporter, task, exportedData, importedData);
+        return null;
+      });
+      checkThread = new Thread(check, "export-counter-check");
+      checkThread.start();
+      assertTrue(checkStarted.await(30, SECONDS), "the transfer check did not start");
+      try
+      {
+        check.get(500, MILLISECONDS);
+        fail("the transfer check completed while the exporter was paused");
+      }
+      catch (TimeoutException expected)
+      {
+        // It must wait for the exporter, even though the receiver has all data.
+      }
+      resume.countDown();
+      assertNull(task.waitForCompletion(90, SECONDS), "the import failed");
+      assertTrue(exportReturned.await(90, SECONDS), "the exporter did not finish its operation");
+      try
+      {
+        check.get(500, MILLISECONDS);
+        fail("the transfer check completed before export completion was recorded");
+      }
+      catch (TimeoutException expected)
+      {
+        // The importer succeeded, but the export result is still withheld.
+      }
+      finishExport.countDown();
+      check.get(90, SECONDS);
+    }
+    finally
+    {
+      brokerReady.countDown();
+      resume.countDown();
+      finishExport.countDown();
+      if (check != null && !check.isDone())
+      {
+        check.cancel(true);
+      }
+      if (exporter != null && originalBroker != null)
+      {
+        exporter.broker = originalBroker;
+      }
+      if (checkThread != null)
+      {
+        checkThread.join(30000);
+      }
+      disable(exporter, importer);
+      remove(rs1, rs2);
+      assertTrue(checkThread == null || !checkThread.isAlive(), "the transfer check thread did not stop");
+    }
+  }
+
+  /** Receiving every byte is not success when the importer fails afterwards. */
+  @Test
+  public void exportChecksRejectImportFailureAfterDataArrives() throws Exception
+  {
+    DN baseDN = DN.valueOf("o=test");
+    ReplicationServer rs = null;
+    RecordingExportDomain exporter = null;
+    FakeReplicationDomain importer = null;
+    try
+    {
+      int port = TestCaseUtils.findFreePort();
+      rs = createReplicationServer(11, port, "exportCounterFailedImport", 100);
+      SortedSet<String> servers = newTreeSet("localhost:" + port);
+      String exportedData = buildExportedData(10);
+      StringBuffer importedData = new StringBuffer();
+      exporter = new RecordingExportDomain(baseDN, 1, servers, 0, exportedData, 10);
+      importer = new FakeReplicationDomain(baseDN, 2, servers, 0, null, importedData, 0)
+      {
+        @Override
+        protected void importBackend(InputStream input) throws DirectoryException
+        {
+          super.importBackend(input);
+          throw new DirectoryException(org.forgerock.opendj.ldap.ResultCode.OTHER,
+              LocalizableMessage.raw("injected import failure after receiving data"));
+        }
+      };
+      waitTopologyKnowsReplica(importer, 1);
+      waitTopologyKnowsReplica(exporter, 2);
+      RecordingInitializeTask task = new RecordingInitializeTask();
+      importer.initializeFromRemote(1, task);
+      assertNotNull(task.waitForCompletion(90, SECONDS), "the injected failure was not reported");
+      assertTrue(importedData.toString().startsWith(exportedData), "the importer did not receive the data");
+      try
+      {
+        assertExportSuccessful(exporter, importer, task, exportedData, importedData);
+        fail("the transfer check accepted a failed import");
+      }
+      catch (AssertionError expected)
+      {
+        assertTrue(expected.getMessage().contains("the import failed"), expected.toString());
+      }
+    }
+    finally
+    {
+      disable(exporter, importer);
+      remove(rs);
+    }
+  }
+
   @DataProvider(name = "exportAndImportData")
   public Object[][] createExportAndimportData()
   {
@@ -438,7 +634,7 @@
     DN testService = DN.valueOf("o=test");
     ReplicationServer replServer = null;
     int replServerID = 11;
-    FakeReplicationDomain domain1 = null;
+    RecordingExportDomain domain1 = null;
     FakeReplicationDomain domain2 = null;
 
     try
@@ -450,8 +646,8 @@
       SortedSet<String> servers = newTreeSet("localhost:" + replServerPort);
 
       String exportedData = buildExportedData(ENTRYCOUNT);
-      domain1 = new FakeReplicationDomain(
-          testService, serverId1, servers, 0, exportedData, null, ENTRYCOUNT);
+      domain1 = new RecordingExportDomain(
+          testService, serverId1, servers, 0, exportedData, ENTRYCOUNT);
 
       StringBuffer importedData = new StringBuffer();
       domain2 = new FakeReplicationDomain(
@@ -461,9 +657,9 @@
        * Trigger a total update from domain1 to domain2.
        * Check that the exported data is correctly received on domain2.
        */
-      assertTrue(initializeFromRemote(domain2));
-      waitEndExport(exportedData, importedData);
-      assertExportSucessful(domain1, domain2, exportedData, importedData);
+      RecordingInitializeTask task = new RecordingInitializeTask();
+      assertTrue(initializeFromRemote(domain2, task));
+      assertExportSuccessful(domain1, domain2, task, exportedData, importedData);
     }
     finally
     {
@@ -474,6 +670,11 @@
 
   private boolean initializeFromRemote(ReplicationDomain domain) throws DirectoryException
   {
+    return initializeFromRemote(domain, NO_INIT_TASK);
+  }
+
+  private boolean initializeFromRemote(ReplicationDomain domain, Task task) throws DirectoryException
+  {
     try {
       Thread.sleep(2000);
     } catch (InterruptedException e) {
@@ -483,7 +684,7 @@
     {
       if (remoteDS.getDsId() != domain.getServerId())
       {
-        domain.initializeFromRemote(remoteDS.getDsId(), NO_INIT_TASK);
+        domain.initializeFromRemote(remoteDS.getDsId(), task);
         return true;
       }
     }
@@ -503,7 +704,7 @@
     ReplicationServer replServer1 = null;
     int replServerID = 11;
     int replServerID2 = 12;
-    FakeReplicationDomain domain1 = null;
+    RecordingExportDomain domain1 = null;
     FakeReplicationDomain domain2 = null;
 
     try
@@ -521,8 +722,8 @@
       SortedSet<String> servers2 = newTreeSet("localhost:" + replServerPort2);
 
       String exportedData = buildExportedData(ENTRYCOUNT);
-      domain1 = new FakeReplicationDomain(
-          testService, 1, servers1, 0, exportedData, null, ENTRYCOUNT);
+      domain1 = new RecordingExportDomain(
+          testService, 1, servers1, 0, exportedData, ENTRYCOUNT);
 
       StringBuffer importedData = new StringBuffer();
       domain2 = new FakeReplicationDomain(
@@ -545,10 +746,10 @@
         });
       }
 
-      domain2.initializeFromRemote(1, NO_INIT_TASK);
+      RecordingInitializeTask task = new RecordingInitializeTask();
+      domain2.initializeFromRemote(1, task);
 
-      waitEndExport(exportedData, importedData);
-      assertExportSucessful(domain1, domain2, exportedData, importedData);
+      assertExportSuccessful(domain1, domain2, task, exportedData, importedData);
     }
     finally
     {
@@ -582,7 +783,7 @@
     final int ENTRYCOUNT = 100;
     DN testService = DN.valueOf("o=test");
     ReplicationServer replServer = null;
-    FakeReplicationDomain exporter = null;
+    RecordingExportDomain exporter = null;
     FakeReplicationDomain importer = null;
 
     try
@@ -592,16 +793,16 @@
       SortedSet<String> servers = newTreeSet("localhost:" + replServerPort);
 
       String exportedData = buildExportedData(ENTRYCOUNT);
-      exporter = new FakeReplicationDomain(testService, exporterId, servers, 0, exportedData, null, ENTRYCOUNT);
+      exporter = new RecordingExportDomain(testService, exporterId, servers, 0, exportedData, ENTRYCOUNT);
       StringBuffer importedData = new StringBuffer();
       importer = new FakeReplicationDomain(testService, importerId, servers, 0, null, importedData, 0);
       waitTopologyKnowsReplica(importer, exporterId);
       waitTopologyKnowsReplica(exporter, importerId);
 
-      importer.initializeFromRemote(exporterId, NO_INIT_TASK);
+      RecordingInitializeTask task = new RecordingInitializeTask();
+      importer.initializeFromRemote(exporterId, task);
 
-      waitEndExport(exportedData, importedData);
-      assertExportSucessful(exporter, importer, exportedData, importedData);
+      assertExportSuccessful(exporter, importer, task, exportedData, importedData);
     }
     finally
     {
@@ -635,7 +836,7 @@
     DN testService = DN.valueOf("o=test");
     ReplicationServer replServer1 = null;
     ReplicationServer replServer2 = null;
-    FakeReplicationDomain exporter = null;
+    RecordingExportDomain exporter = null;
     FakeReplicationDomain importer = null;
 
     try
@@ -645,18 +846,18 @@
       replServer2 = createReplicationServer(12, ports[1], "replicaWithTheIdOfAnRS2", 100, "localhost:" + ports[0]);
 
       String exportedData = buildExportedData(ENTRYCOUNT);
-      exporter = new FakeReplicationDomain(testService, exporterId, newTreeSet("localhost:" + ports[exporterRs]),
-          0, exportedData, null, ENTRYCOUNT);
+      exporter = new RecordingExportDomain(testService, exporterId, newTreeSet("localhost:" + ports[exporterRs]),
+          0, exportedData, ENTRYCOUNT);
       StringBuffer importedData = new StringBuffer();
       importer = new FakeReplicationDomain(testService, importerId, newTreeSet("localhost:" + ports[importerRs]),
           0, null, importedData, 0);
       waitTopologyKnowsReplica(importer, exporterId);
       waitTopologyKnowsReplica(exporter, importerId);
 
-      importer.initializeFromRemote(exporterId, NO_INIT_TASK);
+      RecordingInitializeTask task = new RecordingInitializeTask();
+      importer.initializeFromRemote(exporterId, task);
 
-      waitEndExport(exportedData, importedData);
-      assertExportSucessful(exporter, importer, exportedData, importedData);
+      assertExportSuccessful(exporter, importer, task, exportedData, importedData);
     }
     finally
     {
@@ -1837,11 +2038,88 @@
     };
   }
 
+  /** Records the export's counters on its own thread and its final outcome. */
+  private static class RecordingExportDomain extends FakeReplicationDomain
+  {
+    private final CountDownLatch completed = new CountDownLatch(1);
+    private Throwable failure;
+    private Long leftAfterWriting;
+
+    RecordingExportDomain(DN baseDN, int serverId, SortedSet<String> servers, long heartbeatInterval,
+        String exportedData, int entryCount) throws org.forgerock.opendj.config.server.ConfigException
+    {
+      super(baseDN, serverId, servers, heartbeatInterval, exportedData, null, entryCount);
+    }
+
+    @Override
+    protected void exportBackend(OutputStream output) throws DirectoryException
+    {
+      super.exportBackend(output);
+      leftAfterWriting = getImportExportContext().getLeftEntryCount();
+    }
+
+    @Override
+    protected void initializeRemote(int target, int taskServer, Task task, int window) throws DirectoryException
+    {
+      try
+      {
+        super.initializeRemote(target, taskServer, task, window);
+        beforeCompletion();
+      }
+      catch (InterruptedException e)
+      {
+        Thread.currentThread().interrupt();
+        failure = e;
+        throw new IllegalStateException(e);
+      }
+      catch (DirectoryException | RuntimeException | Error e)
+      {
+        failure = e;
+        throw e;
+      }
+      finally
+      {
+        completed.countDown();
+      }
+    }
+
+    // Test subclasses can delay publication of the completed export result.
+    void beforeCompletion() throws InterruptedException
+    {
+    }
+
+    void assertExportCompleted() throws InterruptedException
+    {
+      assertTrue(completed.await(90, SECONDS), "the export did not complete");
+      assertNull(failure, "the export failed");
+      assertNotNull(leftAfterWriting, "the export did not write its data");
+      assertEquals(leftAfterWriting.longValue(), 0L, "Wrong LeftEntryCount for export");
+    }
+  }
+
   /** InitializeTask double recording the completion reported by the domain. */
   private static final class RecordingInitializeTask extends InitializeTask
   {
     private final CountDownLatch completed = new CountDownLatch(1);
     private volatile DirectoryException failure;
+    private Long total;
+    private Long left;
+    private int totalUpdates;
+
+    @Override
+    public void setTotal(long total)
+    {
+      super.setTotal(total);
+      this.total = total;
+      totalUpdates++;
+    }
+
+    @Override
+    public void setLeft(long left)
+    {
+      super.setLeft(left);
+      this.left = left;
+    }
 
     @Override
     public void updateTaskCompletionState(DirectoryException de)
@@ -1995,25 +2273,16 @@
     return sb.toString();
   }
 
-  private void waitEndExport(String exportedData, StringBuffer importedData) throws Exception
+  private void assertExportSuccessful(RecordingExportDomain exporter, FakeReplicationDomain importer,
+      RecordingInitializeTask task, String exportedData, StringBuffer importedData) throws Exception
   {
-    int count = 0;
-    while (importedData.length() < exportedData.length() && count < 500*5)
-    {
-      if(count % 100 == 0) { //capture thread dump on start and every 10 seconds
-        logger.debug(LocalizableMessage.raw("waitEndExport: thread dump on count=" + count));
-        logger.debug(LocalizableMessage.raw(generateThreadDump()));
-      }
-      count ++;
-      Thread.sleep(100);
-    }
-  }
-
-  private void assertExportSucessful(FakeReplicationDomain domain1,
-      FakeReplicationDomain domain2, String exportedData, StringBuffer importedData)
-  {
-    assertEquals(getLeftEntryCount(domain2), 0, "Wrong LeftEntryCount for export");
-    assertEquals(getLeftEntryCount(domain1), 0, "Wrong LeftEntryCount for import");
+    assertNull(task.waitForCompletion(90, SECONDS), "the import failed");
+    exporter.assertExportCompleted();
+    assertEquals(task.totalUpdates, 1, "the import unexpectedly retried");
+    assertNotNull(task.total, "the import did not receive an entry count");
+    assertNotNull(task.left, "the import did not update its entry count");
+    assertEquals(task.total.longValue(), exporter.countEntries(), "Wrong total entry count for import");
+    assertEquals(task.left.longValue(), 0L, "Wrong LeftEntryCount for import");
     assertEquals(importedData.length(), exportedData.length());
     assertEquals(importedData.toString(), exportedData);
     /*
@@ -2022,23 +2291,13 @@
      * replica has to: the exporter keeps its data and its ServerState - and keeps replaying
      * into them - while the importer is having both replaced.
      */
-    assertEquals(domain2.importReportedDuringImport(), Boolean.TRUE,
+    assertEquals(importer.importReportedDuringImport(), Boolean.TRUE,
         "the replica whose data is being replaced must report a total update into itself");
-    assertEquals(domain1.importReportedDuringExport(), Boolean.FALSE,
+    assertEquals(exporter.importReportedDuringExport(), Boolean.FALSE,
         "the replica which is exporting must not report a total update into itself:"
             + " its data and its ServerState are left alone");
   }
 
-  private long getLeftEntryCount(ReplicationDomain domain)
-  {
-    final ImportExportContext ieContext = domain.getImportExportContext();
-    if (ieContext != null)
-    {
-      return ieContext.getLeftEntryCount();
-    }
-    return 0; // import/export is finished
-  }
-
   /**
    * Sender side of the Total Update Perf test.
    * The goal of this test is to measure the performance

--
Gitblit v1.10.0