Fix 2911 - dsreplication pre/post-external initialisation fails to clear al...

pgamba

13.14.2008 b3c070bb2107aec085fa3f16ba1c118a71e30a7b

Fix 2911 - dsreplication pre/post-external initialisation fails to clear all the changelogs

The following changes fixes the fact that the 'server-state' information, visible under cn=monitor, is erroneous (not updated) when an OpenDS instance starts and there were some changes done on the topology while 
this server was stopped. This bug can always be seen when running the scenario to reproduce #Issue 2911.

This issue comes from the fact that, at start time, the main thread creates and start the ReplicationDomain thread, that itself creates and starts the replication listener threads.
Then the listener thread start receiving some updates, and the replay thread start replaying them.
While, with a race condition, in the main thread, the multimaster replication registers the post op plugin of the replication.
So the first operations are successfully replayed but the necessary replication post op plugin is not called by the core server and the server-state is not updated.

To fix properly that issue, we need to wait for the Replication Master Synchro provider to be successfully registered in the core, before starting listening for the replication updates. So a new method called 
'completeSynchronizationProvider()' (any better proposal is welcome) is added to the SynchronizationProvider API for that purpose.

A side effect is that a ReplicationDomain must now be started after creation. Some replication tests, that create ReplicationDomain on the fly, are also fixed here.

 opends/src/server/org/opends/server/api/SynchronizationProvider.java

@@ -117,6 +117,17 @@


  /**
   * Performs any necessary final initialization processing for this
   * synchronization provider.
   * This will be called just after the provider has been
   * registered with the server but before it has been unloaded.
   */
  public void completeSynchronizationProvider()
  {
    // No implementation is required by default.
  }

  /**
   * Performs any necessary finalization for this synchronization
   * provider.  This will be called just after the provider has been
   * deregistered with the server but before it has been unloaded.

 opends/src/server/org/opends/server/core/DirectoryServer.java

@@ -7640,6 +7640,8 @@
      SynchronizationProvider<SynchronizationProviderCfg> provider)
  {
    directoryServer.synchronizationProviders.add(provider);

    provider.completeSynchronizationProvider();
  }



 opends/src/server/org/opends/server/replication/plugin/MultimasterReplication.java

@@ -112,6 +112,8 @@
   */
  private static int replayThreadNumber = 10;

  private boolean isRegistered = false;

  /**
   * Finds the domain for a given DN.
   *
@@ -200,7 +202,6 @@
    }

    domains.put(domain.getBaseDN(), domain);
    domain.start();
    return domain;
  }

@@ -321,7 +322,11 @@
  {
    try
    {
      createNewDomain(configuration);
      ReplicationDomain rd = createNewDomain(configuration);
      if (isRegistered)
      {
        rd.start();
      }
      return new ConfigChangeResult(ResultCode.SUCCESS, false);
    } catch (ConfigException e)
    {
@@ -507,6 +512,8 @@
  @Override
  public void finalizeSynchronizationProvider()
  {
    isRegistered = false;

    // shutdown all the domains
    for (ReplicationDomain domain : domains.values())
    {
@@ -731,6 +738,18 @@

    return new ConfigChangeResult(ResultCode.SUCCESS, false);
  }

  /**
   * {@inheritDoc}
   */
  public void completeSynchronizationProvider()
  {
    isRegistered = true;

    // start all the domains
    for (ReplicationDomain domain : domains.values())
    {
      domain.start();
    }
  }
}



 opends/src/server/org/opends/server/replication/plugin/ReplicationDomain.java

@@ -266,7 +266,7 @@
   * A boolean indicating if the thread used to save the persistentServerState
   * is terminated.
   */
  private boolean done = false;
  private boolean done = true;

  /**
   * This class contain the context related to an import or export
@@ -1222,6 +1222,8 @@
  @Override
  public void run()
  {
    done = false;

    // Create the listener thread
    listenerThread = new ListenerThread(this, updateToReplayQueue);
    listenerThread.start();

 opends/tests/unit-tests-testng/src/server/org/opends/server/replication/DependencyTest.java

@@ -179,6 +179,7 @@
      domainConf.setHeartbeatInterval(100000);

      domain = MultimasterReplication.createNewDomain(domainConf);
      replicationPlugin.completeSynchronizationProvider();

      // check that last entry in sequence got added.
      Entry lastEntry = getEntry(DN.decode(addDn), 30000, true);
@@ -304,6 +305,7 @@

      Thread.sleep(2000);
      domain = MultimasterReplication.createNewDomain(domainConf);
      replicationPlugin.completeSynchronizationProvider();
      
      ReplicationBroker broker =
        openReplicationSession(baseDn, brokerId, 1000, replServerPort, 1000,
@@ -502,6 +504,7 @@
        new DomainFakeCfg(baseDn, serverId, replServers);

      domain = MultimasterReplication.createNewDomain(domainConf);
      replicationPlugin.completeSynchronizationProvider();

      // check that all entries have been deleted and added
      // again by checking that they do have the correct entryuuid
@@ -626,6 +629,7 @@
        new DomainFakeCfg(baseDn, serverId, replServers);

      domain = MultimasterReplication.createNewDomain(domainConf);
      replicationPlugin.completeSynchronizationProvider();

      // check that all entries have been renamed
      for (sequence = 1; sequence<=AddSequenceLength; sequence ++)

 opends/tests/unit-tests-testng/src/server/org/opends/server/replication/plugin/ReplicationServerFailoverTest.java

@@ -411,6 +411,7 @@
      //domainConf.setHeartbeatInterval(500);
      ReplicationDomain replicationDomain =
        MultimasterReplication.createNewDomain(domainConf);
      replicationDomain.start();

      // Add other server (doing that after connection insure we connect to
      // the right server)