Fixed a readWriteLock deadlock happening on calling ChangelogDB.removeDB() ...

Jean-Noel Rouvignac

05.50.2013 c642f6a72a4ac8bd65607fc2ffdd29e92a3ab055

Fixed a readWriteLock deadlock happening on calling ChangelogDB.removeDB() which then calls ReplicationDB.shutdown().
Code is blocked acquiring a writeLock on the dbCloseLock, but a readLock is never unlocked.
Debugging showed that the ChangeNumberIndexDB is holding a cursor on a ReplicationDB, said cursor holds the readLock.
The solution is to not try to call ReplicationDB.shutdown() in ReplicationServerDomain.shutdownDomain() because it will be closed by JEChangelogDB.shutdownDB().


ReplicationServerDomain.java:
Removed the call to ReplicationDB.shutdownDomain(DN).

ReplicationDomainDB.java, JEChangelogDB.java:
Removed shutdownDomain(DN) which was never used outside of a complete shutdown of the ReplicationServer.
Inlined innerShutdownDomain().

JEReplicaDBCursor.java:
In next(), extracted a local variable to protect against a concurrent update to the cursor field.

 opends/src/server/org/opends/server/replication/server/ReplicationServerDomain.java

@@ -45,6 +45,7 @@
import org.opends.server.core.DirectoryServer;
import org.opends.server.loggers.debug.DebugTracer;
import org.opends.server.replication.common.*;
import org.opends.server.replication.plugin.MultimasterReplication;
import org.opends.server.replication.protocol.*;
import org.opends.server.replication.server.changelog.api.ChangelogException;
import org.opends.server.replication.server.changelog.api.DBCursor;
@@ -1720,7 +1721,7 @@

    stopAllServers(true);

    domainDB.shutdownDomain(baseDN);
    MultimasterReplication.deleteDomain(baseDN);
  }

  /**

 opends/src/server/org/opends/server/replication/server/changelog/api/ReplicationDomainDB.java

@@ -84,14 +84,6 @@
  long getDomainLatestTrimDate(DN baseDN);

  /**
   * Shutdown all the replica databases for the specified replication domain.
   *
   * @param baseDN
   *          the replication domain baseDN
   */
  void shutdownDomain(DN baseDN);

  /**
   * Removes all the data relating to the specified replication domain and
   * shutdown all its replica databases. In particular, it will:
   * <ol>

 opends/src/server/org/opends/server/replication/server/changelog/je/JEChangelogDB.java

@@ -397,7 +397,10 @@
      synchronized (domainMap)
      {
        it.remove();
        innerShutdownDomain(domainMap);
        for (JEReplicaDB replicaDB : domainMap.values())
        {
          replicaDB.shutdown();
        }
      }
    }

@@ -513,37 +516,6 @@

  /** {@inheritDoc} */
  @Override
  public void shutdownDomain(DN baseDN)
  {
    if (this.shutdown.get())
    { // shutdown has already been initiated
      return;
    }

    final Map<Integer, JEReplicaDB> domainMap = domainToReplicaDBs.get(baseDN);
    if (domainMap != null)
    {
      synchronized (domainMap)
      {
        innerShutdownDomain(domainToReplicaDBs.remove(baseDN));
      }
    }
  }

  /**
   * This method assumes the domainMap is synchronized by calling code and that
   * the domainMap is not null.
   */
  private void innerShutdownDomain(final Map<Integer, JEReplicaDB> domainMap)
  {
    for (JEReplicaDB replicaDB : domainMap.values())
    {
      replicaDB.shutdown();
    }
  }

  /** {@inheritDoc} */
  @Override
  public ServerState getDomainOldestCSNs(DN baseDN)
  {
    final ServerState result = new ServerState();

 opends/src/server/org/opends/server/replication/server/changelog/je/JEReplicaDBCursor.java

@@ -96,9 +96,10 @@
  @Override
  public boolean next() throws ChangelogException
  {
    if (cursor != null)
    final ReplServerDBCursor localCursor = cursor;
    if (localCursor != null)
    {
      currentChange = cursor.next();
      currentChange = localCursor.next();
    }
    else
    {