- Added assured replication monitoring in replication server - Assured repl...

mrossign

15.45.2009 9569dcea7f21d0ecf76da6b73636df74f501ca15

- Added assured replication monitoring in replication server
- Assured replication monitoring attributes in directory server have consistent names
- Some existing monitoring attributes renamed in replication server to be consistent with directory server existing ones

 opends/src/server/org/opends/server/replication/server/ExpectedAcksInfo.java

@@ -22,7 +22,7 @@
 * CDDL HEADER END
 *
 *
 *      Copyright 2008 Sun Microsystems, Inc.
 *      Copyright 2008-2009 Sun Microsystems, Inc.
 */

package org.opends.server.replication.server;
@@ -79,6 +79,17 @@
    new HashMap<Short,Boolean>();

  /**
   * Facility for monitoring:
   * If the timeout occurs for the original update, we call createAck(true)
   * in the timeout code for sending back an error ack to the original server.
   * We use this call to also save the list of server ids for server we did not
   * have time to receive an ack from. For updating its counters, the timeout
   * code can then call getTimeoutServers() method to now which servers did not
   * respond in time.
   */
  protected List<Short> serversInTimeout = null;

  /**
   * Creates a new ExpectedAcksInfo.
   * @param changeNumber The change number of the assured update message
   * @param requesterServerHandler The server handler of the server that sent
@@ -112,6 +123,15 @@
  }

  /**
   * Gets the list of expected servers that did not respond in time.
   * @return The list of expected servers that did not respond in time.
   */
  public List<Short> getTimeoutServers()
  {
    return serversInTimeout;
  }

  /**
   * Gets the requested assured mode for the matching update message.
   * @return The requested assured mode for the matching update message.
   */

 opends/src/server/org/opends/server/replication/server/ReplicationServerDomain.java

@@ -244,9 +244,11 @@
        AssuredMode assuredMode = update.getAssuredMode();
        if (assuredMode == AssuredMode.SAFE_DATA_MODE)
        {
          sourceHandler.incrementAssuredSdReceivedUpdates();
          preparedAssuredInfo = processSafeDataUpdateMsg(update, sourceHandler);
        } else if (assuredMode == AssuredMode.SAFE_READ_MODE)
        {
          sourceHandler.incrementAssuredSrReceivedUpdates();
          preparedAssuredInfo = processSafeReadUpdateMsg(update, sourceHandler);
        } else
        {
@@ -867,6 +869,58 @@
            logError(mb.toMessage());
            stopServer(origServer);
          }
          // Increment assured counters
          boolean safeRead =
            (expectedAcksInfo instanceof SafeReadExpectedAcksInfo)
            ? true : false;
          if (safeRead)
          {
            origServer.incrementAssuredSrReceivedUpdatesTimeout();
          } else
          {
            if (origServer.isLDAPserver())
            {
              origServer.incrementAssuredSdReceivedUpdatesTimeout();
            }
          }
          //   retrieve expected servers in timeout to increment their counter
          List<Short> serversInTimeout = expectedAcksInfo.getTimeoutServers();
          for (Short serverId : serversInTimeout)
          {
            ServerHandler expectedServerInTimeout =
              directoryServers.get(serverId);
            if (expectedServerInTimeout != null)
            {
              // Was a DS
              if (safeRead)
              {
                expectedServerInTimeout.incrementAssuredSrSentUpdatesTimeout();
              } else
              {
                // No SD update sent to a DS (meaningless)
              }
            } else
            {
              expectedServerInTimeout =
                replicationServers.get(serverId);
              if (expectedServerInTimeout != null)
              {
                // Was a RS
                if (safeRead)
                {
                  expectedServerInTimeout.
                    incrementAssuredSrSentUpdatesTimeout();
                } else
                {
                  expectedServerInTimeout.
                    incrementAssuredSdSentUpdatesTimeout();
                }
              } else
              {
                // Server disappeared ? Let's forget about it.
              }
            }
          }
          // Mark the ack info object as completed to prevent potential
          // processAck() code parallel run
          expectedAcksInfo.completed();

 opends/src/server/org/opends/server/replication/server/SafeDataExpectedAcksInfo.java

@@ -22,7 +22,7 @@
 * CDDL HEADER END
 *
 *
 *      Copyright 2008 Sun Microsystems, Inc.
 *      Copyright 2008-2009 Sun Microsystems, Inc.
 */

package org.opends.server.replication.server;
@@ -131,11 +131,15 @@
      // Tell wich servers did not send an ack in time
      List<Short> failedServers = new ArrayList<Short>();
      Set<Short> serverIds = expectedServersAckStatus.keySet();
      serversInTimeout = new ArrayList<Short>(); // Use next loop to fill it
      for (Short serverId : serverIds)
      {
        boolean ackReceived = expectedServersAckStatus.get(serverId);
        if (!ackReceived)
        {
          failedServers.add(serverId);
          serversInTimeout.add(serverId);
        }
      }
      ack.setFailedServers(failedServers);
    }

 opends/src/server/org/opends/server/replication/server/SafeReadExpectedAcksInfo.java

@@ -220,13 +220,17 @@

      // Add servers that did not respond in time
      Set<Short> serverIds = expectedServersAckStatus.keySet();
      serversInTimeout = new ArrayList<Short>(); // Use next loop to fill it
      for (Short serverId : serverIds)
      {
        boolean ackReceived = expectedServersAckStatus.get(serverId);
        if (!ackReceived)
        {
          if (!failedServers.contains(serverId))
          {
            failedServers.add(serverId);
            serversInTimeout.add(serverId);
          }
        }
      }
    }

 opends/src/server/org/opends/server/replication/server/ServerHandler.java

@@ -50,6 +50,7 @@
import java.util.concurrent.TimeUnit;

import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import org.opends.server.admin.std.server.MonitorProviderCfg;
import org.opends.server.api.MonitorProvider;
import org.opends.server.config.ConfigException;
@@ -95,9 +96,32 @@
  private MsgQueue lateQueue = new MsgQueue();
  private ReplicationServerDomain replicationServerDomain = null;
  private String serverURL;
  private int outCount = 0; // number of update sent to the server

  private int inCount = 0;  // number of updates received from the server
  // Number of update sent to the server
  private int outCount = 0;
  // Number of updates received from the server
  private int inCount = 0;

  // Number of updates received from the server in assured safe read mode
  private int assuredSrReceivedUpdates = 0;
  // Number of updates received from the server in assured safe read mode that
  // timed out
  private AtomicInteger assuredSrReceivedUpdatesTimeout = new AtomicInteger();
  // Number of updates sent to the server in assured safe read mode
  private int assuredSrSentUpdates = 0;
  // Number of updates sent to the server in assured safe read mode that timed
  // out
  private AtomicInteger assuredSrSentUpdatesTimeout = new AtomicInteger();
  // Number of updates received from the server in assured safe data mode
  private int assuredSdReceivedUpdates = 0;
  // Number of updates received from the server in assured safe data mode that
  // timed out
  private AtomicInteger assuredSdReceivedUpdatesTimeout = new AtomicInteger();
  // Number of updates sent to the server in assured safe data mode
  private int assuredSdSentUpdates = 0;
  // Number of updates sent to the server in assured safe data mode that timed
  // out
  private AtomicInteger assuredSdSentUpdatesTimeout = new AtomicInteger();

  private int maxReceiveQueue = 0;
  private int maxSendQueue = 0;
@@ -1186,6 +1210,162 @@
  }

  /**
   * Get the number of updates received from the server in assured safe read
   * mode.
   * @return The number of updates received from the server in assured safe read
   * mode
   */
  public int getAssuredSrReceivedUpdates()
  {
    return assuredSrReceivedUpdates;
  }

  /**
   * Get the number of updates received from the server in assured safe read
   * mode that timed out.
   * @return The number of updates received from the server in assured safe read
   * mode that timed out.
   */
  public AtomicInteger getAssuredSrReceivedUpdatesTimeout()
  {
    return assuredSrReceivedUpdatesTimeout;
  }

  /**
   * Get the number of updates sent to the server in assured safe read mode.
   * @return The number of updates sent to the server in assured safe read mode
   */
  public int getAssuredSrSentUpdates()
  {
    return assuredSrSentUpdates;
  }

  /**
   * Get the number of updates sent to the server in assured safe read mode that
   * timed out.
   * @return The number of updates sent to the server in assured safe read mode
   * that timed out.
   */
  public AtomicInteger getAssuredSrSentUpdatesTimeout()
  {
    return assuredSrSentUpdatesTimeout;
  }

    /**
   * Get the number of updates received from the server in assured safe data
   * mode.
   * @return The number of updates received from the server in assured safe data
   * mode
   */
  public int getAssuredSdReceivedUpdates()
  {
    return assuredSdReceivedUpdates;
  }

  /**
   * Get the number of updates received from the server in assured safe data
   * mode that timed out.
   * @return The number of updates received from the server in assured safe data
   * mode that timed out.
   */
  public AtomicInteger getAssuredSdReceivedUpdatesTimeout()
  {
    return assuredSdReceivedUpdatesTimeout;
  }

  /**
   * Get the number of updates sent to the server in assured safe data mode.
   * @return The number of updates sent to the server in assured safe data mode
   */
  public int getAssuredSdSentUpdates()
  {
    return assuredSdSentUpdates;
  }

  /**
   * Get the number of updates sent to the server in assured safe data mode that
   * timed out.
   * @return The number of updates sent to the server in assured safe data mode
   * that timed out.
   */
  public AtomicInteger getAssuredSdSentUpdatesTimeout()
  {
    return assuredSdSentUpdatesTimeout;
  }

  /**
   * Increment the number of updates received from the server in assured safe
   * read mode.
   */
  public void incrementAssuredSrReceivedUpdates()
  {
    assuredSrReceivedUpdates++;
  }

  /**
   * Increment the number of updates received from the server in assured safe
   * read mode that timed out.
   */
  public void incrementAssuredSrReceivedUpdatesTimeout()
  {
    assuredSrReceivedUpdatesTimeout.incrementAndGet();
  }

  /**
   * Increment the number of updates sent to the server in assured safe read
   * mode.
   */
  public void incrementAssuredSrSentUpdates()
  {
    assuredSrSentUpdates++;
  }

  /**
   * Increment the number of updates sent to the server in assured safe read
   * mode that timed out.
   */
  public void incrementAssuredSrSentUpdatesTimeout()
  {
    assuredSrSentUpdatesTimeout.incrementAndGet();
  }

  /**
   * Increment the number of updates received from the server in assured safe
   * data mode.
   */
  public void incrementAssuredSdReceivedUpdates()
  {
    assuredSdReceivedUpdates++;
  }

  /**
   * Increment the number of updates received from the server in assured safe
   * data mode that timed out.
   */
  public void incrementAssuredSdReceivedUpdatesTimeout()
  {
    assuredSdReceivedUpdatesTimeout.incrementAndGet();
  }

  /**
   * Increment the number of updates sent to the server in assured safe data
   * mode.
   */
  public void incrementAssuredSdSentUpdates()
  {
    assuredSdSentUpdates++;
  }

  /**
   * Increment the number of updates sent to the server in assured safe data
   * mode that timed out.
   */
  public void incrementAssuredSdSentUpdatesTimeout()
  {
    assuredSdSentUpdatesTimeout.incrementAndGet();
  }

  /**
   * Check is this server is saturated (this server has already been
   * sent a bunch of updates and has not processed them so they are staying
   * in the message queue for this server an the size of the queue
@@ -1544,7 +1724,21 @@
        // loop until not interrupted
      }
    } while (((interrupted) || (!acquired)) && (!shutdownWriter));
    this.incrementOutCount();
    if (msg != null)
    {
      incrementOutCount();
      if (msg.isAssured())
      {
        if (msg.getAssuredMode() == AssuredMode.SAFE_READ_MODE)
        {
          incrementAssuredSrSentUpdates();
        } else
        {
          if (!isLDAPserver())
            incrementAssuredSdSentUpdates();
        }
      }
    }
    return msg;
  }

@@ -1898,11 +2092,34 @@
    // Deprecated
    attributes.add(Attributes.create("max-waiting-changes", String
        .valueOf(maxQueueSize)));
    attributes.add(Attributes.create("update-sent", String
    attributes.add(Attributes.create("sent-updates", String
        .valueOf(getOutCount())));
    attributes.add(Attributes.create("update-received", String
    attributes.add(Attributes.create("received-updates", String
        .valueOf(getInCount())));

    // Assured counters
    attributes.add(Attributes.create("assured-sr-received-updates", String
        .valueOf(getAssuredSrReceivedUpdates())));
    attributes.add(Attributes.create("assured-sr-received-updates-timeout",
      String .valueOf(getAssuredSrReceivedUpdatesTimeout())));
    attributes.add(Attributes.create("assured-sr-sent-updates", String
        .valueOf(getAssuredSrSentUpdates())));
    attributes.add(Attributes.create("assured-sr-sent-updates-timeout", String
        .valueOf(getAssuredSrSentUpdatesTimeout())));
    attributes.add(Attributes.create("assured-sd-received-updates", String
        .valueOf(getAssuredSdReceivedUpdates())));
    if (!isLDAPserver())
    {
      attributes.add(Attributes.create("assured-sd-sent-updates",
        String.valueOf(getAssuredSdSentUpdates())));
      attributes.add(Attributes.create("assured-sd-sent-updates-timeout",
        String.valueOf(getAssuredSdSentUpdatesTimeout())));
    } else
    {
      attributes.add(Attributes.create("assured-sd-received-updates-timeout",
        String.valueOf(getAssuredSdReceivedUpdatesTimeout())));
    }

    // Window stats
    attributes.add(Attributes.create("max-send-window", String
        .valueOf(sendWindowSize)));

 opends/src/server/org/opends/server/replication/service/ReplicationDomain.java

@@ -226,7 +226,7 @@
  /**
   * A set of counters used for Monitoring.
   */
  private AtomicInteger numProcessedUpdates = new AtomicInteger();
  private AtomicInteger numProcessedUpdates = new AtomicInteger(0);
  private AtomicInteger numRcvdUpdates = new AtomicInteger(0);
  private AtomicInteger numSentUpdates = new AtomicInteger(0);

@@ -258,13 +258,13 @@
  private Map<Short,Integer> assuredSrServerNotAcknowledgedUpdates =
    new HashMap<Short,Integer>();
  // Number of updates received in Assured Mode, Safe Read request
  private AtomicInteger receivedAssuredSrUpdates = new AtomicInteger(0);
  private AtomicInteger assuredSrReceivedUpdates = new AtomicInteger(0);
  // Number of updates received in Assured Mode, Safe Read request that we have
  // acked without errors
  private AtomicInteger receivedAssuredSrUpdatesAcked = new AtomicInteger(0);
  private AtomicInteger assuredSrReceivedUpdatesAcked = new AtomicInteger(0);
  // Number of updates received in Assured Mode, Safe Read request that we have
  // acked with errors
  private AtomicInteger receivedAssuredSrUpdatesNotAcked = new AtomicInteger(0);
  private AtomicInteger assuredSrReceivedUpdatesNotAcked = new AtomicInteger(0);
  // Number of updates sent in Assured Mode, Safe Data
  private AtomicInteger assuredSdSentUpdates = new AtomicInteger(0);
  // Number of updates sent in Assured Mode, Safe Data, that have been
@@ -800,7 +800,7 @@
    if ( update.isAssured() && (update.getAssuredMode() ==
      AssuredMode.SAFE_READ_MODE) && (rsGroupId == groupId) )
    {
      receivedAssuredSrUpdates.incrementAndGet();
      assuredSrReceivedUpdates.incrementAndGet();
    }
    return update;
  }
@@ -1632,12 +1632,14 @@

    if (newStatus != status)
    {
      // Reset status date
      lastStatusChangeDate = new Date();
      // Reset monitoring counters if reconnection
      if (newStatus == ServerStatus.NOT_CONNECTED_STATUS)
        resetMonitoringCounters();

      // Store new status
      status = newStatus;
      lastStatusChangeDate = new Date();
      // Reset assured monitoring counters (they are valid for a session with
      // the same status/RS)
      resetAssuredMonitoringCounters();

      if (debugEnabled())
        TRACER.debugInfo("Replication domain " + serviceID +
@@ -2035,9 +2037,9 @@
   * Gets the number of updates received in assured safe read mode request.
   * @return The number of updates received in assured safe read mode request.
   */
  public int getReceivedAssuredSrUpdates()
  public int getAssuredSrReceivedUpdates()
  {
    return receivedAssuredSrUpdates.get();
    return assuredSrReceivedUpdates.get();
  }

  /**
@@ -2046,9 +2048,9 @@
   * @return The number of updates received in assured safe read mode that we
   * acked without error (no replay error).
   */
  public int getReceivedAssuredSrUpdatesAcked()
  public int getAssuredSrReceivedUpdatesAcked()
  {
    return this.receivedAssuredSrUpdatesAcked.get();
    return this.assuredSrReceivedUpdatesAcked.get();
  }

  /**
@@ -2057,9 +2059,9 @@
   * @return The number of updates received in assured safe read mode that we
   * did not ack due to error (replay error).
   */
  public int getReceivedAssuredSrUpdatesNotAcked()
  public int getAssuredSrReceivedUpdatesNotAcked()
  {
    return this.receivedAssuredSrUpdatesNotAcked.get();
    return this.assuredSrReceivedUpdatesNotAcked.get();
  }

  /**
@@ -2126,10 +2128,14 @@
  }

  /**
   * Resets the values of the monitoring counters for assured replication.
   * Resets the values of the monitoring counters.
   */
  private void resetAssuredMonitoringCounters()
  private void resetMonitoringCounters()
  {
    numProcessedUpdates = new AtomicInteger(0);
    numRcvdUpdates = new AtomicInteger(0);
    numSentUpdates = new AtomicInteger(0);

    assuredSrSentUpdates = new AtomicInteger(0);
    assuredSrAcknowledgedUpdates = new AtomicInteger(0);
    assuredSrNotAcknowledgedUpdates = new AtomicInteger(0);
@@ -2137,9 +2143,9 @@
    assuredSrWrongStatusUpdates = new AtomicInteger(0);
    assuredSrReplayErrorUpdates = new AtomicInteger(0);
    assuredSrServerNotAcknowledgedUpdates = new HashMap<Short,Integer>();
    receivedAssuredSrUpdates = new AtomicInteger(0);
    receivedAssuredSrUpdatesAcked = new AtomicInteger(0);
    receivedAssuredSrUpdatesNotAcked = new AtomicInteger(0);
    assuredSrReceivedUpdates = new AtomicInteger(0);
    assuredSrReceivedUpdatesAcked = new AtomicInteger(0);
    assuredSrReceivedUpdatesNotAcked = new AtomicInteger(0);
    assuredSdSentUpdates = new AtomicInteger(0);
    assuredSdAcknowledgedUpdates = new AtomicInteger(0);
    assuredSdTimeoutUpdates = new AtomicInteger(0);
@@ -2405,10 +2411,10 @@
            broker.publish(ackMsg);
            if (replayErrorMsg != null)
            {
              receivedAssuredSrUpdatesNotAcked.incrementAndGet();
              assuredSrReceivedUpdatesNotAcked.incrementAndGet();
            } else
            {
              receivedAssuredSrUpdatesAcked.incrementAndGet();
              assuredSrReceivedUpdatesAcked.incrementAndGet();
            }
          }
        } else if (assuredMode != AssuredMode.SAFE_DATA_MODE)

 opends/src/server/org/opends/server/replication/service/ReplicationMonitor.java

@@ -22,7 +22,7 @@
 * CDDL HEADER END
 *
 *
 *      Copyright 2006-2008 Sun Microsystems, Inc.
 *      Copyright 2006-2009 Sun Microsystems, Inc.
 */
package org.opends.server.replication.service;

@@ -210,14 +210,14 @@
      attributes.add(builder.toAttribute());
    }

    addMonitorData(attributes, "received-assured-sr-updates",
      domain.getReceivedAssuredSrUpdates());
    addMonitorData(attributes, "assured-sr-received-updates",
      domain.getAssuredSrReceivedUpdates());

    addMonitorData(attributes, "received-assured-sr-updates-acked",
      domain.getReceivedAssuredSrUpdatesAcked());
    addMonitorData(attributes, "assured-sr-received-updates-acked",
      domain.getAssuredSrReceivedUpdatesAcked());

    addMonitorData(attributes, "received-assured-sr-updates-not-acked",
      domain.getReceivedAssuredSrUpdatesNotAcked());
    addMonitorData(attributes, "assured-sr-received-updates-not-acked",
      domain.getAssuredSrReceivedUpdatesNotAcked());

    addMonitorData(attributes, "assured-sd-sent-updates",
      domain.getAssuredSdSentUpdates());

 opends/tests/unit-tests-testng/src/server/org/opends/server/replication/ProtocolWindowTest.java

@@ -22,7 +22,7 @@
 * CDDL HEADER END
 *
 *
 *      Copyright 2006-2008 Sun Microsystems, Inc.
 *      Copyright 2006-2009 Sun Microsystems, Inc.
 */

package org.opends.server.replication;
@@ -265,7 +265,7 @@
    InternalSearchOperation op = connection.processSearch(
        new ASN1OctetString("cn=monitor"),
        SearchScope.WHOLE_SUBTREE,
        LDAPFilter.decode("(update-sent=" + WINDOW_SIZE + ")"));
        LDAPFilter.decode("(sent-updates=" + WINDOW_SIZE + ")"));

    assertEquals(op.getResultCode(), ResultCode.SUCCESS);
    assertEquals(op.getEntriesSent(), 1,

 opends/tests/unit-tests-testng/src/server/org/opends/server/replication/plugin/AssuredReplicationPluginTest.java

@@ -935,9 +935,9 @@
        assertEquals(getMonitorAttrValue(baseDn, "assured-sr-replay-error-updates"), 0);
        Map<Short, Integer> errorsByServer = getErrorsByServers(baseDn, AssuredMode.SAFE_READ_MODE);
        assertTrue(errorsByServer.isEmpty());
        assertEquals(getMonitorAttrValue(baseDn, "received-assured-sr-updates"), 0);
        assertEquals(getMonitorAttrValue(baseDn, "received-assured-sr-updates-acked"), 0);
        assertEquals(getMonitorAttrValue(baseDn, "received-assured-sr-updates-not-acked"), 0);
        assertEquals(getMonitorAttrValue(baseDn, "assured-sr-received-updates"), 0);
        assertEquals(getMonitorAttrValue(baseDn, "assured-sr-received-updates-acked"), 0);
        assertEquals(getMonitorAttrValue(baseDn, "assured-sr-received-updates-not-acked"), 0);
        assertEquals(getMonitorAttrValue(baseDn, "assured-sd-sent-updates"), 1);
        assertEquals(getMonitorAttrValue(baseDn, "assured-sd-acknowledged-updates"), 0);
        assertEquals(getMonitorAttrValue(baseDn, "assured-sd-timeout-updates"), 1);
@@ -964,9 +964,9 @@
        assertEquals(getMonitorAttrValue(baseDn, "assured-sr-replay-error-updates"), 0);
        Map<Short, Integer> errorsByServer = getErrorsByServers(baseDn, AssuredMode.SAFE_READ_MODE);
        assertTrue(errorsByServer.isEmpty());
        assertEquals(getMonitorAttrValue(baseDn, "received-assured-sr-updates"), 0);
        assertEquals(getMonitorAttrValue(baseDn, "received-assured-sr-updates-acked"), 0);
        assertEquals(getMonitorAttrValue(baseDn, "received-assured-sr-updates-not-acked"), 0);
        assertEquals(getMonitorAttrValue(baseDn, "assured-sr-received-updates"), 0);
        assertEquals(getMonitorAttrValue(baseDn, "assured-sr-received-updates-acked"), 0);
        assertEquals(getMonitorAttrValue(baseDn, "assured-sr-received-updates-not-acked"), 0);
        assertEquals(getMonitorAttrValue(baseDn, "assured-sd-sent-updates"), 0);
        assertEquals(getMonitorAttrValue(baseDn, "assured-sd-acknowledged-updates"), 0);
        assertEquals(getMonitorAttrValue(baseDn, "assured-sd-timeout-updates"), 0);
@@ -1038,9 +1038,9 @@
        Integer nError = errorsByServer.get((short)RS_SERVER_ID);
        assertNotNull(nError);
        assertEquals(nError.intValue(), 1);
        assertEquals(getMonitorAttrValue(baseDn, "received-assured-sr-updates"), 0);
        assertEquals(getMonitorAttrValue(baseDn, "received-assured-sr-updates-acked"), 0);
        assertEquals(getMonitorAttrValue(baseDn, "received-assured-sr-updates-not-acked"), 0);
        assertEquals(getMonitorAttrValue(baseDn, "assured-sr-received-updates"), 0);
        assertEquals(getMonitorAttrValue(baseDn, "assured-sr-received-updates-acked"), 0);
        assertEquals(getMonitorAttrValue(baseDn, "assured-sr-received-updates-not-acked"), 0);
        assertEquals(getMonitorAttrValue(baseDn, "assured-sd-sent-updates"), 0);
        assertEquals(getMonitorAttrValue(baseDn, "assured-sd-acknowledged-updates"), 0);
        assertEquals(getMonitorAttrValue(baseDn, "assured-sd-timeout-updates"), 0);
@@ -1063,9 +1063,9 @@
        assertEquals(getMonitorAttrValue(baseDn, "assured-sr-replay-error-updates"), 0);
        Map<Short, Integer> errorsByServer = getErrorsByServers(baseDn, AssuredMode.SAFE_READ_MODE);
        assertTrue(errorsByServer.isEmpty());
        assertEquals(getMonitorAttrValue(baseDn, "received-assured-sr-updates"), 0);
        assertEquals(getMonitorAttrValue(baseDn, "received-assured-sr-updates-acked"), 0);
        assertEquals(getMonitorAttrValue(baseDn, "received-assured-sr-updates-not-acked"), 0);
        assertEquals(getMonitorAttrValue(baseDn, "assured-sr-received-updates"), 0);
        assertEquals(getMonitorAttrValue(baseDn, "assured-sr-received-updates-acked"), 0);
        assertEquals(getMonitorAttrValue(baseDn, "assured-sr-received-updates-not-acked"), 0);
        assertEquals(getMonitorAttrValue(baseDn, "assured-sd-sent-updates"), 0);
        assertEquals(getMonitorAttrValue(baseDn, "assured-sd-acknowledged-updates"), 0);
        assertEquals(getMonitorAttrValue(baseDn, "assured-sd-timeout-updates"), 0);
@@ -1220,9 +1220,9 @@
      assertEquals(getMonitorAttrValue(baseDn, "assured-sr-replay-error-updates"), 0);
      Map<Short, Integer> errorsByServer = getErrorsByServers(baseDn, AssuredMode.SAFE_READ_MODE);
      assertTrue(errorsByServer.isEmpty());
      assertEquals(getMonitorAttrValue(baseDn, "received-assured-sr-updates"), 0);
      assertEquals(getMonitorAttrValue(baseDn, "received-assured-sr-updates-acked"), 0);
      assertEquals(getMonitorAttrValue(baseDn, "received-assured-sr-updates-not-acked"), 0);
      assertEquals(getMonitorAttrValue(baseDn, "assured-sr-received-updates"), 0);
      assertEquals(getMonitorAttrValue(baseDn, "assured-sr-received-updates-acked"), 0);
      assertEquals(getMonitorAttrValue(baseDn, "assured-sr-received-updates-not-acked"), 0);
      assertEquals(getMonitorAttrValue(baseDn, "assured-sd-sent-updates"), 1);
      assertEquals(getMonitorAttrValue(baseDn, "assured-sd-acknowledged-updates"), 1);
      assertEquals(getMonitorAttrValue(baseDn, "assured-sd-timeout-updates"), 0);
@@ -1284,9 +1284,9 @@
      assertEquals(getMonitorAttrValue(baseDn, "assured-sr-replay-error-updates"), 0);
      Map<Short, Integer> errorsByServer = getErrorsByServers(baseDn, AssuredMode.SAFE_READ_MODE);
      assertTrue(errorsByServer.isEmpty());
      assertEquals(getMonitorAttrValue(baseDn, "received-assured-sr-updates"), 0);
      assertEquals(getMonitorAttrValue(baseDn, "received-assured-sr-updates-acked"), 0);
      assertEquals(getMonitorAttrValue(baseDn, "received-assured-sr-updates-not-acked"), 0);
      assertEquals(getMonitorAttrValue(baseDn, "assured-sr-received-updates"), 0);
      assertEquals(getMonitorAttrValue(baseDn, "assured-sr-received-updates-acked"), 0);
      assertEquals(getMonitorAttrValue(baseDn, "assured-sr-received-updates-not-acked"), 0);
      assertEquals(getMonitorAttrValue(baseDn, "assured-sd-sent-updates"), 0);
      assertEquals(getMonitorAttrValue(baseDn, "assured-sd-acknowledged-updates"), 0);
      assertEquals(getMonitorAttrValue(baseDn, "assured-sd-timeout-updates"), 0);
@@ -1418,9 +1418,9 @@
        assertEquals(getMonitorAttrValue(baseDn, "assured-sr-replay-error-updates"), 0);
        Map<Short, Integer> errorsByServer = getErrorsByServers(baseDn, AssuredMode.SAFE_READ_MODE);
        assertTrue(errorsByServer.isEmpty());
        assertEquals(getMonitorAttrValue(baseDn, "received-assured-sr-updates"), 1);
        assertEquals(getMonitorAttrValue(baseDn, "received-assured-sr-updates-acked"), 1);
        assertEquals(getMonitorAttrValue(baseDn, "received-assured-sr-updates-not-acked"), 0);
        assertEquals(getMonitorAttrValue(baseDn, "assured-sr-received-updates"), 1);
        assertEquals(getMonitorAttrValue(baseDn, "assured-sr-received-updates-acked"), 1);
        assertEquals(getMonitorAttrValue(baseDn, "assured-sr-received-updates-not-acked"), 0);
        assertEquals(getMonitorAttrValue(baseDn, "assured-sd-sent-updates"), 0);
        assertEquals(getMonitorAttrValue(baseDn, "assured-sd-acknowledged-updates"), 0);
        assertEquals(getMonitorAttrValue(baseDn, "assured-sd-timeout-updates"), 0);
@@ -1560,9 +1560,9 @@
      assertEquals(getMonitorAttrValue(baseDn, "assured-sr-replay-error-updates"), 0);
      Map<Short, Integer> errorsByServer = getErrorsByServers(baseDn, AssuredMode.SAFE_READ_MODE);
      assertTrue(errorsByServer.isEmpty());
      assertEquals(getMonitorAttrValue(baseDn, "received-assured-sr-updates"), 0);
      assertEquals(getMonitorAttrValue(baseDn, "received-assured-sr-updates-acked"), 0);
      assertEquals(getMonitorAttrValue(baseDn, "received-assured-sr-updates-not-acked"), 0);
      assertEquals(getMonitorAttrValue(baseDn, "assured-sr-received-updates"), 0);
      assertEquals(getMonitorAttrValue(baseDn, "assured-sr-received-updates-acked"), 0);
      assertEquals(getMonitorAttrValue(baseDn, "assured-sr-received-updates-not-acked"), 0);
      assertEquals(getMonitorAttrValue(baseDn, "assured-sd-sent-updates"), 1);
      assertEquals(getMonitorAttrValue(baseDn, "assured-sd-acknowledged-updates"), 0);
      assertEquals(getMonitorAttrValue(baseDn, "assured-sd-timeout-updates"), 1);
@@ -1598,9 +1598,9 @@
      assertEquals(getMonitorAttrValue(baseDn, "assured-sr-replay-error-updates"), 0);
      errorsByServer = getErrorsByServers(baseDn, AssuredMode.SAFE_READ_MODE);
      assertTrue(errorsByServer.isEmpty());
      assertEquals(getMonitorAttrValue(baseDn, "received-assured-sr-updates"), 0);
      assertEquals(getMonitorAttrValue(baseDn, "received-assured-sr-updates-acked"), 0);
      assertEquals(getMonitorAttrValue(baseDn, "received-assured-sr-updates-not-acked"), 0);
      assertEquals(getMonitorAttrValue(baseDn, "assured-sr-received-updates"), 0);
      assertEquals(getMonitorAttrValue(baseDn, "assured-sr-received-updates-acked"), 0);
      assertEquals(getMonitorAttrValue(baseDn, "assured-sr-received-updates-not-acked"), 0);
      assertEquals(getMonitorAttrValue(baseDn, "assured-sd-sent-updates"), 2);
      assertEquals(getMonitorAttrValue(baseDn, "assured-sd-acknowledged-updates"), 0);
      assertEquals(getMonitorAttrValue(baseDn, "assured-sd-timeout-updates"), 2);
@@ -1638,9 +1638,9 @@
      assertEquals(getMonitorAttrValue(baseDn, "assured-sr-replay-error-updates"), 0);
      errorsByServer = getErrorsByServers(baseDn, AssuredMode.SAFE_READ_MODE);
      assertTrue(errorsByServer.isEmpty());
      assertEquals(getMonitorAttrValue(baseDn, "received-assured-sr-updates"), 0);
      assertEquals(getMonitorAttrValue(baseDn, "received-assured-sr-updates-acked"), 0);
      assertEquals(getMonitorAttrValue(baseDn, "received-assured-sr-updates-not-acked"), 0);
      assertEquals(getMonitorAttrValue(baseDn, "assured-sr-received-updates"), 0);
      assertEquals(getMonitorAttrValue(baseDn, "assured-sr-received-updates-acked"), 0);
      assertEquals(getMonitorAttrValue(baseDn, "assured-sr-received-updates-not-acked"), 0);
      assertEquals(getMonitorAttrValue(baseDn, "assured-sd-sent-updates"), 3);
      assertEquals(getMonitorAttrValue(baseDn, "assured-sd-acknowledged-updates"), 0);
      assertEquals(getMonitorAttrValue(baseDn, "assured-sd-timeout-updates"), 3);
@@ -1722,9 +1722,9 @@
      nError = errorsByServer.get((short)20);
      assertNotNull(nError);
      assertEquals(nError.intValue(), 1);
      assertEquals(getMonitorAttrValue(baseDn, "received-assured-sr-updates"), 0);
      assertEquals(getMonitorAttrValue(baseDn, "received-assured-sr-updates-acked"), 0);
      assertEquals(getMonitorAttrValue(baseDn, "received-assured-sr-updates-not-acked"), 0);
      assertEquals(getMonitorAttrValue(baseDn, "assured-sr-received-updates"), 0);
      assertEquals(getMonitorAttrValue(baseDn, "assured-sr-received-updates-acked"), 0);
      assertEquals(getMonitorAttrValue(baseDn, "assured-sr-received-updates-not-acked"), 0);
      assertEquals(getMonitorAttrValue(baseDn, "assured-sd-sent-updates"), 0);
      assertEquals(getMonitorAttrValue(baseDn, "assured-sd-acknowledged-updates"), 0);
      assertEquals(getMonitorAttrValue(baseDn, "assured-sd-timeout-updates"), 0);
@@ -1767,9 +1767,9 @@
      nError = errorsByServer.get((short)30);
      assertNotNull(nError);
      assertEquals(nError.intValue(), 1);
      assertEquals(getMonitorAttrValue(baseDn, "received-assured-sr-updates"), 0);
      assertEquals(getMonitorAttrValue(baseDn, "received-assured-sr-updates-acked"), 0);
      assertEquals(getMonitorAttrValue(baseDn, "received-assured-sr-updates-not-acked"), 0);
      assertEquals(getMonitorAttrValue(baseDn, "assured-sr-received-updates"), 0);
      assertEquals(getMonitorAttrValue(baseDn, "assured-sr-received-updates-acked"), 0);
      assertEquals(getMonitorAttrValue(baseDn, "assured-sr-received-updates-not-acked"), 0);
      assertEquals(getMonitorAttrValue(baseDn, "assured-sd-sent-updates"), 0);
      assertEquals(getMonitorAttrValue(baseDn, "assured-sd-acknowledged-updates"), 0);
      assertEquals(getMonitorAttrValue(baseDn, "assured-sd-timeout-updates"), 0);
@@ -1812,9 +1812,9 @@
      nError = errorsByServer.get((short)RS_SERVER_ID);
      assertNotNull(nError);
      assertEquals(nError.intValue(), 1);
      assertEquals(getMonitorAttrValue(baseDn, "received-assured-sr-updates"), 0);
      assertEquals(getMonitorAttrValue(baseDn, "received-assured-sr-updates-acked"), 0);
      assertEquals(getMonitorAttrValue(baseDn, "received-assured-sr-updates-not-acked"), 0);
      assertEquals(getMonitorAttrValue(baseDn, "assured-sr-received-updates"), 0);
      assertEquals(getMonitorAttrValue(baseDn, "assured-sr-received-updates-acked"), 0);
      assertEquals(getMonitorAttrValue(baseDn, "assured-sr-received-updates-not-acked"), 0);
      assertEquals(getMonitorAttrValue(baseDn, "assured-sd-sent-updates"), 0);
      assertEquals(getMonitorAttrValue(baseDn, "assured-sd-acknowledged-updates"), 0);
      assertEquals(getMonitorAttrValue(baseDn, "assured-sd-timeout-updates"), 0);

 opends/tests/unit-tests-testng/src/server/org/opends/server/replication/server/AssuredReplicationServerTest.java

@@ -2395,9 +2395,9 @@
      assertEquals(fakeRd1.getAssuredSrWrongStatusUpdates(), 0);
      assertEquals(fakeRd1.getAssuredSrReplayErrorUpdates(), 0);
      assertEquals(fakeRd1.getAssuredSrServerNotAcknowledgedUpdates().size(), 0);
      assertEquals(fakeRd1.getReceivedAssuredSrUpdates(), 0);
      assertEquals(fakeRd1.getReceivedAssuredSrUpdatesAcked(), 0);
      assertEquals(fakeRd1.getReceivedAssuredSrUpdatesNotAcked(), 0);
      assertEquals(fakeRd1.getAssuredSrReceivedUpdates(), 0);
      assertEquals(fakeRd1.getAssuredSrReceivedUpdatesAcked(), 0);
      assertEquals(fakeRd1.getAssuredSrReceivedUpdatesNotAcked(), 0);

      // Sanity check
      assertEquals(fakeRd1.getReceivedUpdates(), 0);
@@ -2444,9 +2444,9 @@
      assertEquals(fakeRd1.getAssuredSrWrongStatusUpdates(), 0);
      assertEquals(fakeRd1.getAssuredSrReplayErrorUpdates(), 0);
      assertEquals(fakeRd1.getAssuredSrServerNotAcknowledgedUpdates().size(), 0);
      assertEquals(fakeRd1.getReceivedAssuredSrUpdates(), 0);
      assertEquals(fakeRd1.getReceivedAssuredSrUpdatesAcked(), 0);
      assertEquals(fakeRd1.getReceivedAssuredSrUpdatesNotAcked(), 0);
      assertEquals(fakeRd1.getAssuredSrReceivedUpdates(), 0);
      assertEquals(fakeRd1.getAssuredSrReceivedUpdatesAcked(), 0);
      assertEquals(fakeRd1.getAssuredSrReceivedUpdatesNotAcked(), 0);

      assertEquals(fakeRd2.getAssuredSrSentUpdates(), 0);
      assertEquals(fakeRd2.getAssuredSrAcknowledgedUpdates(), 0);
@@ -2455,9 +2455,9 @@
      assertEquals(fakeRd2.getAssuredSrWrongStatusUpdates(), 0);
      assertEquals(fakeRd2.getAssuredSrReplayErrorUpdates(), 0);
      assertEquals(fakeRd2.getAssuredSrServerNotAcknowledgedUpdates().size(), 0);
      assertEquals(fakeRd2.getReceivedAssuredSrUpdates(), 1);
      assertEquals(fakeRd2.getReceivedAssuredSrUpdatesAcked(), 1);
      assertEquals(fakeRd2.getReceivedAssuredSrUpdatesNotAcked(), 0);
      assertEquals(fakeRd2.getAssuredSrReceivedUpdates(), 1);
      assertEquals(fakeRd2.getAssuredSrReceivedUpdatesAcked(), 1);
      assertEquals(fakeRd2.getAssuredSrReceivedUpdatesNotAcked(), 0);

      // Sanity check
      assertEquals(fakeRd1.getReceivedUpdates(), 0);
@@ -2504,9 +2504,9 @@
      assertEquals(fakeRd1.getAssuredSrWrongStatusUpdates(), 0);
      assertEquals(fakeRd1.getAssuredSrReplayErrorUpdates(), 0);
      assertEquals(fakeRd1.getAssuredSrServerNotAcknowledgedUpdates().size(), 0);
      assertEquals(fakeRd1.getReceivedAssuredSrUpdates(), 0);
      assertEquals(fakeRd1.getReceivedAssuredSrUpdatesAcked(), 0);
      assertEquals(fakeRd1.getReceivedAssuredSrUpdatesNotAcked(), 0);
      assertEquals(fakeRd1.getAssuredSrReceivedUpdates(), 0);
      assertEquals(fakeRd1.getAssuredSrReceivedUpdatesAcked(), 0);
      assertEquals(fakeRd1.getAssuredSrReceivedUpdatesNotAcked(), 0);

      assertEquals(fakeRd2.getAssuredSrSentUpdates(), 0);
      assertEquals(fakeRd2.getAssuredSrAcknowledgedUpdates(), 0);
@@ -2515,9 +2515,9 @@
      assertEquals(fakeRd2.getAssuredSrWrongStatusUpdates(), 0);
      assertEquals(fakeRd2.getAssuredSrReplayErrorUpdates(), 0);
      assertEquals(fakeRd2.getAssuredSrServerNotAcknowledgedUpdates().size(), 0);
      assertEquals(fakeRd2.getReceivedAssuredSrUpdates(), 2);
      assertEquals(fakeRd2.getReceivedAssuredSrUpdatesAcked(), 2);
      assertEquals(fakeRd2.getReceivedAssuredSrUpdatesNotAcked(), 0);
      assertEquals(fakeRd2.getAssuredSrReceivedUpdates(), 2);
      assertEquals(fakeRd2.getAssuredSrReceivedUpdatesAcked(), 2);
      assertEquals(fakeRd2.getAssuredSrReceivedUpdatesNotAcked(), 0);

      // Sanity check
      assertEquals(fakeRd1.getReceivedUpdates(), 0);
@@ -2566,9 +2566,9 @@
      assertEquals(fakeRd1.getAssuredSrWrongStatusUpdates(), 0);
      assertEquals(fakeRd1.getAssuredSrReplayErrorUpdates(), 0);
      assertEquals(fakeRd1.getAssuredSrServerNotAcknowledgedUpdates().size(), 0);
      assertEquals(fakeRd1.getReceivedAssuredSrUpdates(), 0);
      assertEquals(fakeRd1.getReceivedAssuredSrUpdatesAcked(), 0);
      assertEquals(fakeRd1.getReceivedAssuredSrUpdatesNotAcked(), 0);
      assertEquals(fakeRd1.getAssuredSrReceivedUpdates(), 0);
      assertEquals(fakeRd1.getAssuredSrReceivedUpdatesAcked(), 0);
      assertEquals(fakeRd1.getAssuredSrReceivedUpdatesNotAcked(), 0);

      // Sanity check
      assertEquals(fakeRd1.getReceivedUpdates(), 0);
@@ -2614,9 +2614,9 @@
      assertEquals(fakeRd1.getAssuredSrWrongStatusUpdates(), 0);
      assertEquals(fakeRd1.getAssuredSrReplayErrorUpdates(), 0);
      assertEquals(fakeRd1.getAssuredSrServerNotAcknowledgedUpdates().size(), 0);
      assertEquals(fakeRd1.getReceivedAssuredSrUpdates(), 0);
      assertEquals(fakeRd1.getReceivedAssuredSrUpdatesAcked(), 0);
      assertEquals(fakeRd1.getReceivedAssuredSrUpdatesNotAcked(), 0);
      assertEquals(fakeRd1.getAssuredSrReceivedUpdates(), 0);
      assertEquals(fakeRd1.getAssuredSrReceivedUpdatesAcked(), 0);
      assertEquals(fakeRd1.getAssuredSrReceivedUpdatesNotAcked(), 0);

      // Sanity check
      assertEquals(fakeRd1.getReceivedUpdates(), 0);
@@ -2910,9 +2910,9 @@
        expectedErrors.put(DS_FRS2_ID, 1);
      checkServerErrorListsAreEqual(fakeRd1.getAssuredSrServerNotAcknowledgedUpdates(), expectedErrors);
      
      assertEquals(fakeRd1.getReceivedAssuredSrUpdates(), 0);
      assertEquals(fakeRd1.getReceivedAssuredSrUpdatesAcked(), 0);
      assertEquals(fakeRd1.getReceivedAssuredSrUpdatesNotAcked(), 0);
      assertEquals(fakeRd1.getAssuredSrReceivedUpdates(), 0);
      assertEquals(fakeRd1.getAssuredSrReceivedUpdatesAcked(), 0);
      assertEquals(fakeRd1.getAssuredSrReceivedUpdatesNotAcked(), 0);

      // Check monitoring values in DS 2
      //
@@ -2923,9 +2923,9 @@
      assertEquals(fakeRd2.getAssuredSrWrongStatusUpdates(), 0);
      assertEquals(fakeRd2.getAssuredSrReplayErrorUpdates(), 0);
      assertEquals(fakeRd2.getAssuredSrServerNotAcknowledgedUpdates().size(), 0);
      assertEquals(fakeRd2.getReceivedAssuredSrUpdates(), 1);
      assertEquals(fakeRd2.getReceivedAssuredSrUpdatesAcked(), 1);
      assertEquals(fakeRd2.getReceivedAssuredSrUpdatesNotAcked(), 0);
      assertEquals(fakeRd2.getAssuredSrReceivedUpdates(), 1);
      assertEquals(fakeRd2.getAssuredSrReceivedUpdatesAcked(), 1);
      assertEquals(fakeRd2.getAssuredSrReceivedUpdatesNotAcked(), 0);

      // Check monitoring values in DS 3
      //
@@ -2938,30 +2938,30 @@
      assertEquals(fakeRd3.getAssuredSrServerNotAcknowledgedUpdates().size(), 0);
      if (dsIsEligible)
      {
        assertEquals(fakeRd3.getReceivedAssuredSrUpdates(), 1);
        assertEquals(fakeRd3.getAssuredSrReceivedUpdates(), 1);
        if (dsIsExpected)
        {
          assertEquals(fakeRd3.getReceivedAssuredSrUpdatesAcked(), 1);
          assertEquals(fakeRd3.getReceivedAssuredSrUpdatesNotAcked(), 0);
          assertEquals(fakeRd3.getAssuredSrReceivedUpdatesAcked(), 1);
          assertEquals(fakeRd3.getAssuredSrReceivedUpdatesNotAcked(), 0);
        } else
        {
          if (shouldSeeReplayError && (otherFakeDsScen == REPLAY_ERROR_DS_SCENARIO))
          {
            // Replay error for the other DS
            assertEquals(fakeRd3.getReceivedAssuredSrUpdatesAcked(), 0);
            assertEquals(fakeRd3.getReceivedAssuredSrUpdatesNotAcked(), 1);
            assertEquals(fakeRd3.getAssuredSrReceivedUpdatesAcked(), 0);
            assertEquals(fakeRd3.getAssuredSrReceivedUpdatesNotAcked(), 1);
          } else
          {
            assertEquals(fakeRd3.getReceivedAssuredSrUpdatesAcked(), 0);
            assertEquals(fakeRd3.getReceivedAssuredSrUpdatesNotAcked(), 0);
            assertEquals(fakeRd3.getAssuredSrReceivedUpdatesAcked(), 0);
            assertEquals(fakeRd3.getAssuredSrReceivedUpdatesNotAcked(), 0);
          }
        }
      }
      else
      {
        assertEquals(fakeRd3.getReceivedAssuredSrUpdates(), 0);
        assertEquals(fakeRd3.getReceivedAssuredSrUpdatesAcked(), 0);
        assertEquals(fakeRd3.getReceivedAssuredSrUpdatesNotAcked(), 0);
        assertEquals(fakeRd3.getAssuredSrReceivedUpdates(), 0);
        assertEquals(fakeRd3.getAssuredSrReceivedUpdatesAcked(), 0);
        assertEquals(fakeRd3.getAssuredSrReceivedUpdatesNotAcked(), 0);
      }

      // Sanity check
@@ -3358,9 +3358,9 @@
    assertEquals(fakeRd.getAssuredSrWrongStatusUpdates(), 0);
    assertEquals(fakeRd.getAssuredSrReplayErrorUpdates(), 0);
    assertEquals(fakeRd.getAssuredSrServerNotAcknowledgedUpdates().size(), 0);
    assertEquals(fakeRd.getReceivedAssuredSrUpdates(), nPacket);
    assertEquals(fakeRd.getReceivedAssuredSrUpdatesAcked(), nPacket);
    assertEquals(fakeRd.getReceivedAssuredSrUpdatesNotAcked(), 0);
    assertEquals(fakeRd.getAssuredSrReceivedUpdates(), nPacket);
    assertEquals(fakeRd.getAssuredSrReceivedUpdatesAcked(), nPacket);
    assertEquals(fakeRd.getAssuredSrReceivedUpdatesNotAcked(), 0);
  }

  // Helper method for some safe read test methods
@@ -3373,9 +3373,9 @@
    assertEquals(fakeRd.getAssuredSrWrongStatusUpdates(), 0);
    assertEquals(fakeRd.getAssuredSrReplayErrorUpdates(), 0);
    assertEquals(fakeRd.getAssuredSrServerNotAcknowledgedUpdates().size(), 0);
    assertEquals(fakeRd.getReceivedAssuredSrUpdates(), 0);
    assertEquals(fakeRd.getReceivedAssuredSrUpdatesAcked(), 0);
    assertEquals(fakeRd.getReceivedAssuredSrUpdatesNotAcked(), 0);
    assertEquals(fakeRd.getAssuredSrReceivedUpdates(), 0);
    assertEquals(fakeRd.getAssuredSrReceivedUpdatesAcked(), 0);
    assertEquals(fakeRd.getAssuredSrReceivedUpdatesNotAcked(), 0);
  }

  /**
@@ -3478,9 +3478,9 @@
      assertEquals(fakeRd3.getAssuredSrWrongStatusUpdates(), 0);
      assertEquals(fakeRd3.getAssuredSrReplayErrorUpdates(), 0);
      assertEquals(fakeRd3.getAssuredSrServerNotAcknowledgedUpdates().size(), 0);
      assertEquals(fakeRd3.getReceivedAssuredSrUpdates(), 0);
      assertEquals(fakeRd3.getReceivedAssuredSrUpdatesAcked(), 0);
      assertEquals(fakeRd3.getReceivedAssuredSrUpdatesNotAcked(), 0);
      assertEquals(fakeRd3.getAssuredSrReceivedUpdates(), 0);
      assertEquals(fakeRd3.getAssuredSrReceivedUpdatesAcked(), 0);
      assertEquals(fakeRd3.getAssuredSrReceivedUpdatesNotAcked(), 0);

      assertEquals(fakeRd4.getAssuredSrSentUpdates(), 0);
      assertEquals(fakeRd4.getAssuredSrAcknowledgedUpdates(), 0);
@@ -3489,9 +3489,9 @@
      assertEquals(fakeRd4.getAssuredSrWrongStatusUpdates(), 0);
      assertEquals(fakeRd4.getAssuredSrReplayErrorUpdates(), 0);
      assertEquals(fakeRd4.getAssuredSrServerNotAcknowledgedUpdates().size(), 0);
      assertEquals(fakeRd4.getReceivedAssuredSrUpdates(), 0);
      assertEquals(fakeRd4.getReceivedAssuredSrUpdatesAcked(), 0);
      assertEquals(fakeRd4.getReceivedAssuredSrUpdatesNotAcked(), 0);
      assertEquals(fakeRd4.getAssuredSrReceivedUpdates(), 0);
      assertEquals(fakeRd4.getAssuredSrReceivedUpdatesAcked(), 0);
      assertEquals(fakeRd4.getAssuredSrReceivedUpdatesNotAcked(), 0);

      assertEquals(fakeRd1.getReceivedUpdates(), 0);
      assertTrue(fakeRd1.receivedUpdatesOk());
@@ -3622,9 +3622,9 @@
            Integer nError = failedServer.get(FDS2_ID);
            assertNotNull(nError);
            assertEquals(nError.intValue(), 1);
            assertEquals(fakeRd1.getReceivedAssuredSrUpdates(), 0);
            assertEquals(fakeRd1.getReceivedAssuredSrUpdatesAcked(), 0);
            assertEquals(fakeRd1.getReceivedAssuredSrUpdatesNotAcked(), 0);
            assertEquals(fakeRd1.getAssuredSrReceivedUpdates(), 0);
            assertEquals(fakeRd1.getAssuredSrReceivedUpdatesAcked(), 0);
            assertEquals(fakeRd1.getAssuredSrReceivedUpdatesNotAcked(), 0);

            assertEquals(fakeRd2.getAssuredSrSentUpdates(), 0);
            assertEquals(fakeRd2.getAssuredSrAcknowledgedUpdates(), 0);
@@ -3633,9 +3633,9 @@
            assertEquals(fakeRd2.getAssuredSrWrongStatusUpdates(), 0);
            assertEquals(fakeRd2.getAssuredSrReplayErrorUpdates(), 0);
            assertEquals(fakeRd2.getAssuredSrServerNotAcknowledgedUpdates().size(), 0);
            assertEquals(fakeRd2.getReceivedAssuredSrUpdates(), 1);
            assertEquals(fakeRd2.getReceivedAssuredSrUpdatesAcked(), 0);
            assertEquals(fakeRd2.getReceivedAssuredSrUpdatesNotAcked(), 0);
            assertEquals(fakeRd2.getAssuredSrReceivedUpdates(), 1);
            assertEquals(fakeRd2.getAssuredSrReceivedUpdatesAcked(), 0);
            assertEquals(fakeRd2.getAssuredSrReceivedUpdatesNotAcked(), 0);
            break;
          case REPLAY_ERROR_DS_SCENARIO:
            assertEquals(fakeRd1.getAssuredSrSentUpdates(), 1);
@@ -3649,9 +3649,9 @@
            nError = failedServer.get(FDS2_ID);
            assertNotNull(nError);
            assertEquals(nError.intValue(), 1);
            assertEquals(fakeRd1.getReceivedAssuredSrUpdates(), 0);
            assertEquals(fakeRd1.getReceivedAssuredSrUpdatesAcked(), 0);
            assertEquals(fakeRd1.getReceivedAssuredSrUpdatesNotAcked(), 0);
            assertEquals(fakeRd1.getAssuredSrReceivedUpdates(), 0);
            assertEquals(fakeRd1.getAssuredSrReceivedUpdatesAcked(), 0);
            assertEquals(fakeRd1.getAssuredSrReceivedUpdatesNotAcked(), 0);

            assertEquals(fakeRd2.getAssuredSrSentUpdates(), 0);
            assertEquals(fakeRd2.getAssuredSrAcknowledgedUpdates(), 0);
@@ -3660,9 +3660,9 @@
            assertEquals(fakeRd2.getAssuredSrWrongStatusUpdates(), 0);
            assertEquals(fakeRd2.getAssuredSrReplayErrorUpdates(), 0);
            assertEquals(fakeRd2.getAssuredSrServerNotAcknowledgedUpdates().size(), 0);
            assertEquals(fakeRd2.getReceivedAssuredSrUpdates(), 1);
            assertEquals(fakeRd2.getReceivedAssuredSrUpdatesAcked(), 0);
            assertEquals(fakeRd2.getReceivedAssuredSrUpdatesNotAcked(), 1);
            assertEquals(fakeRd2.getAssuredSrReceivedUpdates(), 1);
            assertEquals(fakeRd2.getAssuredSrReceivedUpdatesAcked(), 0);
            assertEquals(fakeRd2.getAssuredSrReceivedUpdatesNotAcked(), 1);
            break;
          default:
            fail("Unknown scenario: " + fakeDsScen);
@@ -3785,9 +3785,9 @@
      Integer nError = failedServer.get(FDS2_ID);
      assertNotNull(nError);
      assertEquals(nError.intValue(), 4);
      assertEquals(fakeRd1.getReceivedAssuredSrUpdates(), 0);
      assertEquals(fakeRd1.getReceivedAssuredSrUpdatesAcked(), 0);
      assertEquals(fakeRd1.getReceivedAssuredSrUpdatesNotAcked(), 0);
      assertEquals(fakeRd1.getAssuredSrReceivedUpdates(), 0);
      assertEquals(fakeRd1.getAssuredSrReceivedUpdatesAcked(), 0);
      assertEquals(fakeRd1.getAssuredSrReceivedUpdatesNotAcked(), 0);

      assertEquals(fakeRd2.getAssuredSrSentUpdates(), 0);
      assertEquals(fakeRd2.getAssuredSrAcknowledgedUpdates(), 0);
@@ -3796,9 +3796,9 @@
      assertEquals(fakeRd2.getAssuredSrWrongStatusUpdates(), 0);
      assertEquals(fakeRd2.getAssuredSrReplayErrorUpdates(), 0);
      assertEquals(fakeRd2.getAssuredSrServerNotAcknowledgedUpdates().size(), 0);
      assertEquals(fakeRd2.getReceivedAssuredSrUpdates(), 0);
      assertEquals(fakeRd2.getReceivedAssuredSrUpdatesAcked(), 0);
      assertEquals(fakeRd2.getReceivedAssuredSrUpdatesNotAcked(), 0);
      assertEquals(fakeRd2.getAssuredSrReceivedUpdates(), 0);
      assertEquals(fakeRd2.getAssuredSrReceivedUpdatesAcked(), 0);
      assertEquals(fakeRd2.getAssuredSrReceivedUpdatesNotAcked(), 0);

      assertEquals(fakeRd1.getReceivedUpdates(), 0);
      assertEquals(fakeRd1.getWrongReceivedUpdates(), 0);
@@ -3836,9 +3836,9 @@
      nError = failedServer.get(FDS2_ID);
      assertNotNull(nError);
      assertEquals(nError.intValue(), 4);
      assertEquals(fakeRd1.getReceivedAssuredSrUpdates(), 0);
      assertEquals(fakeRd1.getReceivedAssuredSrUpdatesAcked(), 0);
      assertEquals(fakeRd1.getReceivedAssuredSrUpdatesNotAcked(), 0);
      assertEquals(fakeRd1.getAssuredSrReceivedUpdates(), 0);
      assertEquals(fakeRd1.getAssuredSrReceivedUpdatesAcked(), 0);
      assertEquals(fakeRd1.getAssuredSrReceivedUpdatesNotAcked(), 0);

      assertEquals(fakeRd2.getAssuredSrSentUpdates(), 0);
      assertEquals(fakeRd2.getAssuredSrAcknowledgedUpdates(), 0);
@@ -3847,9 +3847,9 @@
      assertEquals(fakeRd2.getAssuredSrWrongStatusUpdates(), 0);
      assertEquals(fakeRd2.getAssuredSrReplayErrorUpdates(), 0);
      assertEquals(fakeRd2.getAssuredSrServerNotAcknowledgedUpdates().size(), 0);
      assertEquals(fakeRd2.getReceivedAssuredSrUpdates(), 0);
      assertEquals(fakeRd2.getReceivedAssuredSrUpdatesAcked(), 0);
      assertEquals(fakeRd2.getReceivedAssuredSrUpdatesNotAcked(), 0);
      assertEquals(fakeRd2.getAssuredSrReceivedUpdates(), 0);
      assertEquals(fakeRd2.getAssuredSrReceivedUpdatesAcked(), 0);
      assertEquals(fakeRd2.getAssuredSrReceivedUpdatesNotAcked(), 0);

      assertEquals(fakeRd1.getReceivedUpdates(), 0);
      assertEquals(fakeRd1.getWrongReceivedUpdates(), 0);
@@ -3901,9 +3901,9 @@
      nError = failedServer.get(FDS2_ID);
      assertNotNull(nError);
      assertEquals(nError.intValue(), 4);
      assertEquals(fakeRd1.getReceivedAssuredSrUpdates(), 0);
      assertEquals(fakeRd1.getReceivedAssuredSrUpdatesAcked(), 0);
      assertEquals(fakeRd1.getReceivedAssuredSrUpdatesNotAcked(), 0);
      assertEquals(fakeRd1.getAssuredSrReceivedUpdates(), 0);
      assertEquals(fakeRd1.getAssuredSrReceivedUpdatesAcked(), 0);
      assertEquals(fakeRd1.getAssuredSrReceivedUpdatesNotAcked(), 0);

      assertEquals(fakeRd2.getAssuredSrSentUpdates(), 0);
      assertEquals(fakeRd2.getAssuredSrAcknowledgedUpdates(), 0);
@@ -3912,9 +3912,9 @@
      assertEquals(fakeRd2.getAssuredSrWrongStatusUpdates(), 0);
      assertEquals(fakeRd2.getAssuredSrReplayErrorUpdates(), 0);
      assertEquals(fakeRd2.getAssuredSrServerNotAcknowledgedUpdates().size(), 0);
      assertEquals(fakeRd2.getReceivedAssuredSrUpdates(), 0); // status changed to normal so reset of monitoring data
      assertEquals(fakeRd2.getReceivedAssuredSrUpdatesAcked(), 0);
      assertEquals(fakeRd2.getReceivedAssuredSrUpdatesNotAcked(), 0);
      assertEquals(fakeRd2.getAssuredSrReceivedUpdates(), 0); // status changed to normal so reset of monitoring data
      assertEquals(fakeRd2.getAssuredSrReceivedUpdatesAcked(), 0);
      assertEquals(fakeRd2.getAssuredSrReceivedUpdatesNotAcked(), 0);

      assertEquals(fakeRd1.getReceivedUpdates(), 0);
      assertEquals(fakeRd1.getWrongReceivedUpdates(), 0);
@@ -3952,9 +3952,9 @@
      nError = failedServer.get(FDS2_ID);
      assertNotNull(nError);
      assertEquals(nError.intValue(), 4);
      assertEquals(fakeRd1.getReceivedAssuredSrUpdates(), 0);
      assertEquals(fakeRd1.getReceivedAssuredSrUpdatesAcked(), 0);
      assertEquals(fakeRd1.getReceivedAssuredSrUpdatesNotAcked(), 0);
      assertEquals(fakeRd1.getAssuredSrReceivedUpdates(), 0);
      assertEquals(fakeRd1.getAssuredSrReceivedUpdatesAcked(), 0);
      assertEquals(fakeRd1.getAssuredSrReceivedUpdatesNotAcked(), 0);

      assertEquals(fakeRd2.getAssuredSrSentUpdates(), 0);
      assertEquals(fakeRd2.getAssuredSrAcknowledgedUpdates(), 0);
@@ -3963,9 +3963,9 @@
      assertEquals(fakeRd2.getAssuredSrWrongStatusUpdates(), 0);
      assertEquals(fakeRd2.getAssuredSrReplayErrorUpdates(), 0);
      assertEquals(fakeRd2.getAssuredSrServerNotAcknowledgedUpdates().size(), 0);
      assertEquals(fakeRd2.getReceivedAssuredSrUpdates(), 1);
      assertEquals(fakeRd2.getReceivedAssuredSrUpdatesAcked(), 1);
      assertEquals(fakeRd2.getReceivedAssuredSrUpdatesNotAcked(), 0);
      assertEquals(fakeRd2.getAssuredSrReceivedUpdates(), 1);
      assertEquals(fakeRd2.getAssuredSrReceivedUpdatesAcked(), 1);
      assertEquals(fakeRd2.getAssuredSrReceivedUpdatesNotAcked(), 0);

      assertEquals(fakeRd1.getReceivedUpdates(), 0);
      assertEquals(fakeRd1.getWrongReceivedUpdates(), 0);