Trying to fix random failures for HistoricalTest.testRecurringPurgeIn1Run().

Jean-Noel Rouvignac

11.29.2013 ea16c22c6b725367e499b31054497246b67ca703

Trying to fix random failures for HistoricalTest.testRecurringPurgeIn1Run().


ReplicationTestCase.java:
In executeTask() and waitTaskState() added maxWaitTimeInMillis parameter.

HistoricalTest.java:
In testRecurringPurgeIn1Run(), added a sleep before configuring conflicts purge delay + waited purge for a maximum of 120 seconds (was approximatively 20 seconds before): as long as task max duration.

InitOnLineTest.java, ReplicationServerTest.java:
Consequence of the change to ReplicationTestCase.

GenerationIdTest.java:
Consequence of the change to ReplicationTestCase.
Extracted method createSetGenerationIdTask().

 opends/tests/unit-tests-testng/src/server/org/opends/server/replication/GenerationIdTest.java

@@ -721,15 +721,8 @@
      connectServer1ToChangelog(changelog1ID);

      debugInfo("Create Reset task on DS1 to propagate the new gen ID as the reference");
      Entry taskReset = TestCaseUtils.makeEntry(
          "dn: ds-task-id=resetgenid"+genId+ UUID.randomUUID() +
          ",cn=Scheduled Tasks,cn=Tasks",
          "objectclass: top",
          "objectclass: ds-task",
          "objectclass: ds-task-reset-generation-id",
          "ds-task-class-name: org.opends.server.tasks.SetGenerationIdTask",
          "ds-task-reset-generation-id-domain-base-dn: " + baseDnStr);
      executeTask(taskReset);
      Entry taskReset = createSetGenerationIdTask(genId, "");
      executeTask(taskReset, 20000);

      // Broker 2 and 3 should receive 1 change status message to order them
      // to enter the bad gen id status
@@ -818,15 +811,8 @@
          server3ID, 100, getChangelogPort(changelog1ID), 1000, emptyOldChanges, genId);

      debugInfo("Adding reset task to DS1");
      taskReset = TestCaseUtils.makeEntry(
          "dn: ds-task-id=resetgenid"+ UUID.randomUUID() +
          ",cn=Scheduled Tasks,cn=Tasks",
          "objectclass: top",
          "objectclass: ds-task",
          "objectclass: ds-task-reset-generation-id",
          "ds-task-class-name: org.opends.server.tasks.SetGenerationIdTask",
          "ds-task-reset-generation-id-domain-base-dn: " + baseDnStr);
      executeTask(taskReset);
      taskReset = createSetGenerationIdTask(null, "");
      executeTask(taskReset, 20000);

      debugInfo("Verify that RS1 has still the right genID");
      assertEquals(replServer1.getGenerationId(baseDN), rgenId);
@@ -877,6 +863,18 @@
    }
  }

  private Entry createSetGenerationIdTask(Long genId, String additionalAttribute) throws Exception
  {
    String genIdString = genId != null ? genId.toString() : "";
    return TestCaseUtils.makeEntry(
        "dn: ds-task-id=resetgenid" + genIdString + UUID.randomUUID() + ",cn=Scheduled Tasks,cn=Tasks",
        "objectclass: top", "objectclass: ds-task",
        "objectclass: ds-task-reset-generation-id",
        "ds-task-class-name: org.opends.server.tasks.SetGenerationIdTask",
        "ds-task-reset-generation-id-domain-base-dn: " + baseDnStr,
        additionalAttribute);
  }

  private void assertNoMessageReceived(ReplicationBroker broker,
      String brokerName, String reason)
  {
@@ -968,7 +966,6 @@

      debugInfo("Expecting that broker3 is in bad gen id since it has a bad genId");
      assertTrue(isDegradedDueToGenerationId(replServer1, server3ID));

      assertEquals(countUpdatedEntriesInDb(), updatedEntries.length);

      debugInfo("Connecting DS to replServer1.");
@@ -976,15 +973,8 @@


      debugInfo("Adding reset task to DS.");
      Entry taskReset = TestCaseUtils.makeEntry(
        "dn: ds-task-id=resetgenid" + UUID.randomUUID() +
        ",cn=Scheduled Tasks,cn=Tasks",
        "objectclass: top",
        "objectclass: ds-task",
        "objectclass: ds-task-reset-generation-id",
        "ds-task-class-name: org.opends.server.tasks.SetGenerationIdTask",
        "ds-task-reset-generation-id-domain-base-dn: " + baseDnStr);
      executeTask(taskReset);
      Entry taskReset = createSetGenerationIdTask(null, "");
      executeTask(taskReset, 20000);
      Thread.sleep(500);

      debugInfo("Verifying that all replservers genIds have been reset.");
@@ -994,16 +984,8 @@
      assertEquals(replServer3.getGenerationId(baseDN), genId);

      debugInfo("Adding reset task to DS." + genId);
      taskReset = TestCaseUtils.makeEntry(
        "dn: ds-task-id=resetgenid" + UUID.randomUUID() +
        ",cn=Scheduled Tasks,cn=Tasks",
        "objectclass: top",
        "objectclass: ds-task",
        "objectclass: ds-task-reset-generation-id",
        "ds-task-class-name: org.opends.server.tasks.SetGenerationIdTask",
        "ds-task-reset-generation-id-domain-base-dn: " + baseDnStr,
        "ds-task-reset-generation-id-new-value: -1");
      executeTask(taskReset);
      taskReset = createSetGenerationIdTask(null, "ds-task-reset-generation-id-new-value: -1");
      executeTask(taskReset, 20000);

      debugInfo("Verifying that all replservers genIds have been reset.");
      int waitRes = 0;

 opends/tests/unit-tests-testng/src/server/org/opends/server/replication/InitOnLineTest.java

@@ -56,6 +56,7 @@

import static org.opends.messages.ReplicationMessages.*;
import static org.opends.messages.TaskMessages.*;
import static org.opends.server.backends.task.TaskState.*;
import static org.opends.server.config.ConfigConstants.*;
import static org.opends.server.loggers.ErrorLogger.*;
import static org.opends.server.loggers.debug.DebugLogger.*;
@@ -680,8 +681,7 @@
      // Tests that entries have been received by S2
      receiveUpdatedEntries(server2);

      // Wait for task completion
      waitTaskState(taskInitTargetS2, TaskState.COMPLETED_SUCCESSFULLY, null);
      waitTaskState(taskInitTargetS2, COMPLETED_SUCCESSFULLY, 20000, null);

      log("Successfully ending " + testCase);
    } finally
@@ -738,8 +738,7 @@
      receiveUpdatedEntries(server2);
      receiveUpdatedEntries(server3);

      // Wait for task completion
      waitTaskState(taskInitTargetAll, TaskState.COMPLETED_SUCCESSFULLY, null);
      waitTaskState(taskInitTargetAll, COMPLETED_SUCCESSFULLY, 20000, null);

      log("Successfully ending " + testCase);
    } finally
@@ -1028,9 +1027,7 @@
      log(testCase + " receive entries");
      receiveUpdatedEntries(server2);

      // Wait for task completion
      log(testCase + " wait task completed");
      waitTaskState(taskInitTargetS2, TaskState.COMPLETED_SUCCESSFULLY, null);
      waitTaskState(taskInitTargetS2, COMPLETED_SUCCESSFULLY, 20000, null);

      log("Successfully ending " + testCase);
    }
@@ -1138,8 +1135,7 @@
        "ds-task-initialize-replica-server-id: " + 20);

      addTask(taskInit, ResultCode.SUCCESS, null);

      waitTaskState(taskInit, TaskState.STOPPED_BY_ERROR,
      waitTaskState(taskInit, TaskState.STOPPED_BY_ERROR, 20000,
          ERR_NO_REACHABLE_PEER_IN_THE_DOMAIN.get(baseDN.toString(), "20"));

      // Test 2
@@ -1195,8 +1191,7 @@
        "ds-task-initialize-replica-server-id: " + 0);

      addTask(taskInit, ResultCode.SUCCESS, null);

      waitTaskState(taskInit, TaskState.STOPPED_BY_ERROR,
      waitTaskState(taskInit, TaskState.STOPPED_BY_ERROR, 20000,
        ERR_NO_REACHABLE_PEER_IN_THE_DOMAIN.get(baseDN.toString(), "0"));

      if (replDomain != null)
@@ -1285,12 +1280,11 @@

      // Second task is expected to be rejected
      addTask(taskInit2, ResultCode.SUCCESS, null);

      waitTaskState(taskInit2, TaskState.STOPPED_BY_ERROR,
      waitTaskState(taskInit2, STOPPED_BY_ERROR, 20000,
        ERR_SIMULTANEOUS_IMPORT_EXPORT_REJECTED.get());

      // First task is stilll running
      waitTaskState(taskInit, TaskState.RUNNING, null);
      // First task is still running
      waitTaskState(taskInit, RUNNING, 20000, null);

      // External request is supposed to be rejected

@@ -1299,7 +1293,7 @@
      ErrorMsg msg = new ErrorMsg(server1ID, 1, Message.EMPTY);
      server2.publish(msg);

      waitTaskState(taskInit, TaskState.STOPPED_BY_ERROR, null);
      waitTaskState(taskInit, STOPPED_BY_ERROR, 20000, null);

      log("Successfully ending " + testCase);
    } finally

 opends/tests/unit-tests-testng/src/server/org/opends/server/replication/ReplicationTestCase.java

@@ -721,10 +721,10 @@
    return new ReplSessionSecurity(null, null, null, true);
  }

  protected void executeTask(Entry taskEntry) throws Exception
  protected void executeTask(Entry taskEntry, long maxWaitTimeInMillis) throws Exception
  {
    addTask(taskEntry, ResultCode.SUCCESS, null);
    waitTaskState(taskEntry, TaskState.COMPLETED_SUCCESSFULLY, null);
    waitTaskState(taskEntry, TaskState.COMPLETED_SUCCESSFULLY, maxWaitTimeInMillis, null);
  }

  /**
@@ -757,7 +757,7 @@
    }
    else
    {
      waitTaskState(taskEntry, TaskState.RUNNING, null);
      waitTaskState(taskEntry, TaskState.RUNNING, 20000, null);
    }

    // Entry will be removed at the end of the test
@@ -767,13 +767,13 @@
  }

  protected void waitTaskState(Entry taskEntry, TaskState expectedTaskState,
      Message expectedMessage) throws Exception
      long maxWaitTimeInMillis, Message expectedMessage) throws Exception
  {
    TaskState taskState = null;
    int cpt=40;
    long startTime = System.currentTimeMillis();

    SearchFilter filter = SearchFilter.createFilterFromString("(objectclass=*)");
    Entry resultEntry = null;
    TaskState taskState = null;
    do
    {
      InternalSearchOperation searchOperation =
@@ -786,13 +786,12 @@
      String stateString = resultEntry.getAttributeValue(taskStateType, DECODER);
      taskState = TaskState.fromString(stateString);

      Thread.sleep(500);
      cpt--;
      Thread.sleep(100);
    }
    while (taskState != expectedTaskState
        && taskState != TaskState.STOPPED_BY_ERROR
        && taskState != TaskState.COMPLETED_SUCCESSFULLY
        && cpt > 0);
        && (System.currentTimeMillis() - startTime < maxWaitTimeInMillis));

    // Check that the task contains some log messages.
    AttributeType logMessagesType =

 opends/tests/unit-tests-testng/src/server/org/opends/server/replication/plugin/HistoricalTest.java

@@ -479,7 +479,7 @@
   * - verify that all historical has been purged
   *
   * TODO: another test should be written that configures the task no NOT have
   * the time to purge everything in 1 run .. and thus to relauch it to finish
   * the time to purge everything in 1 run .. and thus to relaunch it to finish
   * the purge. And verify that the second run starts on the CSN where
   * the previous task run had stopped.
   */
@@ -489,6 +489,9 @@
    int entryCnt = 10;

    addEntriesWithHistorical(1, entryCnt);
    // leave a little delay between adding/modifying test entries
    // and configuring the purge delay. 
    Thread.sleep(10);

    // set the purge delay to 1 minute
    // FIXME could we change this setting to also accept seconds?
@@ -500,20 +503,20 @@
        "--domain-name",testName,
        "--set","conflicts-historical-purge-delay:1m");

    // Let's go past the purge delay
    Thread.sleep(60*1000);

    // launch the purge
    Entry taskInit = TestCaseUtils.makeEntry(
        "dn: ds-task-id=" + UUID.randomUUID() +
        ",cn=Scheduled Tasks,cn=Tasks",
    final int maxWaitTimeInSeconds = 120;
    Entry purgeConflictsHistoricalTask = TestCaseUtils.makeEntry(
        "dn: ds-task-id=" + UUID.randomUUID() + ",cn=Scheduled Tasks,cn=Tasks",
        "objectclass: top",
        "objectclass: ds-task",
        "objectclass: ds-task-purge-conflicts-historical",
        "ds-task-class-name: org.opends.server.tasks.PurgeConflictsHistoricalTask",
        "ds-task-purge-conflicts-historical-domain-dn: "+TEST_ROOT_DN_STRING,
    "ds-task-purge-conflicts-historical-maximum-duration: 120"); // 120 sec

    executeTask(taskInit);
        "ds-task-purge-conflicts-historical-maximum-duration: " + maxWaitTimeInSeconds);
    executeTask(purgeConflictsHistoricalTask, maxWaitTimeInSeconds * 1000);

    // every entry should be purged from its hist
      // Search for matching entries in config backend

 opends/tests/unit-tests-testng/src/server/org/opends/server/replication/server/ReplicationServerTest.java

@@ -1103,11 +1103,8 @@
   {
     debugInfo("Starting backupRestore");

     Entry backupTask = createBackupTask();
     Entry restoreTask = createRestoreTask();

    executeTask(backupTask);
    executeTask(restoreTask);
    executeTask(createBackupTask(), 20000);
    executeTask(createRestoreTask(), 20000);

     debugInfo("Ending backupRestore");
   }
@@ -1141,12 +1138,12 @@
        publishAll(server2, createChanges("dc=domain2,dc=com",  2));

        debugInfo("Export all");
      executeTask(createExportAllTask());
      executeTask(createExportAllTask(), 20000);
      // Not doing anything with the export file, let's delete it
      new File(DirectoryServer.getInstanceRoot(), exportLDIFAllFile).delete();

        debugInfo("Export domain");
      executeTask(createExportDomainTask("dc=domain2,dc=com"));
      executeTask(createExportDomainTask("dc=domain2,dc=com"), 20000);
      if (exportLDIFDomainFile != null)
      {
        // Not doing anything with the export file, let's delete it