OPENDJ-2183 ECL: changes for new suffix are missing OPENDJ-2184 ECL: wrong ...

Jean-Noel Rouvignac

02.34.2015 a10168c9109c212c5fb12d86221420a11290f0d1

OPENDJ-2183 ECL: changes for new suffix are missing
OPENDJ-2184 ECL: wrong number of changes after adding a new replica

Code review: CR-7431



This problem is linked to OPENDJ-2141, where we introduced a "default" value for a CSN when starting from a changeNumberIndex record.

Explanation: when the server starts up, the newest record is read from the change number index db, so the change number indexer knows where to start.
From there, we must start all the cursors, and we are using the newest CSN as a starting point.
The triplet (LESS_THAN_OR_EQUAL_TO_KEY, ON_MATCHING_KEY, newestCsn) is then used to start all cursors, including the cursors for replica DBs created dynamically much after server startup.
When opening a cursor for any new replica, the provided default CSN cannot be found in the replica DB. Because of this condition and the provided triplet, the file-based changelog code creates an empty cursor.
Empty cursors cannot be recycled to iterate over the populated replica DB (And in that case, the replica DB already has the change csn3!!).

I think this is incorrect. This change fixes this.



LogTest.java, LogFileTest.java, BlockLogReaderWriterTest:
Changed the unit tests, so the returned values match what we need.

Log.java:
In getCursor(), changed the condition that creates a valid cursor to also include LESS_THAN_OR_EQUAL_TO_KEY.

BlockLogReader.java:
In positionToKeySequentially() and getMatchingRecord(), removed the code handling LESS_THAN_OR_EQUAL_TO_KEY and AFTER_MATCHING_KEY.
Renamed positionToKeySequentially() to positionToKey().

 opendj-sdk/opendj-server-legacy/src/main/java/org/opends/server/replication/server/changelog/file/BlockLogReader.java

@@ -167,7 +167,7 @@
    final long markerPosition = searchClosestBlockStartToKey(key);
    if (markerPosition >= 0)
    {
      return positionToKeySequentially(markerPosition, key, matchStrategy, positionStrategy);
      return positionToKey(markerPosition, key, matchStrategy, positionStrategy);
    }
    return Pair.of(false, null);
  }
@@ -447,8 +447,7 @@

  /**
   * Position before, at or after provided key, starting from provided block
   * start position and reading sequentially until key is found according to
   * matching and positioning strategies.
   * start position and reading until key is found according to matching and positioning strategies.
   *
   * @param blockStartPosition
   *          Position of read pointer in the file, expected to be the start of
@@ -465,25 +464,20 @@
   * @throws ChangelogException
   *           If an error occurs.
   */
   Pair<Boolean, Record<K,V>> positionToKeySequentially(final long blockStartPosition, final K key,
   Pair<Boolean, Record<K,V>> positionToKey(final long blockStartPosition, final K key,
       final KeyMatchingStrategy matchStrategy, final PositionStrategy positionStrategy) throws ChangelogException
   {
    Record<K,V> record = readRecord(blockStartPosition);
    Record<K,V> previousRecord = null;
    long previousPosition = blockStartPosition;
    boolean matchingKeyIsLowerThanAnyRecord = true;
    while (record != null)
    {
      final int keysComparison = record.getKey().compareTo(key);
      if (keysComparison <= 0)
      {
        matchingKeyIsLowerThanAnyRecord = false;
      }
      if ((keysComparison == 0 && matchStrategy == EQUAL_TO_KEY)
          || (keysComparison >= 0 && matchStrategy != EQUAL_TO_KEY))
      {
        return getMatchingRecord(matchStrategy, positionStrategy, keysComparison, matchingKeyIsLowerThanAnyRecord,
            record, previousRecord, previousPosition);
        return getMatchingRecord(
            matchStrategy, positionStrategy, keysComparison, record, previousRecord, previousPosition);
      }
      previousRecord = record;
      previousPosition = getFilePosition();
@@ -497,19 +491,14 @@
    return Pair.of(false, null);
  }

  private Pair<Boolean,Record<K,V>> getMatchingRecord(KeyMatchingStrategy matchStrategy,
      PositionStrategy positionStrategy, int keysComparison, boolean matchKeyIsLowerThanAnyRecord,
      Record<K, V> currentRecord, Record<K, V> previousRecord, long previousPosition)
          throws ChangelogException
  private Pair<Boolean, Record<K, V>> getMatchingRecord(KeyMatchingStrategy matchStrategy,
      PositionStrategy positionStrategy, int keysComparison, Record<K, V> currentRecord, Record<K, V> previousRecord,
      long previousPosition) throws ChangelogException
  {
    Record<K, V> record = currentRecord;

    if (positionStrategy == AFTER_MATCHING_KEY)
    {
      if (matchStrategy == LESS_THAN_OR_EQUAL_TO_KEY && matchKeyIsLowerThanAnyRecord)
      {
        return Pair.of(false, null);
      }
      if (keysComparison == 0)
      {
        // skip matching key

 opendj-sdk/opendj-server-legacy/src/main/java/org/opends/server/replication/server/changelog/file/Log.java

@@ -604,8 +604,9 @@
      }
      cursor = new AbortableLogCursor<K, V>(this, new InternalLogCursor<K, V>(this));
      final boolean isSuccessfullyPositioned = cursor.positionTo(key, matchingStrategy, positionStrategy);
      // Allow for cursor re-initialization after exhaustion in case of GREATER_THAN_OR_EQUAL_TO_KEY strategy
      if (isSuccessfullyPositioned || matchingStrategy == GREATER_THAN_OR_EQUAL_TO_KEY)
      // Allow for cursor re-initialization after exhaustion in case of
      // LESS_THAN_OR_EQUAL_TO_KEY ands GREATER_THAN_OR_EQUAL_TO_KEY strategies
      if (isSuccessfullyPositioned || matchingStrategy != EQUAL_TO_KEY)
      {
        registerCursor(cursor);
        return cursor;
@@ -627,7 +628,6 @@
    }
  }


  /**
   * Returns the oldest (first) record from this log.
   *

 opendj-sdk/opendj-server-legacy/src/test/java/org/opends/server/replication/server/changelog/file/BlockLogReaderWriterTest.java

@@ -37,6 +37,7 @@
import java.util.Collections;
import java.util.List;

import org.assertj.core.api.SoftAssertions;
import org.forgerock.opendj.ldap.ByteSequenceReader;
import org.forgerock.opendj.ldap.ByteString;
import org.forgerock.opendj.ldap.ByteStringBuilder;
@@ -161,7 +162,7 @@
      { records(1,2,3), 3, LESS_THAN_OR_EQUAL_TO_KEY, ON_MATCHING_KEY, record(3), true },
      { records(1,2,3), 4, LESS_THAN_OR_EQUAL_TO_KEY, ON_MATCHING_KEY, record(3), true },

      { records(1,2,3), 0, LESS_THAN_OR_EQUAL_TO_KEY, AFTER_MATCHING_KEY, null, false },
      { records(1,2,3), 0, LESS_THAN_OR_EQUAL_TO_KEY, AFTER_MATCHING_KEY, record(1), true },
      { records(1,2,3), 1, LESS_THAN_OR_EQUAL_TO_KEY, AFTER_MATCHING_KEY, record(2), true },
      { records(1,2,3), 2, LESS_THAN_OR_EQUAL_TO_KEY, AFTER_MATCHING_KEY, record(3), true },
      { records(1,2,3), 3, LESS_THAN_OR_EQUAL_TO_KEY, AFTER_MATCHING_KEY, null, true },
@@ -201,7 +202,7 @@
      { records(1,2,3,4,5,6,7,8,9,10), 10, LESS_THAN_OR_EQUAL_TO_KEY, ON_MATCHING_KEY, record(10), true },
      { records(1,2,3,4,5,6,7,8,9,10), 11, LESS_THAN_OR_EQUAL_TO_KEY, ON_MATCHING_KEY, record(10), true },

      { records(1,2,3,4,5,6,7,8,9,10), 0, LESS_THAN_OR_EQUAL_TO_KEY, AFTER_MATCHING_KEY, null, false },
      { records(1,2,3,4,5,6,7,8,9,10), 0, LESS_THAN_OR_EQUAL_TO_KEY, AFTER_MATCHING_KEY, record(1), true },
      { records(1,2,3,4,5,6,7,8,9,10), 1, LESS_THAN_OR_EQUAL_TO_KEY, AFTER_MATCHING_KEY, record(2), true },
      { records(1,2,3,4,5,6,7,8,9,10), 5, LESS_THAN_OR_EQUAL_TO_KEY, AFTER_MATCHING_KEY, record(6), true },
      { records(1,2,3,4,5,7,8,9,10), 6, LESS_THAN_OR_EQUAL_TO_KEY, AFTER_MATCHING_KEY, record(7), true },
@@ -251,8 +252,10 @@
    {
      Pair<Boolean, Record<Integer, Integer>> result = reader.seekToRecord(key, matchingStrategy, positionStrategy);

      assertThat(result.getFirst()).isEqualTo(shouldBeFound);
      assertThat(result.getSecond()).isEqualTo(expectedRecord);
      final SoftAssertions softly = new SoftAssertions();
      softly.assertThat(result.getFirst()).isEqualTo(shouldBeFound);
      softly.assertThat(result.getSecond()).isEqualTo(expectedRecord);
      softly.assertAll();
    }
  }

@@ -389,7 +392,7 @@
      {
        long ts = System.nanoTime();
        Pair<Boolean, Record<Integer, Integer>> result =
            reader.positionToKeySequentially(0, val, GREATER_THAN_OR_EQUAL_TO_KEY, AFTER_MATCHING_KEY);
            reader.positionToKey(0, val, GREATER_THAN_OR_EQUAL_TO_KEY, AFTER_MATCHING_KEY);
        assertThat(result.getSecond()).isEqualTo(Record.from(val, val));
        long te = System.nanoTime() - ts;
        if (te < minTime) minTime = te;

 opendj-sdk/opendj-server-legacy/src/test/java/org/opends/server/replication/server/changelog/file/LogFileTest.java

@@ -142,8 +142,8 @@
      { "key11", LESS_THAN_OR_EQUAL_TO_KEY, ON_MATCHING_KEY, true, 10, 10},

      // key00 is a special case : position is not found but cursor is positioned on beginning
      // so it is possible to iterate on it from 2 to end
      { "key00", LESS_THAN_OR_EQUAL_TO_KEY, AFTER_MATCHING_KEY, false, 2, 10},
      // so it is possible to iterate on it from 1 to end
      { "key00", LESS_THAN_OR_EQUAL_TO_KEY, AFTER_MATCHING_KEY, true, 1, 10 },
      { "key02", LESS_THAN_OR_EQUAL_TO_KEY, AFTER_MATCHING_KEY, true, 3, 10},
      { "key05", LESS_THAN_OR_EQUAL_TO_KEY, AFTER_MATCHING_KEY, true, 6, 10},
      { "key050", LESS_THAN_OR_EQUAL_TO_KEY, AFTER_MATCHING_KEY, true, 6, 10},

 opendj-sdk/opendj-server-legacy/src/test/java/org/opends/server/replication/server/changelog/file/LogTest.java

@@ -159,7 +159,7 @@
      { "key010", EQUAL_TO_KEY, AFTER_MATCHING_KEY, -1, -1 },
      { "key011", EQUAL_TO_KEY, AFTER_MATCHING_KEY, -1, -1 },

      { "key000", LESS_THAN_OR_EQUAL_TO_KEY, ON_MATCHING_KEY, -1, -1 },
      { "key000", LESS_THAN_OR_EQUAL_TO_KEY, ON_MATCHING_KEY, 1, 10 },
      { "key001", LESS_THAN_OR_EQUAL_TO_KEY, ON_MATCHING_KEY, 1, 10 },
      { "key004", LESS_THAN_OR_EQUAL_TO_KEY, ON_MATCHING_KEY, 4, 10 },
      { "key005", LESS_THAN_OR_EQUAL_TO_KEY, ON_MATCHING_KEY, 5, 10 },
@@ -169,7 +169,7 @@
      { "key010", LESS_THAN_OR_EQUAL_TO_KEY, ON_MATCHING_KEY, 10, 10 },
      { "key011", LESS_THAN_OR_EQUAL_TO_KEY, ON_MATCHING_KEY, 10, 10 },

      { "key000", LESS_THAN_OR_EQUAL_TO_KEY, AFTER_MATCHING_KEY, -1, -1 },
      { "key000", LESS_THAN_OR_EQUAL_TO_KEY, AFTER_MATCHING_KEY, 1, 10 },
      { "key001", LESS_THAN_OR_EQUAL_TO_KEY, AFTER_MATCHING_KEY, 2, 10 },
      { "key004", LESS_THAN_OR_EQUAL_TO_KEY, AFTER_MATCHING_KEY, 5, 10 },
      { "key005", LESS_THAN_OR_EQUAL_TO_KEY, AFTER_MATCHING_KEY, 6, 10 },