Updated indexes to order the keys before inserting them into the database. ...

boli

20.45.2007 9bac7b4f5bd4c3fa6abd420e17b90418663faec6

 opendj-sdk/opends/src/server/org/opends/server/backends/jeb/ApproximateIndexer.java

@@ -31,7 +31,6 @@
import java.util.*;

import org.opends.server.types.*;
import org.opends.server.protocols.asn1.ASN1OctetString;
import static org.opends.server.loggers.debug.DebugLogger.*;
import org.opends.server.loggers.debug.DebugTracer;
import org.opends.server.api.ApproximateMatchingRule;
@@ -106,7 +105,7 @@
   * @param keys The set into which the generated keys will be inserted.
   */
  public void indexEntry(Transaction txn, Entry entry,
                       Set<ASN1OctetString> keys)
                       Set<byte[]> keys)
  {
    List<Attribute> attrList =
         entry.getAttribute(attributeType);
@@ -129,31 +128,37 @@
   */
  public void replaceEntry(Transaction txn,
                           Entry oldEntry, Entry newEntry,
                           Set<ASN1OctetString> addKeys,
                           Set<ASN1OctetString> delKeys)
                           Set<byte[]> addKeys,
                           Set<byte[]> delKeys)
  {
    List<Attribute> attrList;
    List<Attribute> newAttributes = newEntry.getAttribute(attributeType, true);
    List<Attribute> oldAttributes = oldEntry.getAttribute(attributeType, true);

    attrList = oldEntry.getAttribute(attributeType);
    Set<ASN1OctetString> oldSet = new HashSet<ASN1OctetString>();
    indexAttribute(attrList, oldSet);

    attrList = newEntry.getAttribute(attributeType);
    Set<ASN1OctetString> newSet = new HashSet<ASN1OctetString>();
    indexAttribute(attrList, newSet);

    HashSet<ASN1OctetString> removeSet = new HashSet<ASN1OctetString>(oldSet);
    removeSet.removeAll(newSet);
    for (ASN1OctetString k : removeSet)
    if(newAttributes == null)
    {
      delKeys.add(k);
      indexAttribute(oldAttributes, delKeys);
    }

    HashSet<ASN1OctetString> addSet = new HashSet<ASN1OctetString>(newSet);
    addSet.removeAll(oldSet);
    for (ASN1OctetString k : addSet)
    else
    {
      addKeys.add(k);
      if(oldAttributes == null)
      {
        indexAttribute(newAttributes, addKeys);
      }
      else
      {
        TreeSet<byte[]> newKeys =
            new TreeSet<byte[]>(comparator);
        TreeSet<byte[]> oldKeys =
            new TreeSet<byte[]>(comparator);
        indexAttribute(newAttributes, newKeys);
        indexAttribute(oldAttributes, oldKeys);

        addKeys.addAll(newKeys);
        addKeys.removeAll(oldKeys);

        delKeys.addAll(oldKeys);
        delKeys.removeAll(newKeys);
      }
    }
  }

@@ -173,13 +178,11 @@
   */
  public void modifyEntry(Transaction txn, Entry oldEntry, Entry newEntry,
                          List<Modification> mods,
                          Set<ASN1OctetString> addKeys,
                          Set<ASN1OctetString> delKeys)
                          Set<byte[]> addKeys,
                          Set<byte[]> delKeys)
  {
    List<Attribute> newAttributes = newEntry.getAttribute(attributeType, true);
    List<Attribute> oldAttributes = oldEntry.getAttribute(attributeType, true);
    HashSet<AttributeValue> newValues;
    HashSet<AttributeValue> oldValues;

    if(newAttributes == null)
    {
@@ -193,10 +196,10 @@
      }
      else
      {
        HashSet<ASN1OctetString> newKeys =
            new HashSet<ASN1OctetString>();
        HashSet<ASN1OctetString> oldKeys =
            new HashSet<ASN1OctetString>();
        TreeSet<byte[]> newKeys =
            new TreeSet<byte[]>(comparator);
        TreeSet<byte[]> oldKeys =
            new TreeSet<byte[]>(comparator);
        indexAttribute(newAttributes, newKeys);
        indexAttribute(oldAttributes, oldKeys);

@@ -215,7 +218,7 @@
   * @param keys The set into which the keys will be inserted.
   */
  private void indexAttribute(List<Attribute> attrList,
                              Set<ASN1OctetString> keys)
                              Set<byte[]> keys)
  {
    if (attrList == null) return;

@@ -231,7 +234,7 @@
   * @param keys The set into which the keys will be inserted.
   */
  private void indexValues(Set<AttributeValue> values,
                           Set<ASN1OctetString> keys)
                           Set<byte[]> keys)
  {
    if (values == null) return;

@@ -242,7 +245,7 @@
        byte[] keyBytes =
             approximateRule.normalizeValue(value.getValue()).value();

        keys.add(new ASN1OctetString(keyBytes));
        keys.add(keyBytes);
      }
      catch (DirectoryException e)
      {

 opendj-sdk/opends/src/server/org/opends/server/backends/jeb/AttributeIndex.java

@@ -27,11 +27,7 @@
package org.opends.server.backends.jeb;
import org.opends.messages.Message;

import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.*;

import com.sleepycat.je.*;

@@ -161,6 +157,7 @@
                                     state,
                                     indexEntryLimit,
                                     cursorEntryLimit,
                                     false,
                                     env,
                                     entryContainer);
    }
@@ -174,6 +171,7 @@
                                     state,
                                     indexEntryLimit,
                                     cursorEntryLimit,
                                     false,
                                     env,
                                     entryContainer);
    }
@@ -195,6 +193,7 @@
                                     state,
                                     indexEntryLimit,
                                     cursorEntryLimit,
                                     false,
                                     env,
                                     entryContainer);
    }
@@ -215,6 +214,7 @@
                                     state,
                                     indexEntryLimit,
                                     cursorEntryLimit,
                                     false,
                                     env,
                                     entryContainer);
    }
@@ -234,6 +234,7 @@
                                        state,
                                        indexEntryLimit,
                                        cursorEntryLimit,
                                        false,
                                        env,
                                        entryContainer);
    }
@@ -571,7 +572,8 @@
      // index substring length, and read those keys.

      // Eliminate duplicates by putting the keys into a set.
      Set<byte[]> set = new HashSet<byte[]>();
      Set<byte[]> set =
          new TreeSet<byte[]>(substringIndex.indexer.getComparator());

      // Example: The value is ABCDE and the substring length is 3.
      // We produce the keys ABC BCD CDE.
@@ -1238,6 +1240,7 @@
                                    state,
                                    indexEntryLimit,
                                    cursorEntryLimit,
                                    false,
                                    env,
                                    entryContainer);
          equalityIndex.open();
@@ -1297,6 +1300,7 @@
                                    state,
                                    indexEntryLimit,
                                    cursorEntryLimit,
                                    false,
                                    env,
                                    entryContainer);
          presenceIndex.open();
@@ -1356,6 +1360,7 @@
                                     state,
                                     indexEntryLimit,
                                     cursorEntryLimit,
                                     false,
                                     env,
                                     entryContainer);
          substringIndex.open();
@@ -1420,6 +1425,7 @@
                                    state,
                                    indexEntryLimit,
                                    cursorEntryLimit,
                                    false,
                                    env,
                                    entryContainer);
          orderingIndex.open();
@@ -1478,6 +1484,7 @@
                                       state,
                                       indexEntryLimit,
                                       cursorEntryLimit,
                                       false,
                                       env,
                                       entryContainer);
          approximateIndex.open();

 opendj-sdk/opends/src/server/org/opends/server/backends/jeb/AttributeIndexBuilder.java

@@ -26,18 +26,13 @@
 */
package org.opends.server.backends.jeb;

import org.opends.server.protocols.asn1.ASN1OctetString;
import org.opends.server.types.Entry;
import static org.opends.server.util.StaticUtils.getFileForPath;

import com.sleepycat.je.DatabaseException;
import com.sleepycat.je.Transaction;

import java.util.Arrays;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
import java.util.*;
import java.io.ByteArrayOutputStream;
import java.io.BufferedOutputStream;
import java.io.DataOutputStream;
@@ -180,29 +175,29 @@
    if (oldEntry != null)
    {
      // This is an entry being replaced.
      Set<ASN1OctetString> addKeys = new HashSet<ASN1OctetString>();
      Set<ASN1OctetString> delKeys = new HashSet<ASN1OctetString>();
      TreeSet<byte[]> addKeys = new TreeSet<byte[]>(indexer.getComparator());
      TreeSet<byte[]> delKeys = new TreeSet<byte[]>(indexer.getComparator());

      indexer.replaceEntry(txn, oldEntry, newEntry, addKeys, delKeys);

      for (ASN1OctetString k : delKeys)
      for (byte[] k : delKeys)
      {
        removeID(k.value(), entryID);
        removeID(k, entryID);
      }

      for (ASN1OctetString k : addKeys)
      for (byte[] k : addKeys)
      {
        insertID(k.value(), entryID);
        insertID(k, entryID);
      }
    }
    else
    {
      // This is a new entry.
      Set<ASN1OctetString> addKeys = new HashSet<ASN1OctetString>();
      TreeSet<byte[]> addKeys = new TreeSet<byte[]>(indexer.getComparator());
      indexer.indexEntry(txn, newEntry, addKeys);
      for (ASN1OctetString k : addKeys)
      for (byte[] k : addKeys)
      {
        insertID(k.value(), entryID);
        insertID(k, entryID);
      }
    }


 opendj-sdk/opends/src/server/org/opends/server/backends/jeb/EntryContainer.java

@@ -498,12 +498,12 @@

      id2children = new Index(databasePrefix + "_" + ID2CHILDREN_DATABASE_NAME,
                              new ID2CIndexer(), state,
                              indexEntryLimit, 0,
                              indexEntryLimit, 0, true,
                              env,this);
      id2children.open();
      id2subtree = new Index(databasePrefix + "_" + ID2SUBTREE_DATABASE_NAME,
                             new ID2SIndexer(), state,
                             indexEntryLimit, 0,
                             indexEntryLimit, 0, true,
                             env, this);
      id2subtree.open();

@@ -1810,7 +1810,10 @@
     */
    public Transaction beginOperationTransaction() throws DatabaseException
    {
      return beginTransaction();
      Transaction txn =  beginTransaction();
      // Multiple adds should never encounter a deadlock.
      txn.setLockTimeout(0);
      return txn;
    }

    /**
@@ -2300,7 +2303,10 @@
     */
    public Transaction beginOperationTransaction() throws DatabaseException
    {
      return beginTransaction();
      Transaction txn =  beginTransaction();
      // Multiple deletes should never encounter a deadlock.
      txn.setLockTimeout(0);
      return txn;
    }

    /**
@@ -2843,7 +2849,10 @@
     */
    public Transaction beginOperationTransaction() throws DatabaseException
    {
      return beginTransaction();
      Transaction txn =  beginTransaction();
      // Multiple replace operations should never encounter a deadlock.
      txn.setLockTimeout(0);
      return txn;
    }

    /**
@@ -3214,7 +3223,8 @@
     */
    public Transaction beginOperationTransaction() throws DatabaseException
    {
      return beginTransaction();
      Transaction txn =  beginTransaction();
      return txn;
    }

    /**

 opendj-sdk/opends/src/server/org/opends/server/backends/jeb/EqualityIndexer.java

@@ -33,7 +33,6 @@
import com.sleepycat.je.Transaction;
import com.sleepycat.je.DatabaseException;

import org.opends.server.protocols.asn1.ASN1OctetString;
import org.opends.server.types.Attribute;
import org.opends.server.types.AttributeType;
import org.opends.server.types.AttributeValue;
@@ -41,10 +40,7 @@
import org.opends.server.types.Entry;
import org.opends.server.types.Modification;

import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.*;

/**
 * An implementation of an Indexer for attribute equality.
@@ -114,7 +110,7 @@
   * @throws DatabaseException If an error occurs in the JE database.
   */
  public void indexEntry(Transaction txn, Entry entry,
                         Set<ASN1OctetString> keys) throws DatabaseException
                         Set<byte[]> keys) throws DatabaseException
  {
    List<Attribute> attrList =
         entry.getAttribute(attributeType);
@@ -139,32 +135,38 @@
   * @throws DatabaseException If an error occurs in the JE database.
   */
  public void replaceEntry(Transaction txn, Entry oldEntry, Entry newEntry,
                           Set<ASN1OctetString> addKeys,
                           Set<ASN1OctetString> delKeys)
       throws DatabaseException
                           Set<byte[]> addKeys,
                           Set<byte[]> delKeys)
      throws DatabaseException
  {
    List<Attribute> attrList;
    List<Attribute> newAttributes = newEntry.getAttribute(attributeType, true);
    List<Attribute> oldAttributes = oldEntry.getAttribute(attributeType, true);

    attrList = oldEntry.getAttribute(attributeType);
    Set<ASN1OctetString> oldSet = new HashSet<ASN1OctetString>();
    indexAttribute(attrList, oldSet);

    attrList = newEntry.getAttribute(attributeType);
    Set<ASN1OctetString> newSet = new HashSet<ASN1OctetString>();
    indexAttribute(attrList, newSet);

    HashSet<ASN1OctetString> removeSet = new HashSet<ASN1OctetString>(oldSet);
    removeSet.removeAll(newSet);
    for (ASN1OctetString k : removeSet)
    if(newAttributes == null)
    {
      delKeys.add(k);
      indexAttribute(oldAttributes, delKeys);
    }

    HashSet<ASN1OctetString> addSet = new HashSet<ASN1OctetString>(newSet);
    addSet.removeAll(oldSet);
    for (ASN1OctetString k : addSet)
    else
    {
      addKeys.add(k);
      if(oldAttributes == null)
      {
        indexAttribute(newAttributes, addKeys);
      }
      else
      {
        TreeSet<byte[]> newKeys =
            new TreeSet<byte[]>(comparator);
        TreeSet<byte[]> oldKeys =
            new TreeSet<byte[]>(comparator);
        indexAttribute(newAttributes, newKeys);
        indexAttribute(oldAttributes, oldKeys);

        addKeys.addAll(newKeys);
        addKeys.removeAll(oldKeys);

        delKeys.addAll(oldKeys);
        delKeys.removeAll(newKeys);
      }
    }
  }

@@ -186,14 +188,12 @@
   */
  public void modifyEntry(Transaction txn, Entry oldEntry, Entry newEntry,
                          List<Modification> mods,
                          Set<ASN1OctetString> addKeys,
                          Set<ASN1OctetString> delKeys)
       throws DatabaseException
                          Set<byte[]> addKeys,
                          Set<byte[]> delKeys)
      throws DatabaseException
  {
    List<Attribute> newAttributes = newEntry.getAttribute(attributeType, true);
    List<Attribute> oldAttributes = oldEntry.getAttribute(attributeType, true);
    HashSet<AttributeValue> newValues;
    HashSet<AttributeValue> oldValues;

    if(newAttributes == null)
    {
@@ -207,26 +207,18 @@
      }
      else
      {
        newValues = new HashSet<AttributeValue>();
        oldValues = new HashSet<AttributeValue>();
        for(Attribute a : newAttributes)
        {
          newValues.addAll(a.getValues());
        }
        for(Attribute a : oldAttributes)
        {
          oldValues.addAll(a.getValues());
        }
        TreeSet<byte[]> newKeys =
            new TreeSet<byte[]>(comparator);
        TreeSet<byte[]> oldKeys =
            new TreeSet<byte[]>(comparator);
        indexAttribute(newAttributes, newKeys);
        indexAttribute(oldAttributes, oldKeys);

        HashSet<AttributeValue> valuesToAdd =
            new HashSet<AttributeValue>(newValues);
        HashSet<AttributeValue> valuesToDel =
            new HashSet<AttributeValue>(oldValues);
        valuesToAdd.removeAll(oldValues);
        valuesToDel.removeAll(newValues);
        addKeys.addAll(newKeys);
        addKeys.removeAll(oldKeys);

        indexValues(valuesToDel, delKeys);
        indexValues(valuesToAdd, addKeys);
        delKeys.addAll(oldKeys);
        delKeys.removeAll(newKeys);
      }
    }
  }
@@ -237,7 +229,7 @@
   * @param keys The set into which the keys will be inserted.
   */
  private void indexValues(Set<AttributeValue> values,
                           Set<ASN1OctetString> keys)
                           Set<byte[]> keys)
  {
    if (values == null) return;

@@ -247,7 +239,7 @@
      {
        byte[] keyBytes = value.getNormalizedValue().value();

        keys.add(new ASN1OctetString(keyBytes));
        keys.add(keyBytes);
      }
      catch (DirectoryException e)
      {
@@ -265,7 +257,7 @@
   * @param keys The set into which the keys will be inserted.
   */
  private void indexAttribute(List<Attribute> attrList,
                              Set<ASN1OctetString> keys)
                              Set<byte[]> keys)
  {
    if (attrList == null) return;


 opendj-sdk/opends/src/server/org/opends/server/backends/jeb/ID2CIndexer.java

@@ -26,7 +26,6 @@
 */
package org.opends.server.backends.jeb;

import org.opends.server.protocols.asn1.ASN1OctetString;
import org.opends.server.types.Entry;
import org.opends.server.types.Modification;

@@ -90,7 +89,7 @@
   * @throws DatabaseException If an error occurs in the JE database.
   */
  public void indexEntry(Transaction txn, Entry entry,
                       Set<ASN1OctetString> addKeys)
                       Set<byte[]> addKeys)
       throws DatabaseException
  {
    // The superior entry IDs are in the entry attachment.
@@ -104,7 +103,7 @@
    if (iter.hasNext())
    {
      DatabaseEntry nodeIDData = ((EntryID)iter.next()).getDatabaseEntry();
      addKeys.add(new ASN1OctetString(nodeIDData.getData()));
      addKeys.add(nodeIDData.getData());
    }
  }

@@ -120,8 +119,8 @@
   * @param delKeys The set into which the keys to be deleted will be inserted.
   */
  public void replaceEntry(Transaction txn, Entry oldEntry, Entry newEntry,
                           Set<ASN1OctetString> addKeys,
                           Set<ASN1OctetString> delKeys)
                           Set<byte[]> addKeys,
                           Set<byte[]> delKeys)
  {
    // Nothing to do.
  }
@@ -143,8 +142,8 @@
   */
  public void modifyEntry(Transaction txn, Entry oldEntry, Entry newEntry,
                          List<Modification> mods,
                          Set<ASN1OctetString> addKeys,
                          Set<ASN1OctetString> delKeys)
                          Set<byte[]> addKeys,
                          Set<byte[]> delKeys)
       throws DatabaseException
  {
    // Nothing to do.

 opendj-sdk/opends/src/server/org/opends/server/backends/jeb/ID2SIndexer.java

@@ -26,7 +26,6 @@
 */
package org.opends.server.backends.jeb;

import org.opends.server.protocols.asn1.ASN1OctetString;
import org.opends.server.types.Entry;
import org.opends.server.types.Modification;

@@ -89,7 +88,7 @@
   * @throws DatabaseException If an error occurs in the JE database.
   */
  public void indexEntry(Transaction txn, Entry entry,
                       Set<ASN1OctetString> addKeys)
                       Set<byte[]> addKeys)
       throws DatabaseException
  {
    // The superior entry IDs are in the entry attachment.
@@ -103,7 +102,7 @@
    while (iter.hasNext())
    {
      DatabaseEntry nodeIDData = ((EntryID)iter.next()).getDatabaseEntry();
      addKeys.add(new ASN1OctetString(nodeIDData.getData()));
      addKeys.add(nodeIDData.getData());
    }
  }

@@ -121,8 +120,8 @@
   */
  public void replaceEntry(Transaction txn,
                           Entry oldEntry, Entry newEntry,
                           Set<ASN1OctetString> addKeys,
                           Set<ASN1OctetString> delKeys)
                           Set<byte[]> addKeys,
                           Set<byte[]> delKeys)
       throws DatabaseException
  {
    // Nothing to do.
@@ -145,8 +144,8 @@
   */
  public void modifyEntry(Transaction txn, Entry oldEntry, Entry newEntry,
                          List<Modification> mods,
                          Set<ASN1OctetString> addKeys,
                          Set<ASN1OctetString> delKeys)
                          Set<byte[]> addKeys,
                          Set<byte[]> delKeys)
       throws DatabaseException
  {
    // Nothing to do.

 opendj-sdk/opends/src/server/org/opends/server/backends/jeb/Index.java

@@ -32,7 +32,6 @@

import com.sleepycat.je.*;

import org.opends.server.protocols.asn1.ASN1OctetString;
import org.opends.server.types.*;
import org.opends.server.util.StaticUtils;
import static org.opends.messages.JebMessages.*;
@@ -79,6 +78,17 @@
   */
  private int entryLimitExceededCount;

  /**
   * The max number of tries to rewrite phantom records.
   */
  final int phantomWriteRetires = 3;

  /**
   * Whether to maintain a count of IDs for a key once the entry limit
   * has exceeded.
   */
  boolean maintainCount;

  private State state;

  /**
@@ -111,13 +121,15 @@
   * @param indexEntryLimit The configured limit on the number of entry IDs
   * that may be indexed by one key.
   * @param cursorEntryLimit The configured limit on the number of entry IDs
   * @param maintainCount Whether to maintain a count of IDs for a key once
   * the entry limit has exceeded.
   * @param env The JE Environemnt
   * @param entryContainer The database entryContainer holding this index.
   * @throws DatabaseException If an error occurs in the JE database.
   */
  public Index(String name, Indexer indexer, State state,
        int indexEntryLimit, int cursorEntryLimit, Environment env,
        EntryContainer entryContainer)
        int indexEntryLimit, int cursorEntryLimit, boolean maintainCount,
        Environment env, EntryContainer entryContainer)
      throws DatabaseException
  {
    super(name, env, entryContainer);
@@ -125,6 +137,7 @@
    this.comparator = indexer.getComparator();
    this.indexEntryLimit = indexEntryLimit;
    this.cursorEntryLimit = cursorEntryLimit;
    this.maintainCount = maintainCount;

    DatabaseConfig dbNodupsConfig = new DatabaseConfig();

@@ -183,60 +196,61 @@
       throws DatabaseException
  {
    OperationStatus status;
    LockMode lockMode = LockMode.RMW;
    DatabaseEntry entryIDData = entryID.getDatabaseEntry();
    DatabaseEntry data = new DatabaseEntry();
    boolean success = true;
    boolean done = false;
    boolean success = false;

    while(!done)
    if(maintainCount)
    {
      status = read(txn, key, data, lockMode);

      if (status == OperationStatus.SUCCESS)
      for(int i = 0; i < phantomWriteRetires; i++)
      {
        if(insertIDWithRMW(txn, key, data, entryIDData, entryID) ==
            OperationStatus.SUCCESS)
        {
          return true;
        }
      }
    }
    else
    {
      status = read(txn, key, data, LockMode.READ_COMMITTED);
      if(status == OperationStatus.SUCCESS)
      {
        EntryIDSet entryIDList =
            new EntryIDSet(key.getData(), data.getData());

        if (entryIDList.isDefined())
        {
          if (indexEntryLimit > 0 && entryIDList.size() >= indexEntryLimit)
          for(int i = 0; i < phantomWriteRetires; i++)
          {
            entryIDList = new EntryIDSet(entryIDList.size());
            entryLimitExceededCount++;

            if(debugEnabled())
            if(insertIDWithRMW(txn, key, data, entryIDData, entryID) ==
                OperationStatus.SUCCESS)
            {
              StringBuilder builder = new StringBuilder();
              StaticUtils.byteArrayToHexPlusAscii(builder, key.getData(), 4);
              TRACER.debugInfo("Index entry exceeded in index %s. " +
                  "Limit: %d. ID list size: %d.\nKey:",
                               name, indexEntryLimit, entryIDList.size(),
                               builder);

              return true;
            }
          }
        }

        success = entryIDList.add(entryID);

        byte[] after = entryIDList.toDatabase();
        data.setData(after);
        put(txn, key, data);
        done = true;
      }
      else
      {
        if(rebuildRunning || trusted)
        {
          status = insert(txn, key, entryIDData);
          if(status == OperationStatus.SUCCESS)
          if(status == OperationStatus.KEYEXIST)
          {
            done = true;
            for(int i = 1; i < phantomWriteRetires; i++)
            {
              if(insertIDWithRMW(txn, key, data, entryIDData, entryID) ==
                  OperationStatus.SUCCESS)
              {
                return true;
              }
            }
          }
        }
        else
        {
          done = true;
          return true;
        }
      }
    }
@@ -244,6 +258,63 @@
    return success;
  }

  private OperationStatus insertIDWithRMW(Transaction txn, DatabaseEntry key,
                                          DatabaseEntry data,
                                          DatabaseEntry entryIDData,
                                          EntryID entryID)
      throws DatabaseException
  {
    OperationStatus status;

    status = read(txn, key, data, LockMode.RMW);
    if(status == OperationStatus.SUCCESS)
    {
      EntryIDSet entryIDList =
          new EntryIDSet(key.getData(), data.getData());
      if (entryIDList.isDefined() && indexEntryLimit > 0 &&
          entryIDList.size() >= indexEntryLimit)
      {
        if(maintainCount)
        {
          entryIDList = new EntryIDSet(entryIDList.size());
        }
        else
        {
          entryIDList = new EntryIDSet();
        }
        entryLimitExceededCount++;

        if(debugEnabled())
        {
          StringBuilder builder = new StringBuilder();
          StaticUtils.byteArrayToHexPlusAscii(builder, key.getData(), 4);
          TRACER.debugInfo("Index entry exceeded in index %s. " +
              "Limit: %d. ID list size: %d.\nKey:",
              name, indexEntryLimit, entryIDList.size(),
              builder);

        }
      }

      entryIDList.add(entryID);

      byte[] after = entryIDList.toDatabase();
      data.setData(after);
      return put(txn, key, data);
    }
    else
    {
      if(rebuildRunning || trusted)
      {
        return insert(txn, key, entryIDData);
      }
      else
      {
        return OperationStatus.SUCCESS;
      }
    }
  }

  /**
   * Remove an entry ID from the set of IDs indexed by a given key.
   *
@@ -256,10 +327,51 @@
      throws DatabaseException
  {
    OperationStatus status;
    LockMode lockMode = LockMode.RMW;
    DatabaseEntry data = new DatabaseEntry();

    status = read(txn, key, data, lockMode);
    if(maintainCount)
    {
      removeIDWithRMW(txn, key, data, entryID);
    }
    else
    {
      status = read(txn, key, data, LockMode.READ_COMMITTED);
      if(status == OperationStatus.SUCCESS)
      {
        EntryIDSet entryIDList = new EntryIDSet(key.getData(), data.getData());
        if(entryIDList.isDefined())
        {
          removeIDWithRMW(txn, key, data, entryID);
        }
      }
      else
      {
        // Ignore failures if rebuild is running since a empty entryIDset
        // will probably not be rebuilt.
        if(trusted && !rebuildRunning)
        {
          setTrusted(txn, false);

          if(debugEnabled())
          {
            StringBuilder builder = new StringBuilder();
            StaticUtils.byteArrayToHexPlusAscii(builder, key.getData(), 4);
            TRACER.debugError("The expected key does not exist in the " +
                "index %s.\nKey:%s", name, builder.toString());
          }

          logError(ERR_JEB_INDEX_CORRUPT_REQUIRES_REBUILD.get(name));
        }
      }
    }
  }

  private void removeIDWithRMW(Transaction txn, DatabaseEntry key,
                               DatabaseEntry data, EntryID entryID)
      throws DatabaseException
  {
    OperationStatus status;
    status = read(txn, key, data, LockMode.RMW);

    if (status == OperationStatus.SUCCESS)
    {
@@ -272,15 +384,13 @@
        {
          setTrusted(txn, false);



          if(debugEnabled())
          {
            StringBuilder builder = new StringBuilder();
            StaticUtils.byteArrayToHexPlusAscii(builder, key.getData(), 4);
            TRACER.debugError("The expected entry ID does not exist in " +
                "the entry ID list for index %s.\nKey:%s",
                              name, builder.toString());
                name, builder.toString());
          }

          logError(ERR_JEB_INDEX_CORRUPT_REQUIRES_REBUILD.get(name));
@@ -612,15 +722,15 @@
  public boolean addEntry(Transaction txn, EntryID entryID, Entry entry)
       throws DatabaseException, DirectoryException
  {
    HashSet<ASN1OctetString> addKeys = new HashSet<ASN1OctetString>();
    TreeSet<byte[]> addKeys = new TreeSet<byte[]>(indexer.getComparator());
    boolean success = true;

    indexer.indexEntry(txn, entry, addKeys);

    DatabaseEntry key = new DatabaseEntry();
    for (ASN1OctetString keyBytes : addKeys)
    for (byte[] keyBytes : addKeys)
    {
      key.setData(keyBytes.value());
      key.setData(keyBytes);
      if(!insertID(txn, key, entryID))
      {
        success = false;
@@ -643,14 +753,14 @@
  public void removeEntry(Transaction txn, EntryID entryID, Entry entry)
       throws DatabaseException, DirectoryException
  {
    HashSet<ASN1OctetString> delKeys = new HashSet<ASN1OctetString>();
    TreeSet<byte[]> delKeys = new TreeSet<byte[]>(indexer.getComparator());

    indexer.indexEntry(txn, entry, delKeys);

    DatabaseEntry key = new DatabaseEntry();
    for (ASN1OctetString keyBytes : delKeys)
    for (byte[] keyBytes : delKeys)
    {
      key.setData(keyBytes.value());
      key.setData(keyBytes);
      removeID(txn, key, entryID);
    }
  }
@@ -674,21 +784,21 @@
                          List<Modification> mods)
       throws DatabaseException
  {
    HashSet<ASN1OctetString> addKeys = new HashSet<ASN1OctetString>();
    HashSet<ASN1OctetString> delKeys = new HashSet<ASN1OctetString>();
    TreeSet<byte[]> addKeys = new TreeSet<byte[]>(indexer.getComparator());
    TreeSet<byte[]> delKeys = new TreeSet<byte[]>(indexer.getComparator());

    indexer.modifyEntry(txn, oldEntry, newEntry, mods, addKeys, delKeys);

    DatabaseEntry key = new DatabaseEntry();
    for (ASN1OctetString keyBytes : delKeys)
    for (byte[] keyBytes : delKeys)
    {
      key.setData(keyBytes.value());
      key.setData(keyBytes);
      removeID(txn, key, entryID);
    }

    for (ASN1OctetString keyBytes : addKeys)
    for (byte[] keyBytes : addKeys)
    {
      key.setData(keyBytes.value());
      key.setData(keyBytes);
      insertID(txn, key, entryID);
    }
  }
@@ -754,4 +864,15 @@
  {
    this.rebuildRunning = rebuildRunning;
  }

  /**
   * Whether this index maintains a count of IDs for keys once the
   * entry limit has exceeded.
   * @return <code>true</code> if this index maintains court of IDs
   * or <code>false</code> otherwise
   */
  public boolean getMaintainCount()
  {
    return maintainCount;
  }
}

 opendj-sdk/opends/src/server/org/opends/server/backends/jeb/IndexMergeThread.java

@@ -305,10 +305,17 @@
            if (merged.size() > entryLimit)
            {
              index.incEntryLimitExceededCount();
              byte[] undefinedSizeBytes =
                  JebFormat.entryIDUndefinedSizeToDatabase(merged.size());
              dbData.setData(undefinedSizeBytes);
              index.put(txn, dbKey, dbData);
              if(index.getMaintainCount())
              {
                byte[] undefinedSizeBytes =
                    JebFormat.entryIDUndefinedSizeToDatabase(merged.size());
                dbData.setData(undefinedSizeBytes);
                index.put(txn, dbKey, dbData);
              }
              else
              {
                index.writeKey(txn, dbKey, new EntryIDSet());
              }
            }
            else
            {

 opendj-sdk/opends/src/server/org/opends/server/backends/jeb/Indexer.java

@@ -26,7 +26,6 @@
 */
package org.opends.server.backends.jeb;

import org.opends.server.protocols.asn1.ASN1OctetString;
import org.opends.server.types.Entry;
import org.opends.server.types.Modification;
import com.sleepycat.je.DatabaseException;
@@ -60,7 +59,7 @@
   * @throws DatabaseException If an error occurs in the JE database.
   */
  public abstract void indexEntry(Transaction txn, Entry entry,
                                Set<ASN1OctetString> keys)
                                Set<byte[]> keys)
       throws DatabaseException;

  /**
@@ -77,8 +76,8 @@
   */
  public abstract void replaceEntry(Transaction txn,
                                    Entry oldEntry, Entry newEntry,
                                    Set<ASN1OctetString> addKeys,
                                    Set<ASN1OctetString> delKeys)
                                    Set<byte[]> addKeys,
                                    Set<byte[]> delKeys)
       throws DatabaseException;

  /**
@@ -97,7 +96,7 @@
  public abstract void modifyEntry(Transaction txn,
                                   Entry oldEntry, Entry newEntry,
                                   List<Modification> mods,
                                   Set<ASN1OctetString> addKeys,
                                   Set<ASN1OctetString> delKeys)
                                   Set<byte[]> addKeys,
                                   Set<byte[]> delKeys)
       throws DatabaseException;
}

 opendj-sdk/opends/src/server/org/opends/server/backends/jeb/OrderingIndexer.java

@@ -31,7 +31,6 @@
import org.opends.server.types.DebugLogLevel;

import org.opends.server.api.OrderingMatchingRule;
import org.opends.server.protocols.asn1.ASN1OctetString;
import org.opends.server.types.Attribute;
import org.opends.server.types.AttributeType;
import org.opends.server.types.AttributeValue;
@@ -110,7 +109,7 @@
   * @param keys The set into which the generated keys will be inserted.
   */
  public void indexEntry(Transaction txn, Entry entry,
                       Set<ASN1OctetString> keys)
                       Set<byte[]> keys)
  {
    List<Attribute> attrList =
         entry.getAttribute(attributeType);
@@ -133,31 +132,37 @@
   */
  public void replaceEntry(Transaction txn,
                           Entry oldEntry, Entry newEntry,
                           Set<ASN1OctetString> addKeys,
                           Set<ASN1OctetString> delKeys)
                           Set<byte[]> addKeys,
                           Set<byte[]> delKeys)
  {
    List<Attribute> attrList;
    List<Attribute> newAttributes = newEntry.getAttribute(attributeType, true);
    List<Attribute> oldAttributes = oldEntry.getAttribute(attributeType, true);

    attrList = oldEntry.getAttribute(attributeType);
    Set<ASN1OctetString> oldSet = new HashSet<ASN1OctetString>();
    indexAttribute(attrList, oldSet);

    attrList = newEntry.getAttribute(attributeType);
    Set<ASN1OctetString> newSet = new HashSet<ASN1OctetString>();
    indexAttribute(attrList, newSet);

    HashSet<ASN1OctetString> removeSet = new HashSet<ASN1OctetString>(oldSet);
    removeSet.removeAll(newSet);
    for (ASN1OctetString k : removeSet)
    if(newAttributes == null)
    {
      delKeys.add(k);
      indexAttribute(oldAttributes, delKeys);
    }

    HashSet<ASN1OctetString> addSet = new HashSet<ASN1OctetString>(newSet);
    addSet.removeAll(oldSet);
    for (ASN1OctetString k : addSet)
    else
    {
      addKeys.add(k);
      if(oldAttributes == null)
      {
        indexAttribute(newAttributes, addKeys);
      }
      else
      {
        TreeSet<byte[]> newKeys =
            new TreeSet<byte[]>(orderingRule);
        TreeSet<byte[]> oldKeys =
            new TreeSet<byte[]>(orderingRule);
        indexAttribute(newAttributes, newKeys);
        indexAttribute(oldAttributes, oldKeys);

        addKeys.addAll(newKeys);
        addKeys.removeAll(oldKeys);

        delKeys.addAll(oldKeys);
        delKeys.removeAll(newKeys);
      }
    }
  }

@@ -176,14 +181,12 @@
   */
  public void modifyEntry(Transaction txn, Entry oldEntry, Entry newEntry,
                          List<Modification> mods,
                          Set<ASN1OctetString> addKeys,
                          Set<ASN1OctetString> delKeys)
       throws DatabaseException
                          Set<byte[]> addKeys,
                          Set<byte[]> delKeys)
      throws DatabaseException
  {
    List<Attribute> newAttributes = newEntry.getAttribute(attributeType, true);
    List<Attribute> oldAttributes = oldEntry.getAttribute(attributeType, true);
    HashSet<AttributeValue> newValues;
    HashSet<AttributeValue> oldValues;

    if(newAttributes == null)
    {
@@ -197,26 +200,18 @@
      }
      else
      {
        newValues = new HashSet<AttributeValue>();
        oldValues = new HashSet<AttributeValue>();
        for(Attribute a : newAttributes)
        {
          newValues.addAll(a.getValues());
        }
        for(Attribute a : oldAttributes)
        {
          oldValues.addAll(a.getValues());
        }
        TreeSet<byte[]> newKeys =
            new TreeSet<byte[]>(orderingRule);
        TreeSet<byte[]> oldKeys =
            new TreeSet<byte[]>(orderingRule);
        indexAttribute(newAttributes, newKeys);
        indexAttribute(oldAttributes, oldKeys);

        HashSet<AttributeValue> valuesToAdd =
            new HashSet<AttributeValue>(newValues);
        HashSet<AttributeValue> valuesToDel =
            new HashSet<AttributeValue>(oldValues);
        valuesToAdd.removeAll(oldValues);
        valuesToDel.removeAll(newValues);
        addKeys.addAll(newKeys);
        addKeys.removeAll(oldKeys);

        indexValues(valuesToDel, delKeys);
        indexValues(valuesToAdd, addKeys);
        delKeys.addAll(oldKeys);
        delKeys.removeAll(newKeys);
      }
    }
  }
@@ -228,7 +223,7 @@
   * @param keys The set into which the keys will be inserted.
   */
  private void indexValues(Set<AttributeValue> values,
                           Set<ASN1OctetString> keys)
                           Set<byte[]> keys)
  {
    if (values == null) return;

@@ -239,7 +234,7 @@
        byte[] keyBytes =
             orderingRule.normalizeValue(value.getValue()).value();

        keys.add(new ASN1OctetString(keyBytes));
        keys.add(keyBytes);
      }
      catch (DirectoryException e)
      {
@@ -257,7 +252,7 @@
   * @param keys The set into which the keys will be inserted.
   */
  private void indexAttribute(List<Attribute> attrList,
                              Set<ASN1OctetString> keys)
                              Set<byte[]> keys)
  {
    if (attrList == null) return;


 opendj-sdk/opends/src/server/org/opends/server/backends/jeb/PresenceIndexer.java

@@ -26,7 +26,6 @@
 */
package org.opends.server.backends.jeb;

import org.opends.server.protocols.asn1.ASN1OctetString;
import org.opends.server.types.Attribute;
import org.opends.server.types.Entry;
import org.opends.server.types.Modification;
@@ -100,7 +99,7 @@
   * @throws DatabaseException If an error occurs in the JE database.
   */
  public void indexEntry(Transaction txn, Entry entry,
                       Set<ASN1OctetString> keys) throws DatabaseException
                       Set<byte[]> keys) throws DatabaseException
  {
    List<Attribute> attrList =
         entry.getAttribute(attributeType);
@@ -108,7 +107,7 @@
    {
      if (!attrList.isEmpty())
      {
        keys.add(new ASN1OctetString(AttributeIndex.presenceKey.getData()));
        keys.add(AttributeIndex.presenceKey.getData());
      }
    }
  }
@@ -128,25 +127,25 @@
   * @throws DatabaseException If an error occurs in the JE database.
   */
  public void replaceEntry(Transaction txn, Entry oldEntry, Entry newEntry,
                           Set<ASN1OctetString> addKeys,
                           Set<ASN1OctetString> delKeys)
       throws DatabaseException
                           Set<byte[]> addKeys,
                           Set<byte[]> delKeys)
      throws DatabaseException
  {
    List<Attribute> beforeList, afterList;

    beforeList = oldEntry.getAttribute(attributeType);
    afterList = newEntry.getAttribute(attributeType);

    if (beforeList == null || beforeList.isEmpty())
    List<Attribute> newAttributes = newEntry.getAttribute(attributeType, true);
    List<Attribute> oldAttributes = oldEntry.getAttribute(attributeType, true);
    if(oldAttributes == null)
    {
      if (afterList != null && !afterList.isEmpty())
      if(newAttributes != null)
      {
        addKeys.add(new ASN1OctetString(AttributeIndex.presenceKey.getData()));
        addKeys.add(AttributeIndex.presenceKey.getData());
      }
    }
    else if (afterList == null || afterList.isEmpty())
    else
    {
      delKeys.add(new ASN1OctetString(AttributeIndex.presenceKey.getData()));
      if(newAttributes == null)
      {
        delKeys.add(AttributeIndex.presenceKey.getData());
      }
    }
  }

@@ -167,8 +166,8 @@
   */
  public void modifyEntry(Transaction txn, Entry oldEntry, Entry newEntry,
                          List<Modification> mods,
                          Set<ASN1OctetString> addKeys,
                          Set<ASN1OctetString> delKeys)
                          Set<byte[]> addKeys,
                          Set<byte[]> delKeys)
       throws DatabaseException
  {
    List<Attribute> newAttributes = newEntry.getAttribute(attributeType, true);
@@ -177,16 +176,14 @@
    {
      if(newAttributes != null)
      {
        addKeys.add(
              new ASN1OctetString(AttributeIndex.presenceKey.getData()));
        addKeys.add(AttributeIndex.presenceKey.getData());
      }
    }
    else
    {
      if(newAttributes == null)
      {
        delKeys.add(
              new ASN1OctetString(AttributeIndex.presenceKey.getData()));
        delKeys.add(AttributeIndex.presenceKey.getData());
      }
    }
  }

 opendj-sdk/opends/src/server/org/opends/server/backends/jeb/SubstringIndexer.java

@@ -29,13 +29,9 @@
import static org.opends.server.loggers.debug.DebugLogger.*;
import org.opends.server.loggers.debug.DebugTracer;
import com.sleepycat.je.Transaction;
import org.opends.server.protocols.asn1.ASN1OctetString;
import org.opends.server.types.*;

import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.*;

/**
 * An implementation of an Indexer for attribute substrings.
@@ -108,7 +104,7 @@
   * @param keys The set into which the generated keys will be inserted.
   */
  public void indexEntry(Transaction txn, Entry entry,
                       Set<ASN1OctetString> keys)
                       Set<byte[]> keys)
  {
    List<Attribute> attrList =
         entry.getAttribute(attributeType);
@@ -131,31 +127,37 @@
   */
  public void replaceEntry(Transaction txn,
                           Entry oldEntry, Entry newEntry,
                           Set<ASN1OctetString> addKeys,
                           Set<ASN1OctetString> delKeys)
                           Set<byte[]> addKeys,
                           Set<byte[]> delKeys)
  {
    List<Attribute> attrList;
    List<Attribute> newAttributes = newEntry.getAttribute(attributeType, true);
    List<Attribute> oldAttributes = oldEntry.getAttribute(attributeType, true);

    attrList = oldEntry.getAttribute(attributeType);
    Set<ASN1OctetString> oldSet = new HashSet<ASN1OctetString>();
    indexAttribute(attrList, oldSet);

    attrList = newEntry.getAttribute(attributeType);
    Set<ASN1OctetString> newSet = new HashSet<ASN1OctetString>();
    indexAttribute(attrList, newSet);

    HashSet<ASN1OctetString> removeSet = new HashSet<ASN1OctetString>(oldSet);
    removeSet.removeAll(newSet);
    for (ASN1OctetString k : removeSet)
    if(newAttributes == null)
    {
      delKeys.add(k);
      indexAttribute(oldAttributes, delKeys);
    }

    HashSet<ASN1OctetString> addSet = new HashSet<ASN1OctetString>(newSet);
    addSet.removeAll(oldSet);
    for (ASN1OctetString k : addSet)
    else
    {
      addKeys.add(k);
      if(oldAttributes == null)
      {
        indexAttribute(newAttributes, addKeys);
      }
      else
      {
        TreeSet<byte[]> newKeys =
            new TreeSet<byte[]>(comparator);
        TreeSet<byte[]> oldKeys =
            new TreeSet<byte[]>(comparator);
        indexAttribute(newAttributes, newKeys);
        indexAttribute(oldAttributes, oldKeys);

        addKeys.addAll(newKeys);
        addKeys.removeAll(oldKeys);

        delKeys.addAll(oldKeys);
        delKeys.removeAll(newKeys);
      }
    }
  }

@@ -175,13 +177,11 @@
   */
  public void modifyEntry(Transaction txn, Entry oldEntry, Entry newEntry,
                          List<Modification> mods,
                          Set<ASN1OctetString> addKeys,
                          Set<ASN1OctetString> delKeys)
                          Set<byte[]> addKeys,
                          Set<byte[]> delKeys)
  {
    List<Attribute> newAttributes = newEntry.getAttribute(attributeType, true);
    List<Attribute> oldAttributes = oldEntry.getAttribute(attributeType, true);
    HashSet<AttributeValue> newValues;
    HashSet<AttributeValue> oldValues;

    if(newAttributes == null)
    {
@@ -195,10 +195,10 @@
      }
      else
      {
        HashSet<ASN1OctetString> newKeys =
            new HashSet<ASN1OctetString>();
        HashSet<ASN1OctetString> oldKeys =
            new HashSet<ASN1OctetString>();
        TreeSet<byte[]> newKeys =
            new TreeSet<byte[]>(comparator);
        TreeSet<byte[]> oldKeys =
            new TreeSet<byte[]>(comparator);
        indexAttribute(newAttributes, newKeys);
        indexAttribute(oldAttributes, oldKeys);

@@ -219,7 +219,7 @@
   * @param keys The set into which the generated keys will be inserted.
   */
  private void indexAttribute(List<Attribute> attrList,
                              Set<ASN1OctetString> keys)
                              Set<byte[]> keys)
  {
    if (attrList == null) return;

@@ -235,7 +235,7 @@
   * @param keys The set into which the keys will be inserted.
   */
  private void indexValues(Set<AttributeValue> values,
                           Set<ASN1OctetString> keys)
                           Set<byte[]> keys)
  {
    if (values == null) return;

@@ -265,7 +265,7 @@
   * @param value A byte array containing the normalized attribute value
   * @param set A set into which the keys will be inserted.
   */
  private void substringKeys(byte[] value, Set<ASN1OctetString> set)
  private void substringKeys(byte[] value, Set<byte[]> set)
  {
    byte[] keyBytes;

@@ -279,7 +279,7 @@
    {
      int len = Math.min(substrLength, remain);
      keyBytes = makeSubstringKey(value, i, len);
      set.add(new ASN1OctetString(keyBytes));
      set.add(keyBytes);
    }
  }


 opendj-sdk/opends/tests/unit-tests-testng/src/server/org/opends/server/backends/jeb/TestBackendImpl.java

@@ -804,7 +804,7 @@
    EntryID entryID;
    AttributeType attribute;
    AttributeIndex index;
    HashSet<ASN1OctetString> addKeys;
    HashSet<byte[]> addKeys;
    DatabaseEntry key;
    PresenceIndexer presenceIndexer;
    EqualityIndexer equalityIndexer;
@@ -843,47 +843,47 @@
      index = ec.getAttributeIndex(attribute);


      addKeys = new HashSet<ASN1OctetString>();
      addKeys = new HashSet<byte[]>();
      presenceIndexer = new PresenceIndexer(index.getAttributeType());
      presenceIndexer.indexEntry(null, entry, addKeys);

      key = new DatabaseEntry();
      for (ASN1OctetString keyBytes : addKeys) {
        key.setData(keyBytes.value());
      for (byte[] keyBytes : addKeys) {
        key.setData(keyBytes);
      }
      assertEquals(index.presenceIndex.containsID(null, key, entryID),
          ConditionResult.FALSE);

      addKeys = new HashSet<ASN1OctetString>();
      addKeys = new HashSet<byte[]>();
      equalityIndexer = new EqualityIndexer(index.getAttributeType());
      equalityIndexer.indexEntry(null, entry, addKeys);

      key = new DatabaseEntry();
      for (ASN1OctetString keyBytes : addKeys) {
        key.setData(keyBytes.value());
      for (byte[] keyBytes : addKeys) {
        key.setData(keyBytes);
      }
      assertEquals(index.equalityIndex.containsID(null, key, entryID),
          ConditionResult.FALSE);

      addKeys = new HashSet<ASN1OctetString>();
      addKeys = new HashSet<byte[]>();
      substringIndexer = new SubstringIndexer(index.getAttributeType(),
                   index.getConfiguration().getSubstringLength());
      substringIndexer.indexEntry(null, entry, addKeys);

      key = new DatabaseEntry();
      for (ASN1OctetString keyBytes : addKeys) {
        key.setData(keyBytes.value());
      for (byte[] keyBytes : addKeys) {
        key.setData(keyBytes);
      }
      assertEquals(index.substringIndex.containsID(null, key, entryID),
          ConditionResult.FALSE);

      addKeys = new HashSet<ASN1OctetString>();
      addKeys = new HashSet<byte[]>();
      orderingIndexer = new OrderingIndexer(index.getAttributeType());
      orderingIndexer.indexEntry(null, entry, addKeys);

      key = new DatabaseEntry();
      for (ASN1OctetString keyBytes : addKeys) {
        key.setData(keyBytes.value());
      for (byte[] keyBytes : addKeys) {
        key.setData(keyBytes);
      }
      assertEquals(index.orderingIndex.containsID(null, key, entryID),
          ConditionResult.FALSE);
@@ -903,7 +903,7 @@
    EntryID entryID;
    AttributeType attribute;
    AttributeIndex index;
    HashSet<ASN1OctetString> addKeys;
    HashSet<byte[]> addKeys;
    DatabaseEntry key;
    EqualityIndexer equalityIndexer;
    SubstringIndexer substringIndexer;
@@ -943,66 +943,66 @@
      attribute = entry.getAttribute("cn").get(0).getAttributeType();
      index = ec.getAttributeIndex(attribute);

      addKeys = new HashSet<ASN1OctetString>();
      addKeys = new HashSet<byte[]>();
      orderingIndexer = new OrderingIndexer(index.getAttributeType());
      orderingIndexer.indexEntry(null, entry, addKeys);

      key = new DatabaseEntry();
      for (ASN1OctetString keyBytes : addKeys) {
        key.setData(keyBytes.value());
      for (byte[] keyBytes : addKeys) {
        key.setData(keyBytes);
      }
      assertEquals(index.orderingIndex.containsID(null, key, entryID),
          ConditionResult.TRUE);

      addKeys = new HashSet<ASN1OctetString>();
      addKeys = new HashSet<byte[]>();
      orderingIndexer.indexEntry(null, oldEntry, addKeys);

      key = new DatabaseEntry();
      for (ASN1OctetString keyBytes : addKeys) {
        key.setData(keyBytes.value());
      for (byte[] keyBytes : addKeys) {
        key.setData(keyBytes);
      }
      assertEquals(index.orderingIndex.containsID(null, key, entryID),
                   ConditionResult.FALSE);

      addKeys = new HashSet<ASN1OctetString>();
      addKeys = new HashSet<byte[]>();
      substringIndexer = new SubstringIndexer(index.getAttributeType(),
                                              index.getConfiguration().getSubstringLength());
      substringIndexer.indexEntry(null, entry, addKeys);

      key = new DatabaseEntry();
      for (ASN1OctetString keyBytes : addKeys) {
        key.setData(keyBytes.value());
      for (byte[] keyBytes : addKeys) {
        key.setData(keyBytes);
      }
      assertEquals(index.substringIndex.containsID(null, key, entryID),
                   ConditionResult.TRUE);

      addKeys = new HashSet<ASN1OctetString>();
      addKeys = new HashSet<byte[]>();
      substringIndexer.indexEntry(null, oldEntry, addKeys);

      key = new DatabaseEntry();
      for (ASN1OctetString keyBytes : addKeys) {
        key.setData(keyBytes.value());
      for (byte[] keyBytes : addKeys) {
        key.setData(keyBytes);
      }
      assertEquals(index.substringIndex.containsID(null, key, entryID),
                   ConditionResult.FALSE);

      addKeys = new HashSet<ASN1OctetString>();
      addKeys = new HashSet<byte[]>();
      equalityIndexer = new EqualityIndexer(index.getAttributeType());
      equalityIndexer.indexEntry(null, entry, addKeys);

      key = new DatabaseEntry();
      for (ASN1OctetString keyBytes : addKeys) {
        key.setData(keyBytes.value());
      for (byte[] keyBytes : addKeys) {
        key.setData(keyBytes);
      }
      assertEquals(index.equalityIndex.containsID(null, key, entryID),
          ConditionResult.TRUE);

      addKeys = new HashSet<ASN1OctetString>();
      addKeys = new HashSet<byte[]>();
      equalityIndexer.indexEntry(null, oldEntry, addKeys);

      key = new DatabaseEntry();
      for (ASN1OctetString keyBytes : addKeys) {
        key.setData(keyBytes.value());
      for (byte[] keyBytes : addKeys) {
        key.setData(keyBytes);
      }
      assertEquals(index.equalityIndex.containsID(null, key, entryID),
          ConditionResult.FALSE);
@@ -1024,7 +1024,7 @@
    AttributeType attribute;
    AttributeIndex titleIndex;
    AttributeIndex nameIndex;
    HashSet<ASN1OctetString> addKeys;
    HashSet<byte[]> addKeys;
    DatabaseEntry key;
    PresenceIndexer presenceIndexer;
    EqualityIndexer equalityIndexer;
@@ -1097,21 +1097,21 @@
      nameIndex = ec.getAttributeIndex(attribute);

      //This current entry in the DB shouldn't be in the presence titleIndex.
      addKeys = new HashSet<ASN1OctetString>();
      addKeys.add(new ASN1OctetString(AttributeIndex.presenceKey.getData()));
      addKeys = new HashSet<byte[]>();
      addKeys.add(AttributeIndex.presenceKey.getData());
      key = new DatabaseEntry();
      for (ASN1OctetString keyBytes : addKeys) {
        key.setData(keyBytes.value());
      for (byte[] keyBytes : addKeys) {
        key.setData(keyBytes);
      }
      assertEquals(titleIndex.presenceIndex.containsID(null, key, entryID),
          ConditionResult.FALSE);

      //This current entry should be in the presence nameIndex.
      addKeys = new HashSet<ASN1OctetString>();
      addKeys.add(new ASN1OctetString(AttributeIndex.presenceKey.getData()));
      addKeys = new HashSet<byte[]>();
      addKeys.add(AttributeIndex.presenceKey.getData());
      key = new DatabaseEntry();
      for (ASN1OctetString keyBytes : addKeys) {
        key.setData(keyBytes.value());
      for (byte[] keyBytes : addKeys) {
        key.setData(keyBytes);
      }
      assertEquals(nameIndex.presenceIndex.containsID(null, key, entryID),
          ConditionResult.TRUE);
@@ -1176,92 +1176,92 @@
      assertFalse(entry.getAttribute("employeenumber").contains(new
          Attribute("employeenumber", "1")));

      addKeys = new HashSet<ASN1OctetString>();
      addKeys = new HashSet<byte[]>();
      presenceIndexer = new PresenceIndexer(titleIndex.getAttributeType());
      presenceIndexer.indexEntry(null, entry, addKeys);

      key = new DatabaseEntry();
      for (ASN1OctetString keyBytes : addKeys) {
        key.setData(keyBytes.value());
      for (byte[] keyBytes : addKeys) {
        key.setData(keyBytes);
        assertEquals(titleIndex.presenceIndex.containsID(null, key, entryID),
          ConditionResult.TRUE);
      }

      addKeys = new HashSet<ASN1OctetString>();
      addKeys = new HashSet<byte[]>();
      presenceIndexer = new PresenceIndexer(nameIndex.getAttributeType());
      presenceIndexer.indexEntry(null, entry, addKeys);

      key = new DatabaseEntry();
      for (ASN1OctetString keyBytes : addKeys) {
        key.setData(keyBytes.value());
      for (byte[] keyBytes : addKeys) {
        key.setData(keyBytes);
        assertEquals(nameIndex.presenceIndex.containsID(null, key, entryID),
          ConditionResult.TRUE);
      }

      addKeys = new HashSet<ASN1OctetString>();
      addKeys = new HashSet<byte[]>();
      orderingIndexer = new OrderingIndexer(titleIndex.getAttributeType());
      orderingIndexer.indexEntry(null, entry, addKeys);

      key = new DatabaseEntry();
      for (ASN1OctetString keyBytes : addKeys) {
        key.setData(keyBytes.value());
      for (byte[] keyBytes : addKeys) {
        key.setData(keyBytes);
        assertEquals(titleIndex.orderingIndex.containsID(null, key, entryID),
          ConditionResult.TRUE);
      }

      addKeys = new HashSet<ASN1OctetString>();
      addKeys = new HashSet<byte[]>();
      orderingIndexer = new OrderingIndexer(nameIndex.getAttributeType());
      orderingIndexer.indexEntry(null, entry, addKeys);

      key = new DatabaseEntry();
      for (ASN1OctetString keyBytes : addKeys) {
        key.setData(keyBytes.value());
      for (byte[] keyBytes : addKeys) {
        key.setData(keyBytes);
        assertEquals(nameIndex.orderingIndex.containsID(null, key, entryID),
          ConditionResult.TRUE);
      }

      addKeys = new HashSet<ASN1OctetString>();
      addKeys = new HashSet<byte[]>();
      equalityIndexer = new EqualityIndexer(titleIndex.getAttributeType());
      equalityIndexer.indexEntry(null, entry, addKeys);

      key = new DatabaseEntry();
      for (ASN1OctetString keyBytes : addKeys) {
        key.setData(keyBytes.value());
      for (byte[] keyBytes : addKeys) {
        key.setData(keyBytes);
        assertEquals(titleIndex.equalityIndex.containsID(null, key, entryID),
          ConditionResult.TRUE);
      }

      addKeys = new HashSet<ASN1OctetString>();
      addKeys = new HashSet<byte[]>();
      equalityIndexer = new EqualityIndexer(nameIndex.getAttributeType());
      equalityIndexer.indexEntry(null, entry, addKeys);

      key = new DatabaseEntry();
      for (ASN1OctetString keyBytes : addKeys) {
        key.setData(keyBytes.value());
      for (byte[] keyBytes : addKeys) {
        key.setData(keyBytes);
        assertEquals(nameIndex.equalityIndex.containsID(null, key, entryID),
          ConditionResult.TRUE);
      }

      addKeys = new HashSet<ASN1OctetString>();
      addKeys = new HashSet<byte[]>();
      substringIndexer = new SubstringIndexer(titleIndex.getAttributeType(),
                   titleIndex.getConfiguration().getSubstringLength());
      substringIndexer.indexEntry(null, entry, addKeys);

      key = new DatabaseEntry();
      for (ASN1OctetString keyBytes : addKeys) {
        key.setData(keyBytes.value());
      for (byte[] keyBytes : addKeys) {
        key.setData(keyBytes);
        assertEquals(titleIndex.substringIndex.containsID(null, key, entryID),
          ConditionResult.TRUE);
      }

      addKeys = new HashSet<ASN1OctetString>();
      addKeys = new HashSet<byte[]>();
      substringIndexer = new SubstringIndexer(nameIndex.getAttributeType(),
                   nameIndex.getConfiguration().getSubstringLength());
      substringIndexer.indexEntry(null, entry, addKeys);

      key = new DatabaseEntry();
      for (ASN1OctetString keyBytes : addKeys) {
        key.setData(keyBytes.value());
      for (byte[] keyBytes : addKeys) {
        key.setData(keyBytes);
        assertEquals(nameIndex.substringIndex.containsID(null, key, entryID),
          ConditionResult.TRUE);
      }