Updated indexes to order the keys before inserting them into the database. ...

boli

20.45.2007 a72ae6523fc66a21ced5b5ab04c23e1629ae4d20

Updated indexes to order the keys before inserting them into the database. This assures no deadlocks will occur between multiple adds and mods. 
Disabled lock timeouts for add and mod operations since deadlocks can not occur. This prevents txn aborts and op retry expiration due to lock timeouts of add and mod operations when the server is under high write load. 

Fix for issue 2186 

 opends/src/server/org/opends/server/backends/jeb/ApproximateIndexer.java

@@ -31,7 +31,6 @@
import java.util.*;

import org.opends.server.types.*;
import org.opends.server.protocols.asn1.ASN1OctetString;
import static org.opends.server.loggers.debug.DebugLogger.*;
import org.opends.server.loggers.debug.DebugTracer;
import org.opends.server.api.ApproximateMatchingRule;
@@ -106,7 +105,7 @@
   * @param keys The set into which the generated keys will be inserted.
   */
  public void indexEntry(Transaction txn, Entry entry,
                       Set<ASN1OctetString> keys)
                       Set<byte[]> keys)
  {
    List<Attribute> attrList =
         entry.getAttribute(attributeType);
@@ -129,31 +128,37 @@
   */
  public void replaceEntry(Transaction txn,
                           Entry oldEntry, Entry newEntry,
                           Set<ASN1OctetString> addKeys,
                           Set<ASN1OctetString> delKeys)
                           Set<byte[]> addKeys,
                           Set<byte[]> delKeys)
  {
    List<Attribute> attrList;
    List<Attribute> newAttributes = newEntry.getAttribute(attributeType, true);
    List<Attribute> oldAttributes = oldEntry.getAttribute(attributeType, true);

    attrList = oldEntry.getAttribute(attributeType);
    Set<ASN1OctetString> oldSet = new HashSet<ASN1OctetString>();
    indexAttribute(attrList, oldSet);

    attrList = newEntry.getAttribute(attributeType);
    Set<ASN1OctetString> newSet = new HashSet<ASN1OctetString>();
    indexAttribute(attrList, newSet);

    HashSet<ASN1OctetString> removeSet = new HashSet<ASN1OctetString>(oldSet);
    removeSet.removeAll(newSet);
    for (ASN1OctetString k : removeSet)
    if(newAttributes == null)
    {
      delKeys.add(k);
      indexAttribute(oldAttributes, delKeys);
    }

    HashSet<ASN1OctetString> addSet = new HashSet<ASN1OctetString>(newSet);
    addSet.removeAll(oldSet);
    for (ASN1OctetString k : addSet)
    else
    {
      addKeys.add(k);
      if(oldAttributes == null)
      {
        indexAttribute(newAttributes, addKeys);
      }
      else
      {
        TreeSet<byte[]> newKeys =
            new TreeSet<byte[]>(comparator);
        TreeSet<byte[]> oldKeys =
            new TreeSet<byte[]>(comparator);
        indexAttribute(newAttributes, newKeys);
        indexAttribute(oldAttributes, oldKeys);

        addKeys.addAll(newKeys);
        addKeys.removeAll(oldKeys);

        delKeys.addAll(oldKeys);
        delKeys.removeAll(newKeys);
      }
    }
  }

@@ -173,13 +178,11 @@
   */
  public void modifyEntry(Transaction txn, Entry oldEntry, Entry newEntry,
                          List<Modification> mods,
                          Set<ASN1OctetString> addKeys,
                          Set<ASN1OctetString> delKeys)
                          Set<byte[]> addKeys,
                          Set<byte[]> delKeys)
  {
    List<Attribute> newAttributes = newEntry.getAttribute(attributeType, true);
    List<Attribute> oldAttributes = oldEntry.getAttribute(attributeType, true);
    HashSet<AttributeValue> newValues;
    HashSet<AttributeValue> oldValues;

    if(newAttributes == null)
    {
@@ -193,10 +196,10 @@
      }
      else
      {
        HashSet<ASN1OctetString> newKeys =
            new HashSet<ASN1OctetString>();
        HashSet<ASN1OctetString> oldKeys =
            new HashSet<ASN1OctetString>();
        TreeSet<byte[]> newKeys =
            new TreeSet<byte[]>(comparator);
        TreeSet<byte[]> oldKeys =
            new TreeSet<byte[]>(comparator);
        indexAttribute(newAttributes, newKeys);
        indexAttribute(oldAttributes, oldKeys);

@@ -215,7 +218,7 @@
   * @param keys The set into which the keys will be inserted.
   */
  private void indexAttribute(List<Attribute> attrList,
                              Set<ASN1OctetString> keys)
                              Set<byte[]> keys)
  {
    if (attrList == null) return;

@@ -231,7 +234,7 @@
   * @param keys The set into which the keys will be inserted.
   */
  private void indexValues(Set<AttributeValue> values,
                           Set<ASN1OctetString> keys)
                           Set<byte[]> keys)
  {
    if (values == null) return;

@@ -242,7 +245,7 @@
        byte[] keyBytes =
             approximateRule.normalizeValue(value.getValue()).value();

        keys.add(new ASN1OctetString(keyBytes));
        keys.add(keyBytes);
      }
      catch (DirectoryException e)
      {

 opends/src/server/org/opends/server/backends/jeb/AttributeIndex.java

@@ -27,11 +27,7 @@
package org.opends.server.backends.jeb;
import org.opends.messages.Message;

import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.*;

import com.sleepycat.je.*;

@@ -161,6 +157,7 @@
                                     state,
                                     indexEntryLimit,
                                     cursorEntryLimit,
                                     false,
                                     env,
                                     entryContainer);
    }
@@ -174,6 +171,7 @@
                                     state,
                                     indexEntryLimit,
                                     cursorEntryLimit,
                                     false,
                                     env,
                                     entryContainer);
    }
@@ -195,6 +193,7 @@
                                     state,
                                     indexEntryLimit,
                                     cursorEntryLimit,
                                     false,
                                     env,
                                     entryContainer);
    }
@@ -215,6 +214,7 @@
                                     state,
                                     indexEntryLimit,
                                     cursorEntryLimit,
                                     false,
                                     env,
                                     entryContainer);
    }
@@ -234,6 +234,7 @@
                                        state,
                                        indexEntryLimit,
                                        cursorEntryLimit,
                                        false,
                                        env,
                                        entryContainer);
    }
@@ -571,7 +572,8 @@
      // index substring length, and read those keys.

      // Eliminate duplicates by putting the keys into a set.
      Set<byte[]> set = new HashSet<byte[]>();
      Set<byte[]> set =
          new TreeSet<byte[]>(substringIndex.indexer.getComparator());

      // Example: The value is ABCDE and the substring length is 3.
      // We produce the keys ABC BCD CDE.
@@ -1238,6 +1240,7 @@
                                    state,
                                    indexEntryLimit,
                                    cursorEntryLimit,
                                    false,
                                    env,
                                    entryContainer);
          equalityIndex.open();
@@ -1297,6 +1300,7 @@
                                    state,
                                    indexEntryLimit,
                                    cursorEntryLimit,
                                    false,
                                    env,
                                    entryContainer);
          presenceIndex.open();
@@ -1356,6 +1360,7 @@
                                     state,
                                     indexEntryLimit,
                                     cursorEntryLimit,
                                     false,
                                     env,
                                     entryContainer);
          substringIndex.open();
@@ -1420,6 +1425,7 @@
                                    state,
                                    indexEntryLimit,
                                    cursorEntryLimit,
                                    false,
                                    env,
                                    entryContainer);
          orderingIndex.open();
@@ -1478,6 +1484,7 @@
                                       state,
                                       indexEntryLimit,
                                       cursorEntryLimit,
                                       false,
                                       env,
                                       entryContainer);
          approximateIndex.open();

 opends/src/server/org/opends/server/backends/jeb/AttributeIndexBuilder.java

@@ -26,18 +26,13 @@
 */
package org.opends.server.backends.jeb;

import org.opends.server.protocols.asn1.ASN1OctetString;
import org.opends.server.types.Entry;
import static org.opends.server.util.StaticUtils.getFileForPath;

import com.sleepycat.je.DatabaseException;
import com.sleepycat.je.Transaction;

import java.util.Arrays;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
import java.util.*;
import java.io.ByteArrayOutputStream;
import java.io.BufferedOutputStream;
import java.io.DataOutputStream;
@@ -180,29 +175,29 @@
    if (oldEntry != null)
    {
      // This is an entry being replaced.
      Set<ASN1OctetString> addKeys = new HashSet<ASN1OctetString>();
      Set<ASN1OctetString> delKeys = new HashSet<ASN1OctetString>();
      TreeSet<byte[]> addKeys = new TreeSet<byte[]>(indexer.getComparator());
      TreeSet<byte[]> delKeys = new TreeSet<byte[]>(indexer.getComparator());

      indexer.replaceEntry(txn, oldEntry, newEntry, addKeys, delKeys);

      for (ASN1OctetString k : delKeys)
      for (byte[] k : delKeys)
      {
        removeID(k.value(), entryID);
        removeID(k, entryID);
      }

      for (ASN1OctetString k : addKeys)
      for (byte[] k : addKeys)
      {
        insertID(k.value(), entryID);
        insertID(k, entryID);
      }
    }
    else
    {
      // This is a new entry.
      Set<ASN1OctetString> addKeys = new HashSet<ASN1OctetString>();
      TreeSet<byte[]> addKeys = new TreeSet<byte[]>(indexer.getComparator());
      indexer.indexEntry(txn, newEntry, addKeys);
      for (ASN1OctetString k : addKeys)
      for (byte[] k : addKeys)
      {
        insertID(k.value(), entryID);
        insertID(k, entryID);
      }
    }


 opends/src/server/org/opends/server/backends/jeb/EntryContainer.java

@@ -498,12 +498,12 @@

      id2children = new Index(databasePrefix + "_" + ID2CHILDREN_DATABASE_NAME,
                              new ID2CIndexer(), state,
                              indexEntryLimit, 0,
                              indexEntryLimit, 0, true,
                              env,this);
      id2children.open();
      id2subtree = new Index(databasePrefix + "_" + ID2SUBTREE_DATABASE_NAME,
                             new ID2SIndexer(), state,
                             indexEntryLimit, 0,
                             indexEntryLimit, 0, true,
                             env, this);
      id2subtree.open();

@@ -1810,7 +1810,10 @@
     */
    public Transaction beginOperationTransaction() throws DatabaseException
    {
      return beginTransaction();
      Transaction txn =  beginTransaction();
      // Multiple adds should never encounter a deadlock.
      txn.setLockTimeout(0);
      return txn;
    }

    /**
@@ -2300,7 +2303,10 @@
     */
    public Transaction beginOperationTransaction() throws DatabaseException
    {
      return beginTransaction();
      Transaction txn =  beginTransaction();
      // Multiple deletes should never encounter a deadlock.
      txn.setLockTimeout(0);
      return txn;
    }

    /**
@@ -2843,7 +2849,10 @@
     */
    public Transaction beginOperationTransaction() throws DatabaseException
    {
      return beginTransaction();
      Transaction txn =  beginTransaction();
      // Multiple replace operations should never encounter a deadlock.
      txn.setLockTimeout(0);
      return txn;
    }

    /**
@@ -3214,7 +3223,8 @@
     */
    public Transaction beginOperationTransaction() throws DatabaseException
    {
      return beginTransaction();
      Transaction txn =  beginTransaction();
      return txn;
    }

    /**

 opends/src/server/org/opends/server/backends/jeb/EqualityIndexer.java

@@ -33,7 +33,6 @@
import com.sleepycat.je.Transaction;
import com.sleepycat.je.DatabaseException;

import org.opends.server.protocols.asn1.ASN1OctetString;
import org.opends.server.types.Attribute;
import org.opends.server.types.AttributeType;
import org.opends.server.types.AttributeValue;
@@ -41,10 +40,7 @@
import org.opends.server.types.Entry;
import org.opends.server.types.Modification;

import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.*;

/**
 * An implementation of an Indexer for attribute equality.
@@ -114,7 +110,7 @@
   * @throws DatabaseException If an error occurs in the JE database.
   */
  public void indexEntry(Transaction txn, Entry entry,
                         Set<ASN1OctetString> keys) throws DatabaseException
                         Set<byte[]> keys) throws DatabaseException
  {
    List<Attribute> attrList =
         entry.getAttribute(attributeType);
@@ -139,32 +135,38 @@
   * @throws DatabaseException If an error occurs in the JE database.
   */
  public void replaceEntry(Transaction txn, Entry oldEntry, Entry newEntry,
                           Set<ASN1OctetString> addKeys,
                           Set<ASN1OctetString> delKeys)
                           Set<byte[]> addKeys,
                           Set<byte[]> delKeys)
       throws DatabaseException
  {
    List<Attribute> attrList;
    List<Attribute> newAttributes = newEntry.getAttribute(attributeType, true);
    List<Attribute> oldAttributes = oldEntry.getAttribute(attributeType, true);

    attrList = oldEntry.getAttribute(attributeType);
    Set<ASN1OctetString> oldSet = new HashSet<ASN1OctetString>();
    indexAttribute(attrList, oldSet);

    attrList = newEntry.getAttribute(attributeType);
    Set<ASN1OctetString> newSet = new HashSet<ASN1OctetString>();
    indexAttribute(attrList, newSet);

    HashSet<ASN1OctetString> removeSet = new HashSet<ASN1OctetString>(oldSet);
    removeSet.removeAll(newSet);
    for (ASN1OctetString k : removeSet)
    if(newAttributes == null)
    {
      delKeys.add(k);
      indexAttribute(oldAttributes, delKeys);
    }

    HashSet<ASN1OctetString> addSet = new HashSet<ASN1OctetString>(newSet);
    addSet.removeAll(oldSet);
    for (ASN1OctetString k : addSet)
    else
    {
      addKeys.add(k);
      if(oldAttributes == null)
      {
        indexAttribute(newAttributes, addKeys);
      }
      else
      {
        TreeSet<byte[]> newKeys =
            new TreeSet<byte[]>(comparator);
        TreeSet<byte[]> oldKeys =
            new TreeSet<byte[]>(comparator);
        indexAttribute(newAttributes, newKeys);
        indexAttribute(oldAttributes, oldKeys);

        addKeys.addAll(newKeys);
        addKeys.removeAll(oldKeys);

        delKeys.addAll(oldKeys);
        delKeys.removeAll(newKeys);
      }
    }
  }

@@ -186,14 +188,12 @@
   */
  public void modifyEntry(Transaction txn, Entry oldEntry, Entry newEntry,
                          List<Modification> mods,
                          Set<ASN1OctetString> addKeys,
                          Set<ASN1OctetString> delKeys)
                          Set<byte[]> addKeys,
                          Set<byte[]> delKeys)
       throws DatabaseException
  {
    List<Attribute> newAttributes = newEntry.getAttribute(attributeType, true);
    List<Attribute> oldAttributes = oldEntry.getAttribute(attributeType, true);
    HashSet<AttributeValue> newValues;
    HashSet<AttributeValue> oldValues;

    if(newAttributes == null)
    {
@@ -207,26 +207,18 @@
      }
      else
      {
        newValues = new HashSet<AttributeValue>();
        oldValues = new HashSet<AttributeValue>();
        for(Attribute a : newAttributes)
        {
          newValues.addAll(a.getValues());
        }
        for(Attribute a : oldAttributes)
        {
          oldValues.addAll(a.getValues());
        }
        TreeSet<byte[]> newKeys =
            new TreeSet<byte[]>(comparator);
        TreeSet<byte[]> oldKeys =
            new TreeSet<byte[]>(comparator);
        indexAttribute(newAttributes, newKeys);
        indexAttribute(oldAttributes, oldKeys);

        HashSet<AttributeValue> valuesToAdd =
            new HashSet<AttributeValue>(newValues);
        HashSet<AttributeValue> valuesToDel =
            new HashSet<AttributeValue>(oldValues);
        valuesToAdd.removeAll(oldValues);
        valuesToDel.removeAll(newValues);
        addKeys.addAll(newKeys);
        addKeys.removeAll(oldKeys);

        indexValues(valuesToDel, delKeys);
        indexValues(valuesToAdd, addKeys);
        delKeys.addAll(oldKeys);
        delKeys.removeAll(newKeys);
      }
    }
  }
@@ -237,7 +229,7 @@
   * @param keys The set into which the keys will be inserted.
   */
  private void indexValues(Set<AttributeValue> values,
                           Set<ASN1OctetString> keys)
                           Set<byte[]> keys)
  {
    if (values == null) return;

@@ -247,7 +239,7 @@
      {
        byte[] keyBytes = value.getNormalizedValue().value();

        keys.add(new ASN1OctetString(keyBytes));
        keys.add(keyBytes);
      }
      catch (DirectoryException e)
      {
@@ -265,7 +257,7 @@
   * @param keys The set into which the keys will be inserted.
   */
  private void indexAttribute(List<Attribute> attrList,
                              Set<ASN1OctetString> keys)
                              Set<byte[]> keys)
  {
    if (attrList == null) return;


 opends/src/server/org/opends/server/backends/jeb/ID2CIndexer.java

@@ -26,7 +26,6 @@
 */
package org.opends.server.backends.jeb;

import org.opends.server.protocols.asn1.ASN1OctetString;
import org.opends.server.types.Entry;
import org.opends.server.types.Modification;

@@ -90,7 +89,7 @@
   * @throws DatabaseException If an error occurs in the JE database.
   */
  public void indexEntry(Transaction txn, Entry entry,
                       Set<ASN1OctetString> addKeys)
                       Set<byte[]> addKeys)
       throws DatabaseException
  {
    // The superior entry IDs are in the entry attachment.
@@ -104,7 +103,7 @@
    if (iter.hasNext())
    {
      DatabaseEntry nodeIDData = ((EntryID)iter.next()).getDatabaseEntry();
      addKeys.add(new ASN1OctetString(nodeIDData.getData()));
      addKeys.add(nodeIDData.getData());
    }
  }

@@ -120,8 +119,8 @@
   * @param delKeys The set into which the keys to be deleted will be inserted.
   */
  public void replaceEntry(Transaction txn, Entry oldEntry, Entry newEntry,
                           Set<ASN1OctetString> addKeys,
                           Set<ASN1OctetString> delKeys)
                           Set<byte[]> addKeys,
                           Set<byte[]> delKeys)
  {
    // Nothing to do.
  }
@@ -143,8 +142,8 @@
   */
  public void modifyEntry(Transaction txn, Entry oldEntry, Entry newEntry,
                          List<Modification> mods,
                          Set<ASN1OctetString> addKeys,
                          Set<ASN1OctetString> delKeys)
                          Set<byte[]> addKeys,
                          Set<byte[]> delKeys)
       throws DatabaseException
  {
    // Nothing to do.

 opends/src/server/org/opends/server/backends/jeb/ID2SIndexer.java

@@ -26,7 +26,6 @@
 */
package org.opends.server.backends.jeb;

import org.opends.server.protocols.asn1.ASN1OctetString;
import org.opends.server.types.Entry;
import org.opends.server.types.Modification;

@@ -89,7 +88,7 @@
   * @throws DatabaseException If an error occurs in the JE database.
   */
  public void indexEntry(Transaction txn, Entry entry,
                       Set<ASN1OctetString> addKeys)
                       Set<byte[]> addKeys)
       throws DatabaseException
  {
    // The superior entry IDs are in the entry attachment.
@@ -103,7 +102,7 @@
    while (iter.hasNext())
    {
      DatabaseEntry nodeIDData = ((EntryID)iter.next()).getDatabaseEntry();
      addKeys.add(new ASN1OctetString(nodeIDData.getData()));
      addKeys.add(nodeIDData.getData());
    }
  }

@@ -121,8 +120,8 @@
   */
  public void replaceEntry(Transaction txn,
                           Entry oldEntry, Entry newEntry,
                           Set<ASN1OctetString> addKeys,
                           Set<ASN1OctetString> delKeys)
                           Set<byte[]> addKeys,
                           Set<byte[]> delKeys)
       throws DatabaseException
  {
    // Nothing to do.
@@ -145,8 +144,8 @@
   */
  public void modifyEntry(Transaction txn, Entry oldEntry, Entry newEntry,
                          List<Modification> mods,
                          Set<ASN1OctetString> addKeys,
                          Set<ASN1OctetString> delKeys)
                          Set<byte[]> addKeys,
                          Set<byte[]> delKeys)
       throws DatabaseException
  {
    // Nothing to do.

 opends/src/server/org/opends/server/backends/jeb/Index.java

@@ -32,7 +32,6 @@

import com.sleepycat.je.*;

import org.opends.server.protocols.asn1.ASN1OctetString;
import org.opends.server.types.*;
import org.opends.server.util.StaticUtils;
import static org.opends.messages.JebMessages.*;
@@ -79,6 +78,17 @@
   */
  private int entryLimitExceededCount;

  /**
   * The max number of tries to rewrite phantom records.
   */
  final int phantomWriteRetires = 3;

  /**
   * Whether to maintain a count of IDs for a key once the entry limit
   * has exceeded.
   */
  boolean maintainCount;

  private State state;

  /**
@@ -111,13 +121,15 @@
   * @param indexEntryLimit The configured limit on the number of entry IDs
   * that may be indexed by one key.
   * @param cursorEntryLimit The configured limit on the number of entry IDs
   * @param maintainCount Whether to maintain a count of IDs for a key once
   * the entry limit has exceeded.
   * @param env The JE Environemnt
   * @param entryContainer The database entryContainer holding this index.
   * @throws DatabaseException If an error occurs in the JE database.
   */
  public Index(String name, Indexer indexer, State state,
        int indexEntryLimit, int cursorEntryLimit, Environment env,
        EntryContainer entryContainer)
        int indexEntryLimit, int cursorEntryLimit, boolean maintainCount,
        Environment env, EntryContainer entryContainer)
      throws DatabaseException
  {
    super(name, env, entryContainer);
@@ -125,6 +137,7 @@
    this.comparator = indexer.getComparator();
    this.indexEntryLimit = indexEntryLimit;
    this.cursorEntryLimit = cursorEntryLimit;
    this.maintainCount = maintainCount;

    DatabaseConfig dbNodupsConfig = new DatabaseConfig();

@@ -183,25 +196,92 @@
       throws DatabaseException
  {
    OperationStatus status;
    LockMode lockMode = LockMode.RMW;
    DatabaseEntry entryIDData = entryID.getDatabaseEntry();
    DatabaseEntry data = new DatabaseEntry();
    boolean success = true;
    boolean done = false;
    boolean success = false;

    while(!done)
    if(maintainCount)
    {
      status = read(txn, key, data, lockMode);

      for(int i = 0; i < phantomWriteRetires; i++)
      {
        if(insertIDWithRMW(txn, key, data, entryIDData, entryID) ==
            OperationStatus.SUCCESS)
        {
          return true;
        }
      }
    }
    else
    {
      status = read(txn, key, data, LockMode.READ_COMMITTED);
      if (status == OperationStatus.SUCCESS)
      {
        EntryIDSet entryIDList =
            new EntryIDSet(key.getData(), data.getData());

        if (entryIDList.isDefined())
        {
          if (indexEntryLimit > 0 && entryIDList.size() >= indexEntryLimit)
          for(int i = 0; i < phantomWriteRetires; i++)
          {
            if(insertIDWithRMW(txn, key, data, entryIDData, entryID) ==
                OperationStatus.SUCCESS)
            {
              return true;
            }
          }
        }
      }
      else
      {
        if(rebuildRunning || trusted)
        {
          status = insert(txn, key, entryIDData);
          if(status == OperationStatus.KEYEXIST)
          {
            for(int i = 1; i < phantomWriteRetires; i++)
            {
              if(insertIDWithRMW(txn, key, data, entryIDData, entryID) ==
                  OperationStatus.SUCCESS)
              {
                return true;
              }
            }
          }
        }
        else
        {
          return true;
        }
      }
    }

    return success;
  }

  private OperationStatus insertIDWithRMW(Transaction txn, DatabaseEntry key,
                                          DatabaseEntry data,
                                          DatabaseEntry entryIDData,
                                          EntryID entryID)
      throws DatabaseException
  {
    OperationStatus status;

    status = read(txn, key, data, LockMode.RMW);
    if(status == OperationStatus.SUCCESS)
    {
      EntryIDSet entryIDList =
          new EntryIDSet(key.getData(), data.getData());
      if (entryIDList.isDefined() && indexEntryLimit > 0 &&
          entryIDList.size() >= indexEntryLimit)
      {
        if(maintainCount)
          {
            entryIDList = new EntryIDSet(entryIDList.size());
        }
        else
        {
          entryIDList = new EntryIDSet();
        }
            entryLimitExceededCount++;

            if(debugEnabled())
@@ -215,35 +295,26 @@

            }
          }
        }

        success = entryIDList.add(entryID);
      entryIDList.add(entryID);

        byte[] after = entryIDList.toDatabase();
        data.setData(after);
        put(txn, key, data);
        done = true;
      return put(txn, key, data);
      }
      else
      {
        if(rebuildRunning || trusted)
        {
          status = insert(txn, key, entryIDData);
          if(status == OperationStatus.SUCCESS)
          {
            done = true;
          }
        return insert(txn, key, entryIDData);
        }
        else
        {
          done = true;
        return OperationStatus.SUCCESS;
        }
      }
    }

    return success;
  }

  /**
   * Remove an entry ID from the set of IDs indexed by a given key.
   *
@@ -256,10 +327,51 @@
      throws DatabaseException
  {
    OperationStatus status;
    LockMode lockMode = LockMode.RMW;
    DatabaseEntry data = new DatabaseEntry();

    status = read(txn, key, data, lockMode);
    if(maintainCount)
    {
      removeIDWithRMW(txn, key, data, entryID);
    }
    else
    {
      status = read(txn, key, data, LockMode.READ_COMMITTED);
      if(status == OperationStatus.SUCCESS)
      {
        EntryIDSet entryIDList = new EntryIDSet(key.getData(), data.getData());
        if(entryIDList.isDefined())
        {
          removeIDWithRMW(txn, key, data, entryID);
        }
      }
      else
      {
        // Ignore failures if rebuild is running since a empty entryIDset
        // will probably not be rebuilt.
        if(trusted && !rebuildRunning)
        {
          setTrusted(txn, false);

          if(debugEnabled())
          {
            StringBuilder builder = new StringBuilder();
            StaticUtils.byteArrayToHexPlusAscii(builder, key.getData(), 4);
            TRACER.debugError("The expected key does not exist in the " +
                "index %s.\nKey:%s", name, builder.toString());
          }

          logError(ERR_JEB_INDEX_CORRUPT_REQUIRES_REBUILD.get(name));
        }
      }
    }
  }

  private void removeIDWithRMW(Transaction txn, DatabaseEntry key,
                               DatabaseEntry data, EntryID entryID)
      throws DatabaseException
  {
    OperationStatus status;
    status = read(txn, key, data, LockMode.RMW);

    if (status == OperationStatus.SUCCESS)
    {
@@ -272,8 +384,6 @@
        {
          setTrusted(txn, false);



          if(debugEnabled())
          {
            StringBuilder builder = new StringBuilder();
@@ -612,15 +722,15 @@
  public boolean addEntry(Transaction txn, EntryID entryID, Entry entry)
       throws DatabaseException, DirectoryException
  {
    HashSet<ASN1OctetString> addKeys = new HashSet<ASN1OctetString>();
    TreeSet<byte[]> addKeys = new TreeSet<byte[]>(indexer.getComparator());
    boolean success = true;

    indexer.indexEntry(txn, entry, addKeys);

    DatabaseEntry key = new DatabaseEntry();
    for (ASN1OctetString keyBytes : addKeys)
    for (byte[] keyBytes : addKeys)
    {
      key.setData(keyBytes.value());
      key.setData(keyBytes);
      if(!insertID(txn, key, entryID))
      {
        success = false;
@@ -643,14 +753,14 @@
  public void removeEntry(Transaction txn, EntryID entryID, Entry entry)
       throws DatabaseException, DirectoryException
  {
    HashSet<ASN1OctetString> delKeys = new HashSet<ASN1OctetString>();
    TreeSet<byte[]> delKeys = new TreeSet<byte[]>(indexer.getComparator());

    indexer.indexEntry(txn, entry, delKeys);

    DatabaseEntry key = new DatabaseEntry();
    for (ASN1OctetString keyBytes : delKeys)
    for (byte[] keyBytes : delKeys)
    {
      key.setData(keyBytes.value());
      key.setData(keyBytes);
      removeID(txn, key, entryID);
    }
  }
@@ -674,21 +784,21 @@
                          List<Modification> mods)
       throws DatabaseException
  {
    HashSet<ASN1OctetString> addKeys = new HashSet<ASN1OctetString>();
    HashSet<ASN1OctetString> delKeys = new HashSet<ASN1OctetString>();
    TreeSet<byte[]> addKeys = new TreeSet<byte[]>(indexer.getComparator());
    TreeSet<byte[]> delKeys = new TreeSet<byte[]>(indexer.getComparator());

    indexer.modifyEntry(txn, oldEntry, newEntry, mods, addKeys, delKeys);

    DatabaseEntry key = new DatabaseEntry();
    for (ASN1OctetString keyBytes : delKeys)
    for (byte[] keyBytes : delKeys)
    {
      key.setData(keyBytes.value());
      key.setData(keyBytes);
      removeID(txn, key, entryID);
    }

    for (ASN1OctetString keyBytes : addKeys)
    for (byte[] keyBytes : addKeys)
    {
      key.setData(keyBytes.value());
      key.setData(keyBytes);
      insertID(txn, key, entryID);
    }
  }
@@ -754,4 +864,15 @@
  {
    this.rebuildRunning = rebuildRunning;
  }

  /**
   * Whether this index maintains a count of IDs for keys once the
   * entry limit has exceeded.
   * @return <code>true</code> if this index maintains court of IDs
   * or <code>false</code> otherwise
   */
  public boolean getMaintainCount()
  {
    return maintainCount;
  }
}

 opends/src/server/org/opends/server/backends/jeb/IndexMergeThread.java

@@ -305,6 +305,8 @@
            if (merged.size() > entryLimit)
            {
              index.incEntryLimitExceededCount();
              if(index.getMaintainCount())
              {
              byte[] undefinedSizeBytes =
                  JebFormat.entryIDUndefinedSizeToDatabase(merged.size());
              dbData.setData(undefinedSizeBytes);
@@ -312,6 +314,11 @@
            }
            else
            {
                index.writeKey(txn, dbKey, new EntryIDSet());
              }
            }
            else
            {
              mergedBytes = merged.encode(mergedBytes);

              dbData.setData(mergedBytes);

 opends/src/server/org/opends/server/backends/jeb/Indexer.java

@@ -26,7 +26,6 @@
 */
package org.opends.server.backends.jeb;

import org.opends.server.protocols.asn1.ASN1OctetString;
import org.opends.server.types.Entry;
import org.opends.server.types.Modification;
import com.sleepycat.je.DatabaseException;
@@ -60,7 +59,7 @@
   * @throws DatabaseException If an error occurs in the JE database.
   */
  public abstract void indexEntry(Transaction txn, Entry entry,
                                Set<ASN1OctetString> keys)
                                Set<byte[]> keys)
       throws DatabaseException;

  /**
@@ -77,8 +76,8 @@
   */
  public abstract void replaceEntry(Transaction txn,
                                    Entry oldEntry, Entry newEntry,
                                    Set<ASN1OctetString> addKeys,
                                    Set<ASN1OctetString> delKeys)
                                    Set<byte[]> addKeys,
                                    Set<byte[]> delKeys)
       throws DatabaseException;

  /**
@@ -97,7 +96,7 @@
  public abstract void modifyEntry(Transaction txn,
                                   Entry oldEntry, Entry newEntry,
                                   List<Modification> mods,
                                   Set<ASN1OctetString> addKeys,
                                   Set<ASN1OctetString> delKeys)
                                   Set<byte[]> addKeys,
                                   Set<byte[]> delKeys)
       throws DatabaseException;
}

 opends/src/server/org/opends/server/backends/jeb/OrderingIndexer.java

@@ -31,7 +31,6 @@
import org.opends.server.types.DebugLogLevel;

import org.opends.server.api.OrderingMatchingRule;
import org.opends.server.protocols.asn1.ASN1OctetString;
import org.opends.server.types.Attribute;
import org.opends.server.types.AttributeType;
import org.opends.server.types.AttributeValue;
@@ -110,7 +109,7 @@
   * @param keys The set into which the generated keys will be inserted.
   */
  public void indexEntry(Transaction txn, Entry entry,
                       Set<ASN1OctetString> keys)
                       Set<byte[]> keys)
  {
    List<Attribute> attrList =
         entry.getAttribute(attributeType);
@@ -133,31 +132,37 @@
   */
  public void replaceEntry(Transaction txn,
                           Entry oldEntry, Entry newEntry,
                           Set<ASN1OctetString> addKeys,
                           Set<ASN1OctetString> delKeys)
                           Set<byte[]> addKeys,
                           Set<byte[]> delKeys)
  {
    List<Attribute> attrList;
    List<Attribute> newAttributes = newEntry.getAttribute(attributeType, true);
    List<Attribute> oldAttributes = oldEntry.getAttribute(attributeType, true);

    attrList = oldEntry.getAttribute(attributeType);
    Set<ASN1OctetString> oldSet = new HashSet<ASN1OctetString>();
    indexAttribute(attrList, oldSet);

    attrList = newEntry.getAttribute(attributeType);
    Set<ASN1OctetString> newSet = new HashSet<ASN1OctetString>();
    indexAttribute(attrList, newSet);

    HashSet<ASN1OctetString> removeSet = new HashSet<ASN1OctetString>(oldSet);
    removeSet.removeAll(newSet);
    for (ASN1OctetString k : removeSet)
    if(newAttributes == null)
    {
      delKeys.add(k);
      indexAttribute(oldAttributes, delKeys);
    }

    HashSet<ASN1OctetString> addSet = new HashSet<ASN1OctetString>(newSet);
    addSet.removeAll(oldSet);
    for (ASN1OctetString k : addSet)
    else
    {
      addKeys.add(k);
      if(oldAttributes == null)
      {
        indexAttribute(newAttributes, addKeys);
      }
      else
      {
        TreeSet<byte[]> newKeys =
            new TreeSet<byte[]>(orderingRule);
        TreeSet<byte[]> oldKeys =
            new TreeSet<byte[]>(orderingRule);
        indexAttribute(newAttributes, newKeys);
        indexAttribute(oldAttributes, oldKeys);

        addKeys.addAll(newKeys);
        addKeys.removeAll(oldKeys);

        delKeys.addAll(oldKeys);
        delKeys.removeAll(newKeys);
      }
    }
  }

@@ -176,14 +181,12 @@
   */
  public void modifyEntry(Transaction txn, Entry oldEntry, Entry newEntry,
                          List<Modification> mods,
                          Set<ASN1OctetString> addKeys,
                          Set<ASN1OctetString> delKeys)
                          Set<byte[]> addKeys,
                          Set<byte[]> delKeys)
       throws DatabaseException
  {
    List<Attribute> newAttributes = newEntry.getAttribute(attributeType, true);
    List<Attribute> oldAttributes = oldEntry.getAttribute(attributeType, true);
    HashSet<AttributeValue> newValues;
    HashSet<AttributeValue> oldValues;

    if(newAttributes == null)
    {
@@ -197,26 +200,18 @@
      }
      else
      {
        newValues = new HashSet<AttributeValue>();
        oldValues = new HashSet<AttributeValue>();
        for(Attribute a : newAttributes)
        {
          newValues.addAll(a.getValues());
        }
        for(Attribute a : oldAttributes)
        {
          oldValues.addAll(a.getValues());
        }
        TreeSet<byte[]> newKeys =
            new TreeSet<byte[]>(orderingRule);
        TreeSet<byte[]> oldKeys =
            new TreeSet<byte[]>(orderingRule);
        indexAttribute(newAttributes, newKeys);
        indexAttribute(oldAttributes, oldKeys);

        HashSet<AttributeValue> valuesToAdd =
            new HashSet<AttributeValue>(newValues);
        HashSet<AttributeValue> valuesToDel =
            new HashSet<AttributeValue>(oldValues);
        valuesToAdd.removeAll(oldValues);
        valuesToDel.removeAll(newValues);
        addKeys.addAll(newKeys);
        addKeys.removeAll(oldKeys);

        indexValues(valuesToDel, delKeys);
        indexValues(valuesToAdd, addKeys);
        delKeys.addAll(oldKeys);
        delKeys.removeAll(newKeys);
      }
    }
  }
@@ -228,7 +223,7 @@
   * @param keys The set into which the keys will be inserted.
   */
  private void indexValues(Set<AttributeValue> values,
                           Set<ASN1OctetString> keys)
                           Set<byte[]> keys)
  {
    if (values == null) return;

@@ -239,7 +234,7 @@
        byte[] keyBytes =
             orderingRule.normalizeValue(value.getValue()).value();

        keys.add(new ASN1OctetString(keyBytes));
        keys.add(keyBytes);
      }
      catch (DirectoryException e)
      {
@@ -257,7 +252,7 @@
   * @param keys The set into which the keys will be inserted.
   */
  private void indexAttribute(List<Attribute> attrList,
                              Set<ASN1OctetString> keys)
                              Set<byte[]> keys)
  {
    if (attrList == null) return;


 opends/src/server/org/opends/server/backends/jeb/PresenceIndexer.java

@@ -26,7 +26,6 @@
 */
package org.opends.server.backends.jeb;

import org.opends.server.protocols.asn1.ASN1OctetString;
import org.opends.server.types.Attribute;
import org.opends.server.types.Entry;
import org.opends.server.types.Modification;
@@ -100,7 +99,7 @@
   * @throws DatabaseException If an error occurs in the JE database.
   */
  public void indexEntry(Transaction txn, Entry entry,
                       Set<ASN1OctetString> keys) throws DatabaseException
                       Set<byte[]> keys) throws DatabaseException
  {
    List<Attribute> attrList =
         entry.getAttribute(attributeType);
@@ -108,7 +107,7 @@
    {
      if (!attrList.isEmpty())
      {
        keys.add(new ASN1OctetString(AttributeIndex.presenceKey.getData()));
        keys.add(AttributeIndex.presenceKey.getData());
      }
    }
  }
@@ -128,25 +127,25 @@
   * @throws DatabaseException If an error occurs in the JE database.
   */
  public void replaceEntry(Transaction txn, Entry oldEntry, Entry newEntry,
                           Set<ASN1OctetString> addKeys,
                           Set<ASN1OctetString> delKeys)
                           Set<byte[]> addKeys,
                           Set<byte[]> delKeys)
       throws DatabaseException
  {
    List<Attribute> beforeList, afterList;

    beforeList = oldEntry.getAttribute(attributeType);
    afterList = newEntry.getAttribute(attributeType);

    if (beforeList == null || beforeList.isEmpty())
    List<Attribute> newAttributes = newEntry.getAttribute(attributeType, true);
    List<Attribute> oldAttributes = oldEntry.getAttribute(attributeType, true);
    if(oldAttributes == null)
    {
      if (afterList != null && !afterList.isEmpty())
      if(newAttributes != null)
      {
        addKeys.add(new ASN1OctetString(AttributeIndex.presenceKey.getData()));
        addKeys.add(AttributeIndex.presenceKey.getData());
      }
    }
    else if (afterList == null || afterList.isEmpty())
    else
    {
      delKeys.add(new ASN1OctetString(AttributeIndex.presenceKey.getData()));
      if(newAttributes == null)
      {
        delKeys.add(AttributeIndex.presenceKey.getData());
      }
    }
  }

@@ -167,8 +166,8 @@
   */
  public void modifyEntry(Transaction txn, Entry oldEntry, Entry newEntry,
                          List<Modification> mods,
                          Set<ASN1OctetString> addKeys,
                          Set<ASN1OctetString> delKeys)
                          Set<byte[]> addKeys,
                          Set<byte[]> delKeys)
       throws DatabaseException
  {
    List<Attribute> newAttributes = newEntry.getAttribute(attributeType, true);
@@ -177,16 +176,14 @@
    {
      if(newAttributes != null)
      {
        addKeys.add(
              new ASN1OctetString(AttributeIndex.presenceKey.getData()));
        addKeys.add(AttributeIndex.presenceKey.getData());
      }
    }
    else
    {
      if(newAttributes == null)
      {
        delKeys.add(
              new ASN1OctetString(AttributeIndex.presenceKey.getData()));
        delKeys.add(AttributeIndex.presenceKey.getData());
      }
    }
  }

 opends/src/server/org/opends/server/backends/jeb/SubstringIndexer.java

@@ -29,13 +29,9 @@
import static org.opends.server.loggers.debug.DebugLogger.*;
import org.opends.server.loggers.debug.DebugTracer;
import com.sleepycat.je.Transaction;
import org.opends.server.protocols.asn1.ASN1OctetString;
import org.opends.server.types.*;

import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.*;

/**
 * An implementation of an Indexer for attribute substrings.
@@ -108,7 +104,7 @@
   * @param keys The set into which the generated keys will be inserted.
   */
  public void indexEntry(Transaction txn, Entry entry,
                       Set<ASN1OctetString> keys)
                       Set<byte[]> keys)
  {
    List<Attribute> attrList =
         entry.getAttribute(attributeType);
@@ -131,31 +127,37 @@
   */
  public void replaceEntry(Transaction txn,
                           Entry oldEntry, Entry newEntry,
                           Set<ASN1OctetString> addKeys,
                           Set<ASN1OctetString> delKeys)
                           Set<byte[]> addKeys,
                           Set<byte[]> delKeys)
  {
    List<Attribute> attrList;
    List<Attribute> newAttributes = newEntry.getAttribute(attributeType, true);
    List<Attribute> oldAttributes = oldEntry.getAttribute(attributeType, true);

    attrList = oldEntry.getAttribute(attributeType);
    Set<ASN1OctetString> oldSet = new HashSet<ASN1OctetString>();
    indexAttribute(attrList, oldSet);

    attrList = newEntry.getAttribute(attributeType);
    Set<ASN1OctetString> newSet = new HashSet<ASN1OctetString>();
    indexAttribute(attrList, newSet);

    HashSet<ASN1OctetString> removeSet = new HashSet<ASN1OctetString>(oldSet);
    removeSet.removeAll(newSet);
    for (ASN1OctetString k : removeSet)
    if(newAttributes == null)
    {
      delKeys.add(k);
      indexAttribute(oldAttributes, delKeys);
    }

    HashSet<ASN1OctetString> addSet = new HashSet<ASN1OctetString>(newSet);
    addSet.removeAll(oldSet);
    for (ASN1OctetString k : addSet)
    else
    {
      addKeys.add(k);
      if(oldAttributes == null)
      {
        indexAttribute(newAttributes, addKeys);
      }
      else
      {
        TreeSet<byte[]> newKeys =
            new TreeSet<byte[]>(comparator);
        TreeSet<byte[]> oldKeys =
            new TreeSet<byte[]>(comparator);
        indexAttribute(newAttributes, newKeys);
        indexAttribute(oldAttributes, oldKeys);

        addKeys.addAll(newKeys);
        addKeys.removeAll(oldKeys);

        delKeys.addAll(oldKeys);
        delKeys.removeAll(newKeys);
      }
    }
  }

@@ -175,13 +177,11 @@
   */
  public void modifyEntry(Transaction txn, Entry oldEntry, Entry newEntry,
                          List<Modification> mods,
                          Set<ASN1OctetString> addKeys,
                          Set<ASN1OctetString> delKeys)
                          Set<byte[]> addKeys,
                          Set<byte[]> delKeys)
  {
    List<Attribute> newAttributes = newEntry.getAttribute(attributeType, true);
    List<Attribute> oldAttributes = oldEntry.getAttribute(attributeType, true);
    HashSet<AttributeValue> newValues;
    HashSet<AttributeValue> oldValues;

    if(newAttributes == null)
    {
@@ -195,10 +195,10 @@
      }
      else
      {
        HashSet<ASN1OctetString> newKeys =
            new HashSet<ASN1OctetString>();
        HashSet<ASN1OctetString> oldKeys =
            new HashSet<ASN1OctetString>();
        TreeSet<byte[]> newKeys =
            new TreeSet<byte[]>(comparator);
        TreeSet<byte[]> oldKeys =
            new TreeSet<byte[]>(comparator);
        indexAttribute(newAttributes, newKeys);
        indexAttribute(oldAttributes, oldKeys);

@@ -219,7 +219,7 @@
   * @param keys The set into which the generated keys will be inserted.
   */
  private void indexAttribute(List<Attribute> attrList,
                              Set<ASN1OctetString> keys)
                              Set<byte[]> keys)
  {
    if (attrList == null) return;

@@ -235,7 +235,7 @@
   * @param keys The set into which the keys will be inserted.
   */
  private void indexValues(Set<AttributeValue> values,
                           Set<ASN1OctetString> keys)
                           Set<byte[]> keys)
  {
    if (values == null) return;

@@ -265,7 +265,7 @@
   * @param value A byte array containing the normalized attribute value
   * @param set A set into which the keys will be inserted.
   */
  private void substringKeys(byte[] value, Set<ASN1OctetString> set)
  private void substringKeys(byte[] value, Set<byte[]> set)
  {
    byte[] keyBytes;

@@ -279,7 +279,7 @@
    {
      int len = Math.min(substrLength, remain);
      keyBytes = makeSubstringKey(value, i, len);
      set.add(new ASN1OctetString(keyBytes));
      set.add(keyBytes);
    }
  }


 opends/tests/unit-tests-testng/src/server/org/opends/server/backends/jeb/TestBackendImpl.java

@@ -804,7 +804,7 @@
    EntryID entryID;
    AttributeType attribute;
    AttributeIndex index;
    HashSet<ASN1OctetString> addKeys;
    HashSet<byte[]> addKeys;
    DatabaseEntry key;
    PresenceIndexer presenceIndexer;
    EqualityIndexer equalityIndexer;
@@ -843,47 +843,47 @@
      index = ec.getAttributeIndex(attribute);


      addKeys = new HashSet<ASN1OctetString>();
      addKeys = new HashSet<byte[]>();
      presenceIndexer = new PresenceIndexer(index.getAttributeType());
      presenceIndexer.indexEntry(null, entry, addKeys);

      key = new DatabaseEntry();
      for (ASN1OctetString keyBytes : addKeys) {
        key.setData(keyBytes.value());
      for (byte[] keyBytes : addKeys) {
        key.setData(keyBytes);
      }
      assertEquals(index.presenceIndex.containsID(null, key, entryID),
          ConditionResult.FALSE);

      addKeys = new HashSet<ASN1OctetString>();
      addKeys = new HashSet<byte[]>();
      equalityIndexer = new EqualityIndexer(index.getAttributeType());
      equalityIndexer.indexEntry(null, entry, addKeys);

      key = new DatabaseEntry();
      for (ASN1OctetString keyBytes : addKeys) {
        key.setData(keyBytes.value());
      for (byte[] keyBytes : addKeys) {
        key.setData(keyBytes);
      }
      assertEquals(index.equalityIndex.containsID(null, key, entryID),
          ConditionResult.FALSE);

      addKeys = new HashSet<ASN1OctetString>();
      addKeys = new HashSet<byte[]>();
      substringIndexer = new SubstringIndexer(index.getAttributeType(),
                   index.getConfiguration().getSubstringLength());
      substringIndexer.indexEntry(null, entry, addKeys);

      key = new DatabaseEntry();
      for (ASN1OctetString keyBytes : addKeys) {
        key.setData(keyBytes.value());
      for (byte[] keyBytes : addKeys) {
        key.setData(keyBytes);
      }
      assertEquals(index.substringIndex.containsID(null, key, entryID),
          ConditionResult.FALSE);

      addKeys = new HashSet<ASN1OctetString>();
      addKeys = new HashSet<byte[]>();
      orderingIndexer = new OrderingIndexer(index.getAttributeType());
      orderingIndexer.indexEntry(null, entry, addKeys);

      key = new DatabaseEntry();
      for (ASN1OctetString keyBytes : addKeys) {
        key.setData(keyBytes.value());
      for (byte[] keyBytes : addKeys) {
        key.setData(keyBytes);
      }
      assertEquals(index.orderingIndex.containsID(null, key, entryID),
          ConditionResult.FALSE);
@@ -903,7 +903,7 @@
    EntryID entryID;
    AttributeType attribute;
    AttributeIndex index;
    HashSet<ASN1OctetString> addKeys;
    HashSet<byte[]> addKeys;
    DatabaseEntry key;
    EqualityIndexer equalityIndexer;
    SubstringIndexer substringIndexer;
@@ -943,66 +943,66 @@
      attribute = entry.getAttribute("cn").get(0).getAttributeType();
      index = ec.getAttributeIndex(attribute);

      addKeys = new HashSet<ASN1OctetString>();
      addKeys = new HashSet<byte[]>();
      orderingIndexer = new OrderingIndexer(index.getAttributeType());
      orderingIndexer.indexEntry(null, entry, addKeys);

      key = new DatabaseEntry();
      for (ASN1OctetString keyBytes : addKeys) {
        key.setData(keyBytes.value());
      for (byte[] keyBytes : addKeys) {
        key.setData(keyBytes);
      }
      assertEquals(index.orderingIndex.containsID(null, key, entryID),
          ConditionResult.TRUE);

      addKeys = new HashSet<ASN1OctetString>();
      addKeys = new HashSet<byte[]>();
      orderingIndexer.indexEntry(null, oldEntry, addKeys);

      key = new DatabaseEntry();
      for (ASN1OctetString keyBytes : addKeys) {
        key.setData(keyBytes.value());
      for (byte[] keyBytes : addKeys) {
        key.setData(keyBytes);
      }
      assertEquals(index.orderingIndex.containsID(null, key, entryID),
                   ConditionResult.FALSE);

      addKeys = new HashSet<ASN1OctetString>();
      addKeys = new HashSet<byte[]>();
      substringIndexer = new SubstringIndexer(index.getAttributeType(),
                                              index.getConfiguration().getSubstringLength());
      substringIndexer.indexEntry(null, entry, addKeys);

      key = new DatabaseEntry();
      for (ASN1OctetString keyBytes : addKeys) {
        key.setData(keyBytes.value());
      for (byte[] keyBytes : addKeys) {
        key.setData(keyBytes);
      }
      assertEquals(index.substringIndex.containsID(null, key, entryID),
                   ConditionResult.TRUE);

      addKeys = new HashSet<ASN1OctetString>();
      addKeys = new HashSet<byte[]>();
      substringIndexer.indexEntry(null, oldEntry, addKeys);

      key = new DatabaseEntry();
      for (ASN1OctetString keyBytes : addKeys) {
        key.setData(keyBytes.value());
      for (byte[] keyBytes : addKeys) {
        key.setData(keyBytes);
      }
      assertEquals(index.substringIndex.containsID(null, key, entryID),
                   ConditionResult.FALSE);

      addKeys = new HashSet<ASN1OctetString>();
      addKeys = new HashSet<byte[]>();
      equalityIndexer = new EqualityIndexer(index.getAttributeType());
      equalityIndexer.indexEntry(null, entry, addKeys);

      key = new DatabaseEntry();
      for (ASN1OctetString keyBytes : addKeys) {
        key.setData(keyBytes.value());
      for (byte[] keyBytes : addKeys) {
        key.setData(keyBytes);
      }
      assertEquals(index.equalityIndex.containsID(null, key, entryID),
          ConditionResult.TRUE);

      addKeys = new HashSet<ASN1OctetString>();
      addKeys = new HashSet<byte[]>();
      equalityIndexer.indexEntry(null, oldEntry, addKeys);

      key = new DatabaseEntry();
      for (ASN1OctetString keyBytes : addKeys) {
        key.setData(keyBytes.value());
      for (byte[] keyBytes : addKeys) {
        key.setData(keyBytes);
      }
      assertEquals(index.equalityIndex.containsID(null, key, entryID),
          ConditionResult.FALSE);
@@ -1024,7 +1024,7 @@
    AttributeType attribute;
    AttributeIndex titleIndex;
    AttributeIndex nameIndex;
    HashSet<ASN1OctetString> addKeys;
    HashSet<byte[]> addKeys;
    DatabaseEntry key;
    PresenceIndexer presenceIndexer;
    EqualityIndexer equalityIndexer;
@@ -1097,21 +1097,21 @@
      nameIndex = ec.getAttributeIndex(attribute);

      //This current entry in the DB shouldn't be in the presence titleIndex.
      addKeys = new HashSet<ASN1OctetString>();
      addKeys.add(new ASN1OctetString(AttributeIndex.presenceKey.getData()));
      addKeys = new HashSet<byte[]>();
      addKeys.add(AttributeIndex.presenceKey.getData());
      key = new DatabaseEntry();
      for (ASN1OctetString keyBytes : addKeys) {
        key.setData(keyBytes.value());
      for (byte[] keyBytes : addKeys) {
        key.setData(keyBytes);
      }
      assertEquals(titleIndex.presenceIndex.containsID(null, key, entryID),
          ConditionResult.FALSE);

      //This current entry should be in the presence nameIndex.
      addKeys = new HashSet<ASN1OctetString>();
      addKeys.add(new ASN1OctetString(AttributeIndex.presenceKey.getData()));
      addKeys = new HashSet<byte[]>();
      addKeys.add(AttributeIndex.presenceKey.getData());
      key = new DatabaseEntry();
      for (ASN1OctetString keyBytes : addKeys) {
        key.setData(keyBytes.value());
      for (byte[] keyBytes : addKeys) {
        key.setData(keyBytes);
      }
      assertEquals(nameIndex.presenceIndex.containsID(null, key, entryID),
          ConditionResult.TRUE);
@@ -1176,92 +1176,92 @@
      assertFalse(entry.getAttribute("employeenumber").contains(new
          Attribute("employeenumber", "1")));

      addKeys = new HashSet<ASN1OctetString>();
      addKeys = new HashSet<byte[]>();
      presenceIndexer = new PresenceIndexer(titleIndex.getAttributeType());
      presenceIndexer.indexEntry(null, entry, addKeys);

      key = new DatabaseEntry();
      for (ASN1OctetString keyBytes : addKeys) {
        key.setData(keyBytes.value());
      for (byte[] keyBytes : addKeys) {
        key.setData(keyBytes);
        assertEquals(titleIndex.presenceIndex.containsID(null, key, entryID),
          ConditionResult.TRUE);
      }

      addKeys = new HashSet<ASN1OctetString>();
      addKeys = new HashSet<byte[]>();
      presenceIndexer = new PresenceIndexer(nameIndex.getAttributeType());
      presenceIndexer.indexEntry(null, entry, addKeys);

      key = new DatabaseEntry();
      for (ASN1OctetString keyBytes : addKeys) {
        key.setData(keyBytes.value());
      for (byte[] keyBytes : addKeys) {
        key.setData(keyBytes);
        assertEquals(nameIndex.presenceIndex.containsID(null, key, entryID),
          ConditionResult.TRUE);
      }

      addKeys = new HashSet<ASN1OctetString>();
      addKeys = new HashSet<byte[]>();
      orderingIndexer = new OrderingIndexer(titleIndex.getAttributeType());
      orderingIndexer.indexEntry(null, entry, addKeys);

      key = new DatabaseEntry();
      for (ASN1OctetString keyBytes : addKeys) {
        key.setData(keyBytes.value());
      for (byte[] keyBytes : addKeys) {
        key.setData(keyBytes);
        assertEquals(titleIndex.orderingIndex.containsID(null, key, entryID),
          ConditionResult.TRUE);
      }

      addKeys = new HashSet<ASN1OctetString>();
      addKeys = new HashSet<byte[]>();
      orderingIndexer = new OrderingIndexer(nameIndex.getAttributeType());
      orderingIndexer.indexEntry(null, entry, addKeys);

      key = new DatabaseEntry();
      for (ASN1OctetString keyBytes : addKeys) {
        key.setData(keyBytes.value());
      for (byte[] keyBytes : addKeys) {
        key.setData(keyBytes);
        assertEquals(nameIndex.orderingIndex.containsID(null, key, entryID),
          ConditionResult.TRUE);
      }

      addKeys = new HashSet<ASN1OctetString>();
      addKeys = new HashSet<byte[]>();
      equalityIndexer = new EqualityIndexer(titleIndex.getAttributeType());
      equalityIndexer.indexEntry(null, entry, addKeys);

      key = new DatabaseEntry();
      for (ASN1OctetString keyBytes : addKeys) {
        key.setData(keyBytes.value());
      for (byte[] keyBytes : addKeys) {
        key.setData(keyBytes);
        assertEquals(titleIndex.equalityIndex.containsID(null, key, entryID),
          ConditionResult.TRUE);
      }

      addKeys = new HashSet<ASN1OctetString>();
      addKeys = new HashSet<byte[]>();
      equalityIndexer = new EqualityIndexer(nameIndex.getAttributeType());
      equalityIndexer.indexEntry(null, entry, addKeys);

      key = new DatabaseEntry();
      for (ASN1OctetString keyBytes : addKeys) {
        key.setData(keyBytes.value());
      for (byte[] keyBytes : addKeys) {
        key.setData(keyBytes);
        assertEquals(nameIndex.equalityIndex.containsID(null, key, entryID),
          ConditionResult.TRUE);
      }

      addKeys = new HashSet<ASN1OctetString>();
      addKeys = new HashSet<byte[]>();
      substringIndexer = new SubstringIndexer(titleIndex.getAttributeType(),
                   titleIndex.getConfiguration().getSubstringLength());
      substringIndexer.indexEntry(null, entry, addKeys);

      key = new DatabaseEntry();
      for (ASN1OctetString keyBytes : addKeys) {
        key.setData(keyBytes.value());
      for (byte[] keyBytes : addKeys) {
        key.setData(keyBytes);
        assertEquals(titleIndex.substringIndex.containsID(null, key, entryID),
          ConditionResult.TRUE);
      }

      addKeys = new HashSet<ASN1OctetString>();
      addKeys = new HashSet<byte[]>();
      substringIndexer = new SubstringIndexer(nameIndex.getAttributeType(),
                   nameIndex.getConfiguration().getSubstringLength());
      substringIndexer.indexEntry(null, entry, addKeys);

      key = new DatabaseEntry();
      for (ASN1OctetString keyBytes : addKeys) {
        key.setData(keyBytes.value());
      for (byte[] keyBytes : addKeys) {
        key.setData(keyBytes);
        assertEquals(nameIndex.substringIndex.containsID(null, key, entryID),
          ConditionResult.TRUE);
      }