OPENDJ-3230-3223: upgrade to 3.5.0 should rebuild indexes using DN syntax

Yannick Lecaillez

05.38.2016 eeed545264e26d4f8d417047adc77432a30e968a

OPENDJ-3230-3223: upgrade to 3.5.0 should rebuild indexes using DN
syntax

Allow more flexibility on how to specify the indexes to rebuild.
Rebuild-index is now accepting the format
(attributeType)((.indexType)|(.matchingRuleName)).
This allows to rebuild all indexes for a given attributeType (i.e: "cn")
but also all indexes for a given index type (i.e: ".substring) or
matching rule (i.e: ".caseIgnoreSubstringMatch) and also a combination
of both (i.e: "cn.substring" or "cn.caseIgnoreSubstringsMatch")

Update upgrade tasks so that the rebuild-index performed during upgrades
only reference matchingRules rather than hard-coded attributes.

 opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/AttributeIndex.java

@@ -486,33 +486,37 @@
   */
  boolean isIndexed(org.opends.server.types.IndexType indexType)
  {
    Set<IndexType> indexTypes = config.getIndexType();
    switch (indexType)
    {
    case PRESENCE:
      return indexTypes.contains(IndexType.PRESENCE);
      return isIndexed(IndexType.PRESENCE);

    case EQUALITY:
      return indexTypes.contains(IndexType.EQUALITY);
      return isIndexed(IndexType.EQUALITY);

    case SUBSTRING:
    case SUBINITIAL:
    case SUBANY:
    case SUBFINAL:
      return indexTypes.contains(IndexType.SUBSTRING);
      return isIndexed(IndexType.SUBSTRING);

    case GREATER_OR_EQUAL:
    case LESS_OR_EQUAL:
      return indexTypes.contains(IndexType.ORDERING);
      return isIndexed(IndexType.ORDERING);

    case APPROXIMATE:
      return indexTypes.contains(IndexType.APPROXIMATE);
      return isIndexed(IndexType.APPROXIMATE);

    default:
      return false;
    }
  }

  boolean isIndexed(IndexType indexType)
  {
    return config.getIndexType().contains(indexType);
  }

  /**
   * Update the attribute index for a new entry.
   *

 opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/BackendImpl.java

@@ -682,9 +682,9 @@
    }
  }

  private ImportStrategy getImportStrategy(RootContainer rootContainer)
  private ImportStrategy getImportStrategy(final RootContainer rootContainer)
  {
    return new OnDiskMergeImporter.StrategyImpl(rootContainer, cfg);
    return new OnDiskMergeImporter.StrategyImpl(serverContext, rootContainer, cfg);
  }

  @Override

 opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/OnDiskMergeImporter.java

@@ -99,6 +99,8 @@
import org.forgerock.opendj.ldap.ByteSequence;
import org.forgerock.opendj.ldap.ByteString;
import org.forgerock.opendj.ldap.DN;
import org.forgerock.opendj.ldap.schema.MatchingRule;
import org.forgerock.opendj.ldap.schema.UnknownSchemaElementException;
import org.forgerock.opendj.ldap.spi.Indexer;
import org.forgerock.opendj.server.config.meta.BackendIndexCfgDefn.IndexType;
import org.forgerock.opendj.server.config.server.BackendIndexCfg;
@@ -124,6 +126,8 @@
import org.opends.server.backends.pluggable.spi.WriteOperation;
import org.opends.server.backends.pluggable.spi.WriteableTransaction;
import org.opends.server.core.DirectoryServer;
import org.opends.server.core.ServerContext;
import org.opends.server.schema.SchemaConstants;
import org.opends.server.types.DirectoryException;
import org.opends.server.types.Entry;
import org.opends.server.types.InitializationException;
@@ -132,6 +136,7 @@

import com.forgerock.opendj.util.OperatingSystem;
import com.forgerock.opendj.util.PackedLong;
import com.forgerock.opendj.util.Predicate;

import net.jcip.annotations.NotThreadSafe;

@@ -170,11 +175,23 @@
    /** Small heap threshold used to give more memory to JVM to attempt OOM errors. */
    private static final int SMALL_HEAP_SIZE = 256 * MB;

    private static final Predicate<Tree, Void> IS_VLV = new Predicate<Tree, Void>()
    {
      @Override
      public boolean matches(Tree value, Void p)
      {
        return value.getName().getIndexId().startsWith("vlv.");
      }
    };

    private final ServerContext serverContext;
    private final RootContainer rootContainer;
    private final PluggableBackendCfg backendCfg;

    StrategyImpl(RootContainer rootContainer, PluggableBackendCfg backendCfg)
    StrategyImpl(final ServerContext serverContext, final RootContainer rootContainer,
        final PluggableBackendCfg backendCfg)
    {
      this.serverContext = serverContext;
      this.rootContainer = rootContainer;
      this.backendCfg = backendCfg;
    }
@@ -303,7 +320,7 @@
      }
    }

    private void clearDegradedState(final EntryContainer entryContainer, final Set<String> indexes)
    private void clearDegradedState(final EntryContainer entryContainer, final Set<String> indexIds)
        throws ExecutionException
    {
      try
@@ -313,7 +330,7 @@
          @Override
          public void run(WriteableTransaction txn)
          {
            visitIndexes(entryContainer, visitOnlyIndexes(indexes, setTrust(true, txn)));
            visitIndexes(entryContainer, visitOnlyIndexes(indexIdIn(indexIds), setTrust(true, txn)));
          }
        });
      }
@@ -449,130 +466,140 @@
      return new BufferPool(nbBuffers, bufferSize, false);
    }

    private static final Set<String> selectIndexesToRebuild(EntryContainer entryContainer, RebuildConfig rebuildConfig,
        long totalEntries) throws InitializationException
    private final Set<String> selectIndexesToRebuild(final EntryContainer entryContainer,
        final RebuildConfig rebuildConfig, final long totalEntries) throws InitializationException
    {
      final SelectIndexName selector = new SelectIndexName();
      final Set<String> indexesToRebuild;
      switch (rebuildConfig.getRebuildMode())
      {
      case ALL:
        visitIndexes(entryContainer, selector);
        indexesToRebuild = selector.getSelectedIndexNames();
        logger.info(NOTE_REBUILD_ALL_START, totalEntries);
        break;
      case DEGRADED:
        visitIndexes(entryContainer, visitOnlyDegraded(selector));
        indexesToRebuild = selector.getSelectedIndexNames();
        logger.info(NOTE_REBUILD_DEGRADED_START, totalEntries);
        break;
      case USER_DEFINED:
        // User defined format is attributeType(.indexType)
        visitIndexes(entryContainer,
            visitOnlyIndexes(buildUserDefinedIndexNames(entryContainer, rebuildConfig.getRebuildList()), selector));
        // User defined format is (attributeType)((.indexType)|(.matchingRule))
        indexesToRebuild = expandIndexNames(entryContainer, rebuildConfig.getRebuildList());
        if (!rebuildConfig.isClearDegradedState())
        {
          logger.info(NOTE_REBUILD_START, Utils.joinAsString(", ", rebuildConfig.getRebuildList()), totalEntries);
          logger.info(NOTE_REBUILD_START, Utils.joinAsString(", ", indexesToRebuild), totalEntries);
        }
        break;
      default:
        throw new UnsupportedOperationException("Unsupported rebuild mode " + rebuildConfig.getRebuildMode());
      }
      final Set<String> indexesToRebuild = selector.getSelectedIndexNames();
      if (indexesToRebuild.contains(SuffixContainer.DN2ID_INDEX_NAME))
      if (indexesToRebuild.contains(entryContainer.getDN2ID().getName().getIndexId()))
      {
        // Always rebuild id2childrencount with dn2id.
        indexesToRebuild.add(SuffixContainer.ID2CHILDREN_COUNT_NAME);
        indexesToRebuild.add(entryContainer.getID2ChildrenCount().getName().getIndexId());
      }
      return selector.getSelectedIndexNames();
      return indexesToRebuild;
    }

    /**
     * Translate attributeType(.indexType|matchingRuleOid) into attributeType.matchingRuleOid index name.
     * Translate (attributeType)((.indexType)|(.matchingRuleNameOrOid)) into attributeType.matchingRuleOid index name.
     *
     * @throws InitializationException
     *           if rebuildList contains an invalid/non-existing attribute/index name.
     */
    private static final Set<String> buildUserDefinedIndexNames(EntryContainer entryContainer,
        Collection<String> rebuildList) throws InitializationException
    final Set<String> expandIndexNames(final EntryContainer entryContainer, final Collection<String> rebuildList)
        throws InitializationException
    {
      final Set<String> indexNames = new HashSet<>();
      for (String name : rebuildList)
      for (final String name : rebuildList)
      {
        final String parts[] = name.split("\\.");
        if (parts.length == 1)
        {
          // Add all indexes of this attribute
          // for example: "cn" or "sn"
          final AttributeIndex attrIndex = findAttributeIndex(entryContainer, parts[0]);
          for (Tree index : attrIndex.getNameToIndexes().values())
          {
            indexNames.add(index.getName().getIndexId());
          }
        }
        else if (parts.length == 2)
        {
          // First, assume the supplied name is a valid index name ...
          // for example: "cn.substring", "vlv.someVlvIndex", "cn.caseIgnoreMatch"
          // or "cn.caseIgnoreSubstringsMatch:6"
        // Name could be a (attributeType)((.indexType)|(.matchingRule))
        // Add a trailing "." to ensure that resulting parts has always at least two parts.
        final String parts[] = (name + ".").split("\\.");

        // Is name represents all VLV index ?
          final SelectIndexName selector = new SelectIndexName();
          visitIndexes(entryContainer, visitOnlyIndexes(Arrays.asList(name), selector));
          indexNames.addAll(selector.getSelectedIndexNames());
        if (parts[0].equalsIgnoreCase("vlv") && isWildcard(parts[1]))
        {
          visitIndexes(entryContainer, visitOnlyIndexes(IS_VLV, selector));
        }
        else
        {
          // Is name a fully qualified index id ? i.e: "dn2id", "sn.caseIgnoreMatch:6"
          visitIndexes(entryContainer, visitOnlyIndexes(indexIdIn(Arrays.asList(name)), selector));
        }

          if (selector.getSelectedIndexNames().isEmpty())
          {
            // ... if not, assume the supplied name identify an attributeType.indexType
            // for example: aliases like "cn.substring" could not be found by the previous step
            try
          if (isWildcard(parts[1]))
            {
              final AttributeIndex attrIndex = findAttributeIndex(entryContainer, parts[0]);
              indexNames.addAll(getIndexNames(IndexType.valueOf(parts[1].toUpperCase()), attrIndex));
            }
            catch (IllegalArgumentException e)
            // Name is "attributeType", "attributeType." or "attributeType.*"
            visitAttributeIndexes(entryContainer.getAttributeIndexes(), attributeTypeIs(parts[0]), selector);
            if (selector.getSelectedIndexNames().isEmpty())
            {
              throw new InitializationException(ERR_ATTRIBUTE_INDEX_NOT_CONFIGURED.get(name), e);
            }
              throw new InitializationException(ERR_ATTRIBUTE_INDEX_NOT_CONFIGURED.get(name));
          }
        }
        else
        {
          throw new InitializationException(ERR_ATTRIBUTE_INDEX_NOT_CONFIGURED.get(name));
            final Collection<Predicate<MatchingRuleIndex, AttributeIndex>> attributeIndexFilters = new ArrayList<>();
            if (!isWildcard(parts[0]))
            {
              // Name is attributeType.matchingRule or attributeType.indexType
              attributeIndexFilters.add(attributeTypeIs(parts[0]));
        }
            // Filter on index type or matching rule.
            final IndexType indexType = indexTypeForNameOrNull(parts[1]);
            if (indexType == null)
            {
              // Name contains a matching rule
              try
              {
                attributeIndexFilters.add(matchingRuleIs(serverContext.getSchema().getMatchingRule(parts[1])));
              }
              catch (UnknownSchemaElementException usee)
              {
                throw new InitializationException(usee.getMessageObject(), usee);
              }

              if (parts[1].equalsIgnoreCase(SchemaConstants.EMR_DN_NAME))
              {
                indexNames.add(entryContainer.getDN2ID().getName().getIndexId());
                indexNames.add(entryContainer.getDN2URI().getName().getIndexId());
              }
            }
            else
            {
              // Name contains an index type
              attributeIndexFilters.add(indexTypeIs(indexType));
            }
            visitAttributeIndexes(entryContainer.getAttributeIndexes(), matchingAllOf(attributeIndexFilters), selector);
          }
        }
        indexNames.addAll(selector.getSelectedIndexNames());
      }
      return indexNames;
    }

    private static final AttributeIndex findAttributeIndex(EntryContainer entryContainer, String name)
        throws InitializationException
    private static boolean isWildcard(final String part)
    {
      for (AttributeIndex index : entryContainer.getAttributeIndexes())
      {
        if (index.getAttributeType().hasNameOrOID(name.toLowerCase()))
        {
          return index;
        }
      }
      throw new InitializationException(ERR_ATTRIBUTE_INDEX_NOT_CONFIGURED.get(name));
      return part.isEmpty() || "*".equals(part);
    }

    private static Collection<String> getIndexNames(IndexType indexType, AttributeIndex attrIndex)
    /**
     * Retrieves the index type for the specified name, or {@code null} if there is no such index type.
     */
    private static IndexType indexTypeForNameOrNull(final String indexName)
    {
      if (indexType.equals(IndexType.PRESENCE))
      for (final IndexType type : IndexType.values())
      {
        if (!attrIndex.isIndexed(org.opends.server.types.IndexType.PRESENCE))
        if (type.name().equalsIgnoreCase(indexName))
        {
          throw new IllegalArgumentException("No index found for type " + indexType);
          return type;
        }
        return Collections.singletonList(IndexType.PRESENCE.toString());
      }
      final Set<String> indexNames = new HashSet<>();
      for (Indexer indexer : AttributeIndex.getMatchingRule(indexType, attrIndex.getAttributeType())
                                           .createIndexers(attrIndex.getIndexingOptions()))
      {
        final Tree indexTree = attrIndex.getNameToIndexes().get(indexer.getIndexID());
        if (indexTree == null)
        {
          throw new IllegalArgumentException("No index found for type " + indexType);
        }
        indexNames.add(indexTree.getName().getIndexId());
      }
      return indexNames;
      return null;
    }

    private static File prepareTempDir(PluggableBackendCfg backendCfg, String tmpDirectory)
@@ -1150,7 +1177,7 @@
    @Override
    public Chunk newChunk(TreeName treeName) throws Exception
    {
      if (isID2Entry(treeName))
      if (isID2Entry(entryContainers.get(treeName.getBaseDN()), treeName))
      {
        return new MostlyOrderedChunk(asChunk(treeName, importer));
      }
@@ -1163,11 +1190,11 @@
    {
      final EntryContainer entryContainer = entryContainers.get(treeName.getBaseDN());

      if (isID2Entry(treeName))
      if (isID2Entry(entryContainer, treeName))
      {
        return newFlushTask(source);
      }
      else if (isDN2ID(treeName))
      else if (isDN2ID(entryContainer, treeName))
      {
        return newDN2IDImporterTask(treeName, source, progressReporter);
      }
@@ -1185,33 +1212,29 @@
    private final Set<String> indexesToRebuild;

    RebuildIndexStrategy(Collection<EntryContainer> entryContainers, Importer importer, File tempDir,
        BufferPool bufferPool, Executor sorter, Collection<String> indexNames)
        BufferPool bufferPool, Executor sorter, Set<String> indexNames)
    {
      super(entryContainers, importer, tempDir, bufferPool, sorter);
      this.indexesToRebuild = new HashSet<>(indexNames.size());
      for (String indexName : indexNames)
      {
        this.indexesToRebuild.add(indexName.toLowerCase());
      }
      this.indexesToRebuild = indexNames;
    }

    @Override
    void beforePhaseOne(EntryContainer entryContainer)
    {
      visitIndexes(entryContainer, visitOnlyIndexes(indexesToRebuild, setTrust(false, importer)));
      visitIndexes(entryContainer, visitOnlyIndexes(indexesToRebuild, deleteDatabase(importer)));
      visitIndexes(entryContainer, visitOnlyIndexes(indexIdIn(indexesToRebuild), setTrust(false, importer)));
      visitIndexes(entryContainer, visitOnlyIndexes(indexIdIn(indexesToRebuild), deleteDatabase(importer)));
    }

    @Override
    void afterPhaseTwo(EntryContainer entryContainer)
    {
      visitIndexes(entryContainer, visitOnlyIndexes(indexesToRebuild, setTrust(true, importer)));
      visitIndexes(entryContainer, visitOnlyIndexes(indexIdIn(indexesToRebuild), setTrust(true, importer)));
    }

    @Override
    public Chunk newChunk(TreeName treeName) throws Exception
    {
      if (indexesToRebuild.contains(treeName.getIndexId().toLowerCase()))
      if (indexesToRebuild.contains(treeName.getIndexId()))
      {
        return newExternalSortChunk(treeName);
      }
@@ -1224,9 +1247,12 @@
    {
      final EntryContainer entryContainer = entryContainers.get(treeName.getBaseDN());

      if (indexesToRebuild.contains(treeName.getIndexId().toLowerCase()))
      if (!indexesToRebuild.contains(treeName.getIndexId()))
      {
        if (isDN2ID(treeName))
        // Do nothing (flush null chunk)
        return newFlushTask(source);
      }
      else if (isDN2ID(entryContainer, treeName))
        {
          return newDN2IDImporterTask(treeName, source, progressReporter);
        }
@@ -1236,9 +1262,6 @@
        }
        return newChunkCopierTask(treeName, source, progressReporter);
      }
      // Do nothing (flush null chunk)
      return newFlushTask(source);
    }
  }

  private static <V> List<V> invokeParallel(String threadNameTemplate, Collection<Callable<V>> tasks)
@@ -3081,17 +3104,17 @@
      // key conflicts == merge EntryIDSets
      return new EntryIDSetsCollector(index);
    }
    else if (isID2ChildrenCount(treeName))
    else if (isID2ChildrenCount(entryContainer, treeName))
    {
      // key conflicts == sum values
      return ID2ChildrenCount.getSumLongCollectorInstance();
    }
    else if (isDN2ID(treeName))
    else if (isDN2ID(entryContainer, treeName))
    {
      // Detection of duplicate DN will be performed during phase 2 by the DNImporterTask
      return null;
    }
    else if (isDN2URI(treeName) || isVLVIndex(entryContainer, treeName))
    else if (isDN2URI(entryContainer, treeName) || isVLVIndex(entryContainer, treeName))
    {
      // key conflicts == exception
      return UniqueValueCollector.getInstance();
@@ -3111,24 +3134,24 @@
    return newPhaseTwoCollector(entryContainer, treeName);
  }

  private static boolean isDN2ID(TreeName treeName)
  private static boolean isDN2ID(final EntryContainer entryContainer, final TreeName treeName)
  {
    return SuffixContainer.DN2ID_INDEX_NAME.equals(treeName.getIndexId());
    return entryContainer.getDN2ID().getName().equals(treeName);
  }

  private static boolean isDN2URI(TreeName treeName)
  private static boolean isDN2URI(final EntryContainer entryContainer, TreeName treeName)
  {
    return SuffixContainer.DN2URI_INDEX_NAME.equals(treeName.getIndexId());
    return entryContainer.getDN2URI().getName().equals(treeName);
  }

  private static boolean isID2Entry(TreeName treeName)
  private static boolean isID2Entry(final EntryContainer entryContainer, final TreeName treeName)
  {
    return SuffixContainer.ID2ENTRY_INDEX_NAME.equals(treeName.getIndexId());
    return entryContainer.getID2Entry().getName().equals(treeName);
  }

  private static boolean isID2ChildrenCount(TreeName treeName)
  private static boolean isID2ChildrenCount(final EntryContainer entryContainer, final TreeName treeName)
  {
    return SuffixContainer.ID2CHILDREN_COUNT_NAME.equals(treeName.getIndexId());
    return entryContainer.getID2ChildrenCount().getName().equals(treeName);
  }

  private static boolean isVLVIndex(EntryContainer entryContainer, TreeName treeName)
@@ -3494,11 +3517,26 @@
    }
  }

  private static void visitIndexes(final EntryContainer entryContainer, IndexVisitor visitor)
  private static void visitAttributeIndexes(final Collection<AttributeIndex> attributeIndexes,
      final Predicate<MatchingRuleIndex, AttributeIndex> predicate, final IndexVisitor visitor)
  {
    for (AttributeIndex attribute : entryContainer.getAttributeIndexes())
    for (final AttributeIndex attribute : attributeIndexes)
    {
      for (MatchingRuleIndex index : attribute.getNameToIndexes().values())
      for (final MatchingRuleIndex index : attribute.getNameToIndexes().values())
      {
        if (predicate.matches(index, attribute))
        {
          visitor.visitAttributeIndex(index);
        }
      }
    }
  }

  private static void visitIndexes(final EntryContainer entryContainer, final IndexVisitor visitor)
  {
    for (final AttributeIndex attribute : entryContainer.getAttributeIndexes())
    {
      for (final MatchingRuleIndex index : attribute.getNameToIndexes().values())
      {
        visitor.visitAttributeIndex(index);
      }
@@ -3681,31 +3719,121 @@
    }
  }

  private static final IndexVisitor visitOnlyIndexes(Collection<String> indexNames, IndexVisitor delegate)
  private static final IndexVisitor visitOnlyIndexes(final Predicate<Tree, Void> predicate, final IndexVisitor delegate)
  {
    return new SpecificIndexFilter(delegate, indexNames);
    return new SpecificIndexFilter(delegate, predicate);
  }

  /** Visit indexes only if their name match one contained in a list. */
  private static final Predicate<Tree, Void> indexIdIn(final Collection<String> caseIgnoredNames) {
    final Set<String> indexNames = new HashSet<>(caseIgnoredNames.size());
    for (final String indexName : caseIgnoredNames)
    {
      indexNames.add(indexName.toLowerCase());
    }
    return new Predicate<Tree, Void>()
    {
      @Override
      public boolean matches(final Tree tree, final Void p)
      {
        return indexNames.contains(tree.getName().getIndexId().toLowerCase());
      }
    };
  }

  private static final Predicate<MatchingRuleIndex, AttributeIndex> attributeTypeIs(final String caseIgnoredName)
  {
    return new Predicate<MatchingRuleIndex, AttributeIndex>()
    {
      @Override
      public boolean matches(final MatchingRuleIndex index, final AttributeIndex attribute)
      {
        return caseIgnoredName.equalsIgnoreCase(attribute.getAttributeType().getNameOrOID());
      }
    };
  }

  private static final Predicate<MatchingRuleIndex, AttributeIndex> matchingRuleIs(final MatchingRule matchingRule)
  {
    return new Predicate<MatchingRuleIndex, AttributeIndex>()
    {
      @Override
      public boolean matches(final MatchingRuleIndex index, final AttributeIndex attribute)
      {
        for (final Indexer indexer : matchingRule.createIndexers(attribute.getIndexingOptions()))
        {
          if (index.equals(attribute.getNameToIndexes().get(indexer.getIndexID())))
          {
            return true;
          }
        }
        return false;
      }
    };
  }

  private static final Predicate<MatchingRuleIndex, AttributeIndex> indexTypeIs(final IndexType type)
  {
    return new Predicate<MatchingRuleIndex, AttributeIndex>()
    {
      @Override
      public boolean matches(final MatchingRuleIndex index, final AttributeIndex attribute)
      {
        if (IndexType.PRESENCE.equals(type))
        {
          return index.equals(attribute.getNameToIndexes().get(IndexType.PRESENCE.toString()));
        }
        try
        {
          final MatchingRule matchingRule = AttributeIndex.getMatchingRule(type, attribute.getAttributeType());
          if (matchingRule != null && matchingRuleIs(matchingRule).matches(index, attribute))
          {
            return true;
          }
        }
        catch (IllegalArgumentException iae)
        {
          // getMatchingRule() not implemented for the specified index type. Ignore silently.
        }
        return false;
      }
    };
  }

  private static final <M, A> Predicate<M, A> matchingAllOf(final Iterable<Predicate<M, A>> predicates)
  {
    return new Predicate<M, A>()
    {
      @Override
      public boolean matches(M value, A p)
      {
        for (Predicate<M, A> predicate : predicates)
        {
          if (!predicate.matches(value, p))
          {
            return false;
          }
        }
        return true;
      }
    };
  }

  /** Visit indexes only if they match the provided predicate. */
  private static final class SpecificIndexFilter implements IndexVisitor
  {
    private final IndexVisitor delegate;
    private final Collection<String> indexNames;
    private final Predicate<Tree, Void> predicate;

    SpecificIndexFilter(IndexVisitor delegate, Collection<String> names)
    SpecificIndexFilter(IndexVisitor delegate, Predicate<Tree, Void> predicate)
    {
      this.delegate = delegate;
      this.indexNames = new HashSet<>(names.size());
      for(String indexName : names)
      {
        this.indexNames.add(indexName.toLowerCase());
      }
      this.predicate = predicate;
    }

    @Override
    public void visitAttributeIndex(Index index)
    {
      if (indexIncluded(index))
      if (predicate.matches(index, null))
      {
        delegate.visitAttributeIndex(index);
      }
@@ -3714,7 +3842,7 @@
    @Override
    public void visitVLVIndex(VLVIndex index)
    {
      if (indexIncluded(index))
      if (predicate.matches(index, null))
      {
        delegate.visitVLVIndex(index);
      }
@@ -3723,16 +3851,11 @@
    @Override
    public void visitSystemIndex(Tree index)
    {
      if (indexIncluded(index))
      if (predicate.matches(index, null))
      {
        delegate.visitSystemIndex(index);
      }
    }

    private boolean indexIncluded(Tree index)
    {
      return indexNames.contains(index.getName().getIndexId().toLowerCase());
    }
  }

  private static WriteableTransaction asWriteableTransaction(Importer importer)

 opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/SuffixContainer.java

@@ -29,8 +29,6 @@
{
  /** The name of the index associating normalized DNs to ids. LDAP DNs uniquely identify entries. */
  String DN2ID_INDEX_NAME = "dn2id";
  /** The name of the index associating normalized DNs to URIs. */
  String DN2URI_INDEX_NAME = "dn2uri";
  /**
   * The name of the index associating entry ids to entries. Entry ids are
   * monotonically increasing unique longs and entries are serialized versions

 opendj-server-legacy/src/main/java/org/opends/server/tools/upgrade/Upgrade.java

@@ -38,6 +38,7 @@
import static javax.security.auth.callback.TextOutputCallback.WARNING;

import static org.opends.messages.ToolMessages.*;
import static org.opends.server.schema.SchemaConstants.*;
import static org.opends.server.tools.upgrade.FormattedNotificationCallback.*;
import static org.opends.server.tools.upgrade.LicenseFile.*;
import static org.opends.server.tools.upgrade.UpgradeTasks.*;
@@ -379,7 +380,7 @@

    register("3.5.0",
        rebuildIndexesNamed(INFO_UPGRADE_REBUILD_INDEXES_DISTINGUISHED_NAME.get(),
            "distinguishedName", "member", "owner", "roleOccupant", "seeAlso"));
            "." + EMR_DN_NAME, "." + EMR_OID_NAME, "." + EMR_UNIQUE_MEMBER_NAME, "." + EMR_CERTIFICATE_EXACT_NAME));

    register("3.5.0",
        deleteConfigEntry(INFO_UPGRADE_TASK_CONFIGURATION_BACKEND_NOT_CONFIGURABLE.get(),

 opendj-server-legacy/src/main/java/org/opends/server/tools/upgrade/UpgradeTasks.java

@@ -37,6 +37,7 @@
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -87,7 +88,7 @@
  static int countErrors;

  /** Contains all the indexes to rebuild. */
  private static final Set<String> indexesToRebuild = new HashSet<>();
  private static final Set<String> indexesToRebuild = new LinkedHashSet<>();

  /** A flag to avoid rebuild single indexes if 'rebuild all' is selected. */
  private static boolean isRebuildAllIndexesIsPresent;
@@ -729,21 +730,20 @@
          for (final Map.Entry<String, Set<String>> backendEntry : baseDNsForBackends.entrySet())
          {
            final String backend = backendEntry.getKey();
            final List<String> filteredIndexes = filterExistingIndexes(indexesToRebuild, backend);
            if (filteredIndexes.isEmpty())
            if (indexesToRebuild.isEmpty())
            {
              logger.debug(INFO_UPGRADE_NO_INDEX_TO_REBUILD_FOR_BACKEND.get(backend));
              continue;
            }

            final List<String> args = new ArrayList<>();
            for (final String indexToRebuild : filteredIndexes)
            for (final String indexToRebuild : indexesToRebuild)
            {
              args.add("--index");
              args.add(indexToRebuild);
            }
            final Set<String> baseDNs = backendEntry.getValue();
            rebuildIndex(INFO_UPGRADE_REBUILD_INDEX_STARTS.get(filteredIndexes, baseDNs), context, baseDNs, args);
            rebuildIndex(INFO_UPGRADE_REBUILD_INDEX_STARTS.get(indexesToRebuild, baseDNs), context, baseDNs, args);
          }
        }
      }

 opendj-server-legacy/src/main/java/org/opends/server/tools/upgrade/UpgradeUtils.java

@@ -23,10 +23,8 @@
import java.io.FileReader;
import java.io.FilenameFilter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

@@ -730,36 +728,6 @@
    return modifiedLines;
  }

  /** Filter provided list of indexes name to return indexes which are present in the provided backend. */
  static List<String> filterExistingIndexes(final Set<String> candidateIndexes, final String backendID)
  {
    final List<String> indexesToRebuild = new ArrayList<>();
    try (final LDIFEntryReader entryReader = new LDIFEntryReader(new FileInputStream(configFile)))
    {
      while (entryReader.hasNext())
      {
        final Entry entry = entryReader.readEntry();
        if (entry.containsAttribute("objectClass", "ds-cfg-backend-index")
            && entry.getName().toString().contains("ds-cfg-backend-id=" + backendID))
        {
          for (final String indexName : candidateIndexes)
          {
            if (entry.containsAttribute("ds-cfg-attribute", indexName))
            {
              indexesToRebuild.add(indexName);
            }
          }
        }
      }
    }
    catch (final IOException unlikely)
    {
      logger.error(ERR_UPGRADE_READING_CONF_FILE.get(unlikely.getMessage()));
    }

    return indexesToRebuild;
  }

  /** Returns {@code true} if the installed instance contains at least one JE backend. */
  static boolean instanceContainsJeBackends()
  {

 opendj-server-legacy/src/test/java/org/opends/server/backends/pluggable/OnDiskMergeImporterTest.java

@@ -16,11 +16,16 @@
package org.opends.server.backends.pluggable;

import static org.assertj.core.api.Assertions.*;
import static org.opends.server.backends.pluggable.EntryIDSet.*;
import static org.forgerock.opendj.config.ConfigurationMock.mockCfg;
import static org.forgerock.opendj.ldap.ResultCode.*;
import static org.forgerock.opendj.server.config.meta.BackendIndexCfgDefn.IndexType.*;
import static java.util.Arrays.asList;
import static org.forgerock.util.Pair.of;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.*;
import static org.opends.server.backends.pluggable.DnKeyFormat.dnToDNKey;
import static org.forgerock.opendj.ldap.ResultCode.*;
import static org.opends.server.backends.pluggable.EntryIDSet.*;
import static org.opends.server.util.CollectionUtils.newTreeSet;

import java.io.File;
import java.nio.ByteBuffer;
@@ -33,9 +38,11 @@
import java.util.Collection;
import java.util.Collections;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.concurrent.ForkJoinPool;

@@ -44,10 +51,18 @@
import org.forgerock.opendj.ldap.ByteStringBuilder;
import org.forgerock.opendj.ldap.DN;
import org.forgerock.opendj.ldap.ResultCode;
import org.forgerock.opendj.ldap.schema.AttributeType;
import org.forgerock.opendj.server.config.meta.BackendIndexCfgDefn.IndexType;
import org.forgerock.opendj.server.config.meta.BackendVLVIndexCfgDefn.Scope;
import org.forgerock.opendj.server.config.server.BackendIndexCfg;
import org.forgerock.opendj.server.config.server.BackendVLVIndexCfg;
import org.forgerock.opendj.server.config.server.JEBackendCfg;
import org.forgerock.util.Pair;
import org.forgerock.util.Reject;
import org.mockito.Mockito;
import org.opends.server.DirectoryServerTestCase;
import org.opends.server.TestCaseUtils;
import org.opends.server.backends.jeb.JEBackend;
import org.opends.server.backends.pluggable.OnDiskMergeImporter.BufferPool;
import org.opends.server.backends.pluggable.OnDiskMergeImporter.BufferPool.MemoryBuffer;
import org.opends.server.backends.pluggable.OnDiskMergeImporter.Chunk;
@@ -60,20 +75,162 @@
import org.opends.server.backends.pluggable.OnDiskMergeImporter.ExternalSortChunk.FileRegion;
import org.opends.server.backends.pluggable.OnDiskMergeImporter.ExternalSortChunk.InMemorySortedChunk;
import org.opends.server.backends.pluggable.OnDiskMergeImporter.MeteredCursor;
import org.opends.server.backends.pluggable.OnDiskMergeImporter.StrategyImpl;
import org.opends.server.backends.pluggable.OnDiskMergeImporter.UniqueValueCollector;
import org.opends.server.backends.pluggable.spi.ReadableTransaction;
import org.opends.server.backends.pluggable.spi.SequentialCursor;
import org.opends.server.backends.pluggable.spi.StorageRuntimeException;
import org.opends.server.backends.pluggable.spi.TreeName;
import org.opends.server.backends.pluggable.spi.WriteableTransaction;
import org.opends.server.core.DirectoryServer;
import org.opends.server.core.ServerContext;
import org.opends.server.crypto.CryptoSuite;
import org.opends.server.types.DirectoryException;
import org.opends.server.types.InitializationException;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;

import com.forgerock.opendj.util.PackedLong;

public class OnDiskMergeImporterTest extends DirectoryServerTestCase
{
  private BackendImpl<JEBackendCfg> backend;
  private ServerContext serverContext;
  private JEBackendCfg backendCfg;
  private EntryContainer entryContainer;
  private final DN testBaseDN = DN.valueOf("dc=test,dc=com");
  private final Map<String, IndexType[]> backendIndexes = new HashMap<>();
  {
    backendIndexes.put("entryUUID", new IndexType[] { EQUALITY });
    backendIndexes.put("cn", new IndexType[] { SUBSTRING });
    backendIndexes.put("sn", new IndexType[] { PRESENCE, EQUALITY, SUBSTRING });
    backendIndexes.put("uid", new IndexType[] { EQUALITY });
    backendIndexes.put("telephoneNumber", new IndexType[] { EQUALITY, SUBSTRING });
    backendIndexes.put("mail", new IndexType[] { SUBSTRING });
  }
  private final String[] backendVlvIndexes = { "people" };

  @BeforeClass
  public void setUp() throws Exception
  {
    // Need the schema to be available, so make sure the server is started.
    TestCaseUtils.startServer();

    serverContext = DirectoryServer.getInstance().getServerContext();

    backendCfg = mockCfg(JEBackendCfg.class);
    when(backendCfg.getBackendId()).thenReturn("OnDiskMergeImporterTest");
    when(backendCfg.getDBDirectory()).thenReturn("OnDiskMergeImporterTest");
    when(backendCfg.getDBDirectoryPermissions()).thenReturn("755");
    when(backendCfg.getDBCacheSize()).thenReturn(0L);
    when(backendCfg.getDBCachePercent()).thenReturn(20);
    when(backendCfg.getDBNumCleanerThreads()).thenReturn(2);
    when(backendCfg.getDBNumLockTables()).thenReturn(63);
    when(backendCfg.dn()).thenReturn(testBaseDN);
    when(backendCfg.getBaseDN()).thenReturn(newTreeSet(testBaseDN));
    when(backendCfg.listBackendIndexes()).thenReturn(backendIndexes.keySet().toArray(new String[0]));
    when(backendCfg.listBackendVLVIndexes()).thenReturn(backendVlvIndexes);

    for (Map.Entry<String, IndexType[]> index : backendIndexes.entrySet())
    {
      final String attributeName = index.getKey();
      final AttributeType attribute = DirectoryServer.getSchema().getAttributeType(attributeName);
      Reject.ifNull(attribute, "Attribute type '" + attributeName + "' doesn't exists.");

      BackendIndexCfg indexCfg = mock(BackendIndexCfg.class);
      when(indexCfg.getIndexType()).thenReturn(newTreeSet(index.getValue()));
      when(indexCfg.getAttribute()).thenReturn(attribute);
      when(indexCfg.getIndexEntryLimit()).thenReturn(4000);
      when(indexCfg.getSubstringLength()).thenReturn(6);
      when(backendCfg.getBackendIndex(index.getKey())).thenReturn(indexCfg);
      if (backendCfg.isConfidentialityEnabled())
      {
        when(indexCfg.isConfidentialityEnabled()).thenReturn(true);
      }
    }

    BackendVLVIndexCfg vlvIndexCfg = mock(BackendVLVIndexCfg.class);
    when(vlvIndexCfg.getName()).thenReturn("people");
    when(vlvIndexCfg.getBaseDN()).thenReturn(testBaseDN);
    when(vlvIndexCfg.getFilter()).thenReturn("(objectClass=person)");
    when(vlvIndexCfg.getScope()).thenReturn(Scope.WHOLE_SUBTREE);
    when(vlvIndexCfg.getSortOrder()).thenReturn("sn -employeeNumber +uid");
    when(backendCfg.getBackendVLVIndex(backendVlvIndexes[0])).thenReturn(vlvIndexCfg);

    backend = new JEBackend();
    backend.setBackendID(backendCfg.getBackendId());
    backend.configureBackend(backendCfg, DirectoryServer.getInstance().getServerContext());
    backend.openBackend();

    entryContainer = backend.getRootContainer().getEntryContainer(testBaseDN);
  }

  @AfterClass
  public void cleanUp() throws Exception
  {
    backend.finalizeBackend();
    backend = null;
  }

  @DataProvider(name="expandIndexData")
  private Object[][] expandIndexData() {
    return new Object[][] {
      { new String[] { "dn2id", "referral" }, new String[] { "dn2id", "referral" } },
      { new String[] { "vlv.people" }, new String[] { "vlv.people" } },
      { new String[] { "vlv." }, new String[] { "vlv.people" } },
      { new String[] { "vlv.*" }, new String[] { "vlv.people" } },
      { new String[] { "vlv" }, new String[] { "vlv.people" } },
      { new String[] { "sn", "cn.*" },
        new String[] { "sn.presence", "sn.caseIgnoreMatch", "sn.caseIgnoreSubstringsMatch:6",
                       "cn.caseIgnoreSubstringsMatch:6" } },
      { new String[] { "sn.", "cn." },
        new String[] { "sn.presence", "sn.caseIgnoreMatch", "sn.caseIgnoreSubstringsMatch:6",
                       "cn.caseIgnoreSubstringsMatch:6" } },
      { new String[] { ".substring", "*.presence" },
        new String[] { "sn.presence", "sn.caseIgnoreSubstringsMatch:6", "cn.caseIgnoreSubstringsMatch:6",
                       "mail.caseIgnoreIA5SubstringsMatch:6", "telephoneNumber.telephoneNumberSubstringsMatch:6" } },
      { new String[] { ".caseIgnoreSubstringsMatch" },
        new String[] { "sn.caseIgnoreSubstringsMatch:6", "cn.caseIgnoreSubstringsMatch:6" } },
      { new String[] { "dn2id", "uid.caseIgnoreMatch", ".caseIgnoreSubstringsMatch", ".presence" },
        new String[] { "dn2id", "uid.caseIgnoreMatch", "sn.caseIgnoreSubstringsMatch:6", "sn.presence",
                       "cn.caseIgnoreSubstringsMatch:6" } }
    };
  }

  @Test(dataProvider="expandIndexData")
  public void testCanExpandIndexNames(final String[] indexNames, final String[] expectedExpandedIndexNames)
      throws InitializationException {
    final StrategyImpl strategy = new StrategyImpl(serverContext, backend.getRootContainer(), backendCfg);
    assertThat(strategy.expandIndexNames(entryContainer, asList(indexNames)))
      .containsExactlyInAnyOrder(expectedExpandedIndexNames);
  }

  @Test(expectedExceptions=InitializationException.class)
  public void testThrowOnUnindexedType() throws InitializationException {
    final StrategyImpl strategy = new StrategyImpl(serverContext, backend.getRootContainer(), backendCfg);
    strategy.expandIndexNames(entryContainer, asList(".approximate"));
  }

  @Test(expectedExceptions=InitializationException.class)
  public void testThrowOnUnrecognizedMatchingRule() throws InitializationException {
    final StrategyImpl strategy = new StrategyImpl(serverContext, backend.getRootContainer(), backendCfg);
    strategy.expandIndexNames(entryContainer, asList(".unknown"));
  }

  @Test(expectedExceptions=InitializationException.class)
  public void testThrowOnUnrecognizedAttribute() throws InitializationException {
    final StrategyImpl strategy = new StrategyImpl(serverContext, backend.getRootContainer(), backendCfg);
    strategy.expandIndexNames(entryContainer, asList("unknown"));
  }

  @Test(expectedExceptions=InitializationException.class)
  public void testThrowOnUnrecognizedVlvIndex() throws InitializationException {
    final StrategyImpl strategy = new StrategyImpl(serverContext, backend.getRootContainer(), backendCfg);
    strategy.expandIndexNames(entryContainer, asList("vlv.unknown"));
  }

  @Test
  public void testHeapBuffer()
  {