From eeed545264e26d4f8d417047adc77432a30e968a Mon Sep 17 00:00:00 2001
From: Yannick Lecaillez <yannick.lecaillez@forgerock.com>
Date: Tue, 11 Oct 2016 13:27:25 +0000
Subject: [PATCH] OPENDJ-3230-3223: upgrade to 3.5.0 should rebuild indexes using DN syntax
---
opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/OnDiskMergeImporter.java | 393 +++++++++++++++++++++++------------
opendj-server-legacy/src/test/java/org/opends/server/backends/pluggable/OnDiskMergeImporterTest.java | 161 ++++++++++++++
opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/BackendImpl.java | 4
opendj-server-legacy/src/main/java/org/opends/server/tools/upgrade/Upgrade.java | 3
opendj-server-legacy/src/main/java/org/opends/server/tools/upgrade/UpgradeTasks.java | 10
opendj-server-legacy/src/main/java/org/opends/server/tools/upgrade/UpgradeUtils.java | 32 --
opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/AttributeIndex.java | 16
opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/SuffixContainer.java | 2
8 files changed, 436 insertions(+), 185 deletions(-)
diff --git a/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/AttributeIndex.java b/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/AttributeIndex.java
index b84dc63..8a0e9f7 100644
--- a/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/AttributeIndex.java
+++ b/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/AttributeIndex.java
@@ -486,33 +486,37 @@
*/
boolean isIndexed(org.opends.server.types.IndexType indexType)
{
- Set<IndexType> indexTypes = config.getIndexType();
switch (indexType)
{
case PRESENCE:
- return indexTypes.contains(IndexType.PRESENCE);
+ return isIndexed(IndexType.PRESENCE);
case EQUALITY:
- return indexTypes.contains(IndexType.EQUALITY);
+ return isIndexed(IndexType.EQUALITY);
case SUBSTRING:
case SUBINITIAL:
case SUBANY:
case SUBFINAL:
- return indexTypes.contains(IndexType.SUBSTRING);
+ return isIndexed(IndexType.SUBSTRING);
case GREATER_OR_EQUAL:
case LESS_OR_EQUAL:
- return indexTypes.contains(IndexType.ORDERING);
+ return isIndexed(IndexType.ORDERING);
case APPROXIMATE:
- return indexTypes.contains(IndexType.APPROXIMATE);
+ return isIndexed(IndexType.APPROXIMATE);
default:
return false;
}
}
+ boolean isIndexed(IndexType indexType)
+ {
+ return config.getIndexType().contains(indexType);
+ }
+
/**
* Update the attribute index for a new entry.
*
diff --git a/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/BackendImpl.java b/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/BackendImpl.java
index 8c0f919..38464c5 100644
--- a/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/BackendImpl.java
+++ b/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/BackendImpl.java
@@ -682,9 +682,9 @@
}
}
- private ImportStrategy getImportStrategy(RootContainer rootContainer)
+ private ImportStrategy getImportStrategy(final RootContainer rootContainer)
{
- return new OnDiskMergeImporter.StrategyImpl(rootContainer, cfg);
+ return new OnDiskMergeImporter.StrategyImpl(serverContext, rootContainer, cfg);
}
@Override
diff --git a/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/OnDiskMergeImporter.java b/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/OnDiskMergeImporter.java
index 0683c9d..60874a7 100644
--- a/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/OnDiskMergeImporter.java
+++ b/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/OnDiskMergeImporter.java
@@ -99,6 +99,8 @@
import org.forgerock.opendj.ldap.ByteSequence;
import org.forgerock.opendj.ldap.ByteString;
import org.forgerock.opendj.ldap.DN;
+import org.forgerock.opendj.ldap.schema.MatchingRule;
+import org.forgerock.opendj.ldap.schema.UnknownSchemaElementException;
import org.forgerock.opendj.ldap.spi.Indexer;
import org.forgerock.opendj.server.config.meta.BackendIndexCfgDefn.IndexType;
import org.forgerock.opendj.server.config.server.BackendIndexCfg;
@@ -124,6 +126,8 @@
import org.opends.server.backends.pluggable.spi.WriteOperation;
import org.opends.server.backends.pluggable.spi.WriteableTransaction;
import org.opends.server.core.DirectoryServer;
+import org.opends.server.core.ServerContext;
+import org.opends.server.schema.SchemaConstants;
import org.opends.server.types.DirectoryException;
import org.opends.server.types.Entry;
import org.opends.server.types.InitializationException;
@@ -132,6 +136,7 @@
import com.forgerock.opendj.util.OperatingSystem;
import com.forgerock.opendj.util.PackedLong;
+import com.forgerock.opendj.util.Predicate;
import net.jcip.annotations.NotThreadSafe;
@@ -170,11 +175,23 @@
/** Small heap threshold used to give more memory to JVM to attempt OOM errors. */
private static final int SMALL_HEAP_SIZE = 256 * MB;
+ private static final Predicate<Tree, Void> IS_VLV = new Predicate<Tree, Void>()
+ {
+ @Override
+ public boolean matches(Tree value, Void p)
+ {
+ return value.getName().getIndexId().startsWith("vlv.");
+ }
+ };
+
+ private final ServerContext serverContext;
private final RootContainer rootContainer;
private final PluggableBackendCfg backendCfg;
- StrategyImpl(RootContainer rootContainer, PluggableBackendCfg backendCfg)
+ StrategyImpl(final ServerContext serverContext, final RootContainer rootContainer,
+ final PluggableBackendCfg backendCfg)
{
+ this.serverContext = serverContext;
this.rootContainer = rootContainer;
this.backendCfg = backendCfg;
}
@@ -303,7 +320,7 @@
}
}
- private void clearDegradedState(final EntryContainer entryContainer, final Set<String> indexes)
+ private void clearDegradedState(final EntryContainer entryContainer, final Set<String> indexIds)
throws ExecutionException
{
try
@@ -313,7 +330,7 @@
@Override
public void run(WriteableTransaction txn)
{
- visitIndexes(entryContainer, visitOnlyIndexes(indexes, setTrust(true, txn)));
+ visitIndexes(entryContainer, visitOnlyIndexes(indexIdIn(indexIds), setTrust(true, txn)));
}
});
}
@@ -449,130 +466,140 @@
return new BufferPool(nbBuffers, bufferSize, false);
}
- private static final Set<String> selectIndexesToRebuild(EntryContainer entryContainer, RebuildConfig rebuildConfig,
- long totalEntries) throws InitializationException
+ private final Set<String> selectIndexesToRebuild(final EntryContainer entryContainer,
+ final RebuildConfig rebuildConfig, final long totalEntries) throws InitializationException
{
final SelectIndexName selector = new SelectIndexName();
+ final Set<String> indexesToRebuild;
switch (rebuildConfig.getRebuildMode())
{
case ALL:
visitIndexes(entryContainer, selector);
+ indexesToRebuild = selector.getSelectedIndexNames();
logger.info(NOTE_REBUILD_ALL_START, totalEntries);
break;
case DEGRADED:
visitIndexes(entryContainer, visitOnlyDegraded(selector));
+ indexesToRebuild = selector.getSelectedIndexNames();
logger.info(NOTE_REBUILD_DEGRADED_START, totalEntries);
break;
case USER_DEFINED:
- // User defined format is attributeType(.indexType)
- visitIndexes(entryContainer,
- visitOnlyIndexes(buildUserDefinedIndexNames(entryContainer, rebuildConfig.getRebuildList()), selector));
+ // User defined format is (attributeType)((.indexType)|(.matchingRule))
+ indexesToRebuild = expandIndexNames(entryContainer, rebuildConfig.getRebuildList());
if (!rebuildConfig.isClearDegradedState())
{
- logger.info(NOTE_REBUILD_START, Utils.joinAsString(", ", rebuildConfig.getRebuildList()), totalEntries);
+ logger.info(NOTE_REBUILD_START, Utils.joinAsString(", ", indexesToRebuild), totalEntries);
}
break;
default:
throw new UnsupportedOperationException("Unsupported rebuild mode " + rebuildConfig.getRebuildMode());
}
- final Set<String> indexesToRebuild = selector.getSelectedIndexNames();
- if (indexesToRebuild.contains(SuffixContainer.DN2ID_INDEX_NAME))
+ if (indexesToRebuild.contains(entryContainer.getDN2ID().getName().getIndexId()))
{
// Always rebuild id2childrencount with dn2id.
- indexesToRebuild.add(SuffixContainer.ID2CHILDREN_COUNT_NAME);
+ indexesToRebuild.add(entryContainer.getID2ChildrenCount().getName().getIndexId());
}
- return selector.getSelectedIndexNames();
+ return indexesToRebuild;
}
/**
- * Translate attributeType(.indexType|matchingRuleOid) into attributeType.matchingRuleOid index name.
+ * Translate (attributeType)((.indexType)|(.matchingRuleNameOrOid)) into attributeType.matchingRuleOid index name.
*
* @throws InitializationException
* if rebuildList contains an invalid/non-existing attribute/index name.
*/
- private static final Set<String> buildUserDefinedIndexNames(EntryContainer entryContainer,
- Collection<String> rebuildList) throws InitializationException
- {
+ final Set<String> expandIndexNames(final EntryContainer entryContainer, final Collection<String> rebuildList)
+ throws InitializationException
+ {
final Set<String> indexNames = new HashSet<>();
- for (String name : rebuildList)
+ for (final String name : rebuildList)
{
- final String parts[] = name.split("\\.");
- if (parts.length == 1)
+ // Name could be a (attributeType)((.indexType)|(.matchingRule))
+ // Add a trailing "." to ensure that resulting parts has always at least two parts.
+ final String parts[] = (name + ".").split("\\.");
+
+ // Is name represents all VLV index ?
+ final SelectIndexName selector = new SelectIndexName();
+ if (parts[0].equalsIgnoreCase("vlv") && isWildcard(parts[1]))
{
- // Add all indexes of this attribute
- // for example: "cn" or "sn"
- final AttributeIndex attrIndex = findAttributeIndex(entryContainer, parts[0]);
- for (Tree index : attrIndex.getNameToIndexes().values())
- {
- indexNames.add(index.getName().getIndexId());
- }
- }
- else if (parts.length == 2)
- {
- // First, assume the supplied name is a valid index name ...
- // for example: "cn.substring", "vlv.someVlvIndex", "cn.caseIgnoreMatch"
- // or "cn.caseIgnoreSubstringsMatch:6"
- final SelectIndexName selector = new SelectIndexName();
- visitIndexes(entryContainer, visitOnlyIndexes(Arrays.asList(name), selector));
- indexNames.addAll(selector.getSelectedIndexNames());
- if (selector.getSelectedIndexNames().isEmpty())
- {
- // ... if not, assume the supplied name identify an attributeType.indexType
- // for example: aliases like "cn.substring" could not be found by the previous step
- try
- {
- final AttributeIndex attrIndex = findAttributeIndex(entryContainer, parts[0]);
- indexNames.addAll(getIndexNames(IndexType.valueOf(parts[1].toUpperCase()), attrIndex));
- }
- catch (IllegalArgumentException e)
- {
- throw new InitializationException(ERR_ATTRIBUTE_INDEX_NOT_CONFIGURED.get(name), e);
- }
- }
+ visitIndexes(entryContainer, visitOnlyIndexes(IS_VLV, selector));
}
else
{
- throw new InitializationException(ERR_ATTRIBUTE_INDEX_NOT_CONFIGURED.get(name));
+ // Is name a fully qualified index id ? i.e: "dn2id", "sn.caseIgnoreMatch:6"
+ visitIndexes(entryContainer, visitOnlyIndexes(indexIdIn(Arrays.asList(name)), selector));
}
+
+ if (selector.getSelectedIndexNames().isEmpty())
+ {
+ if (isWildcard(parts[1]))
+ {
+ // Name is "attributeType", "attributeType." or "attributeType.*"
+ visitAttributeIndexes(entryContainer.getAttributeIndexes(), attributeTypeIs(parts[0]), selector);
+ if (selector.getSelectedIndexNames().isEmpty())
+ {
+ throw new InitializationException(ERR_ATTRIBUTE_INDEX_NOT_CONFIGURED.get(name));
+ }
+ }
+ else
+ {
+ final Collection<Predicate<MatchingRuleIndex, AttributeIndex>> attributeIndexFilters = new ArrayList<>();
+ if (!isWildcard(parts[0]))
+ {
+ // Name is attributeType.matchingRule or attributeType.indexType
+ attributeIndexFilters.add(attributeTypeIs(parts[0]));
+ }
+ // Filter on index type or matching rule.
+ final IndexType indexType = indexTypeForNameOrNull(parts[1]);
+ if (indexType == null)
+ {
+ // Name contains a matching rule
+ try
+ {
+ attributeIndexFilters.add(matchingRuleIs(serverContext.getSchema().getMatchingRule(parts[1])));
+ }
+ catch (UnknownSchemaElementException usee)
+ {
+ throw new InitializationException(usee.getMessageObject(), usee);
+ }
+
+ if (parts[1].equalsIgnoreCase(SchemaConstants.EMR_DN_NAME))
+ {
+ indexNames.add(entryContainer.getDN2ID().getName().getIndexId());
+ indexNames.add(entryContainer.getDN2URI().getName().getIndexId());
+ }
+ }
+ else
+ {
+ // Name contains an index type
+ attributeIndexFilters.add(indexTypeIs(indexType));
+ }
+ visitAttributeIndexes(entryContainer.getAttributeIndexes(), matchingAllOf(attributeIndexFilters), selector);
+ }
+ }
+ indexNames.addAll(selector.getSelectedIndexNames());
}
return indexNames;
}
- private static final AttributeIndex findAttributeIndex(EntryContainer entryContainer, String name)
- throws InitializationException
+ private static boolean isWildcard(final String part)
{
- for (AttributeIndex index : entryContainer.getAttributeIndexes())
- {
- if (index.getAttributeType().hasNameOrOID(name.toLowerCase()))
- {
- return index;
- }
- }
- throw new InitializationException(ERR_ATTRIBUTE_INDEX_NOT_CONFIGURED.get(name));
+ return part.isEmpty() || "*".equals(part);
}
- private static Collection<String> getIndexNames(IndexType indexType, AttributeIndex attrIndex)
+ /**
+ * Retrieves the index type for the specified name, or {@code null} if there is no such index type.
+ */
+ private static IndexType indexTypeForNameOrNull(final String indexName)
{
- if (indexType.equals(IndexType.PRESENCE))
+ for (final IndexType type : IndexType.values())
{
- if (!attrIndex.isIndexed(org.opends.server.types.IndexType.PRESENCE))
+ if (type.name().equalsIgnoreCase(indexName))
{
- throw new IllegalArgumentException("No index found for type " + indexType);
+ return type;
}
- return Collections.singletonList(IndexType.PRESENCE.toString());
}
- final Set<String> indexNames = new HashSet<>();
- for (Indexer indexer : AttributeIndex.getMatchingRule(indexType, attrIndex.getAttributeType())
- .createIndexers(attrIndex.getIndexingOptions()))
- {
- final Tree indexTree = attrIndex.getNameToIndexes().get(indexer.getIndexID());
- if (indexTree == null)
- {
- throw new IllegalArgumentException("No index found for type " + indexType);
- }
- indexNames.add(indexTree.getName().getIndexId());
- }
- return indexNames;
+ return null;
}
private static File prepareTempDir(PluggableBackendCfg backendCfg, String tmpDirectory)
@@ -1150,7 +1177,7 @@
@Override
public Chunk newChunk(TreeName treeName) throws Exception
{
- if (isID2Entry(treeName))
+ if (isID2Entry(entryContainers.get(treeName.getBaseDN()), treeName))
{
return new MostlyOrderedChunk(asChunk(treeName, importer));
}
@@ -1163,11 +1190,11 @@
{
final EntryContainer entryContainer = entryContainers.get(treeName.getBaseDN());
- if (isID2Entry(treeName))
+ if (isID2Entry(entryContainer, treeName))
{
return newFlushTask(source);
}
- else if (isDN2ID(treeName))
+ else if (isDN2ID(entryContainer, treeName))
{
return newDN2IDImporterTask(treeName, source, progressReporter);
}
@@ -1185,33 +1212,29 @@
private final Set<String> indexesToRebuild;
RebuildIndexStrategy(Collection<EntryContainer> entryContainers, Importer importer, File tempDir,
- BufferPool bufferPool, Executor sorter, Collection<String> indexNames)
+ BufferPool bufferPool, Executor sorter, Set<String> indexNames)
{
super(entryContainers, importer, tempDir, bufferPool, sorter);
- this.indexesToRebuild = new HashSet<>(indexNames.size());
- for (String indexName : indexNames)
- {
- this.indexesToRebuild.add(indexName.toLowerCase());
- }
+ this.indexesToRebuild = indexNames;
}
@Override
void beforePhaseOne(EntryContainer entryContainer)
{
- visitIndexes(entryContainer, visitOnlyIndexes(indexesToRebuild, setTrust(false, importer)));
- visitIndexes(entryContainer, visitOnlyIndexes(indexesToRebuild, deleteDatabase(importer)));
+ visitIndexes(entryContainer, visitOnlyIndexes(indexIdIn(indexesToRebuild), setTrust(false, importer)));
+ visitIndexes(entryContainer, visitOnlyIndexes(indexIdIn(indexesToRebuild), deleteDatabase(importer)));
}
@Override
void afterPhaseTwo(EntryContainer entryContainer)
{
- visitIndexes(entryContainer, visitOnlyIndexes(indexesToRebuild, setTrust(true, importer)));
+ visitIndexes(entryContainer, visitOnlyIndexes(indexIdIn(indexesToRebuild), setTrust(true, importer)));
}
@Override
public Chunk newChunk(TreeName treeName) throws Exception
{
- if (indexesToRebuild.contains(treeName.getIndexId().toLowerCase()))
+ if (indexesToRebuild.contains(treeName.getIndexId()))
{
return newExternalSortChunk(treeName);
}
@@ -1224,20 +1247,20 @@
{
final EntryContainer entryContainer = entryContainers.get(treeName.getBaseDN());
- if (indexesToRebuild.contains(treeName.getIndexId().toLowerCase()))
+ if (!indexesToRebuild.contains(treeName.getIndexId()))
{
- if (isDN2ID(treeName))
- {
- return newDN2IDImporterTask(treeName, source, progressReporter);
- }
- else if (isVLVIndex(entryContainer, treeName))
- {
- return newVLVIndexImporterTask(getVLVIndex(entryContainer, treeName), source, progressReporter);
- }
- return newChunkCopierTask(treeName, source, progressReporter);
+ // Do nothing (flush null chunk)
+ return newFlushTask(source);
}
- // Do nothing (flush null chunk)
- return newFlushTask(source);
+ else if (isDN2ID(entryContainer, treeName))
+ {
+ return newDN2IDImporterTask(treeName, source, progressReporter);
+ }
+ else if (isVLVIndex(entryContainer, treeName))
+ {
+ return newVLVIndexImporterTask(getVLVIndex(entryContainer, treeName), source, progressReporter);
+ }
+ return newChunkCopierTask(treeName, source, progressReporter);
}
}
@@ -3081,17 +3104,17 @@
// key conflicts == merge EntryIDSets
return new EntryIDSetsCollector(index);
}
- else if (isID2ChildrenCount(treeName))
+ else if (isID2ChildrenCount(entryContainer, treeName))
{
// key conflicts == sum values
return ID2ChildrenCount.getSumLongCollectorInstance();
}
- else if (isDN2ID(treeName))
+ else if (isDN2ID(entryContainer, treeName))
{
// Detection of duplicate DN will be performed during phase 2 by the DNImporterTask
return null;
}
- else if (isDN2URI(treeName) || isVLVIndex(entryContainer, treeName))
+ else if (isDN2URI(entryContainer, treeName) || isVLVIndex(entryContainer, treeName))
{
// key conflicts == exception
return UniqueValueCollector.getInstance();
@@ -3111,24 +3134,24 @@
return newPhaseTwoCollector(entryContainer, treeName);
}
- private static boolean isDN2ID(TreeName treeName)
+ private static boolean isDN2ID(final EntryContainer entryContainer, final TreeName treeName)
{
- return SuffixContainer.DN2ID_INDEX_NAME.equals(treeName.getIndexId());
+ return entryContainer.getDN2ID().getName().equals(treeName);
}
- private static boolean isDN2URI(TreeName treeName)
+ private static boolean isDN2URI(final EntryContainer entryContainer, TreeName treeName)
{
- return SuffixContainer.DN2URI_INDEX_NAME.equals(treeName.getIndexId());
+ return entryContainer.getDN2URI().getName().equals(treeName);
}
- private static boolean isID2Entry(TreeName treeName)
+ private static boolean isID2Entry(final EntryContainer entryContainer, final TreeName treeName)
{
- return SuffixContainer.ID2ENTRY_INDEX_NAME.equals(treeName.getIndexId());
+ return entryContainer.getID2Entry().getName().equals(treeName);
}
- private static boolean isID2ChildrenCount(TreeName treeName)
+ private static boolean isID2ChildrenCount(final EntryContainer entryContainer, final TreeName treeName)
{
- return SuffixContainer.ID2CHILDREN_COUNT_NAME.equals(treeName.getIndexId());
+ return entryContainer.getID2ChildrenCount().getName().equals(treeName);
}
private static boolean isVLVIndex(EntryContainer entryContainer, TreeName treeName)
@@ -3494,11 +3517,26 @@
}
}
- private static void visitIndexes(final EntryContainer entryContainer, IndexVisitor visitor)
+ private static void visitAttributeIndexes(final Collection<AttributeIndex> attributeIndexes,
+ final Predicate<MatchingRuleIndex, AttributeIndex> predicate, final IndexVisitor visitor)
{
- for (AttributeIndex attribute : entryContainer.getAttributeIndexes())
+ for (final AttributeIndex attribute : attributeIndexes)
{
- for (MatchingRuleIndex index : attribute.getNameToIndexes().values())
+ for (final MatchingRuleIndex index : attribute.getNameToIndexes().values())
+ {
+ if (predicate.matches(index, attribute))
+ {
+ visitor.visitAttributeIndex(index);
+ }
+ }
+ }
+ }
+
+ private static void visitIndexes(final EntryContainer entryContainer, final IndexVisitor visitor)
+ {
+ for (final AttributeIndex attribute : entryContainer.getAttributeIndexes())
+ {
+ for (final MatchingRuleIndex index : attribute.getNameToIndexes().values())
{
visitor.visitAttributeIndex(index);
}
@@ -3681,31 +3719,121 @@
}
}
- private static final IndexVisitor visitOnlyIndexes(Collection<String> indexNames, IndexVisitor delegate)
+ private static final IndexVisitor visitOnlyIndexes(final Predicate<Tree, Void> predicate, final IndexVisitor delegate)
{
- return new SpecificIndexFilter(delegate, indexNames);
+ return new SpecificIndexFilter(delegate, predicate);
}
- /** Visit indexes only if their name match one contained in a list. */
+ private static final Predicate<Tree, Void> indexIdIn(final Collection<String> caseIgnoredNames) {
+ final Set<String> indexNames = new HashSet<>(caseIgnoredNames.size());
+ for (final String indexName : caseIgnoredNames)
+ {
+ indexNames.add(indexName.toLowerCase());
+ }
+ return new Predicate<Tree, Void>()
+ {
+ @Override
+ public boolean matches(final Tree tree, final Void p)
+ {
+ return indexNames.contains(tree.getName().getIndexId().toLowerCase());
+ }
+ };
+ }
+
+ private static final Predicate<MatchingRuleIndex, AttributeIndex> attributeTypeIs(final String caseIgnoredName)
+ {
+ return new Predicate<MatchingRuleIndex, AttributeIndex>()
+ {
+ @Override
+ public boolean matches(final MatchingRuleIndex index, final AttributeIndex attribute)
+ {
+ return caseIgnoredName.equalsIgnoreCase(attribute.getAttributeType().getNameOrOID());
+ }
+ };
+ }
+
+ private static final Predicate<MatchingRuleIndex, AttributeIndex> matchingRuleIs(final MatchingRule matchingRule)
+ {
+ return new Predicate<MatchingRuleIndex, AttributeIndex>()
+ {
+ @Override
+ public boolean matches(final MatchingRuleIndex index, final AttributeIndex attribute)
+ {
+ for (final Indexer indexer : matchingRule.createIndexers(attribute.getIndexingOptions()))
+ {
+ if (index.equals(attribute.getNameToIndexes().get(indexer.getIndexID())))
+ {
+ return true;
+ }
+ }
+ return false;
+ }
+ };
+ }
+
+ private static final Predicate<MatchingRuleIndex, AttributeIndex> indexTypeIs(final IndexType type)
+ {
+ return new Predicate<MatchingRuleIndex, AttributeIndex>()
+ {
+ @Override
+ public boolean matches(final MatchingRuleIndex index, final AttributeIndex attribute)
+ {
+ if (IndexType.PRESENCE.equals(type))
+ {
+ return index.equals(attribute.getNameToIndexes().get(IndexType.PRESENCE.toString()));
+ }
+ try
+ {
+ final MatchingRule matchingRule = AttributeIndex.getMatchingRule(type, attribute.getAttributeType());
+ if (matchingRule != null && matchingRuleIs(matchingRule).matches(index, attribute))
+ {
+ return true;
+ }
+ }
+ catch (IllegalArgumentException iae)
+ {
+ // getMatchingRule() not implemented for the specified index type. Ignore silently.
+ }
+ return false;
+ }
+ };
+ }
+
+ private static final <M, A> Predicate<M, A> matchingAllOf(final Iterable<Predicate<M, A>> predicates)
+ {
+ return new Predicate<M, A>()
+ {
+ @Override
+ public boolean matches(M value, A p)
+ {
+ for (Predicate<M, A> predicate : predicates)
+ {
+ if (!predicate.matches(value, p))
+ {
+ return false;
+ }
+ }
+ return true;
+ }
+ };
+ }
+
+ /** Visit indexes only if they match the provided predicate. */
private static final class SpecificIndexFilter implements IndexVisitor
{
private final IndexVisitor delegate;
- private final Collection<String> indexNames;
+ private final Predicate<Tree, Void> predicate;
- SpecificIndexFilter(IndexVisitor delegate, Collection<String> names)
+ SpecificIndexFilter(IndexVisitor delegate, Predicate<Tree, Void> predicate)
{
this.delegate = delegate;
- this.indexNames = new HashSet<>(names.size());
- for(String indexName : names)
- {
- this.indexNames.add(indexName.toLowerCase());
- }
+ this.predicate = predicate;
}
@Override
public void visitAttributeIndex(Index index)
{
- if (indexIncluded(index))
+ if (predicate.matches(index, null))
{
delegate.visitAttributeIndex(index);
}
@@ -3714,7 +3842,7 @@
@Override
public void visitVLVIndex(VLVIndex index)
{
- if (indexIncluded(index))
+ if (predicate.matches(index, null))
{
delegate.visitVLVIndex(index);
}
@@ -3723,16 +3851,11 @@
@Override
public void visitSystemIndex(Tree index)
{
- if (indexIncluded(index))
+ if (predicate.matches(index, null))
{
delegate.visitSystemIndex(index);
}
}
-
- private boolean indexIncluded(Tree index)
- {
- return indexNames.contains(index.getName().getIndexId().toLowerCase());
- }
}
private static WriteableTransaction asWriteableTransaction(Importer importer)
diff --git a/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/SuffixContainer.java b/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/SuffixContainer.java
index 52672da..203377e 100644
--- a/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/SuffixContainer.java
+++ b/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/SuffixContainer.java
@@ -29,8 +29,6 @@
{
/** The name of the index associating normalized DNs to ids. LDAP DNs uniquely identify entries. */
String DN2ID_INDEX_NAME = "dn2id";
- /** The name of the index associating normalized DNs to URIs. */
- String DN2URI_INDEX_NAME = "dn2uri";
/**
* The name of the index associating entry ids to entries. Entry ids are
* monotonically increasing unique longs and entries are serialized versions
diff --git a/opendj-server-legacy/src/main/java/org/opends/server/tools/upgrade/Upgrade.java b/opendj-server-legacy/src/main/java/org/opends/server/tools/upgrade/Upgrade.java
index 2506897..ba63409 100644
--- a/opendj-server-legacy/src/main/java/org/opends/server/tools/upgrade/Upgrade.java
+++ b/opendj-server-legacy/src/main/java/org/opends/server/tools/upgrade/Upgrade.java
@@ -38,6 +38,7 @@
import static javax.security.auth.callback.TextOutputCallback.WARNING;
import static org.opends.messages.ToolMessages.*;
+import static org.opends.server.schema.SchemaConstants.*;
import static org.opends.server.tools.upgrade.FormattedNotificationCallback.*;
import static org.opends.server.tools.upgrade.LicenseFile.*;
import static org.opends.server.tools.upgrade.UpgradeTasks.*;
@@ -379,7 +380,7 @@
register("3.5.0",
rebuildIndexesNamed(INFO_UPGRADE_REBUILD_INDEXES_DISTINGUISHED_NAME.get(),
- "distinguishedName", "member", "owner", "roleOccupant", "seeAlso"));
+ "." + EMR_DN_NAME, "." + EMR_OID_NAME, "." + EMR_UNIQUE_MEMBER_NAME, "." + EMR_CERTIFICATE_EXACT_NAME));
register("3.5.0",
deleteConfigEntry(INFO_UPGRADE_TASK_CONFIGURATION_BACKEND_NOT_CONFIGURABLE.get(),
diff --git a/opendj-server-legacy/src/main/java/org/opends/server/tools/upgrade/UpgradeTasks.java b/opendj-server-legacy/src/main/java/org/opends/server/tools/upgrade/UpgradeTasks.java
index 0c809d0..25b8767 100644
--- a/opendj-server-legacy/src/main/java/org/opends/server/tools/upgrade/UpgradeTasks.java
+++ b/opendj-server-legacy/src/main/java/org/opends/server/tools/upgrade/UpgradeTasks.java
@@ -37,6 +37,7 @@
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
+import java.util.LinkedHashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -87,7 +88,7 @@
static int countErrors;
/** Contains all the indexes to rebuild. */
- private static final Set<String> indexesToRebuild = new HashSet<>();
+ private static final Set<String> indexesToRebuild = new LinkedHashSet<>();
/** A flag to avoid rebuild single indexes if 'rebuild all' is selected. */
private static boolean isRebuildAllIndexesIsPresent;
@@ -729,21 +730,20 @@
for (final Map.Entry<String, Set<String>> backendEntry : baseDNsForBackends.entrySet())
{
final String backend = backendEntry.getKey();
- final List<String> filteredIndexes = filterExistingIndexes(indexesToRebuild, backend);
- if (filteredIndexes.isEmpty())
+ if (indexesToRebuild.isEmpty())
{
logger.debug(INFO_UPGRADE_NO_INDEX_TO_REBUILD_FOR_BACKEND.get(backend));
continue;
}
final List<String> args = new ArrayList<>();
- for (final String indexToRebuild : filteredIndexes)
+ for (final String indexToRebuild : indexesToRebuild)
{
args.add("--index");
args.add(indexToRebuild);
}
final Set<String> baseDNs = backendEntry.getValue();
- rebuildIndex(INFO_UPGRADE_REBUILD_INDEX_STARTS.get(filteredIndexes, baseDNs), context, baseDNs, args);
+ rebuildIndex(INFO_UPGRADE_REBUILD_INDEX_STARTS.get(indexesToRebuild, baseDNs), context, baseDNs, args);
}
}
}
diff --git a/opendj-server-legacy/src/main/java/org/opends/server/tools/upgrade/UpgradeUtils.java b/opendj-server-legacy/src/main/java/org/opends/server/tools/upgrade/UpgradeUtils.java
index 1ede286..b3072f9 100644
--- a/opendj-server-legacy/src/main/java/org/opends/server/tools/upgrade/UpgradeUtils.java
+++ b/opendj-server-legacy/src/main/java/org/opends/server/tools/upgrade/UpgradeUtils.java
@@ -23,10 +23,8 @@
import java.io.FileReader;
import java.io.FilenameFilter;
import java.io.IOException;
-import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -730,36 +728,6 @@
return modifiedLines;
}
- /** Filter provided list of indexes name to return indexes which are present in the provided backend. */
- static List<String> filterExistingIndexes(final Set<String> candidateIndexes, final String backendID)
- {
- final List<String> indexesToRebuild = new ArrayList<>();
- try (final LDIFEntryReader entryReader = new LDIFEntryReader(new FileInputStream(configFile)))
- {
- while (entryReader.hasNext())
- {
- final Entry entry = entryReader.readEntry();
- if (entry.containsAttribute("objectClass", "ds-cfg-backend-index")
- && entry.getName().toString().contains("ds-cfg-backend-id=" + backendID))
- {
- for (final String indexName : candidateIndexes)
- {
- if (entry.containsAttribute("ds-cfg-attribute", indexName))
- {
- indexesToRebuild.add(indexName);
- }
- }
- }
- }
- }
- catch (final IOException unlikely)
- {
- logger.error(ERR_UPGRADE_READING_CONF_FILE.get(unlikely.getMessage()));
- }
-
- return indexesToRebuild;
- }
-
/** Returns {@code true} if the installed instance contains at least one JE backend. */
static boolean instanceContainsJeBackends()
{
diff --git a/opendj-server-legacy/src/test/java/org/opends/server/backends/pluggable/OnDiskMergeImporterTest.java b/opendj-server-legacy/src/test/java/org/opends/server/backends/pluggable/OnDiskMergeImporterTest.java
index 947b3f4..41bdc43 100644
--- a/opendj-server-legacy/src/test/java/org/opends/server/backends/pluggable/OnDiskMergeImporterTest.java
+++ b/opendj-server-legacy/src/test/java/org/opends/server/backends/pluggable/OnDiskMergeImporterTest.java
@@ -16,11 +16,16 @@
package org.opends.server.backends.pluggable;
import static org.assertj.core.api.Assertions.*;
-import static org.opends.server.backends.pluggable.EntryIDSet.*;
+import static org.forgerock.opendj.config.ConfigurationMock.mockCfg;
+import static org.forgerock.opendj.ldap.ResultCode.*;
+import static org.forgerock.opendj.server.config.meta.BackendIndexCfgDefn.IndexType.*;
+import static java.util.Arrays.asList;
import static org.forgerock.util.Pair.of;
+import static org.mockito.Matchers.any;
import static org.mockito.Mockito.*;
import static org.opends.server.backends.pluggable.DnKeyFormat.dnToDNKey;
-import static org.forgerock.opendj.ldap.ResultCode.*;
+import static org.opends.server.backends.pluggable.EntryIDSet.*;
+import static org.opends.server.util.CollectionUtils.newTreeSet;
import java.io.File;
import java.nio.ByteBuffer;
@@ -33,9 +38,11 @@
import java.util.Collection;
import java.util.Collections;
import java.util.EnumSet;
+import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
+import java.util.Map;
import java.util.NoSuchElementException;
import java.util.concurrent.ForkJoinPool;
@@ -44,10 +51,18 @@
import org.forgerock.opendj.ldap.ByteStringBuilder;
import org.forgerock.opendj.ldap.DN;
import org.forgerock.opendj.ldap.ResultCode;
+import org.forgerock.opendj.ldap.schema.AttributeType;
+import org.forgerock.opendj.server.config.meta.BackendIndexCfgDefn.IndexType;
+import org.forgerock.opendj.server.config.meta.BackendVLVIndexCfgDefn.Scope;
+import org.forgerock.opendj.server.config.server.BackendIndexCfg;
+import org.forgerock.opendj.server.config.server.BackendVLVIndexCfg;
+import org.forgerock.opendj.server.config.server.JEBackendCfg;
import org.forgerock.util.Pair;
+import org.forgerock.util.Reject;
import org.mockito.Mockito;
import org.opends.server.DirectoryServerTestCase;
import org.opends.server.TestCaseUtils;
+import org.opends.server.backends.jeb.JEBackend;
import org.opends.server.backends.pluggable.OnDiskMergeImporter.BufferPool;
import org.opends.server.backends.pluggable.OnDiskMergeImporter.BufferPool.MemoryBuffer;
import org.opends.server.backends.pluggable.OnDiskMergeImporter.Chunk;
@@ -60,20 +75,162 @@
import org.opends.server.backends.pluggable.OnDiskMergeImporter.ExternalSortChunk.FileRegion;
import org.opends.server.backends.pluggable.OnDiskMergeImporter.ExternalSortChunk.InMemorySortedChunk;
import org.opends.server.backends.pluggable.OnDiskMergeImporter.MeteredCursor;
+import org.opends.server.backends.pluggable.OnDiskMergeImporter.StrategyImpl;
import org.opends.server.backends.pluggable.OnDiskMergeImporter.UniqueValueCollector;
import org.opends.server.backends.pluggable.spi.ReadableTransaction;
import org.opends.server.backends.pluggable.spi.SequentialCursor;
import org.opends.server.backends.pluggable.spi.StorageRuntimeException;
import org.opends.server.backends.pluggable.spi.TreeName;
import org.opends.server.backends.pluggable.spi.WriteableTransaction;
+import org.opends.server.core.DirectoryServer;
+import org.opends.server.core.ServerContext;
import org.opends.server.crypto.CryptoSuite;
import org.opends.server.types.DirectoryException;
+import org.opends.server.types.InitializationException;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
import com.forgerock.opendj.util.PackedLong;
public class OnDiskMergeImporterTest extends DirectoryServerTestCase
{
+ private BackendImpl<JEBackendCfg> backend;
+ private ServerContext serverContext;
+ private JEBackendCfg backendCfg;
+ private EntryContainer entryContainer;
+ private final DN testBaseDN = DN.valueOf("dc=test,dc=com");
+ private final Map<String, IndexType[]> backendIndexes = new HashMap<>();
+ {
+ backendIndexes.put("entryUUID", new IndexType[] { EQUALITY });
+ backendIndexes.put("cn", new IndexType[] { SUBSTRING });
+ backendIndexes.put("sn", new IndexType[] { PRESENCE, EQUALITY, SUBSTRING });
+ backendIndexes.put("uid", new IndexType[] { EQUALITY });
+ backendIndexes.put("telephoneNumber", new IndexType[] { EQUALITY, SUBSTRING });
+ backendIndexes.put("mail", new IndexType[] { SUBSTRING });
+ }
+ private final String[] backendVlvIndexes = { "people" };
+
+ @BeforeClass
+ public void setUp() throws Exception
+ {
+ // Need the schema to be available, so make sure the server is started.
+ TestCaseUtils.startServer();
+
+ serverContext = DirectoryServer.getInstance().getServerContext();
+
+ backendCfg = mockCfg(JEBackendCfg.class);
+ when(backendCfg.getBackendId()).thenReturn("OnDiskMergeImporterTest");
+ when(backendCfg.getDBDirectory()).thenReturn("OnDiskMergeImporterTest");
+ when(backendCfg.getDBDirectoryPermissions()).thenReturn("755");
+ when(backendCfg.getDBCacheSize()).thenReturn(0L);
+ when(backendCfg.getDBCachePercent()).thenReturn(20);
+ when(backendCfg.getDBNumCleanerThreads()).thenReturn(2);
+ when(backendCfg.getDBNumLockTables()).thenReturn(63);
+ when(backendCfg.dn()).thenReturn(testBaseDN);
+ when(backendCfg.getBaseDN()).thenReturn(newTreeSet(testBaseDN));
+ when(backendCfg.listBackendIndexes()).thenReturn(backendIndexes.keySet().toArray(new String[0]));
+ when(backendCfg.listBackendVLVIndexes()).thenReturn(backendVlvIndexes);
+
+ for (Map.Entry<String, IndexType[]> index : backendIndexes.entrySet())
+ {
+ final String attributeName = index.getKey();
+ final AttributeType attribute = DirectoryServer.getSchema().getAttributeType(attributeName);
+ Reject.ifNull(attribute, "Attribute type '" + attributeName + "' doesn't exists.");
+
+ BackendIndexCfg indexCfg = mock(BackendIndexCfg.class);
+ when(indexCfg.getIndexType()).thenReturn(newTreeSet(index.getValue()));
+ when(indexCfg.getAttribute()).thenReturn(attribute);
+ when(indexCfg.getIndexEntryLimit()).thenReturn(4000);
+ when(indexCfg.getSubstringLength()).thenReturn(6);
+ when(backendCfg.getBackendIndex(index.getKey())).thenReturn(indexCfg);
+ if (backendCfg.isConfidentialityEnabled())
+ {
+ when(indexCfg.isConfidentialityEnabled()).thenReturn(true);
+ }
+ }
+
+ BackendVLVIndexCfg vlvIndexCfg = mock(BackendVLVIndexCfg.class);
+ when(vlvIndexCfg.getName()).thenReturn("people");
+ when(vlvIndexCfg.getBaseDN()).thenReturn(testBaseDN);
+ when(vlvIndexCfg.getFilter()).thenReturn("(objectClass=person)");
+ when(vlvIndexCfg.getScope()).thenReturn(Scope.WHOLE_SUBTREE);
+ when(vlvIndexCfg.getSortOrder()).thenReturn("sn -employeeNumber +uid");
+ when(backendCfg.getBackendVLVIndex(backendVlvIndexes[0])).thenReturn(vlvIndexCfg);
+
+ backend = new JEBackend();
+ backend.setBackendID(backendCfg.getBackendId());
+ backend.configureBackend(backendCfg, DirectoryServer.getInstance().getServerContext());
+ backend.openBackend();
+
+ entryContainer = backend.getRootContainer().getEntryContainer(testBaseDN);
+ }
+
+ @AfterClass
+ public void cleanUp() throws Exception
+ {
+ backend.finalizeBackend();
+ backend = null;
+ }
+
+ @DataProvider(name="expandIndexData")
+ private Object[][] expandIndexData() {
+ return new Object[][] {
+ { new String[] { "dn2id", "referral" }, new String[] { "dn2id", "referral" } },
+ { new String[] { "vlv.people" }, new String[] { "vlv.people" } },
+ { new String[] { "vlv." }, new String[] { "vlv.people" } },
+ { new String[] { "vlv.*" }, new String[] { "vlv.people" } },
+ { new String[] { "vlv" }, new String[] { "vlv.people" } },
+ { new String[] { "sn", "cn.*" },
+ new String[] { "sn.presence", "sn.caseIgnoreMatch", "sn.caseIgnoreSubstringsMatch:6",
+ "cn.caseIgnoreSubstringsMatch:6" } },
+ { new String[] { "sn.", "cn." },
+ new String[] { "sn.presence", "sn.caseIgnoreMatch", "sn.caseIgnoreSubstringsMatch:6",
+ "cn.caseIgnoreSubstringsMatch:6" } },
+ { new String[] { ".substring", "*.presence" },
+ new String[] { "sn.presence", "sn.caseIgnoreSubstringsMatch:6", "cn.caseIgnoreSubstringsMatch:6",
+ "mail.caseIgnoreIA5SubstringsMatch:6", "telephoneNumber.telephoneNumberSubstringsMatch:6" } },
+ { new String[] { ".caseIgnoreSubstringsMatch" },
+ new String[] { "sn.caseIgnoreSubstringsMatch:6", "cn.caseIgnoreSubstringsMatch:6" } },
+ { new String[] { "dn2id", "uid.caseIgnoreMatch", ".caseIgnoreSubstringsMatch", ".presence" },
+ new String[] { "dn2id", "uid.caseIgnoreMatch", "sn.caseIgnoreSubstringsMatch:6", "sn.presence",
+ "cn.caseIgnoreSubstringsMatch:6" } }
+ };
+ }
+
+ @Test(dataProvider="expandIndexData")
+ public void testCanExpandIndexNames(final String[] indexNames, final String[] expectedExpandedIndexNames)
+ throws InitializationException {
+ final StrategyImpl strategy = new StrategyImpl(serverContext, backend.getRootContainer(), backendCfg);
+ assertThat(strategy.expandIndexNames(entryContainer, asList(indexNames)))
+ .containsExactlyInAnyOrder(expectedExpandedIndexNames);
+ }
+
+ @Test(expectedExceptions=InitializationException.class)
+ public void testThrowOnUnindexedType() throws InitializationException {
+ final StrategyImpl strategy = new StrategyImpl(serverContext, backend.getRootContainer(), backendCfg);
+ strategy.expandIndexNames(entryContainer, asList(".approximate"));
+ }
+
+ @Test(expectedExceptions=InitializationException.class)
+ public void testThrowOnUnrecognizedMatchingRule() throws InitializationException {
+ final StrategyImpl strategy = new StrategyImpl(serverContext, backend.getRootContainer(), backendCfg);
+ strategy.expandIndexNames(entryContainer, asList(".unknown"));
+ }
+
+ @Test(expectedExceptions=InitializationException.class)
+ public void testThrowOnUnrecognizedAttribute() throws InitializationException {
+ final StrategyImpl strategy = new StrategyImpl(serverContext, backend.getRootContainer(), backendCfg);
+ strategy.expandIndexNames(entryContainer, asList("unknown"));
+ }
+
+ @Test(expectedExceptions=InitializationException.class)
+ public void testThrowOnUnrecognizedVlvIndex() throws InitializationException {
+ final StrategyImpl strategy = new StrategyImpl(serverContext, backend.getRootContainer(), backendCfg);
+ strategy.expandIndexNames(entryContainer, asList("vlv.unknown"));
+ }
+
@Test
public void testHeapBuffer()
{
--
Gitblit v1.10.0