From dea70ae4fbbaf9f5343bdf5bce826c5712e748a3 Mon Sep 17 00:00:00 2001
From: Jean-Noel Rouvignac <jean-noel.rouvignac@forgerock.com>
Date: Fri, 12 Dec 2014 15:40:32 +0000
Subject: [PATCH] OPENDJ-1602 (CR-5566) New pluggable storage based backend
---
opendj3-server-dev/src/server/org/opends/server/backends/pluggable/EntryCachePreloader.java | 374 +
opendj3-server-dev/src/server/org/opends/server/backends/pluggable/IndexFilter.java | 394 +
opendj3-server-dev/src/server/org/opends/server/backends/pluggable/NullIndex.java | 266
opendj3-server-dev/src/server/org/opends/server/backends/pluggable/Indexer.java | 86
opendj3-server-dev/src/server/org/opends/server/backends/pluggable/IDSetIterator.java | 133
opendj3-server-dev/src/server/org/opends/server/backends/pluggable/VLVIndex.java | 1428 +++
opendj3-server-dev/src/server/org/opends/server/backends/pluggable/ID2SIndexer.java | 92
opendj3-server-dev/src/server/org/opends/server/backends/pluggable/IndexQueryFactoryImpl.java | 214
opendj3-server-dev/src/server/org/opends/server/backends/pluggable/SortValuesSet.java | 699 +
opendj3-server-dev/src/server/org/opends/server/backends/pluggable/AttributeIndex.java | 1119 +++
opendj3-server-dev/src/server/org/opends/server/backends/pluggable/EntryID.java | 159
opendj3-server-dev/src/server/org/opends/server/backends/pluggable/SortValues.java | 275
opendj3-server-dev/src/server/org/opends/server/backends/pluggable/AttributeIndexer.java | 202
opendj3-server-dev/src/server/org/opends/server/backends/pluggable/PresenceIndexer.java | 117
opendj3-server-dev/src/server/org/opends/server/backends/pluggable/EntryIDSet.java | 676 +
opendj3-server-dev/src/server/org/opends/server/backends/pluggable/ID2Entry.java | 442 +
opendj3-server-dev/src/server/org/opends/server/backends/pluggable/DN2ID.java | 172
opendj3-server-dev/src/server/org/opends/server/backends/pluggable/RootContainer.java | 890 ++
opendj3-server-dev/src/server/org/opends/server/util/StaticUtils.java | 33
opendj3-server-dev/src/server/org/opends/server/backends/pluggable/DataConfig.java | 133
opendj3-server-dev/src/server/org/opends/server/backends/pluggable/ExportJob.java | 311
opendj3-server-dev/src/server/org/opends/server/backends/pluggable/ConfigurableEnvironment.java | 594 +
opendj3-server-dev/src/server/org/opends/server/backends/pluggable/DN2URI.java | 693 +
opendj3-server-dev/src/server/org/opends/server/backends/pluggable/DbPreloadComparator.java | 80
opendj3-server-dev/src/server/org/opends/server/backends/pluggable/JECompressedSchema.java | 311
opendj3-server-dev/src/server/org/opends/server/backends/pluggable/ID2CIndexer.java | 94
opendj3-server-dev/src/server/org/opends/server/backends/pluggable/Index.java | 785 ++
opendj3-server-dev/src/server/org/opends/server/backends/pluggable/State.java | 122
opendj3-server-dev/src/server/org/opends/server/backends/pluggable/VerifyJob.java | 1808 +++++
opendj3-server-dev/src/server/org/opends/server/backends/pluggable/JebFormat.java | 385 +
opendj3-server-dev/src/server/org/opends/server/backends/pluggable/DatabaseContainer.java | 339
opendj3-server-dev/src/server/org/opends/server/backends/pluggable/BackupManager.java | 1255 +++
opendj3-server-dev/src/server/org/opends/server/backends/pluggable/EqualityIndexer.java | 78
opendj3-server-dev/build.xml | 3
opendj3-server-dev/src/server/org/opends/server/backends/pluggable/VerifyConfig.java | 116
opendj3-server-dev/src/server/org/opends/server/backends/pluggable/JebException.java | 90
opendj3-server-dev/src/server/org/opends/server/backends/pluggable/RebuildConfig.java | 295
opendj3-server-dev/src/server/org/opends/server/backends/pluggable/EnvManager.java | 141
opendj3-server-dev/src/server/org/opends/server/backends/pluggable/IndexBuffer.java | 289
opendj3-server-dev/src/server/org/opends/server/backends/pluggable/EntryContainer.java | 3554 +++++++++
opendj3-server-dev/src/server/org/opends/server/backends/pluggable/IndexQuery.java | 213
opendj3-server-dev/src/server/org/opends/server/backends/pluggable/VLVKeyComparator.java | 351
opendj3-server-dev/src/server/org/opends/server/backends/pluggable/BackendImpl.java | 1532 ++++
opendj3-server-dev/src/server/org/opends/server/backends/pluggable/EntryIDSetSorter.java | 271
44 files changed, 21,611 insertions(+), 3 deletions(-)
diff --git a/opendj3-server-dev/build.xml b/opendj3-server-dev/build.xml
index b1220ba..8f319be 100644
--- a/opendj3-server-dev/build.xml
+++ b/opendj3-server-dev/build.xml
@@ -42,6 +42,7 @@
<!-- General server-wide properties -->
<property name="src.dir" location="src/server" />
+ <property name="pluggablebackend.pkg" value="org/opends/server/backends/pluggable" />
<property name="build.dir" location="build" />
<property name="classes.dir" location="${build.dir}/classes" />
<property name="build.lib.dir" location="${build.dir}/lib" />
@@ -678,6 +679,7 @@
<fileset dir="${src.dir}">
<include name="**/*.java"/>
<exclude name="**/PublicAPI.java" />
+ <exclude name="${pluggablebackend.pkg}/*.java" />
</fileset>
<formatter type="plain" />
</checkstyle>
@@ -756,6 +758,7 @@
<mkdir dir="${build.lib.dir}" />
<javac srcdir="${src.dir}:${admin.src.dir}:${msg.src.dir}:${msg.javagen.dir}:${ads.src.dir}:${quicksetup.src.dir}:${guitools.src.dir}"
+ excludes="${pluggablebackend.pkg}/**"
destdir="${classes.dir}">
<classpath>
<fileset refid="opendj.runtime.jars"/>
diff --git a/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/AttributeIndex.java b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/AttributeIndex.java
new file mode 100644
index 0000000..849475b
--- /dev/null
+++ b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/AttributeIndex.java
@@ -0,0 +1,1119 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License"). You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at legal-notices/CDDLv1_0.txt
+ * or http://forgerock.org/license/CDDLv1.0.html.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at legal-notices/CDDLv1_0.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information:
+ * Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ *
+ *
+ * Copyright 2006-2010 Sun Microsystems, Inc.
+ * Portions Copyright 2011-2014 ForgeRock AS
+ * Portions Copyright 2014 Manuel Gaupp
+ */
+package org.opends.server.backends.pluggable;
+
+import java.io.Closeable;
+import java.util.*;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.forgerock.i18n.LocalizableMessage;
+import org.forgerock.i18n.LocalizableMessageBuilder;
+import org.forgerock.i18n.slf4j.LocalizedLogger;
+import org.forgerock.opendj.config.server.ConfigException;
+import org.forgerock.opendj.ldap.Assertion;
+import org.forgerock.opendj.ldap.ByteString;
+import org.forgerock.opendj.ldap.DecodeException;
+import org.forgerock.opendj.ldap.ResultCode;
+import org.forgerock.opendj.ldap.schema.MatchingRule;
+import org.forgerock.opendj.ldap.spi.IndexQueryFactory;
+import org.forgerock.opendj.ldap.spi.IndexingOptions;
+import org.forgerock.util.Utils;
+import org.opends.server.admin.server.ConfigurationChangeListener;
+import org.opends.server.admin.std.meta.LocalDBIndexCfgDefn.IndexType;
+import org.opends.server.admin.std.server.LocalDBIndexCfg;
+import org.opends.server.backends.pluggable.BackendImpl.StorageRuntimeException;
+import org.opends.server.backends.pluggable.BackendImpl.TreeName;
+import org.opends.server.core.DirectoryServer;
+import org.opends.server.monitors.DatabaseEnvironmentMonitor;
+import org.opends.server.types.*;
+import org.opends.server.util.StaticUtils;
+
+import static org.opends.messages.JebMessages.*;
+import static org.opends.server.util.ServerConstants.*;
+import static org.opends.server.util.StaticUtils.*;
+
+/**
+ * Class representing an attribute index.
+ * We have a separate database for each type of indexing, which makes it easy
+ * to tell which attribute indexes are configured. The different types of
+ * indexing are equality, presence, substrings and ordering. The keys in the
+ * ordering index are ordered by setting the btree comparator to the ordering
+ * matching rule comparator.
+ * Note that the values in the equality index are normalized by the equality
+ * matching rule, whereas the values in the ordering index are normalized
+ * by the ordering matching rule. If these could be guaranteed to be identical
+ * then we would not need a separate ordering index.
+ */
+public class AttributeIndex
+ implements ConfigurationChangeListener<LocalDBIndexCfg>, Closeable
+{
+ private static final LocalizedLogger logger = LocalizedLogger.getLoggerForThisClass();
+
+ /** Type of the index filter. */
+ static enum IndexFilterType
+ {
+ /** Equality. */
+ EQUALITY(IndexType.EQUALITY),
+ /** Presence. */
+ PRESENCE(IndexType.PRESENCE),
+ /** Ordering. */
+ GREATER_OR_EQUAL(IndexType.ORDERING),
+ /** Ordering. */
+ LESS_OR_EQUAL(IndexType.ORDERING),
+ /** Substring. */
+ SUBSTRING(IndexType.SUBSTRING),
+ /** Approximate. */
+ APPROXIMATE(IndexType.APPROXIMATE);
+
+ private final IndexType indexType;
+
+ private IndexFilterType(IndexType indexType)
+ {
+ this.indexType = indexType;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public String toString()
+ {
+ return indexType.toString();
+ }
+ }
+
+ /*
+ * FIXME Matthew Swift: Once the matching rules have been migrated we should
+ * revisit this class. All of the evaluateXXX methods should go (the Matcher
+ * class in the SDK could implement the logic, I hope).
+ */
+
+ /** The entryContainer in which this attribute index resides. */
+ private final EntryContainer entryContainer;
+
+ /** The attribute index configuration. */
+ private LocalDBIndexCfg indexConfig;
+
+ /** The mapping from names to indexes. */
+ private final Map<String, Index> nameToIndexes = new HashMap<String, Index>();
+ private final IndexQueryFactory<IndexQuery> indexQueryFactory;
+
+ /**
+ * The mapping from extensible index types (e.g. "substring" or "shared") to list of indexes.
+ */
+ private Map<String, Collection<Index>> extensibleIndexesMapping;
+
+ /**
+ * Create a new attribute index object.
+ *
+ * @param indexConfig The attribute index configuration.
+ * @param entryContainer The entryContainer of this attribute index.
+ * @throws ConfigException if a configuration related error occurs.
+ */
+ public AttributeIndex(LocalDBIndexCfg indexConfig, EntryContainer entryContainer) throws ConfigException
+ {
+ this.entryContainer = entryContainer;
+ this.indexConfig = indexConfig;
+
+ buildPresenceIndex();
+ buildIndexes(IndexType.EQUALITY);
+ buildIndexes(IndexType.SUBSTRING);
+ buildIndexes(IndexType.ORDERING);
+ buildIndexes(IndexType.APPROXIMATE);
+ buildExtensibleIndexes();
+
+ final JEIndexConfig config = new JEIndexConfig(indexConfig.getSubstringLength());
+ indexQueryFactory = new IndexQueryFactoryImpl(nameToIndexes, config);
+ extensibleIndexesMapping = computeExtensibleIndexesMapping();
+ }
+
+ private void buildPresenceIndex()
+ {
+ final IndexType indexType = IndexType.PRESENCE;
+ if (indexConfig.getIndexType().contains(indexType))
+ {
+ String indexID = indexType.toString();
+ nameToIndexes.put(indexID, newPresenceIndex(indexConfig));
+ }
+ }
+
+ private Index newPresenceIndex(LocalDBIndexCfg cfg)
+ {
+ final AttributeType attrType = cfg.getAttribute();
+ final TreeName indexName = getIndexName(attrType, IndexType.PRESENCE.toString());
+ final PresenceIndexer indexer = new PresenceIndexer(attrType);
+ return entryContainer.newIndexForAttribute(indexName, indexer, cfg.getIndexEntryLimit());
+ }
+
+ private void buildExtensibleIndexes() throws ConfigException
+ {
+ final IndexType indexType = IndexType.EXTENSIBLE;
+ if (indexConfig.getIndexType().contains(indexType))
+ {
+ final AttributeType attrType = indexConfig.getAttribute();
+ Set<String> extensibleRules = indexConfig.getIndexExtensibleMatchingRule();
+ if (extensibleRules == null || extensibleRules.isEmpty())
+ {
+ throw new ConfigException(ERR_CONFIG_INDEX_TYPE_NEEDS_MATCHING_RULE.get(attrType, indexType.toString()));
+ }
+
+ // Iterate through the Set and create the index only if necessary.
+ // Collation equality and Ordering matching rules share the same indexer and index
+ // A Collation substring matching rule is treated differently
+ // as it uses a separate indexer and index.
+ for (final String ruleName : extensibleRules)
+ {
+ MatchingRule rule = DirectoryServer.getMatchingRule(toLowerCase(ruleName));
+ if (rule == null)
+ {
+ logger.error(ERR_CONFIG_INDEX_TYPE_NEEDS_VALID_MATCHING_RULE, attrType, ruleName);
+ continue;
+ }
+ for (org.forgerock.opendj.ldap.spi.Indexer indexer : rule.getIndexers())
+ {
+ final String indexId = indexer.getIndexID();
+ if (!nameToIndexes.containsKey(indexId))
+ {
+ // There is no index available for this index id. Create a new index
+ nameToIndexes.put(indexId, newAttributeIndex(indexConfig, indexer));
+ }
+ }
+ }
+ }
+ }
+
+ private void buildIndexes(IndexType indexType) throws ConfigException
+ {
+ if (indexConfig.getIndexType().contains(indexType))
+ {
+ final AttributeType attrType = indexConfig.getAttribute();
+ final String indexID = indexType.toString();
+ final MatchingRule rule = getMatchingRule(indexType, attrType);
+ if (rule == null)
+ {
+ throw new ConfigException(ERR_CONFIG_INDEX_TYPE_NEEDS_MATCHING_RULE.get(attrType, indexID));
+ }
+
+ for (org.forgerock.opendj.ldap.spi.Indexer indexer : rule.getIndexers())
+ {
+ nameToIndexes.put(indexID, newAttributeIndex(indexConfig, indexer));
+ }
+ }
+ }
+
+ private MatchingRule getMatchingRule(IndexType indexType, AttributeType attrType)
+ {
+ switch (indexType)
+ {
+ case APPROXIMATE:
+ return attrType.getApproximateMatchingRule();
+ case EQUALITY:
+ return attrType.getEqualityMatchingRule();
+ case ORDERING:
+ return attrType.getOrderingMatchingRule();
+ case SUBSTRING:
+ return attrType.getSubstringMatchingRule();
+ default:
+ throw new IllegalArgumentException("Not implemented for index type " + indexType);
+ }
+ }
+
+ private Index newAttributeIndex(LocalDBIndexCfg indexConfig, org.forgerock.opendj.ldap.spi.Indexer indexer)
+ {
+ final AttributeType attrType = indexConfig.getAttribute();
+ final TreeName indexName = getIndexName(attrType, indexer.getIndexID());
+ final AttributeIndexer attrIndexer = new AttributeIndexer(attrType, indexer);
+ return entryContainer.newIndexForAttribute(indexName, attrIndexer, indexConfig.getIndexEntryLimit());
+ }
+
+ private TreeName getIndexName(AttributeType attrType, String indexID)
+ {
+ return entryContainer.getDatabasePrefix().child(attrType.getNameOrOID() + "." + indexID);
+ }
+
+ /**
+ * Open the attribute index.
+ *
+ * @throws StorageRuntimeException if a JE database error occurs while
+ * opening the index.
+ */
+ public void open() throws StorageRuntimeException
+ {
+ for (Index index : nameToIndexes.values())
+ {
+ index.open();
+ }
+ indexConfig.addChangeListener(this);
+ }
+
+ /** Closes the attribute index. */
+ @Override
+ public void close()
+ {
+ Utils.closeSilently(nameToIndexes.values());
+ indexConfig.removeChangeListener(this);
+ // The entryContainer is responsible for closing the JE databases.
+ }
+
+ /**
+ * Get the attribute type of this attribute index.
+ * @return The attribute type of this attribute index.
+ */
+ public AttributeType getAttributeType()
+ {
+ return indexConfig.getAttribute();
+ }
+
+ /**
+ * Return the indexing options of this AttributeIndex.
+ *
+ * @return the indexing options of this AttributeIndex.
+ */
+ public IndexingOptions getIndexingOptions()
+ {
+ return indexQueryFactory.getIndexingOptions();
+ }
+
+ /**
+ * Get the JE index configuration used by this index.
+ * @return The configuration in effect.
+ */
+ public LocalDBIndexCfg getConfiguration()
+ {
+ return indexConfig;
+ }
+
+ /**
+ * Update the attribute index for a new entry.
+ *
+ * @param buffer The index buffer to use to store the added keys
+ * @param entryID The entry ID.
+ * @param entry The contents of the new entry.
+ * @throws StorageRuntimeException If an error occurs in the JE database.
+ * @throws DirectoryException If a Directory Server error occurs.
+ */
+ public void addEntry(IndexBuffer buffer, EntryID entryID, Entry entry)
+ throws StorageRuntimeException, DirectoryException
+ {
+ final IndexingOptions options = indexQueryFactory.getIndexingOptions();
+ for (Index index : nameToIndexes.values())
+ {
+ index.addEntry(buffer, entryID, entry, options);
+ }
+ }
+
+ /**
+ * Update the attribute index for a deleted entry.
+ *
+ * @param buffer The index buffer to use to store the deleted keys
+ * @param entryID The entry ID
+ * @param entry The contents of the deleted entry.
+ * @throws StorageRuntimeException If an error occurs in the JE database.
+ * @throws DirectoryException If a Directory Server error occurs.
+ */
+ public void removeEntry(IndexBuffer buffer, EntryID entryID, Entry entry)
+ throws StorageRuntimeException, DirectoryException
+ {
+ final IndexingOptions options = indexQueryFactory.getIndexingOptions();
+ for (Index index : nameToIndexes.values())
+ {
+ index.removeEntry(buffer, entryID, entry, options);
+ }
+ }
+
+ /**
+ * Update the index to reflect a sequence of modifications in a Modify
+ * operation.
+ *
+ * @param buffer The index buffer used to buffer up the index changes.
+ * @param entryID The ID of the entry that was modified.
+ * @param oldEntry The entry before the modifications were applied.
+ * @param newEntry The entry after the modifications were applied.
+ * @param mods The sequence of modifications in the Modify operation.
+ * @throws StorageRuntimeException If an error occurs during an operation on a
+ * JE database.
+ */
+ public void modifyEntry(IndexBuffer buffer,
+ EntryID entryID,
+ Entry oldEntry,
+ Entry newEntry,
+ List<Modification> mods)
+ throws StorageRuntimeException
+ {
+ final IndexingOptions options = indexQueryFactory.getIndexingOptions();
+ for (Index index : nameToIndexes.values())
+ {
+ index.modifyEntry(buffer, entryID, oldEntry, newEntry, mods, options);
+ }
+ }
+
+ /**
+ * Decompose an attribute value into a set of substring index keys.
+ * The ID of the entry containing this value should be inserted
+ * into the list of each of these keys.
+ *
+ * @param normValue A byte array containing the normalized attribute value.
+ * @return A set of index keys.
+ */
+ Set<ByteString> substringKeys(ByteString normValue)
+ { // FIXME replace this code with SDK's
+ // AbstractSubstringMatchingRuleImpl.SubstringIndexer.createKeys()
+
+ // Eliminate duplicates by putting the keys into a set.
+ // Sorting the keys will ensure database record locks are acquired
+ // in a consistent order and help prevent transaction deadlocks between
+ // concurrent writers.
+ Set<ByteString> set = new HashSet<ByteString>();
+
+ int substrLength = indexConfig.getSubstringLength();
+
+ // Example: The value is ABCDE and the substring length is 3.
+ // We produce the keys ABC BCD CDE DE E
+ // To find values containing a short substring such as DE,
+ // iterate through keys with prefix DE. To find values
+ // containing a longer substring such as BCDE, read keys BCD and CDE.
+ for (int i = 0, remain = normValue.length(); remain > 0; i++, remain--)
+ {
+ int len = Math.min(substrLength, remain);
+ set.add(normValue.subSequence(i, i + len));
+ }
+ return set;
+ }
+
+ /**
+ * Retrieve the entry IDs that might match the provided assertion.
+ *
+ * @param indexQuery
+ * The query used to retrieve entries.
+ * @param indexName
+ * The name of index used to retrieve entries.
+ * @param filter
+ * The filter on entries.
+ * @param debugBuffer
+ * If not null, a diagnostic string will be written which will help
+ * determine how the indexes contributed to this search.
+ * @param monitor
+ * The database environment monitor provider that will keep index
+ * filter usage statistics.
+ * @return The candidate entry IDs that might contain the filter assertion
+ * value.
+ */
+ private EntryIDSet evaluateIndexQuery(IndexQuery indexQuery, String indexName, SearchFilter filter,
+ StringBuilder debugBuffer, DatabaseEnvironmentMonitor monitor)
+ {
+ LocalizableMessageBuilder debugMessage = monitor.isFilterUseEnabled() ? new LocalizableMessageBuilder() : null;
+ EntryIDSet results = indexQuery.evaluate(debugMessage);
+
+ if (debugBuffer != null)
+ {
+ debugBuffer.append("[INDEX:").append(indexConfig.getAttribute().getNameOrOID())
+ .append(".").append(indexName).append("]");
+ }
+
+ if (monitor.isFilterUseEnabled())
+ {
+ if (results.isDefined())
+ {
+ monitor.updateStats(filter, results.size());
+ }
+ else
+ {
+ monitor.updateStats(filter, debugMessage.toMessage());
+ }
+ }
+ return results;
+ }
+
+ /**
+ * Retrieve the entry IDs that might match two filters that restrict a value
+ * to both a lower bound and an upper bound.
+ *
+ * @param filter1
+ * The first filter, that is either a less-or-equal filter or a
+ * greater-or-equal filter.
+ * @param filter2
+ * The second filter, that is either a less-or-equal filter or a
+ * greater-or-equal filter. It must not be of the same type than the
+ * first filter.
+ * @param debugBuffer
+ * If not null, a diagnostic string will be written which will help
+ * determine how the indexes contributed to this search.
+ * @param monitor
+ * The database environment monitor provider that will keep index
+ * filter usage statistics.
+ * @return The candidate entry IDs that might contain match both filters.
+ */
+ public EntryIDSet evaluateBoundedRange(SearchFilter filter1, SearchFilter filter2, StringBuilder debugBuffer,
+ DatabaseEnvironmentMonitor monitor)
+ {
+ // TODO : this implementation is not optimal
+ // as it implies two separate evaluations instead of a single one,
+ // thus defeating the purpose of the optimization done
+ // in IndexFilter#evaluateLogicalAndFilter method.
+ // One solution could be to implement a boundedRangeAssertion that combine
+ // the two operations in one.
+ EntryIDSet results = evaluate(filter1, debugBuffer, monitor);
+ EntryIDSet results2 = evaluate(filter2, debugBuffer, monitor);
+ results.retainAll(results2);
+ return results;
+ }
+
+ private EntryIDSet evaluate(SearchFilter filter, StringBuilder debugBuffer, DatabaseEnvironmentMonitor monitor)
+ {
+ boolean isLessOrEqual = filter.getFilterType() == FilterType.LESS_OR_EQUAL;
+ IndexFilterType indexFilterType = isLessOrEqual ? IndexFilterType.LESS_OR_EQUAL : IndexFilterType.GREATER_OR_EQUAL;
+ return evaluateFilter(indexFilterType, filter, debugBuffer, monitor);
+ }
+
+ /**
+ * Retrieve the entry IDs that might match a filter.
+ *
+ * @param indexFilterType the index type filter
+ * @param filter The filter.
+ * @param debugBuffer If not null, a diagnostic string will be written
+ * which will help determine how the indexes contributed
+ * to this search.
+ * @param monitor The database environment monitor provider that will keep
+ * index filter usage statistics.
+ * @return The candidate entry IDs that might contain a value
+ * that matches the filter type.
+ */
+ public EntryIDSet evaluateFilter(IndexFilterType indexFilterType, SearchFilter filter, StringBuilder debugBuffer,
+ DatabaseEnvironmentMonitor monitor)
+ {
+ try
+ {
+ final IndexQuery indexQuery = getIndexQuery(indexFilterType, filter);
+ return evaluateIndexQuery(indexQuery, indexFilterType.toString(), filter, debugBuffer, monitor);
+ }
+ catch (DecodeException e)
+ {
+ logger.traceException(e);
+ return new EntryIDSet();
+ }
+ }
+
+ private IndexQuery getIndexQuery(IndexFilterType indexFilterType, SearchFilter filter) throws DecodeException
+ {
+ MatchingRule rule;
+ Assertion assertion;
+ switch (indexFilterType)
+ {
+ case EQUALITY:
+ rule = filter.getAttributeType().getEqualityMatchingRule();
+ assertion = rule.getAssertion(filter.getAssertionValue());
+ return assertion.createIndexQuery(indexQueryFactory);
+
+ case PRESENCE:
+ return indexQueryFactory.createMatchAllQuery();
+
+ case GREATER_OR_EQUAL:
+ rule = filter.getAttributeType().getOrderingMatchingRule();
+ assertion = rule.getGreaterOrEqualAssertion(filter.getAssertionValue());
+ return assertion.createIndexQuery(indexQueryFactory);
+
+ case LESS_OR_EQUAL:
+ rule = filter.getAttributeType().getOrderingMatchingRule();
+ assertion = rule.getLessOrEqualAssertion(filter.getAssertionValue());
+ return assertion.createIndexQuery(indexQueryFactory);
+
+ case SUBSTRING:
+ rule = filter.getAttributeType().getSubstringMatchingRule();
+ assertion = rule.getSubstringAssertion(
+ filter.getSubInitialElement(), filter.getSubAnyElements(), filter.getSubFinalElement());
+ return assertion.createIndexQuery(indexQueryFactory);
+
+ case APPROXIMATE:
+ rule = filter.getAttributeType().getApproximateMatchingRule();
+ assertion = rule.getAssertion(filter.getAssertionValue());
+ return assertion.createIndexQuery(indexQueryFactory);
+
+ default:
+ return null;
+ }
+ }
+
+ /**
+ * Return the number of values that have exceeded the entry limit since this
+ * object was created.
+ *
+ * @return The number of values that have exceeded the entry limit.
+ */
+ public long getEntryLimitExceededCount()
+ {
+ long entryLimitExceededCount = 0;
+
+ for (Index index : nameToIndexes.values())
+ {
+ entryLimitExceededCount += index.getEntryLimitExceededCount();
+ }
+ return entryLimitExceededCount;
+ }
+
+ /**
+ * Get a list of the databases opened by this attribute index.
+ * @param dbList A list of database containers.
+ */
+ public void listDatabases(List<DatabaseContainer> dbList)
+ {
+ dbList.addAll(nameToIndexes.values());
+ }
+
+ /**
+ * Get a string representation of this object.
+ * @return return A string representation of this object.
+ */
+ @Override
+ public String toString()
+ {
+ return getName();
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public synchronized boolean isConfigurationChangeAcceptable(
+ LocalDBIndexCfg cfg, List<LocalizableMessage> unacceptableReasons)
+ {
+ if (!isIndexAcceptable(cfg, IndexType.EQUALITY, unacceptableReasons)
+ || !isIndexAcceptable(cfg, IndexType.SUBSTRING, unacceptableReasons)
+ || !isIndexAcceptable(cfg, IndexType.ORDERING, unacceptableReasons)
+ || !isIndexAcceptable(cfg, IndexType.APPROXIMATE, unacceptableReasons))
+ {
+ return false;
+ }
+
+ AttributeType attrType = cfg.getAttribute();
+ if (cfg.getIndexType().contains(IndexType.EXTENSIBLE))
+ {
+ Set<String> newRules = cfg.getIndexExtensibleMatchingRule();
+ if (newRules == null || newRules.isEmpty())
+ {
+ unacceptableReasons.add(ERR_CONFIG_INDEX_TYPE_NEEDS_MATCHING_RULE.get(attrType, "extensible"));
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private boolean isIndexAcceptable(LocalDBIndexCfg cfg, IndexType indexType,
+ List<LocalizableMessage> unacceptableReasons)
+ {
+ final String indexId = indexType.toString();
+ final AttributeType attrType = cfg.getAttribute();
+ if (cfg.getIndexType().contains(indexType)
+ && nameToIndexes.get(indexId) == null
+ && getMatchingRule(indexType, attrType) == null)
+ {
+ unacceptableReasons.add(ERR_CONFIG_INDEX_TYPE_NEEDS_MATCHING_RULE.get(attrType, indexId));
+ return false;
+ }
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public synchronized ConfigChangeResult applyConfigurationChange(LocalDBIndexCfg cfg)
+ {
+ // this method is not perf sensitive, using an AtomicBoolean will not hurt
+ AtomicBoolean adminActionRequired = new AtomicBoolean(false);
+ ArrayList<LocalizableMessage> messages = new ArrayList<LocalizableMessage>();
+ try
+ {
+ applyChangeToPresenceIndex(cfg, adminActionRequired, messages);
+ applyChangeToIndex(IndexType.EQUALITY, cfg, adminActionRequired, messages);
+ applyChangeToIndex(IndexType.SUBSTRING, cfg, adminActionRequired, messages);
+ applyChangeToIndex(IndexType.ORDERING, cfg, adminActionRequired, messages);
+ applyChangeToIndex(IndexType.APPROXIMATE, cfg, adminActionRequired, messages);
+ applyChangeToExtensibleIndexes(cfg, adminActionRequired, messages);
+
+ extensibleIndexesMapping = computeExtensibleIndexesMapping();
+ indexConfig = cfg;
+
+ return new ConfigChangeResult(ResultCode.SUCCESS, adminActionRequired.get(), messages);
+ }
+ catch(Exception e)
+ {
+ messages.add(LocalizableMessage.raw(StaticUtils.stackTraceToSingleLineString(e)));
+ return new ConfigChangeResult(
+ DirectoryServer.getServerErrorResultCode(), adminActionRequired.get(), messages);
+ }
+ }
+
+ private void applyChangeToExtensibleIndexes(LocalDBIndexCfg cfg,
+ AtomicBoolean adminActionRequired, ArrayList<LocalizableMessage> messages)
+ {
+ final AttributeType attrType = cfg.getAttribute();
+ if (!cfg.getIndexType().contains(IndexType.EXTENSIBLE))
+ {
+ final Set<MatchingRule> validRules = Collections.emptySet();
+ final Set<String> validIndexIds = Collections.emptySet();
+ removeIndexesForExtensibleMatchingRules(validRules, validIndexIds);
+ return;
+ }
+
+ final Set<String> extensibleRules = cfg.getIndexExtensibleMatchingRule();
+ final Set<MatchingRule> validRules = new HashSet<MatchingRule>();
+ final Set<String> validIndexIds = new HashSet<String>();
+ final int indexEntryLimit = cfg.getIndexEntryLimit();
+
+ for (String ruleName : extensibleRules)
+ {
+ MatchingRule rule = DirectoryServer.getMatchingRule(toLowerCase(ruleName));
+ if (rule == null)
+ {
+ logger.error(ERR_CONFIG_INDEX_TYPE_NEEDS_VALID_MATCHING_RULE, attrType, ruleName);
+ continue;
+ }
+ validRules.add(rule);
+ for (org.forgerock.opendj.ldap.spi.Indexer indexer : rule.getIndexers())
+ {
+ String indexId = indexer.getIndexID();
+ validIndexIds.add(indexId);
+ if (!nameToIndexes.containsKey(indexId))
+ {
+ Index index = newAttributeIndex(cfg, indexer);
+ openIndex(index, adminActionRequired, messages);
+ nameToIndexes.put(indexId, index);
+ }
+ else
+ {
+ Index index = nameToIndexes.get(indexId);
+ if (index.setIndexEntryLimit(indexEntryLimit))
+ {
+ adminActionRequired.set(true);
+ messages.add(NOTE_JEB_CONFIG_INDEX_ENTRY_LIMIT_REQUIRES_REBUILD.get(index.getName()));
+ }
+ if (indexConfig.getSubstringLength() != cfg.getSubstringLength())
+ {
+ index.setIndexer(new AttributeIndexer(attrType, indexer));
+ }
+ }
+ }
+ }
+ removeIndexesForExtensibleMatchingRules(validRules, validIndexIds);
+ }
+
+ /** Remove indexes which do not correspond to valid rules. */
+ private void removeIndexesForExtensibleMatchingRules(Set<MatchingRule> validRules, Set<String> validIndexIds)
+ {
+ final Set<MatchingRule> rulesToDelete = getCurrentExtensibleMatchingRules();
+ rulesToDelete.removeAll(validRules);
+ if (!rulesToDelete.isEmpty())
+ {
+ entryContainer.exclusiveLock.lock();
+ try
+ {
+ for (MatchingRule rule: rulesToDelete)
+ {
+ final List<String> indexIdsToRemove = new ArrayList<String>();
+ for (org.forgerock.opendj.ldap.spi.Indexer indexer : rule.getIndexers())
+ {
+ final String indexId = indexer.getIndexID();
+ if (!validIndexIds.contains(indexId))
+ {
+ indexIdsToRemove.add(indexId);
+ }
+ }
+ // Delete indexes which are not used
+ for (String indexId : indexIdsToRemove)
+ {
+ Index index = nameToIndexes.get(indexId);
+ if (index != null)
+ {
+ entryContainer.deleteDatabase(index);
+ nameToIndexes.remove(index);
+ }
+ }
+ }
+ }
+ finally
+ {
+ entryContainer.exclusiveLock.unlock();
+ }
+ }
+ }
+
+ private Set<MatchingRule> getCurrentExtensibleMatchingRules()
+ {
+ final Set<MatchingRule> rules = new HashSet<MatchingRule>();
+ for (String ruleName : indexConfig.getIndexExtensibleMatchingRule())
+ {
+ final MatchingRule rule = DirectoryServer.getMatchingRule(toLowerCase(ruleName));
+ if (rule != null)
+ {
+ rules.add(rule);
+ }
+ }
+ return rules;
+ }
+
+ private void applyChangeToIndex(IndexType indexType, LocalDBIndexCfg cfg,
+ AtomicBoolean adminActionRequired, ArrayList<LocalizableMessage> messages)
+ {
+ String indexId = indexType.toString();
+ Index index = nameToIndexes.get(indexId);
+ if (!cfg.getIndexType().contains(indexType))
+ {
+ removeIndex(index, indexType);
+ return;
+ }
+
+ if (index == null)
+ {
+ final MatchingRule matchingRule = getMatchingRule(indexType, cfg.getAttribute());
+ for (org.forgerock.opendj.ldap.spi.Indexer indexer : matchingRule.getIndexers())
+ {
+ index = newAttributeIndex(cfg, indexer);
+ openIndex(index, adminActionRequired, messages);
+ nameToIndexes.put(indexId, index);
+ }
+ }
+ else
+ {
+ // already exists. Just update index entry limit.
+ if (index.setIndexEntryLimit(cfg.getIndexEntryLimit()))
+ {
+ adminActionRequired.set(true);
+ messages.add(NOTE_JEB_CONFIG_INDEX_ENTRY_LIMIT_REQUIRES_REBUILD.get(index.getName()));
+ }
+ }
+ }
+
+ private void applyChangeToPresenceIndex(LocalDBIndexCfg cfg, AtomicBoolean adminActionRequired,
+ ArrayList<LocalizableMessage> messages)
+ {
+ final IndexType indexType = IndexType.PRESENCE;
+ final String indexID = indexType.toString();
+ Index index = nameToIndexes.get(indexID);
+ if (!cfg.getIndexType().contains(indexType))
+ {
+ removeIndex(index, indexType);
+ return;
+ }
+
+ if (index == null)
+ {
+ index = newPresenceIndex(cfg);
+ openIndex(index, adminActionRequired, messages);
+ nameToIndexes.put(indexID, index);
+ }
+ else
+ {
+ // already exists. Just update index entry limit.
+ if (index.setIndexEntryLimit(cfg.getIndexEntryLimit()))
+ {
+ adminActionRequired.set(true);
+ messages.add(NOTE_JEB_CONFIG_INDEX_ENTRY_LIMIT_REQUIRES_REBUILD.get(index.getName()));
+ }
+ }
+ }
+
+ private void removeIndex(Index index, IndexType indexType)
+ {
+ if (index != null)
+ {
+ entryContainer.exclusiveLock.lock();
+ try
+ {
+ nameToIndexes.remove(indexType.toString());
+ entryContainer.deleteDatabase(index);
+ }
+ finally
+ {
+ entryContainer.exclusiveLock.unlock();
+ }
+ }
+ }
+
+ private void openIndex(Index index, AtomicBoolean adminActionRequired, ArrayList<LocalizableMessage> messages)
+ {
+ index.open();
+
+ if (!index.isTrusted())
+ {
+ adminActionRequired.set(true);
+ messages.add(NOTE_JEB_INDEX_ADD_REQUIRES_REBUILD.get(index.getName()));
+ }
+ }
+
+ /**
+ * Return true iff this index is trusted.
+ * @return the trusted state of this index
+ */
+ public boolean isTrusted()
+ {
+ for (Index index : nameToIndexes.values())
+ {
+ if (!index.isTrusted())
+ {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Set the rebuild status of this index.
+ * @param rebuildRunning True if a rebuild process on this index
+ * is running or False otherwise.
+ */
+ public synchronized void setRebuildStatus(boolean rebuildRunning)
+ {
+ for (Index index : nameToIndexes.values())
+ {
+ index.setRebuildStatus(rebuildRunning);
+ }
+ }
+
+ /**
+ * Get the JE database name prefix for indexes in this attribute index.
+ *
+ * @return JE database name for this database container.
+ */
+ public String getName()
+ {
+ return entryContainer.getDatabasePrefix()
+ + "_"
+ + indexConfig.getAttribute().getNameOrOID();
+ }
+
+ /**
+ * Return the equality index.
+ *
+ * @return The equality index.
+ */
+ public Index getEqualityIndex() {
+ return nameToIndexes.get(IndexType.EQUALITY.toString());
+ }
+
+ /**
+ * Return the approximate index.
+ *
+ * @return The approximate index.
+ */
+ public Index getApproximateIndex() {
+ return nameToIndexes.get(IndexType.APPROXIMATE.toString());
+ }
+
+ /**
+ * Return the ordering index.
+ *
+ * @return The ordering index.
+ */
+ public Index getOrderingIndex() {
+ return nameToIndexes.get(IndexType.ORDERING.toString());
+ }
+
+ /**
+ * Return the substring index.
+ *
+ * @return The substring index.
+ */
+ public Index getSubstringIndex() {
+ return nameToIndexes.get(IndexType.SUBSTRING.toString());
+ }
+
+ /**
+ * Return the presence index.
+ *
+ * @return The presence index.
+ */
+ public Index getPresenceIndex() {
+ return nameToIndexes.get(IndexType.PRESENCE.toString());
+ }
+
+ /**
+ * Return the mapping of extensible index types and indexes.
+ *
+ * @return The map containing entries (extensible index type, list of indexes)
+ */
+ public Map<String, Collection<Index>> getExtensibleIndexes()
+ {
+ return extensibleIndexesMapping;
+ }
+
+ private Map<String, Collection<Index>> computeExtensibleIndexesMapping()
+ {
+ final Collection<Index> substring = new ArrayList<Index>();
+ final Collection<Index> shared = new ArrayList<Index>();
+ for (Map.Entry<String, Index> entry : nameToIndexes.entrySet())
+ {
+ final String indexId = entry.getKey();
+ if (isDefaultIndex(indexId)) {
+ continue;
+ }
+ if (indexId.endsWith(EXTENSIBLE_INDEXER_ID_SUBSTRING))
+ {
+ substring.add(entry.getValue());
+ }
+ else
+ {
+ shared.add(entry.getValue());
+ }
+ }
+ final Map<String, Collection<Index>> indexMap = new HashMap<String,Collection<Index>>();
+ indexMap.put(EXTENSIBLE_INDEXER_ID_SUBSTRING, substring);
+ indexMap.put(EXTENSIBLE_INDEXER_ID_SHARED, shared);
+ return Collections.unmodifiableMap(indexMap);
+ }
+
+ private boolean isDefaultIndex(String indexId)
+ {
+ return indexId.equals(IndexType.EQUALITY.toString())
+ || indexId.equals(IndexType.PRESENCE.toString())
+ || indexId.equals(IndexType.SUBSTRING.toString())
+ || indexId.equals(IndexType.ORDERING.toString())
+ || indexId.equals(IndexType.APPROXIMATE.toString());
+ }
+
+ /**
+ * Retrieves all the indexes used by this attribute index.
+ *
+ * @return A collection of all indexes in use by this attribute
+ * index.
+ */
+ public Collection<Index> getAllIndexes() {
+ return new LinkedHashSet<Index>(nameToIndexes.values());
+ }
+
+ /**
+ * Retrieve the entry IDs that might match an extensible filter.
+ *
+ * @param filter The extensible filter.
+ * @param debugBuffer If not null, a diagnostic string will be written
+ * which will help determine how the indexes contributed
+ * to this search.
+ * @param monitor The database environment monitor provider that will keep
+ * index filter usage statistics.
+ * @return The candidate entry IDs that might contain the filter
+ * assertion value.
+ */
+ public EntryIDSet evaluateExtensibleFilter(SearchFilter filter,
+ StringBuilder debugBuffer,
+ DatabaseEnvironmentMonitor monitor)
+ {
+ //Get the Matching Rule OID of the filter.
+ String matchRuleOID = filter.getMatchingRuleID();
+ /**
+ * Use the default equality index in two conditions:
+ * 1. There is no matching rule provided
+ * 2. The matching rule specified is actually the default equality.
+ */
+ MatchingRule eqRule = indexConfig.getAttribute().getEqualityMatchingRule();
+ if (matchRuleOID == null
+ || matchRuleOID.equals(eqRule.getOID())
+ || matchRuleOID.equalsIgnoreCase(eqRule.getNameOrOID()))
+ {
+ //No matching rule is defined; use the default equality matching rule.
+ return evaluateFilter(IndexFilterType.EQUALITY, filter, debugBuffer, monitor);
+ }
+
+ MatchingRule rule = DirectoryServer.getMatchingRule(matchRuleOID);
+ if (!ruleHasAtLeasOneIndex(rule))
+ {
+ if (monitor.isFilterUseEnabled())
+ {
+ monitor.updateStats(filter, INFO_JEB_INDEX_FILTER_MATCHING_RULE_NOT_INDEXED.get(
+ matchRuleOID, indexConfig.getAttribute().getNameOrOID()));
+ }
+ return IndexQuery.createNullIndexQuery().evaluate(null);
+ }
+
+ try
+ {
+ if (debugBuffer != null)
+ {
+ debugBuffer.append("[INDEX:");
+ for (org.forgerock.opendj.ldap.spi.Indexer indexer : rule.getIndexers())
+ {
+ debugBuffer.append(" ")
+ .append(filter.getAttributeType().getNameOrOID())
+ .append(".")
+ .append(indexer.getIndexID());
+ }
+ debugBuffer.append("]");
+ }
+
+ final IndexQuery indexQuery = rule.getAssertion(filter.getAssertionValue()).createIndexQuery(indexQueryFactory);
+ LocalizableMessageBuilder debugMessage = monitor.isFilterUseEnabled() ? new LocalizableMessageBuilder() : null;
+ EntryIDSet results = indexQuery.evaluate(debugMessage);
+ if (monitor.isFilterUseEnabled())
+ {
+ if (results.isDefined())
+ {
+ monitor.updateStats(filter, results.size());
+ }
+ else
+ {
+ monitor.updateStats(filter, debugMessage.toMessage());
+ }
+ }
+ return results;
+ }
+ catch (DecodeException e)
+ {
+ logger.traceException(e);
+ return IndexQuery.createNullIndexQuery().evaluate(null);
+ }
+ }
+
+ private boolean ruleHasAtLeasOneIndex(MatchingRule rule)
+ {
+ for (org.forgerock.opendj.ldap.spi.Indexer indexer : rule.getIndexers())
+ {
+ if (nameToIndexes.containsKey(indexer.getIndexID()))
+ {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * This class extends the IndexConfig for JE Backend.
+ */
+ private class JEIndexConfig implements IndexingOptions
+ {
+ /** The length of the substring index. */
+ private int substringLength;
+
+ /**
+ * Creates a new JEIndexConfig instance.
+ * @param substringLength The length of the substring.
+ */
+ private JEIndexConfig(int substringLength)
+ {
+ this.substringLength = substringLength;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public int substringKeySize()
+ {
+ return substringLength;
+ }
+ }
+}
diff --git a/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/AttributeIndexer.java b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/AttributeIndexer.java
new file mode 100644
index 0000000..76cae7a
--- /dev/null
+++ b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/AttributeIndexer.java
@@ -0,0 +1,202 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License"). You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at legal-notices/CDDLv1_0.txt
+ * or http://forgerock.org/license/CDDLv1.0.html.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at legal-notices/CDDLv1_0.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information:
+ * Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ *
+ *
+ * Copyright 2009-2010 Sun Microsystems, Inc.
+ * Portions Copyright 2014 ForgeRock AS
+ */
+package org.opends.server.backends.pluggable;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.forgerock.i18n.slf4j.LocalizedLogger;
+import org.forgerock.opendj.ldap.ByteString;
+import org.forgerock.opendj.ldap.DecodeException;
+import org.forgerock.opendj.ldap.schema.Schema;
+import org.forgerock.opendj.ldap.spi.IndexingOptions;
+import org.opends.server.types.Attribute;
+import org.opends.server.types.AttributeType;
+import org.opends.server.types.Entry;
+import org.opends.server.types.Modification;
+
+/**
+ * This class implements an attribute indexer for matching rules in JE Backend.
+ */
+public final class AttributeIndexer extends Indexer
+{
+ private static final LocalizedLogger logger = LocalizedLogger.getLoggerForThisClass();
+
+ /** The attribute type for which this instance will generate index keys. */
+ private final AttributeType attributeType;
+
+ /**
+ * The indexer which will generate the keys
+ * for the associated extensible matching rule.
+ */
+ private final org.forgerock.opendj.ldap.spi.Indexer indexer;
+
+ /**
+ * Creates a new extensible indexer for JE backend.
+ *
+ * @param attributeType The attribute type for which an indexer is
+ * required.
+ * @param extensibleIndexer The extensible indexer to be used.
+ */
+ public AttributeIndexer(AttributeType attributeType, org.forgerock.opendj.ldap.spi.Indexer extensibleIndexer)
+ {
+ this.attributeType = attributeType;
+ this.indexer = extensibleIndexer;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public String toString()
+ {
+ return attributeType.getNameOrOID() + "." + indexer.getIndexID();
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void indexEntry(Entry entry, Set<ByteString> keys, IndexingOptions options)
+ {
+ List<Attribute> attrList = entry.getAttribute(attributeType);
+ if (attrList != null)
+ {
+ indexAttribute(attrList, keys, options);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void replaceEntry(Entry oldEntry, Entry newEntry,
+ Map<ByteString, Boolean> modifiedKeys, IndexingOptions options)
+ {
+ List<Attribute> newAttributes = newEntry.getAttribute(attributeType, true);
+ List<Attribute> oldAttributes = oldEntry.getAttribute(attributeType, true);
+
+ indexAttribute(oldAttributes, modifiedKeys, false, options);
+ indexAttribute(newAttributes, modifiedKeys, true, options);
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void modifyEntry(Entry oldEntry, Entry newEntry,
+ List<Modification> mods, Map<ByteString, Boolean> modifiedKeys,
+ IndexingOptions options)
+ {
+ List<Attribute> newAttributes = newEntry.getAttribute(attributeType, true);
+ List<Attribute> oldAttributes = oldEntry.getAttribute(attributeType, true);
+
+ indexAttribute(oldAttributes, modifiedKeys, false, options);
+ indexAttribute(newAttributes, modifiedKeys, true, options);
+ }
+
+
+
+ /**
+ * Generates the set of extensible index keys for an attribute.
+ * @param attrList The attribute for which substring keys are required.
+ * @param keys The set into which the generated keys will be inserted.
+ */
+ private void indexAttribute(List<Attribute> attrList, Set<ByteString> keys,
+ IndexingOptions options)
+ {
+ if (attrList == null)
+ {
+ return;
+ }
+
+ for (Attribute attr : attrList)
+ {
+ if (!attr.isVirtual())
+ {
+ for (ByteString value : attr)
+ {
+ try
+ {
+ indexer.createKeys(Schema.getDefaultSchema(), value, options, keys);
+ }
+ catch (DecodeException e)
+ {
+ logger.traceException(e);
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Generates the set of index keys for an attribute.
+ * @param attrList The attribute to be indexed.
+ * @param modifiedKeys The map into which the modified
+ * keys will be inserted.
+ * @param insert <code>true</code> if generated keys should
+ * be inserted or <code>false</code> otherwise.
+ */
+ private void indexAttribute(List<Attribute> attrList,
+ Map<ByteString, Boolean> modifiedKeys, Boolean insert,
+ IndexingOptions options)
+ {
+ if (attrList == null)
+ {
+ return;
+ }
+
+ final Set<ByteString> keys = new HashSet<ByteString>();
+ indexAttribute(attrList, keys, options);
+ computeModifiedKeys(modifiedKeys, insert, keys);
+ }
+
+ /**
+ * Computes a map of index keys and a boolean flag indicating whether the
+ * corresponding key will be inserted or deleted.
+ *
+ * @param modifiedKeys
+ * A map containing the keys and a boolean. Keys corresponding to the
+ * boolean value <code>true</code> should be inserted and
+ * <code>false</code> should be deleted.
+ * @param insert
+ * <code>true</code> if generated keys should be inserted or
+ * <code>false</code> otherwise.
+ * @param keys
+ * The index keys to map.
+ */
+ private static void computeModifiedKeys(Map<ByteString, Boolean> modifiedKeys,
+ Boolean insert, Set<ByteString> keys)
+ {
+ for (ByteString key : keys)
+ {
+ Boolean cInsert = modifiedKeys.get(key);
+ if (cInsert == null)
+ {
+ modifiedKeys.put(key, insert);
+ }
+ else if (!cInsert.equals(insert))
+ {
+ modifiedKeys.remove(key);
+ }
+ }
+ }
+}
diff --git a/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/BackendImpl.java b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/BackendImpl.java
new file mode 100644
index 0000000..718e88d
--- /dev/null
+++ b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/BackendImpl.java
@@ -0,0 +1,1532 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License"). You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at legal-notices/CDDLv1_0.txt
+ * or http://forgerock.org/license/CDDLv1.0.html.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at legal-notices/CDDLv1_0.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information:
+ * Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ *
+ *
+ * Copyright 2007-2010 Sun Microsystems, Inc.
+ * Portions Copyright 2013-2014 ForgeRock AS
+ */
+package org.opends.server.backends.pluggable;
+
+import java.io.Closeable;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.util.*;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.logging.Level;
+import java.util.zip.Adler32;
+import java.util.zip.CheckedInputStream;
+
+import org.forgerock.i18n.LocalizableMessage;
+import org.forgerock.i18n.slf4j.LocalizedLogger;
+import org.forgerock.opendj.config.server.ConfigException;
+import org.forgerock.opendj.ldap.ByteSequence;
+import org.forgerock.opendj.ldap.ByteString;
+import org.forgerock.opendj.ldap.ConditionResult;
+import org.forgerock.opendj.ldap.ResultCode;
+import org.forgerock.util.Reject;
+import org.opends.server.admin.server.ConfigurationChangeListener;
+import org.opends.server.admin.std.meta.LocalDBIndexCfgDefn;
+import org.opends.server.admin.std.server.LocalDBBackendCfg;
+import org.opends.server.api.AlertGenerator;
+import org.opends.server.api.Backend;
+import org.opends.server.api.DiskSpaceMonitorHandler;
+import org.opends.server.api.MonitorProvider;
+import org.opends.server.core.*;
+import org.opends.server.extensions.DiskSpaceMonitor;
+import org.opends.server.types.*;
+import org.opends.server.util.RuntimeInformation;
+
+import com.sleepycat.je.Durability;
+import com.sleepycat.je.EnvironmentConfig;
+
+import static com.sleepycat.je.EnvironmentConfig.*;
+
+import static org.opends.messages.BackendMessages.*;
+import static org.opends.messages.JebMessages.*;
+import static org.opends.server.backends.jeb.ConfigurableEnvironment.*;
+import static org.opends.server.util.ServerConstants.*;
+import static org.opends.server.util.StaticUtils.*;
+
+/**
+ * This is an implementation of a Directory Server Backend which stores entries
+ * locally in a Berkeley DB JE database.
+ */
+public class BackendImpl extends Backend<LocalDBBackendCfg>
+ implements ConfigurationChangeListener<LocalDBBackendCfg>, AlertGenerator,
+ DiskSpaceMonitorHandler
+{
+ private static final LocalizedLogger logger = LocalizedLogger.getLoggerForThisClass();
+
+ interface Importer extends Closeable
+ {
+ void createTree(TreeName name);
+
+ void put(TreeName name, ByteSequence key, ByteSequence value);
+
+ @Override
+ void close();
+ }
+
+ interface ReadOperation<T>
+ {
+ T run(ReadableStorage txn) throws Exception;
+ }
+
+ interface ReadableStorage
+ {
+ ByteString get(TreeName name, ByteSequence key);
+
+ ByteString getRMW(TreeName name, ByteSequence key);
+
+ Cursor openCursor(TreeName name);
+
+ // TODO: contains, etc.
+ }
+
+ interface Cursor extends Closeable
+ {
+ boolean positionToKey(ByteSequence key);
+
+ boolean positionToKeyOrNext(ByteSequence key);
+
+ boolean positionToLastKey();
+
+ boolean next();
+
+ boolean previous();
+
+ ByteString getKey();
+
+ ByteString getValue();
+
+ @Override
+ public void close();
+ }
+
+ interface Storage extends Closeable
+ {
+ void initialize(Map<String, String> options) throws Exception;
+
+ Importer startImport() throws Exception;
+
+ void open() throws Exception;
+
+ void openTree(TreeName name);
+
+ <T> T read(ReadOperation<T> readTransaction) throws Exception;
+
+ void update(WriteOperation updateTransaction) throws Exception;
+
+ Cursor openCursor(TreeName name);
+
+ @Override
+ void close();
+ }
+
+ @SuppressWarnings("serial")
+ static final class StorageRuntimeException extends RuntimeException
+ {
+
+ public StorageRuntimeException(final String message)
+ {
+ super(message);
+ }
+
+ public StorageRuntimeException(final String message, final Throwable cause)
+ {
+ super(message, cause);
+ }
+
+ public StorageRuntimeException(final Throwable cause)
+ {
+ super(cause);
+ }
+ }
+
+ /** Assumes name components don't contain a '/'. */
+ static final class TreeName
+ {
+ public static TreeName of(final String... names)
+ {
+ return new TreeName(Arrays.asList(names));
+ }
+
+ private final List<String> names;
+ private final String s;
+
+ public TreeName(final List<String> names)
+ {
+ this.names = names;
+ final StringBuilder builder = new StringBuilder();
+ for (final String name : names)
+ {
+ builder.append('/');
+ builder.append(name);
+ }
+ this.s = builder.toString();
+ }
+
+ public List<String> getNames()
+ {
+ return names;
+ }
+
+ public TreeName child(final String name)
+ {
+ final List<String> newNames = new ArrayList<String>(names.size() + 1);
+ newNames.addAll(names);
+ newNames.add(name);
+ return new TreeName(newNames);
+ }
+
+ public TreeName getSuffix()
+ {
+ if (names.size() == 0)
+ {
+ throw new IllegalStateException();
+ }
+ return new TreeName(Collections.singletonList(names.get(0)));
+ }
+
+ public boolean isSuffixOf(TreeName tree)
+ {
+ if (names.size() > tree.names.size())
+ {
+ return false;
+ }
+ for (int i = 0; i < names.size(); i++)
+ {
+ if (!tree.names.get(i).equals(names.get(i)))
+ {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ @Override
+ public boolean equals(final Object obj)
+ {
+ if (this == obj)
+ {
+ return true;
+ }
+ else if (obj instanceof TreeName)
+ {
+ return s.equals(((TreeName) obj).s);
+ }
+ else
+ {
+ return false;
+ }
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return s.hashCode();
+ }
+
+ @Override
+ public String toString()
+ {
+ return s;
+ }
+ }
+
+ interface WriteOperation
+ {
+ void run(WriteableStorage txn) throws Exception;
+ }
+
+ interface WriteableStorage extends ReadableStorage
+ {
+ void put(TreeName name, ByteSequence key, ByteSequence value);
+
+ boolean putIfAbsent(TreeName treeName, ByteSequence key, ByteSequence value);
+
+ boolean remove(TreeName name, ByteSequence key);
+
+ boolean remove(TreeName name, ByteSequence key, ByteSequence value);
+ }
+
+ /** The configuration of this JE backend. */
+ private LocalDBBackendCfg cfg;
+ /** The root JE container to use for this backend. */
+ private RootContainer rootContainer;
+ /** A count of the total operation threads currently in the backend. */
+ private final AtomicInteger threadTotalCount = new AtomicInteger(0);
+ /** A count of the write operation threads currently in the backend. */
+ private final AtomicInteger threadWriteCount = new AtomicInteger(0);
+ /** The base DNs defined for this backend instance. */
+ private DN[] baseDNs;
+
+ private MonitorProvider<?> rootContainerMonitor;
+ private DiskSpaceMonitor diskMonitor;
+
+ /**
+ * The controls supported by this backend.
+ */
+ private static final Set<String> supportedControls = new HashSet<String>(Arrays.asList(
+ OID_SUBTREE_DELETE_CONTROL,
+ OID_PAGED_RESULTS_CONTROL,
+ OID_MANAGE_DSAIT_CONTROL,
+ OID_SERVER_SIDE_SORT_REQUEST_CONTROL,
+ OID_VLV_REQUEST_CONTROL));
+
+ /** Begin a Backend API method that reads the database. */
+ private void readerBegin()
+ {
+ threadTotalCount.getAndIncrement();
+ }
+
+ /** End a Backend API method that reads the database. */
+ private void readerEnd()
+ {
+ threadTotalCount.getAndDecrement();
+ }
+
+ /** Begin a Backend API method that writes the database. */
+ private void writerBegin()
+ {
+ threadTotalCount.getAndIncrement();
+ threadWriteCount.getAndIncrement();
+ }
+
+ /** End a Backend API method that writes the database. */
+ private void writerEnd()
+ {
+ threadWriteCount.getAndDecrement();
+ threadTotalCount.getAndDecrement();
+ }
+
+
+
+ /**
+ * Wait until there are no more threads accessing the database. It is assumed
+ * that new threads have been prevented from entering the database at the time
+ * this method is called.
+ */
+ private void waitUntilQuiescent()
+ {
+ while (threadTotalCount.get() > 0)
+ {
+ // Still have threads in the database so sleep a little
+ try
+ {
+ Thread.sleep(500);
+ }
+ catch (InterruptedException e)
+ {
+ logger.traceException(e);
+ }
+ }
+ }
+
+ /**
+ * This method will attempt to checksum the current JE db environment by
+ * computing the Adler-32 checksum on the latest JE log file available.
+ *
+ * @return The checksum of JE db environment or zero if checksum failed.
+ */
+ private long checksumDbEnv() {
+
+ File parentDirectory = getFileForPath(cfg.getDBDirectory());
+ File backendDirectory = new File(parentDirectory, cfg.getBackendId());
+
+ List<File> jdbFiles = new ArrayList<File>();
+ if(backendDirectory.isDirectory())
+ {
+ jdbFiles =
+ Arrays.asList(backendDirectory.listFiles(new FilenameFilter() {
+ @Override
+ public boolean accept(File dir, String name) {
+ return name.endsWith(".jdb");
+ }
+ }));
+ }
+
+ if ( !jdbFiles.isEmpty() ) {
+ Collections.sort(jdbFiles, Collections.reverseOrder());
+ FileInputStream fis = null;
+ try {
+ fis = new FileInputStream(jdbFiles.get(0).toString());
+ CheckedInputStream cis = new CheckedInputStream(fis, new Adler32());
+ byte[] tempBuf = new byte[8192];
+ while (cis.read(tempBuf) >= 0) {
+ }
+
+ return cis.getChecksum().getValue();
+ } catch (Exception e) {
+ logger.traceException(e);
+ } finally {
+ close(fis);
+ }
+ }
+
+ return 0;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void configureBackend(LocalDBBackendCfg cfg) throws ConfigException
+ {
+ Reject.ifNull(cfg);
+
+ this.cfg = cfg;
+ baseDNs = this.cfg.getBaseDN().toArray(new DN[0]);
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void initializeBackend()
+ throws ConfigException, InitializationException
+ {
+ // Checksum this db environment and register its offline state id/checksum.
+ DirectoryServer.registerOfflineBackendStateID(getBackendID(), checksumDbEnv());
+
+ if (mustOpenRootContainer())
+ {
+ rootContainer = initializeRootContainer(parseConfigEntry(cfg));
+ }
+
+ // Preload the database cache.
+ rootContainer.preload(cfg.getPreloadTimeLimit());
+
+ try
+ {
+ // Log an informational message about the number of entries.
+ logger.info(NOTE_JEB_BACKEND_STARTED, cfg.getBackendId(), rootContainer.getEntryCount());
+ }
+ catch (StorageRuntimeException e)
+ {
+ logger.traceException(e);
+ LocalizableMessage message = WARN_JEB_GET_ENTRY_COUNT_FAILED.get(e.getMessage());
+ throw new InitializationException(message, e);
+ }
+
+ for (DN dn : cfg.getBaseDN())
+ {
+ try
+ {
+ DirectoryServer.registerBaseDN(dn, this, false);
+ }
+ catch (Exception e)
+ {
+ logger.traceException(e);
+ throw new InitializationException(ERR_BACKEND_CANNOT_REGISTER_BASEDN.get(dn, e), e);
+ }
+ }
+
+ // Register a monitor provider for the environment.
+ rootContainerMonitor = rootContainer.getMonitorProvider();
+ DirectoryServer.registerMonitorProvider(rootContainerMonitor);
+
+ // Register as disk space monitor handler
+ File parentDirectory = getFileForPath(cfg.getDBDirectory());
+ File backendDirectory =
+ new File(parentDirectory, cfg.getBackendId());
+ diskMonitor = new DiskSpaceMonitor(getBackendID() + " backend",
+ backendDirectory, cfg.getDiskLowThreshold(), cfg.getDiskFullThreshold(),
+ 5, TimeUnit.SECONDS, this);
+ diskMonitor.initializeMonitorProvider(null);
+ DirectoryServer.registerMonitorProvider(diskMonitor);
+
+ //Register as an AlertGenerator.
+ DirectoryServer.registerAlertGenerator(this);
+ // Register this backend as a change listener.
+ cfg.addLocalDBChangeListener(this);
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void finalizeBackend()
+ {
+ super.finalizeBackend();
+ cfg.removeLocalDBChangeListener(this);
+
+ // Deregister our base DNs.
+ for (DN dn : rootContainer.getBaseDNs())
+ {
+ try
+ {
+ DirectoryServer.deregisterBaseDN(dn);
+ }
+ catch (Exception e)
+ {
+ logger.traceException(e);
+ }
+ }
+
+ DirectoryServer.deregisterMonitorProvider(rootContainerMonitor);
+ DirectoryServer.deregisterMonitorProvider(diskMonitor);
+
+ // We presume the server will prevent more operations coming into this
+ // backend, but there may be existing operations already in the
+ // backend. We need to wait for them to finish.
+ waitUntilQuiescent();
+
+ // Close the database.
+ try
+ {
+ rootContainer.close();
+ rootContainer = null;
+ }
+ catch (StorageRuntimeException e)
+ {
+ logger.traceException(e);
+ logger.error(ERR_JEB_DATABASE_EXCEPTION, e.getMessage());
+ }
+
+ // Checksum this db environment and register its offline state id/checksum.
+ DirectoryServer.registerOfflineBackendStateID(getBackendID(), checksumDbEnv());
+ DirectoryServer.deregisterAlertGenerator(this);
+
+ // Make sure the thread counts are zero for next initialization.
+ threadTotalCount.set(0);
+ threadWriteCount.set(0);
+
+ // Log an informational message.
+ logger.info(NOTE_BACKEND_OFFLINE, cfg.getBackendId());
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public boolean isLocal()
+ {
+ return true;
+ }
+
+
+
+ /** {@inheritDoc} */
+ @Override
+ public boolean isIndexed(AttributeType attributeType, IndexType indexType)
+ {
+ try
+ {
+ EntryContainer ec = rootContainer.getEntryContainer(baseDNs[0]);
+ AttributeIndex ai = ec.getAttributeIndex(attributeType);
+ if (ai == null)
+ {
+ return false;
+ }
+
+ Set<LocalDBIndexCfgDefn.IndexType> indexTypes =
+ ai.getConfiguration().getIndexType();
+ switch (indexType)
+ {
+ case PRESENCE:
+ return indexTypes.contains(LocalDBIndexCfgDefn.IndexType.PRESENCE);
+
+ case EQUALITY:
+ return indexTypes.contains(LocalDBIndexCfgDefn.IndexType.EQUALITY);
+
+ case SUBSTRING:
+ case SUBINITIAL:
+ case SUBANY:
+ case SUBFINAL:
+ return indexTypes.contains(LocalDBIndexCfgDefn.IndexType.SUBSTRING);
+
+ case GREATER_OR_EQUAL:
+ case LESS_OR_EQUAL:
+ return indexTypes.contains(LocalDBIndexCfgDefn.IndexType.ORDERING);
+
+ case APPROXIMATE:
+ return indexTypes.contains(LocalDBIndexCfgDefn.IndexType.APPROXIMATE);
+
+ default:
+ return false;
+ }
+ }
+ catch (Exception e)
+ {
+ logger.traceException(e);
+
+ return false;
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public boolean supportsLDIFExport()
+ {
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public boolean supportsLDIFImport()
+ {
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public boolean supportsBackup()
+ {
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public boolean supportsBackup(BackupConfig backupConfig,
+ StringBuilder unsupportedReason)
+ {
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public boolean supportsRestore()
+ {
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public Set<String> getSupportedFeatures()
+ {
+ return Collections.emptySet();
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public Set<String> getSupportedControls()
+ {
+ return supportedControls;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public DN[] getBaseDNs()
+ {
+ return baseDNs;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public long getEntryCount()
+ {
+ if (rootContainer != null)
+ {
+ try
+ {
+ return rootContainer.getEntryCount();
+ }
+ catch (Exception e)
+ {
+ logger.traceException(e);
+ }
+ }
+
+ return -1;
+ }
+
+
+
+ /** {@inheritDoc} */
+ @Override
+ public ConditionResult hasSubordinates(DN entryDN)
+ throws DirectoryException
+ {
+ long ret = numSubordinates(entryDN, false);
+ if(ret < 0)
+ {
+ return ConditionResult.UNDEFINED;
+ }
+ return ConditionResult.valueOf(ret != 0);
+ }
+
+
+
+ /** {@inheritDoc} */
+ @Override
+ public long numSubordinates(DN entryDN, boolean subtree)
+ throws DirectoryException
+ {
+ checkRootContainerInitialized();
+ EntryContainer ec = rootContainer.getEntryContainer(entryDN);
+ if(ec == null)
+ {
+ return -1;
+ }
+
+ readerBegin();
+ ec.sharedLock.lock();
+ try
+ {
+ long count = ec.getNumSubordinates(entryDN, subtree);
+ if(count == Long.MAX_VALUE)
+ {
+ // The index entry limit has exceeded and there is no count maintained.
+ return -1;
+ }
+ return count;
+ }
+ catch (StorageRuntimeException e)
+ {
+ logger.traceException(e);
+ throw createDirectoryException(e);
+ }
+ finally
+ {
+ ec.sharedLock.unlock();
+ readerEnd();
+ }
+ }
+
+
+
+ /** {@inheritDoc} */
+ @Override
+ public Entry getEntry(DN entryDN) throws DirectoryException
+ {
+ readerBegin();
+
+ checkRootContainerInitialized();
+ EntryContainer ec = rootContainer.getEntryContainer(entryDN);
+ ec.sharedLock.lock();
+ Entry entry;
+ try
+ {
+ entry = ec.getEntry(entryDN);
+ }
+ catch (StorageRuntimeException e)
+ {
+ logger.traceException(e);
+ throw createDirectoryException(e);
+ }
+ finally
+ {
+ ec.sharedLock.unlock();
+ readerEnd();
+ }
+
+ return entry;
+ }
+
+
+
+ /** {@inheritDoc} */
+ @Override
+ public void addEntry(Entry entry, AddOperation addOperation)
+ throws DirectoryException, CanceledOperationException
+ {
+ checkDiskSpace(addOperation);
+ writerBegin();
+
+ checkRootContainerInitialized();
+ EntryContainer ec = rootContainer.getEntryContainer(entry.getName());
+ ec.sharedLock.lock();
+ try
+ {
+ ec.addEntry(entry, addOperation);
+ }
+ catch (StorageRuntimeException e)
+ {
+ logger.traceException(e);
+ throw createDirectoryException(e);
+ }
+ finally
+ {
+ ec.sharedLock.unlock();
+ writerEnd();
+ }
+ }
+
+
+
+ /** {@inheritDoc} */
+ @Override
+ public void deleteEntry(DN entryDN, DeleteOperation deleteOperation)
+ throws DirectoryException, CanceledOperationException
+ {
+ checkDiskSpace(deleteOperation);
+ writerBegin();
+
+ checkRootContainerInitialized();
+ EntryContainer ec = rootContainer.getEntryContainer(entryDN);
+ ec.sharedLock.lock();
+ try
+ {
+ ec.deleteEntry(entryDN, deleteOperation);
+ }
+ catch (StorageRuntimeException e)
+ {
+ logger.traceException(e);
+ throw createDirectoryException(e);
+ }
+ finally
+ {
+ ec.sharedLock.unlock();
+ writerEnd();
+ }
+ }
+
+
+
+ /** {@inheritDoc} */
+ @Override
+ public void replaceEntry(Entry oldEntry, Entry newEntry,
+ ModifyOperation modifyOperation) throws DirectoryException,
+ CanceledOperationException
+ {
+ checkDiskSpace(modifyOperation);
+ writerBegin();
+
+ checkRootContainerInitialized();
+ EntryContainer ec = rootContainer.getEntryContainer(newEntry.getName());
+ ec.sharedLock.lock();
+
+ try
+ {
+ ec.replaceEntry(oldEntry, newEntry, modifyOperation);
+ }
+ catch (StorageRuntimeException e)
+ {
+ logger.traceException(e);
+ throw createDirectoryException(e);
+ }
+ finally
+ {
+ ec.sharedLock.unlock();
+ writerEnd();
+ }
+ }
+
+
+
+ /** {@inheritDoc} */
+ @Override
+ public void renameEntry(DN currentDN, Entry entry,
+ ModifyDNOperation modifyDNOperation)
+ throws DirectoryException, CanceledOperationException
+ {
+ checkDiskSpace(modifyDNOperation);
+ writerBegin();
+
+ checkRootContainerInitialized();
+ EntryContainer currentContainer = rootContainer.getEntryContainer(currentDN);
+ EntryContainer container = rootContainer.getEntryContainer(entry.getName());
+
+ if (currentContainer != container)
+ {
+ // FIXME: No reason why we cannot implement a move between containers
+ // since the containers share the same database environment.
+ LocalizableMessage msg = WARN_JEB_FUNCTION_NOT_SUPPORTED.get();
+ throw new DirectoryException(ResultCode.UNWILLING_TO_PERFORM, msg);
+ }
+
+ currentContainer.sharedLock.lock();
+ try
+ {
+ currentContainer.renameEntry(currentDN, entry, modifyDNOperation);
+ }
+ catch (StorageRuntimeException e)
+ {
+ logger.traceException(e);
+ throw createDirectoryException(e);
+ }
+ finally
+ {
+ currentContainer.sharedLock.unlock();
+ writerEnd();
+ }
+ }
+
+
+
+ /** {@inheritDoc} */
+ @Override
+ public void search(SearchOperation searchOperation)
+ throws DirectoryException, CanceledOperationException
+ {
+ readerBegin();
+
+ checkRootContainerInitialized();
+ EntryContainer ec = rootContainer.getEntryContainer(searchOperation.getBaseDN());
+ ec.sharedLock.lock();
+
+ try
+ {
+ ec.search(searchOperation);
+ }
+ catch (StorageRuntimeException e)
+ {
+ logger.traceException(e);
+ throw createDirectoryException(e);
+ }
+ finally
+ {
+ ec.sharedLock.unlock();
+ readerEnd();
+ }
+ }
+
+ private void checkRootContainerInitialized() throws DirectoryException
+ {
+ if (rootContainer == null)
+ {
+ LocalizableMessage msg = ERR_ROOT_CONTAINER_NOT_INITIALIZED.get(getBackendID());
+ throw new DirectoryException(DirectoryServer.getServerErrorResultCode(), msg);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void exportLDIF(LDIFExportConfig exportConfig)
+ throws DirectoryException
+ {
+ // If the backend already has the root container open, we must use the same
+ // underlying root container
+ boolean openRootContainer = mustOpenRootContainer();
+ final ResultCode errorRC = DirectoryServer.getServerErrorResultCode();
+ try
+ {
+ if (openRootContainer)
+ {
+ rootContainer = getReadOnlyRootContainer();
+ }
+
+ ExportJob exportJob = new ExportJob(exportConfig);
+ exportJob.exportLDIF(rootContainer);
+ }
+ catch (IOException ioe)
+ {
+ logger.traceException(ioe);
+ throw new DirectoryException(errorRC, ERR_JEB_EXPORT_IO_ERROR.get(ioe.getMessage()));
+ }
+ catch (StorageRuntimeException de)
+ {
+ logger.traceException(de);
+ throw createDirectoryException(de);
+ }
+ catch (ConfigException ce)
+ {
+ throw new DirectoryException(errorRC, ce.getMessageObject());
+ }
+ catch (IdentifiedException e)
+ {
+ if (e instanceof DirectoryException)
+ {
+ throw (DirectoryException) e;
+ }
+ logger.traceException(e);
+ throw new DirectoryException(errorRC, e.getMessageObject());
+ }
+ finally
+ {
+ closeTemporaryRootContainer(openRootContainer);
+ }
+ }
+
+ private boolean mustOpenRootContainer()
+ {
+ return rootContainer == null;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public LDIFImportResult importLDIF(LDIFImportConfig importConfig)
+ throws DirectoryException
+ {
+ RuntimeInformation.logInfo();
+
+ // If the backend already has the root container open, we must use the same
+ // underlying root container
+ boolean openRootContainer = rootContainer == null;
+
+ // If the rootContainer is open, the backend is initialized by something else.
+ // We can't do import while the backend is online.
+ final ResultCode errorRC = DirectoryServer.getServerErrorResultCode();
+ if(!openRootContainer)
+ {
+ throw new DirectoryException(errorRC, ERR_JEB_IMPORT_BACKEND_ONLINE.get());
+ }
+
+ try
+ {
+ final EnvironmentConfig envConfig = getEnvConfigForImport();
+
+ if (!importConfig.appendToExistingData()
+ && (importConfig.clearBackend() || cfg.getBaseDN().size() <= 1))
+ {
+ // We have the writer lock on the environment, now delete the
+ // environment and re-open it. Only do this when we are
+ // importing to all the base DNs in the backend or if the backend only
+ // have one base DN.
+ File parentDirectory = getFileForPath(cfg.getDBDirectory());
+ File backendDirectory = new File(parentDirectory, cfg.getBackendId());
+ // If the backend does not exist the import will create it.
+ if (backendDirectory.exists())
+ {
+ EnvManager.removeFiles(backendDirectory.getPath());
+ }
+ }
+
+ throw new NotImplementedException();
+// Importer importer = new Importer(importConfig, cfg, envConfig);
+// rootContainer = initializeRootContainer(envConfig);
+// return importer.processImport(rootContainer);
+ }
+ catch (ExecutionException execEx)
+ {
+ logger.traceException(execEx);
+ if (execEx.getCause() instanceof DirectoryException)
+ {
+ throw ((DirectoryException) execEx.getCause());
+ }
+ throw new DirectoryException(errorRC, ERR_EXECUTION_ERROR.get(execEx.getMessage()));
+ }
+ catch (InterruptedException intEx)
+ {
+ logger.traceException(intEx);
+ throw new DirectoryException(errorRC, ERR_INTERRUPTED_ERROR.get(intEx.getMessage()));
+ }
+ catch (JebException je)
+ {
+ logger.traceException(je);
+ throw new DirectoryException(errorRC, je.getMessageObject());
+ }
+ catch (InitializationException ie)
+ {
+ logger.traceException(ie);
+ throw new DirectoryException(errorRC, ie.getMessageObject());
+ }
+ catch (ConfigException ce)
+ {
+ logger.traceException(ce);
+ throw new DirectoryException(errorRC, ce.getMessageObject());
+ }
+ finally
+ {
+ // leave the backend in the same state.
+ try
+ {
+ if (rootContainer != null)
+ {
+ long startTime = System.currentTimeMillis();
+ rootContainer.close();
+ long finishTime = System.currentTimeMillis();
+ long closeTime = (finishTime - startTime) / 1000;
+ logger.info(NOTE_JEB_IMPORT_LDIF_ROOTCONTAINER_CLOSE, closeTime);
+ rootContainer = null;
+ }
+
+ // Sync the environment to disk.
+ logger.info(NOTE_JEB_IMPORT_CLOSING_DATABASE);
+ }
+ catch (StorageRuntimeException de)
+ {
+ logger.traceException(de);
+ }
+ }
+ }
+
+ private EnvironmentConfig getEnvConfigForImport()
+ {
+ final EnvironmentConfig envConfig = new EnvironmentConfig();
+ envConfig.setAllowCreate(true);
+ envConfig.setTransactional(false);
+ envConfig.setDurability(Durability.COMMIT_NO_SYNC);
+ envConfig.setLockTimeout(0, TimeUnit.SECONDS);
+ envConfig.setTxnTimeout(0, TimeUnit.SECONDS);
+ envConfig.setConfigParam(CLEANER_MIN_FILE_UTILIZATION,
+ String.valueOf(cfg.getDBCleanerMinUtilization()));
+ envConfig.setConfigParam(LOG_FILE_MAX,
+ String.valueOf(cfg.getDBLogFileMax()));
+ return envConfig;
+ }
+
+ /**
+ * Verify the integrity of the backend instance.
+ * @param verifyConfig The verify configuration.
+ * @param statEntry Optional entry to save stats into.
+ * @return The error count.
+ * @throws ConfigException If an unrecoverable problem arises during
+ * initialization.
+ * @throws InitializationException If a problem occurs during initialization
+ * that is not related to the server
+ * configuration.
+ * @throws DirectoryException If a Directory Server error occurs.
+ */
+ public long verifyBackend(VerifyConfig verifyConfig, Entry statEntry)
+ throws InitializationException, ConfigException, DirectoryException
+ {
+ // If the backend already has the root container open, we must use the same
+ // underlying root container
+ final boolean openRootContainer = mustOpenRootContainer();
+ try
+ {
+ if (openRootContainer)
+ {
+ rootContainer = getReadOnlyRootContainer();
+ }
+
+ VerifyJob verifyJob = new VerifyJob(verifyConfig);
+ return verifyJob.verifyBackend(rootContainer, statEntry);
+ }
+ catch (StorageRuntimeException e)
+ {
+ logger.traceException(e);
+ throw createDirectoryException(e);
+ }
+ catch (JebException e)
+ {
+ logger.traceException(e);
+ throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
+ e.getMessageObject());
+ }
+ finally
+ {
+ closeTemporaryRootContainer(openRootContainer);
+ }
+ }
+
+
+ /**
+ * Rebuild index(es) in the backend instance. Note that the server will not
+ * explicitly initialize this backend before calling this method.
+ * @param rebuildConfig The rebuild configuration.
+ * @throws ConfigException If an unrecoverable problem arises during
+ * initialization.
+ * @throws InitializationException If a problem occurs during initialization
+ * that is not related to the server
+ * configuration.
+ * @throws DirectoryException If a Directory Server error occurs.
+ */
+ public void rebuildBackend(RebuildConfig rebuildConfig)
+ throws InitializationException, ConfigException, DirectoryException
+ {
+ // If the backend already has the root container open, we must use the same
+ // underlying root container
+ boolean openRootContainer = mustOpenRootContainer();
+
+ /*
+ * If the rootContainer is open, the backend is initialized by something
+ * else. We can't do any rebuild of system indexes while others are using
+ * this backend.
+ */
+ final ResultCode errorRC = DirectoryServer.getServerErrorResultCode();
+ if(!openRootContainer && rebuildConfig.includesSystemIndex())
+ {
+ throw new DirectoryException(errorRC, ERR_JEB_REBUILD_BACKEND_ONLINE.get());
+ }
+
+ try
+ {
+ final EnvironmentConfig envConfig;
+ if (openRootContainer)
+ {
+ envConfig = getEnvConfigForImport();
+ rootContainer = initializeRootContainer(envConfig);
+ }
+ else
+ {
+ envConfig = parseConfigEntry(cfg);
+
+ }
+ throw new NotImplementedException();
+// final Importer importer = new Importer(rebuildConfig, cfg, envConfig);
+// importer.rebuildIndexes(rootContainer);
+ }
+ catch (ExecutionException execEx)
+ {
+ logger.traceException(execEx);
+ throw new DirectoryException(errorRC, ERR_EXECUTION_ERROR.get(execEx.getMessage()));
+ }
+ catch (InterruptedException intEx)
+ {
+ logger.traceException(intEx);
+ throw new DirectoryException(errorRC, ERR_INTERRUPTED_ERROR.get(intEx.getMessage()));
+ }
+ catch (ConfigException ce)
+ {
+ logger.traceException(ce);
+ throw new DirectoryException(errorRC, ce.getMessageObject());
+ }
+ catch (JebException e)
+ {
+ logger.traceException(e);
+ throw new DirectoryException(errorRC, e.getMessageObject());
+ }
+ catch (InitializationException e)
+ {
+ logger.traceException(e);
+ throw new InitializationException(e.getMessageObject());
+ }
+ finally
+ {
+ closeTemporaryRootContainer(openRootContainer);
+ }
+ }
+
+ /**
+ * If a root container was opened in the calling method method as read only,
+ * close it to leave the backend in the same state.
+ */
+ private void closeTemporaryRootContainer(boolean openRootContainer)
+ {
+ if (openRootContainer && rootContainer != null)
+ {
+ try
+ {
+ rootContainer.close();
+ rootContainer = null;
+ }
+ catch (StorageRuntimeException e)
+ {
+ logger.traceException(e);
+ }
+ }
+ }
+
+
+ /** {@inheritDoc} */
+ @Override
+ public void createBackup(BackupConfig backupConfig) throws DirectoryException
+ {
+ BackupManager backupManager = new BackupManager(getBackendID());
+ File parentDir = getFileForPath(cfg.getDBDirectory());
+ File backendDir = new File(parentDir, cfg.getBackendId());
+ backupManager.createBackup(backendDir, backupConfig);
+ }
+
+
+
+ /** {@inheritDoc} */
+ @Override
+ public void removeBackup(BackupDirectory backupDirectory, String backupID)
+ throws DirectoryException
+ {
+ BackupManager backupManager = new BackupManager(getBackendID());
+ backupManager.removeBackup(backupDirectory, backupID);
+ }
+
+
+
+ /** {@inheritDoc} */
+ @Override
+ public void restoreBackup(RestoreConfig restoreConfig)
+ throws DirectoryException
+ {
+ BackupManager backupManager = new BackupManager(getBackendID());
+ File parentDir = getFileForPath(cfg.getDBDirectory());
+ File backendDir = new File(parentDir, cfg.getBackendId());
+ backupManager.restoreBackup(backendDir, restoreConfig);
+ }
+
+
+
+ /** {@inheritDoc} */
+ @Override
+ public boolean isConfigurationAcceptable(LocalDBBackendCfg config,
+ List<LocalizableMessage> unacceptableReasons)
+ {
+ return isConfigurationChangeAcceptable(config, unacceptableReasons);
+ }
+
+
+
+ /** {@inheritDoc} */
+ @Override
+ public boolean isConfigurationChangeAcceptable(
+ LocalDBBackendCfg cfg,
+ List<LocalizableMessage> unacceptableReasons)
+ {
+ // Make sure that the logging level value is acceptable.
+ try {
+ Level.parse(cfg.getDBLoggingLevel());
+ return true;
+ } catch (Exception e) {
+ unacceptableReasons.add(ERR_JEB_INVALID_LOGGING_LEVEL.get(cfg.getDBLoggingLevel(), cfg.dn()));
+ return false;
+ }
+ }
+
+
+
+ /** {@inheritDoc} */
+ @Override
+ public ConfigChangeResult applyConfigurationChange(LocalDBBackendCfg newCfg)
+ {
+ ResultCode resultCode = ResultCode.SUCCESS;
+ ArrayList<LocalizableMessage> messages = new ArrayList<LocalizableMessage>();
+
+ try
+ {
+ if(rootContainer != null)
+ {
+ SortedSet<DN> newBaseDNs = newCfg.getBaseDN();
+ DN[] newBaseDNsArray = newBaseDNs.toArray(new DN[newBaseDNs.size()]);
+
+ // Check for changes to the base DNs.
+ removeDeletedBaseDNs(newBaseDNs);
+ ConfigChangeResult failure = createNewBaseDNs(newBaseDNsArray, messages);
+ if (failure != null)
+ {
+ return failure;
+ }
+
+ baseDNs = newBaseDNsArray;
+ }
+
+ if(cfg.getDiskFullThreshold() != newCfg.getDiskFullThreshold() ||
+ cfg.getDiskLowThreshold() != newCfg.getDiskLowThreshold())
+ {
+ diskMonitor.setFullThreshold(newCfg.getDiskFullThreshold());
+ diskMonitor.setLowThreshold(newCfg.getDiskLowThreshold());
+ }
+
+ // Put the new configuration in place.
+ this.cfg = newCfg;
+ }
+ catch (Exception e)
+ {
+ messages.add(LocalizableMessage.raw(stackTraceToSingleLineString(e)));
+ return new ConfigChangeResult(
+ DirectoryServer.getServerErrorResultCode(), false, messages);
+ }
+
+ return new ConfigChangeResult(resultCode, false, messages);
+ }
+
+ private void removeDeletedBaseDNs(SortedSet<DN> newBaseDNs) throws DirectoryException
+ {
+ for (DN baseDN : cfg.getBaseDN())
+ {
+ if (!newBaseDNs.contains(baseDN))
+ {
+ // The base DN was deleted.
+ DirectoryServer.deregisterBaseDN(baseDN);
+ EntryContainer ec = rootContainer.unregisterEntryContainer(baseDN);
+ ec.close();
+ ec.delete();
+ }
+ }
+ }
+
+ private ConfigChangeResult createNewBaseDNs(DN[] newBaseDNsArray, ArrayList<LocalizableMessage> messages)
+ {
+ for (DN baseDN : newBaseDNsArray)
+ {
+ if (!rootContainer.getBaseDNs().contains(baseDN))
+ {
+ try
+ {
+ // The base DN was added.
+ EntryContainer ec = rootContainer.openEntryContainer(baseDN, null);
+ rootContainer.registerEntryContainer(baseDN, ec);
+ DirectoryServer.registerBaseDN(baseDN, this, false);
+ }
+ catch (Exception e)
+ {
+ logger.traceException(e);
+
+ ResultCode resultCode = DirectoryServer.getServerErrorResultCode();
+ messages.add(ERR_BACKEND_CANNOT_REGISTER_BASEDN.get(baseDN, e));
+ return new ConfigChangeResult(resultCode, false, messages);
+ }
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Returns a handle to the JE root container currently used by this backend.
+ * The rootContainer could be NULL if the backend is not initialized.
+ *
+ * @return The RootContainer object currently used by this backend.
+ */
+ public RootContainer getRootContainer()
+ {
+ return rootContainer;
+ }
+
+ /**
+ * Returns a new read-only handle to the JE root container for this backend.
+ * The caller is responsible for closing the root container after use.
+ *
+ * @return The read-only RootContainer object for this backend.
+ *
+ * @throws ConfigException If an unrecoverable problem arises during
+ * initialization.
+ * @throws InitializationException If a problem occurs during initialization
+ * that is not related to the server
+ * configuration.
+ */
+ public RootContainer getReadOnlyRootContainer()
+ throws ConfigException, InitializationException
+ {
+ EnvironmentConfig envConfig = parseConfigEntry(cfg);
+
+ envConfig.setReadOnly(true);
+ envConfig.setAllowCreate(false);
+ envConfig.setTransactional(false);
+ envConfig.setConfigParam(ENV_IS_LOCKING, "true");
+ envConfig.setConfigParam(ENV_RUN_CHECKPOINTER, "true");
+
+ return initializeRootContainer(envConfig);
+ }
+
+ /**
+ * Clears all the entries from the backend. This method is for test cases
+ * that use the JE backend.
+ *
+ * @throws ConfigException If an unrecoverable problem arises in the
+ * process of performing the initialization.
+ *
+ * @throws JebException If an error occurs while removing the data.
+ */
+ public void clearBackend()
+ throws ConfigException, JebException
+ {
+ // Determine the backend database directory.
+ File parentDirectory = getFileForPath(cfg.getDBDirectory());
+ File backendDirectory = new File(parentDirectory, cfg.getBackendId());
+ EnvManager.removeFiles(backendDirectory.getPath());
+ }
+
+ /**
+ * Creates a customized DirectoryException from the StorageRuntimeException
+ * thrown by JE backend.
+ *
+ * @param e
+ * The StorageRuntimeException to be converted.
+ * @return DirectoryException created from exception.
+ */
+ DirectoryException createDirectoryException(StorageRuntimeException e)
+ {
+ if (true) {
+ throw new NotImplementedException();
+ }
+ if (/*e instanceof EnvironmentFailureException && */ !rootContainer.isValid()) {
+ LocalizableMessage message = NOTE_BACKEND_ENVIRONMENT_UNUSABLE.get(getBackendID());
+ logger.info(message);
+ DirectoryServer.sendAlertNotification(DirectoryServer.getInstance(),
+ ALERT_TYPE_BACKEND_ENVIRONMENT_UNUSABLE, message);
+ }
+
+ String jeMessage = e.getMessage();
+ if (jeMessage == null) {
+ jeMessage = stackTraceToSingleLineString(e);
+ }
+ LocalizableMessage message = ERR_JEB_DATABASE_EXCEPTION.get(jeMessage);
+ return new DirectoryException(
+ DirectoryServer.getServerErrorResultCode(), message, e);
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public String getClassName() {
+ return BackendImpl.class.getName();
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public Map<String, String> getAlerts()
+ {
+ Map<String, String> alerts = new LinkedHashMap<String, String>();
+
+ alerts.put(ALERT_TYPE_BACKEND_ENVIRONMENT_UNUSABLE,
+ ALERT_DESCRIPTION_BACKEND_ENVIRONMENT_UNUSABLE);
+ alerts.put(ALERT_TYPE_DISK_SPACE_LOW,
+ ALERT_DESCRIPTION_DISK_SPACE_LOW);
+ alerts.put(ALERT_TYPE_DISK_FULL,
+ ALERT_DESCRIPTION_DISK_FULL);
+ return alerts;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public DN getComponentEntryDN() {
+ return cfg.dn();
+ }
+
+ private RootContainer initializeRootContainer(EnvironmentConfig envConfig)
+ throws ConfigException, InitializationException {
+ // Open the database environment
+ try {
+ RootContainer rc = new RootContainer(this, cfg);
+ rc.open(envConfig);
+ return rc;
+ }
+ catch (StorageRuntimeException e)
+ {
+ logger.traceException(e);
+ LocalizableMessage message = ERR_JEB_OPEN_ENV_FAIL.get(e.getMessage());
+ throw new InitializationException(message, e);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void preloadEntryCache() throws
+ UnsupportedOperationException {
+ EntryCachePreloader preloader = new EntryCachePreloader(this);
+ preloader.preload();
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void diskLowThresholdReached(DiskSpaceMonitor monitor) {
+ LocalizableMessage msg = ERR_JEB_DISK_LOW_THRESHOLD_REACHED.get(
+ monitor.getDirectory().getPath(), cfg.getBackendId(), monitor.getFreeSpace(),
+ Math.max(monitor.getLowThreshold(), monitor.getFullThreshold()));
+ DirectoryServer.sendAlertNotification(this, ALERT_TYPE_DISK_SPACE_LOW, msg);
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void diskFullThresholdReached(DiskSpaceMonitor monitor) {
+ LocalizableMessage msg = ERR_JEB_DISK_FULL_THRESHOLD_REACHED.get(
+ monitor.getDirectory().getPath(), cfg.getBackendId(), monitor.getFreeSpace(),
+ Math.max(monitor.getLowThreshold(), monitor.getFullThreshold()));
+ DirectoryServer.sendAlertNotification(this, ALERT_TYPE_DISK_FULL, msg);
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void diskSpaceRestored(DiskSpaceMonitor monitor) {
+ logger.error(NOTE_JEB_DISK_SPACE_RESTORED, monitor.getFreeSpace(),
+ monitor.getDirectory().getPath(), cfg.getBackendId(),
+ Math.max(monitor.getLowThreshold(), monitor.getFullThreshold()));
+ }
+
+ private void checkDiskSpace(Operation operation) throws DirectoryException
+ {
+ if(diskMonitor.isFullThresholdReached() ||
+ (diskMonitor.isLowThresholdReached()
+ && operation != null
+ && !operation.getClientConnection().hasPrivilege(
+ Privilege.BYPASS_LOCKDOWN, operation)))
+ {
+ throw new DirectoryException(ResultCode.UNWILLING_TO_PERFORM,
+ WARN_JEB_OUT_OF_DISK_SPACE.get());
+ }
+ }
+}
diff --git a/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/BackupManager.java b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/BackupManager.java
new file mode 100644
index 0000000..0e22e78
--- /dev/null
+++ b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/BackupManager.java
@@ -0,0 +1,1255 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License"). You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at legal-notices/CDDLv1_0.txt
+ * or http://forgerock.org/license/CDDLv1.0.html.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at legal-notices/CDDLv1_0.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information:
+ * Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ *
+ *
+ * Copyright 2006-2009 Sun Microsystems, Inc.
+ * Portions Copyright 2013-2014 ForgeRock AS.
+ */
+package org.opends.server.backends.pluggable;
+import org.forgerock.i18n.LocalizableMessage;
+import org.forgerock.opendj.config.server.ConfigException;
+import org.opends.server.core.DirectoryServer;
+import org.opends.server.util.DynamicConstants;
+import org.opends.server.util.StaticUtils;
+import org.opends.server.types.CryptoManagerException;
+
+import javax.crypto.Mac;
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.FilenameFilter;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.security.MessageDigest;
+import java.util.*;
+import java.util.zip.Deflater;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+import java.util.zip.ZipOutputStream;
+
+import org.opends.server.types.*;
+import org.forgerock.i18n.slf4j.LocalizedLogger;
+import static org.opends.messages.JebMessages.*;
+import static org.opends.server.util.ServerConstants.*;
+import static org.opends.server.util.StaticUtils.*;
+
+/**
+ * A backup manager for JE backends.
+ */
+public class BackupManager
+{
+ private static final LocalizedLogger logger = LocalizedLogger.getLoggerForThisClass();
+
+ /**
+ * The common prefix for archive files.
+ */
+ public static final String BACKUP_BASE_FILENAME = "backup-";
+
+ /**
+ * The name of the property that holds the name of the latest log file
+ * at the time the backup was created.
+ */
+ public static final String PROPERTY_LAST_LOGFILE_NAME = "last_logfile_name";
+
+ /**
+ * The name of the property that holds the size of the latest log file
+ * at the time the backup was created.
+ */
+ public static final String PROPERTY_LAST_LOGFILE_SIZE = "last_logfile_size";
+
+
+ /**
+ * The name of the entry in an incremental backup archive file
+ * containing a list of log files that are unchanged since the
+ * previous backup.
+ */
+ public static final String ZIPENTRY_UNCHANGED_LOGFILES = "unchanged.txt";
+
+ /**
+ * The name of a dummy entry in the backup archive file that will act
+ * as a placeholder in case a backup is done on an empty backend.
+ */
+ public static final String ZIPENTRY_EMPTY_PLACEHOLDER = "empty.placeholder";
+
+
+ /**
+ * The backend ID.
+ */
+ private String backendID;
+
+
+ /**
+ * Construct a backup manager for a JE backend.
+ * @param backendID The ID of the backend instance for which a backup
+ * manager is required.
+ */
+ public BackupManager(String backendID)
+ {
+ this.backendID = backendID;
+ }
+
+ /**
+ * Create a backup of the JE backend. The backup is stored in a single zip
+ * file in the backup directory. If the backup is incremental, then the
+ * first entry in the zip is a text file containing a list of all the JE
+ * log files that are unchanged since the previous backup. The remaining
+ * zip entries are the JE log files themselves, which, for an incremental,
+ * only include those files that have changed.
+ * @param backendDir The directory of the backend instance for
+ * which the backup is required.
+ * @param backupConfig The configuration to use when performing the backup.
+ * @throws DirectoryException If a Directory Server error occurs.
+ */
+ public void createBackup(File backendDir, BackupConfig backupConfig)
+ throws DirectoryException
+ {
+ // Get the properties to use for the backup.
+ String backupID = backupConfig.getBackupID();
+ BackupDirectory backupDir = backupConfig.getBackupDirectory();
+ boolean incremental = backupConfig.isIncremental();
+ String incrBaseID = backupConfig.getIncrementalBaseID();
+ boolean compress = backupConfig.compressData();
+ boolean encrypt = backupConfig.encryptData();
+ boolean hash = backupConfig.hashData();
+ boolean signHash = backupConfig.signHash();
+
+
+ HashMap<String,String> backupProperties = new HashMap<String,String>();
+
+ // Get the crypto manager and use it to obtain references to the message
+ // digest and/or MAC to use for hashing and/or signing.
+ CryptoManager cryptoManager = DirectoryServer.getCryptoManager();
+ Mac mac = null;
+ MessageDigest digest = null;
+ String macKeyID = null;
+
+ if (hash)
+ {
+ if (signHash)
+ {
+ try
+ {
+ macKeyID = cryptoManager.getMacEngineKeyEntryID();
+ backupProperties.put(BACKUP_PROPERTY_MAC_KEY_ID, macKeyID);
+
+ mac = cryptoManager.getMacEngine(macKeyID);
+ }
+ catch (Exception e)
+ {
+ logger.traceException(e);
+
+ LocalizableMessage message = ERR_JEB_BACKUP_CANNOT_GET_MAC.get(
+ macKeyID, stackTraceToSingleLineString(e));
+ throw new DirectoryException(
+ DirectoryServer.getServerErrorResultCode(), message, e);
+ }
+ }
+ else
+ {
+ String digestAlgorithm = cryptoManager
+ .getPreferredMessageDigestAlgorithm();
+ backupProperties.put(BACKUP_PROPERTY_DIGEST_ALGORITHM, digestAlgorithm);
+
+ try
+ {
+ digest = cryptoManager.getPreferredMessageDigest();
+ }
+ catch (Exception e)
+ {
+ logger.traceException(e);
+
+ LocalizableMessage message = ERR_JEB_BACKUP_CANNOT_GET_DIGEST.get(
+ digestAlgorithm, stackTraceToSingleLineString(e));
+ throw new DirectoryException(
+ DirectoryServer.getServerErrorResultCode(), message, e);
+ }
+ }
+ }
+
+
+ // Date the backup.
+ Date backupDate = new Date();
+
+ // If this is an incremental, determine the base backup for this backup.
+ HashSet<String> dependencies = new HashSet<String>();
+ BackupInfo baseBackup = null;
+
+ if (incremental)
+ {
+ if (incrBaseID == null)
+ {
+ // The default is to use the latest backup as base.
+ if (backupDir.getLatestBackup() != null)
+ {
+ incrBaseID = backupDir.getLatestBackup().getBackupID();
+ }
+ }
+
+ if (incrBaseID == null)
+ {
+ // No incremental backup ID: log a message informing that a backup
+ // could not be found and that a normal backup will be done.
+ incremental = false;
+ logger.warn(WARN_BACKUPDB_INCREMENTAL_NOT_FOUND_DOING_NORMAL, backupDir.getPath());
+ }
+ else
+ {
+ baseBackup = getBackupInfo(backupDir, incrBaseID);
+ }
+ }
+
+ // Get information about the latest log file from the base backup.
+ String latestFileName = null;
+ long latestFileSize = 0;
+ if (baseBackup != null)
+ {
+ HashMap<String,String> properties = baseBackup.getBackupProperties();
+ latestFileName = properties.get(PROPERTY_LAST_LOGFILE_NAME);
+ latestFileSize = Long.parseLong(
+ properties.get(PROPERTY_LAST_LOGFILE_SIZE));
+ }
+
+ /*
+ Create an output stream that will be used to write the archive file. At
+ its core, it will be a file output stream to put a file on the disk. If
+ we are to encrypt the data, then that file output stream will be wrapped
+ in a cipher output stream. The resulting output stream will then be
+ wrapped by a zip output stream (which may or may not actually use
+ compression).
+ */
+ String archiveFilename = null;
+ OutputStream outputStream;
+ File archiveFile;
+ try
+ {
+ archiveFilename = BACKUP_BASE_FILENAME + backendID + "-" + backupID;
+ archiveFile = new File(backupDir.getPath(), archiveFilename);
+ if (archiveFile.exists())
+ {
+ int i=1;
+ while (true)
+ {
+ archiveFile = new File(backupDir.getPath(),
+ archiveFilename + "." + i);
+ if (archiveFile.exists())
+ {
+ i++;
+ }
+ else
+ {
+ archiveFilename = archiveFilename + "." + i;
+ break;
+ }
+ }
+ }
+
+ outputStream = new FileOutputStream(archiveFile, false);
+ backupProperties.put(BACKUP_PROPERTY_ARCHIVE_FILENAME, archiveFilename);
+ }
+ catch (Exception e)
+ {
+ logger.traceException(e);
+
+ LocalizableMessage message = ERR_JEB_BACKUP_CANNOT_CREATE_ARCHIVE_FILE.
+ get(archiveFilename, backupDir.getPath(), stackTraceToSingleLineString(e));
+ throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
+ message, e);
+ }
+
+ // If we should encrypt the data, then wrap the output stream in a cipher
+ // output stream.
+ if (encrypt)
+ {
+ try
+ {
+ outputStream
+ = cryptoManager.getCipherOutputStream(outputStream);
+ }
+ catch (CryptoManagerException e)
+ {
+ logger.traceException(e);
+
+ LocalizableMessage message = ERR_JEB_BACKUP_CANNOT_GET_CIPHER.get(
+ stackTraceToSingleLineString(e));
+ throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
+ message, e);
+ }
+ }
+
+
+ // Wrap the file output stream in a zip output stream.
+ ZipOutputStream zipStream = new ZipOutputStream(outputStream);
+
+ LocalizableMessage message = ERR_JEB_BACKUP_ZIP_COMMENT.get(
+ DynamicConstants.PRODUCT_NAME,
+ backupID, backendID);
+ zipStream.setComment(message.toString());
+
+ if (compress)
+ {
+ zipStream.setLevel(Deflater.DEFAULT_COMPRESSION);
+ }
+ else
+ {
+ zipStream.setLevel(Deflater.NO_COMPRESSION);
+ }
+
+ // Get a list of all the log files comprising the database.
+ FilenameFilter filenameFilter = new FilenameFilter()
+ {
+ public boolean accept(File d, String name)
+ {
+ return name.endsWith(".jdb");
+ }
+ };
+
+ File[] logFiles;
+ try
+ {
+ logFiles = backendDir.listFiles(filenameFilter);
+ }
+ catch (Exception e)
+ {
+ logger.traceException(e);
+
+ message = ERR_JEB_BACKUP_CANNOT_LIST_LOG_FILES.get(
+ backendDir.getAbsolutePath(), stackTraceToSingleLineString(e));
+ throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
+ message, e);
+ }
+
+ // Check to see if backend is empty. If so, insert placeholder entry into
+ // archive
+ if(logFiles.length <= 0)
+ {
+ try
+ {
+ ZipEntry emptyPlaceholder = new ZipEntry(ZIPENTRY_EMPTY_PLACEHOLDER);
+ zipStream.putNextEntry(emptyPlaceholder);
+ }
+ catch (IOException e)
+ {
+ logger.traceException(e);
+ message = ERR_JEB_BACKUP_CANNOT_WRITE_ARCHIVE_FILE.get(
+ ZIPENTRY_EMPTY_PLACEHOLDER, stackTraceToSingleLineString(e));
+ throw new DirectoryException(
+ DirectoryServer.getServerErrorResultCode(), message, e);
+ }
+ }
+
+ // Sort the log files from oldest to youngest since this is the order
+ // in which they must be copied.
+ // This is easy since the files are created in alphabetical order by JE.
+ Arrays.sort(logFiles);
+
+ try
+ {
+ // Process log files that are unchanged from the base backup.
+ int indexCurrent = 0;
+ if (latestFileName != null)
+ {
+ ArrayList<String> unchangedList = new ArrayList<String>();
+ while (indexCurrent < logFiles.length &&
+ !backupConfig.isCancelled())
+ {
+ File logFile = logFiles[indexCurrent];
+ String logFileName = logFile.getName();
+
+ // Stop when we get to the first log file that has been
+ // written since the base backup.
+ int compareResult = logFileName.compareTo(latestFileName);
+ if (compareResult > 0 ||
+ (compareResult == 0 && logFile.length() != latestFileSize))
+ {
+ break;
+ }
+
+ logger.info(NOTE_JEB_BACKUP_FILE_UNCHANGED, logFileName);
+
+ unchangedList.add(logFileName);
+
+ indexCurrent++;
+ }
+
+ // Write a file containing the list of unchanged log files.
+ if (!unchangedList.isEmpty())
+ {
+ String zipEntryName = ZIPENTRY_UNCHANGED_LOGFILES;
+ try
+ {
+ archiveList(zipStream, mac, digest, zipEntryName, unchangedList);
+ }
+ catch (IOException e)
+ {
+ logger.traceException(e);
+ message = ERR_JEB_BACKUP_CANNOT_WRITE_ARCHIVE_FILE.get(
+ zipEntryName, stackTraceToSingleLineString(e));
+ throw new DirectoryException(
+ DirectoryServer.getServerErrorResultCode(), message, e);
+ }
+
+ // Set the dependency.
+ dependencies.add(baseBackup.getBackupID());
+ }
+ }
+
+ // Write the new log files to the zip file.
+ do
+ {
+ boolean deletedFiles = false;
+
+ while (indexCurrent < logFiles.length &&
+ !backupConfig.isCancelled())
+ {
+ File logFile = logFiles[indexCurrent];
+
+ try
+ {
+ latestFileSize = archiveFile(zipStream, mac, digest,
+ logFile, backupConfig);
+ latestFileName = logFile.getName();
+ }
+ catch (FileNotFoundException e)
+ {
+ logger.traceException(e);
+
+ // A log file has been deleted by the cleaner since we started.
+ deletedFiles = true;
+ }
+ catch (IOException e)
+ {
+ logger.traceException(e);
+ message = ERR_JEB_BACKUP_CANNOT_WRITE_ARCHIVE_FILE.get(
+ logFile.getName(), stackTraceToSingleLineString(e));
+ throw new DirectoryException(
+ DirectoryServer.getServerErrorResultCode(), message, e);
+ }
+
+ indexCurrent++;
+ }
+
+ if (deletedFiles)
+ {
+ /*
+ The cleaner is active and has deleted one or more of the log files
+ since we started. The in-use data from those log files will have
+ been written to new log files, so we must include those new files.
+ */
+ final String latest = logFiles[logFiles.length-1].getName();
+ FilenameFilter filter = new JELatestFileFilter(latest,
+ latestFileSize);
+
+ try
+ {
+ logFiles = backendDir.listFiles(filter);
+ indexCurrent = 0;
+ }
+ catch (Exception e)
+ {
+ logger.traceException(e);
+
+ message = ERR_JEB_BACKUP_CANNOT_LIST_LOG_FILES.get(
+ backendDir.getAbsolutePath(), stackTraceToSingleLineString(e));
+ throw new DirectoryException(
+ DirectoryServer.getServerErrorResultCode(), message, e);
+ }
+
+ if (logFiles == null)
+ {
+ break;
+ }
+
+ Arrays.sort(logFiles);
+
+ logger.info(NOTE_JEB_BACKUP_CLEANER_ACTIVITY, logFiles.length);
+ }
+ else
+ {
+ // We are done.
+ break;
+ }
+ }
+ while (true);
+
+ }
+ // FIXME: The handling of exception below, plus the lack of finally block
+ // to close the zipStream is clumsy. Needs cleanup and best practice.
+ catch (DirectoryException e)
+ {
+ logger.traceException(e);
+
+ try
+ {
+ zipStream.close();
+ } catch (Exception e2) {}
+ }
+
+ // We're done writing the file, so close the zip stream (which should also
+ // close the underlying stream).
+ try
+ {
+ zipStream.close();
+ }
+ catch (Exception e)
+ {
+ logger.traceException(e);
+
+ message = ERR_JEB_BACKUP_CANNOT_CLOSE_ZIP_STREAM.
+ get(archiveFilename, backupDir.getPath(),
+ stackTraceToSingleLineString(e));
+ throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
+ message, e);
+ }
+
+
+ // Get the digest or MAC bytes if appropriate.
+ byte[] digestBytes = null;
+ byte[] macBytes = null;
+ if (hash)
+ {
+ if (signHash)
+ {
+ macBytes = mac.doFinal();
+ }
+ else
+ {
+ digestBytes = digest.digest();
+ }
+ }
+
+
+ // Create a descriptor for this backup.
+ backupProperties.put(PROPERTY_LAST_LOGFILE_NAME, latestFileName);
+ backupProperties.put(PROPERTY_LAST_LOGFILE_SIZE,
+ String.valueOf(latestFileSize));
+ BackupInfo backupInfo = new BackupInfo(backupDir, backupID,
+ backupDate, incremental, compress,
+ encrypt, digestBytes, macBytes,
+ dependencies, backupProperties);
+
+ try
+ {
+ backupDir.addBackup(backupInfo);
+ backupDir.writeBackupDirectoryDescriptor();
+ }
+ catch (Exception e)
+ {
+ logger.traceException(e);
+
+ message = ERR_JEB_BACKUP_CANNOT_UPDATE_BACKUP_DESCRIPTOR.get(
+ backupDir.getDescriptorPath(), stackTraceToSingleLineString(e));
+ throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
+ message, e);
+ }
+
+ // Remove the backup if this operation was cancelled since the
+ // backup may be incomplete
+ if (backupConfig.isCancelled())
+ {
+ removeBackup(backupDir, backupID);
+ }
+ }
+
+
+
+ /**
+ * Restore a JE backend from backup, or verify the backup.
+ * @param backendDir The configuration of the backend instance to be
+ * restored.
+ * @param restoreConfig The configuration to use when performing the restore.
+ * @throws DirectoryException If a Directory Server error occurs.
+ */
+ public void restoreBackup(File backendDir,
+ RestoreConfig restoreConfig)
+ throws DirectoryException
+ {
+ // Get the properties to use for the restore.
+ String backupID = restoreConfig.getBackupID();
+ BackupDirectory backupDir = restoreConfig.getBackupDirectory();
+ boolean verifyOnly = restoreConfig.verifyOnly();
+
+ BackupInfo backupInfo = getBackupInfo(backupDir, backupID);
+
+ // Create a restore directory with a different name to the backend
+ // directory.
+ File restoreDir = new File(backendDir.getPath() + "-restore-" + backupID);
+ if (!verifyOnly)
+ {
+ // FIXME: It's odd that we try to clean the directory before creating it
+ cleanup(restoreDir);
+ restoreDir.mkdir();
+ }
+
+ // Get the set of restore files that are in dependencies.
+ Set<String> includeFiles;
+ try
+ {
+ includeFiles = getUnchanged(backupDir, backupInfo);
+ }
+ catch (IOException e)
+ {
+ logger.traceException(e);
+ LocalizableMessage message = ERR_JEB_BACKUP_CANNOT_RESTORE.get(
+ backupInfo.getBackupID(), stackTraceToSingleLineString(e));
+ throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
+ message, e);
+ }
+
+ // Restore any dependencies.
+ List<BackupInfo> dependents = getDependents(backupDir, backupInfo);
+ for (BackupInfo dependent : dependents)
+ {
+ try
+ {
+ restoreArchive(restoreDir, restoreConfig, dependent, includeFiles);
+ }
+ catch (IOException e)
+ {
+ logger.traceException(e);
+ LocalizableMessage message = ERR_JEB_BACKUP_CANNOT_RESTORE.get(
+ dependent.getBackupID(), stackTraceToSingleLineString(e));
+ throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
+ message, e);
+ }
+ }
+
+ // Restore the final archive file.
+ try
+ {
+ restoreArchive(restoreDir, restoreConfig, backupInfo, null);
+ }
+ catch (IOException e)
+ {
+ logger.traceException(e);
+ LocalizableMessage message = ERR_JEB_BACKUP_CANNOT_RESTORE.get(
+ backupInfo.getBackupID(), stackTraceToSingleLineString(e));
+ throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
+ message, e);
+ }
+
+ // Delete the current backend directory and rename the restore directory.
+ if (!verifyOnly)
+ {
+ StaticUtils.recursiveDelete(backendDir);
+ if (!restoreDir.renameTo(backendDir))
+ {
+ LocalizableMessage msg = ERR_JEB_CANNOT_RENAME_RESTORE_DIRECTORY.get(
+ restoreDir.getPath(), backendDir.getPath());
+ throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
+ msg);
+ }
+ }
+ }
+
+ private void cleanup(File directory) {
+ File[] files = directory.listFiles();
+ if (files != null)
+ {
+ for (File f : files)
+ {
+ f.delete();
+ }
+ }
+ }
+
+ /**
+ * Removes the specified backup if it is possible to do so.
+ *
+ * @param backupDir The backup directory structure with which the
+ * specified backup is associated.
+ * @param backupID The backup ID for the backup to be removed.
+ *
+ * @throws DirectoryException If it is not possible to remove the specified
+ * backup for some reason (e.g., no such backup
+ * exists or there are other backups that are
+ * dependent upon it).
+ */
+ public void removeBackup(BackupDirectory backupDir,
+ String backupID)
+ throws DirectoryException
+ {
+ try
+ {
+ backupDir.removeBackup(backupID);
+ }
+ catch (ConfigException e)
+ {
+ logger.traceException(e);
+ throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
+ e.getMessageObject());
+ }
+
+ try
+ {
+ backupDir.writeBackupDirectoryDescriptor();
+ }
+ catch (Exception e)
+ {
+ logger.traceException(e);
+
+ LocalizableMessage message = ERR_JEB_BACKUP_CANNOT_UPDATE_BACKUP_DESCRIPTOR.get(
+ backupDir.getDescriptorPath(), stackTraceToSingleLineString(e));
+ throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
+ message, e);
+ }
+
+ // Remove the archive file.
+ BackupInfo backupInfo = getBackupInfo(backupDir, backupID);
+ File archiveFile = getArchiveFile(backupDir, backupInfo);
+ archiveFile.delete();
+
+ }
+
+ private File getArchiveFile(BackupDirectory backupDir,
+ BackupInfo backupInfo) {
+ Map<String,String> backupProperties = backupInfo.getBackupProperties();
+
+ String archiveFilename =
+ backupProperties.get(BACKUP_PROPERTY_ARCHIVE_FILENAME);
+ return new File(backupDir.getPath(), archiveFilename);
+ }
+
+
+ /**
+ * Restore the contents of an archive file. If the archive is being
+ * restored as a dependency, then only files in the specified set
+ * are restored, and the restored files are removed from the set. Otherwise
+ * all files from the archive are restored, and files that are to be found
+ * in dependencies are added to the set.
+ *
+ * @param restoreDir The directory in which files are to be restored.
+ * @param restoreConfig The restore configuration.
+ * @param backupInfo The backup containing the files to be restored.
+ * @param includeFiles The set of files to be restored. If null, then
+ * all files are restored.
+ * @throws DirectoryException If a Directory Server error occurs.
+ * @throws IOException If an I/O exception occurs during the restore.
+ */
+ private void restoreArchive(File restoreDir,
+ RestoreConfig restoreConfig,
+ BackupInfo backupInfo,
+ Set<String> includeFiles)
+ throws DirectoryException,IOException
+ {
+ BackupDirectory backupDir = restoreConfig.getBackupDirectory();
+ boolean verifyOnly = restoreConfig.verifyOnly();
+
+ String backupID = backupInfo.getBackupID();
+ boolean encrypt = backupInfo.isEncrypted();
+ byte[] hash = backupInfo.getUnsignedHash();
+ byte[] signHash = backupInfo.getSignedHash();
+
+ HashMap<String,String> backupProperties = backupInfo.getBackupProperties();
+
+ String archiveFilename =
+ backupProperties.get(BACKUP_PROPERTY_ARCHIVE_FILENAME);
+ File archiveFile = new File(backupDir.getPath(), archiveFilename);
+
+ InputStream inputStream = new FileInputStream(archiveFile);
+
+ // Get the crypto manager and use it to obtain references to the message
+ // digest and/or MAC to use for hashing and/or signing.
+ CryptoManager cryptoManager = DirectoryServer.getCryptoManager();
+ Mac mac = null;
+ MessageDigest digest = null;
+
+ if (signHash != null)
+ {
+ String macKeyID = backupProperties.get(BACKUP_PROPERTY_MAC_KEY_ID);
+
+ try
+ {
+ mac = cryptoManager.getMacEngine(macKeyID);
+ }
+ catch (Exception e)
+ {
+ logger.traceException(e);
+
+ LocalizableMessage message = ERR_JEB_BACKUP_CANNOT_GET_MAC.get(
+ macKeyID, stackTraceToSingleLineString(e));
+ throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
+ message, e);
+ }
+ }
+
+ if (hash != null)
+ {
+ String digestAlgorithm = backupProperties.get(
+ BACKUP_PROPERTY_DIGEST_ALGORITHM);
+
+ try
+ {
+ digest = cryptoManager.getMessageDigest(digestAlgorithm);
+ }
+ catch (Exception e)
+ {
+ logger.traceException(e);
+
+ LocalizableMessage message = ERR_JEB_BACKUP_CANNOT_GET_DIGEST.get(
+ digestAlgorithm, stackTraceToSingleLineString(e));
+ throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
+ message, e);
+ }
+ }
+
+
+ // If the data is encrypted, then wrap the input stream in a cipher
+ // input stream.
+ if (encrypt)
+ {
+ try
+ {
+ inputStream = cryptoManager.getCipherInputStream(inputStream);
+ }
+ catch (CryptoManagerException e)
+ {
+ logger.traceException(e);
+
+ LocalizableMessage message = ERR_JEB_BACKUP_CANNOT_GET_CIPHER.get(
+ stackTraceToSingleLineString(e));
+ throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
+ message, e);
+ }
+ }
+
+
+ // Wrap the file input stream in a zip input stream.
+ ZipInputStream zipStream = new ZipInputStream(inputStream);
+
+ // Iterate through the entries in the zip file.
+ ZipEntry zipEntry = zipStream.getNextEntry();
+ while (zipEntry != null && !restoreConfig.isCancelled())
+ {
+ String name = zipEntry.getName();
+
+ if (name.equals(ZIPENTRY_EMPTY_PLACEHOLDER))
+ {
+ // This entry is treated specially to indicate a backup of an empty
+ // backend was attempted.
+
+ zipEntry = zipStream.getNextEntry();
+ continue;
+ }
+
+ if (name.equals(ZIPENTRY_UNCHANGED_LOGFILES))
+ {
+ // This entry is treated specially. It is never restored,
+ // and its hash is computed on the strings, not the bytes.
+ if (mac != null || digest != null)
+ {
+ // The file name is part of the hash.
+ if (mac != null)
+ {
+ mac.update(getBytes(name));
+ }
+
+ if (digest != null)
+ {
+ digest.update(getBytes(name));
+ }
+
+ InputStreamReader reader = new InputStreamReader(zipStream);
+ BufferedReader bufferedReader = new BufferedReader(reader);
+ String line = bufferedReader.readLine();
+ while (line != null)
+ {
+ if (mac != null)
+ {
+ mac.update(getBytes(line));
+ }
+
+ if (digest != null)
+ {
+ digest.update(getBytes(line));
+ }
+
+ line = bufferedReader.readLine();
+ }
+ }
+
+ zipEntry = zipStream.getNextEntry();
+ continue;
+ }
+
+ // See if we need to restore the file.
+ File file = new File(restoreDir, name);
+ OutputStream outputStream = null;
+ if (includeFiles == null || includeFiles.contains(zipEntry.getName()))
+ {
+ if (!verifyOnly)
+ {
+ outputStream = new FileOutputStream(file);
+ }
+ }
+
+ if (outputStream != null || mac != null || digest != null)
+ {
+ if (verifyOnly)
+ {
+ logger.info(NOTE_JEB_BACKUP_VERIFY_FILE, zipEntry.getName());
+ }
+
+ // The file name is part of the hash.
+ if (mac != null)
+ {
+ mac.update(getBytes(name));
+ }
+
+ if (digest != null)
+ {
+ digest.update(getBytes(name));
+ }
+
+ // Process the file.
+ long totalBytesRead = 0;
+ byte[] buffer = new byte[8192];
+ int bytesRead = zipStream.read(buffer);
+ while (bytesRead > 0 && !restoreConfig.isCancelled())
+ {
+ totalBytesRead += bytesRead;
+
+ if (mac != null)
+ {
+ mac.update(buffer, 0, bytesRead);
+ }
+
+ if (digest != null)
+ {
+ digest.update(buffer, 0, bytesRead);
+ }
+
+ if (outputStream != null)
+ {
+ outputStream.write(buffer, 0, bytesRead);
+ }
+
+ bytesRead = zipStream.read(buffer);
+ }
+
+ if (outputStream != null)
+ {
+ outputStream.close();
+
+ logger.info(NOTE_JEB_BACKUP_RESTORED_FILE, zipEntry.getName(), totalBytesRead);
+ }
+ }
+
+ zipEntry = zipStream.getNextEntry();
+ }
+
+ zipStream.close();
+
+ // Check the hash.
+ if (digest != null)
+ {
+ if (!Arrays.equals(digest.digest(), hash))
+ {
+ LocalizableMessage message = ERR_JEB_BACKUP_UNSIGNED_HASH_ERROR.get(backupID);
+ throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
+ message);
+ }
+ }
+
+ if (mac != null)
+ {
+ byte[] computedSignHash = mac.doFinal();
+
+ if (!Arrays.equals(computedSignHash, signHash))
+ {
+ LocalizableMessage message = ERR_JEB_BACKUP_SIGNED_HASH_ERROR.get(backupID);
+ throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
+ message);
+ }
+ }
+ }
+
+
+
+ /**
+ * Writes a file to an entry in the archive file.
+ * @param zipStream The zip output stream to which the file is to be
+ * written.
+ * @param mac A message authentication code to be updated, if not null.
+ * @param digest A message digest to be updated, if not null.
+ * @param file The file to be written.
+ * @return The number of bytes written from the file.
+ * @throws FileNotFoundException If the file to be archived does not exist.
+ * @throws IOException If an I/O error occurs while archiving the file.
+ */
+ private long archiveFile(ZipOutputStream zipStream,
+ Mac mac, MessageDigest digest, File file,
+ BackupConfig backupConfig)
+ throws IOException, FileNotFoundException
+ {
+ ZipEntry zipEntry = new ZipEntry(file.getName());
+
+ // Open the file for reading.
+ InputStream inputStream = new FileInputStream(file);
+
+ // Start the zip entry.
+ zipStream.putNextEntry(zipEntry);
+
+ // Put the name in the hash.
+ if (mac != null)
+ {
+ mac.update(getBytes(file.getName()));
+ }
+
+ if (digest != null)
+ {
+ digest.update(getBytes(file.getName()));
+ }
+
+ // Write the file.
+ long totalBytesRead = 0;
+ byte[] buffer = new byte[8192];
+ int bytesRead = inputStream.read(buffer);
+ while (bytesRead > 0 && !backupConfig.isCancelled())
+ {
+ if (mac != null)
+ {
+ mac.update(buffer, 0, bytesRead);
+ }
+
+ if (digest != null)
+ {
+ digest.update(buffer, 0, bytesRead);
+ }
+
+ zipStream.write(buffer, 0, bytesRead);
+ totalBytesRead += bytesRead;
+ bytesRead = inputStream.read(buffer);
+ }
+ inputStream.close();
+
+ // Finish the zip entry.
+ zipStream.closeEntry();
+
+ logger.info(NOTE_JEB_BACKUP_ARCHIVED_FILE, zipEntry.getName());
+
+ return totalBytesRead;
+ }
+
+ /**
+ * Write a list of strings to an entry in the archive file.
+ * @param zipStream The zip output stream to which the entry is to be
+ * written.
+ * @param mac An optional MAC to be updated.
+ * @param digest An optional message digest to be updated.
+ * @param fileName The name of the zip entry to be written.
+ * @param list A list of strings to be written. The strings must not
+ * contain newlines.
+ * @throws IOException If an I/O error occurs while writing the archive entry.
+ */
+ private void archiveList(ZipOutputStream zipStream,
+ Mac mac, MessageDigest digest, String fileName,
+ List<String> list)
+ throws IOException
+ {
+ ZipEntry zipEntry = new ZipEntry(fileName);
+
+ // Start the zip entry.
+ zipStream.putNextEntry(zipEntry);
+
+ // Put the name in the hash.
+ if (mac != null)
+ {
+ mac.update(getBytes(fileName));
+ }
+
+ if (digest != null)
+ {
+ digest.update(getBytes(fileName));
+ }
+
+ Writer writer = new OutputStreamWriter(zipStream);
+ for (String s : list)
+ {
+ if (mac != null)
+ {
+ mac.update(getBytes(s));
+ }
+
+ if (digest != null)
+ {
+ digest.update(getBytes(s));
+ }
+
+ writer.write(s);
+ writer.write(EOL);
+ }
+ writer.flush();
+
+ // Finish the zip entry.
+ zipStream.closeEntry();
+ }
+
+ /**
+ * Obtains the set of files in a backup that are unchanged from its
+ * dependent backup or backups. This list is stored as the first entry
+ * in the archive file.
+ * @param backupDir The backup directory.
+ * @param backupInfo The backup info.
+ * @return The set of files that were unchanged.
+ * @throws DirectoryException If an error occurs while trying to get the
+ * appropriate cipher algorithm for an encrypted backup.
+ * @throws IOException If an I/O error occurs while reading the backup
+ * archive file.
+ */
+ private Set<String> getUnchanged(BackupDirectory backupDir,
+ BackupInfo backupInfo)
+ throws DirectoryException, IOException
+ {
+ HashSet<String> hashSet = new HashSet<String>();
+
+ boolean encrypt = backupInfo.isEncrypted();
+
+ File archiveFile = getArchiveFile(backupDir, backupInfo);
+
+ InputStream inputStream = new FileInputStream(archiveFile);
+
+ // Get the crypto manager and use it to obtain references to the message
+ // digest and/or MAC to use for hashing and/or signing.
+ CryptoManager cryptoManager = DirectoryServer.getCryptoManager();
+
+ // If the data is encrypted, then wrap the input stream in a cipher
+ // input stream.
+ if (encrypt)
+ {
+ try
+ {
+ inputStream = cryptoManager.getCipherInputStream(inputStream);
+ }
+ catch (CryptoManagerException e)
+ {
+ logger.traceException(e);
+
+ LocalizableMessage message = ERR_JEB_BACKUP_CANNOT_GET_CIPHER.get(
+ stackTraceToSingleLineString(e));
+ throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
+ message, e);
+ }
+ }
+
+
+ // Wrap the file input stream in a zip input stream.
+ ZipInputStream zipStream = new ZipInputStream(inputStream);
+
+ // Iterate through the entries in the zip file.
+ ZipEntry zipEntry = zipStream.getNextEntry();
+ while (zipEntry != null)
+ {
+ // We are looking for the entry containing the list of unchanged files.
+ if (zipEntry.getName().equals(ZIPENTRY_UNCHANGED_LOGFILES))
+ {
+ InputStreamReader reader = new InputStreamReader(zipStream);
+ BufferedReader bufferedReader = new BufferedReader(reader);
+ String line = bufferedReader.readLine();
+ while (line != null)
+ {
+ hashSet.add(line);
+ line = bufferedReader.readLine();
+ }
+ break;
+ }
+
+ zipEntry = zipStream.getNextEntry();
+ }
+
+ zipStream.close();
+ return hashSet;
+ }
+
+ /**
+ * Obtains a list of the dependencies of a given backup in order from
+ * the oldest (the full backup), to the most recent.
+ * @param backupDir The backup directory.
+ * @param backupInfo The backup for which dependencies are required.
+ * @return A list of dependent backups.
+ * @throws DirectoryException If a Directory Server error occurs.
+ */
+ private ArrayList<BackupInfo> getDependents(BackupDirectory backupDir,
+ BackupInfo backupInfo)
+ throws DirectoryException
+ {
+ ArrayList<BackupInfo> dependents = new ArrayList<BackupInfo>();
+ while (backupInfo != null && !backupInfo.getDependencies().isEmpty())
+ {
+ String backupID = backupInfo.getDependencies().iterator().next();
+ backupInfo = getBackupInfo(backupDir, backupID);
+ if (backupInfo != null)
+ {
+ dependents.add(backupInfo);
+ }
+ }
+ Collections.reverse(dependents);
+ return dependents;
+ }
+
+ /**
+ * Get the information for a given backup ID from the backup directory.
+ * @param backupDir The backup directory.
+ * @param backupID The backup ID.
+ * @return The backup information, never null.
+ * @throws DirectoryException If the backup information cannot be found.
+ */
+ private BackupInfo getBackupInfo(BackupDirectory backupDir,
+ String backupID) throws DirectoryException
+ {
+ BackupInfo backupInfo = backupDir.getBackupInfo(backupID);
+ if (backupInfo == null)
+ {
+ LocalizableMessage message =
+ ERR_JEB_BACKUP_MISSING_BACKUPID.get(backupDir.getPath(), backupID);
+ throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
+ message);
+ }
+ return backupInfo;
+ }
+
+ /**
+ * This class implements a FilenameFilter to detect the last file
+ * from a JE database.
+ */
+ private static class JELatestFileFilter implements FilenameFilter {
+ private final String latest;
+ private final long latestSize;
+
+ public JELatestFileFilter(String latest, long latestSize) {
+ this.latest = latest;
+ this.latestSize = latestSize;
+ }
+
+ public boolean accept(File d, String name)
+ {
+ if (!name.endsWith(".jdb")) return false;
+ int compareTo = name.compareTo(latest);
+ return compareTo > 0 || compareTo == 0 && d.length() > latestSize;
+ }
+ }
+}
diff --git a/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/ConfigurableEnvironment.java b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/ConfigurableEnvironment.java
new file mode 100644
index 0000000..7baa2bf
--- /dev/null
+++ b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/ConfigurableEnvironment.java
@@ -0,0 +1,594 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License"). You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at legal-notices/CDDLv1_0.txt
+ * or http://forgerock.org/license/CDDLv1.0.html.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at legal-notices/CDDLv1_0.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information:
+ * Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ *
+ *
+ * Copyright 2006-2010 Sun Microsystems, Inc.
+ * Portions Copyright 2010-2014 ForgeRock AS.
+ */
+package org.opends.server.backends.pluggable;
+
+import java.lang.reflect.Method;
+import java.math.BigInteger;
+import java.util.*;
+import java.util.concurrent.TimeUnit;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import org.forgerock.i18n.LocalizableMessage;
+import org.forgerock.i18n.slf4j.LocalizedLogger;
+import org.opends.server.admin.BooleanPropertyDefinition;
+import org.opends.server.admin.DurationPropertyDefinition;
+import org.opends.server.admin.PropertyDefinition;
+import org.opends.server.admin.std.meta.LocalDBBackendCfgDefn;
+import org.opends.server.admin.std.server.LocalDBBackendCfg;
+import org.opends.server.config.ConfigConstants;
+import org.forgerock.opendj.config.server.ConfigException;
+
+
+
+
+import static com.sleepycat.je.EnvironmentConfig.*;
+
+import static org.opends.messages.BackendMessages.*;
+import static org.opends.messages.ConfigMessages.*;
+import static org.opends.messages.JebMessages.*;
+
+/**
+ * This class maps JE properties to configuration attributes.
+ */
+public class ConfigurableEnvironment
+{
+ private static final LocalizedLogger logger = LocalizedLogger.getLoggerForThisClass();
+
+ /**
+ * The name of the attribute which configures the database cache size as a
+ * percentage of Java VM heap size.
+ */
+ public static final String ATTR_DATABASE_CACHE_PERCENT =
+ ConfigConstants.NAME_PREFIX_CFG + "db-cache-percent";
+
+ /**
+ * The name of the attribute which configures the database cache size as an
+ * approximate number of bytes.
+ */
+ public static final String ATTR_DATABASE_CACHE_SIZE =
+ ConfigConstants.NAME_PREFIX_CFG + "db-cache-size";
+
+ /**
+ * The name of the attribute which configures whether data updated by a
+ * database transaction is forced to disk.
+ */
+ public static final String ATTR_DATABASE_TXN_NO_SYNC =
+ ConfigConstants.NAME_PREFIX_CFG + "db-txn-no-sync";
+
+ /**
+ * The name of the attribute which configures whether data updated by a
+ * database transaction is written from the Java VM to the O/S.
+ */
+ public static final String ATTR_DATABASE_TXN_WRITE_NO_SYNC =
+ ConfigConstants.NAME_PREFIX_CFG + "db-txn-write-no-sync";
+
+ /**
+ * The name of the attribute which configures whether the database background
+ * cleaner thread runs.
+ */
+ public static final String ATTR_DATABASE_RUN_CLEANER =
+ ConfigConstants.NAME_PREFIX_CFG + "db-run-cleaner";
+
+ /**
+ * The name of the attribute which configures the minimum percentage of log
+ * space that must be used in log files.
+ */
+ public static final String ATTR_CLEANER_MIN_UTILIZATION =
+ ConfigConstants.NAME_PREFIX_CFG + "db-cleaner-min-utilization";
+
+ /**
+ * The name of the attribute which configures the maximum size of each
+ * individual JE log file, in bytes.
+ */
+ public static final String ATTR_DATABASE_LOG_FILE_MAX =
+ ConfigConstants.NAME_PREFIX_CFG + "db-log-file-max";
+
+ /**
+ * The name of the attribute which configures the database cache eviction
+ * algorithm.
+ */
+ public static final String ATTR_EVICTOR_LRU_ONLY =
+ ConfigConstants.NAME_PREFIX_CFG + "db-evictor-lru-only";
+
+ /**
+ * The name of the attribute which configures the number of nodes in one scan
+ * of the database cache evictor.
+ */
+ public static final String ATTR_EVICTOR_NODES_PER_SCAN =
+ ConfigConstants.NAME_PREFIX_CFG + "db-evictor-nodes-per-scan";
+
+ /**
+ * The name of the attribute which configures the minimum number of threads
+ * of the database cache evictor pool.
+ */
+ public static final String ATTR_EVICTOR_CORE_THREADS =
+ ConfigConstants.NAME_PREFIX_CFG + "db-evictor-core-threads";
+ /**
+ * The name of the attribute which configures the maximum number of threads
+ * of the database cache evictor pool.
+ */
+ public static final String ATTR_EVICTOR_MAX_THREADS =
+ ConfigConstants.NAME_PREFIX_CFG + "db-evictor-max-threads";
+
+ /**
+ * The name of the attribute which configures the time excess threads
+ * of the database cache evictor pool are kept alive.
+ */
+ public static final String ATTR_EVICTOR_KEEP_ALIVE =
+ ConfigConstants.NAME_PREFIX_CFG + "db-evictor-keep-alive";
+
+ /**
+ * The name of the attribute which configures whether the logging file
+ * handler will be on or off.
+ */
+ public static final String ATTR_LOGGING_FILE_HANDLER_ON =
+ ConfigConstants.NAME_PREFIX_CFG + "db-logging-file-handler-on";
+
+
+ /**
+ * The name of the attribute which configures the trace logging message level.
+ */
+ public static final String ATTR_LOGGING_LEVEL =
+ ConfigConstants.NAME_PREFIX_CFG + "db-logging-level";
+
+
+ /**
+ * The name of the attribute which configures how many bytes are written to
+ * the log before the checkpointer runs.
+ */
+ public static final String ATTR_CHECKPOINTER_BYTES_INTERVAL =
+ ConfigConstants.NAME_PREFIX_CFG + "db-checkpointer-bytes-interval";
+
+
+ /**
+ * The name of the attribute which configures the amount of time between
+ * runs of the checkpointer.
+ */
+ public static final String ATTR_CHECKPOINTER_WAKEUP_INTERVAL =
+ ConfigConstants.NAME_PREFIX_CFG +
+ "db-checkpointer-wakeup-interval";
+
+
+ /**
+ * The name of the attribute which configures the number of lock tables.
+ */
+ public static final String ATTR_NUM_LOCK_TABLES =
+ ConfigConstants.NAME_PREFIX_CFG + "db-num-lock-tables";
+
+
+ /**
+ * The name of the attribute which configures the number threads
+ * allocated by the cleaner for log file processing.
+ */
+ public static final String ATTR_NUM_CLEANER_THREADS =
+ ConfigConstants.NAME_PREFIX_CFG + "db-num-cleaner-threads";
+
+ /**
+ * The name of the attribute which configures the size of the file
+ * handle cache.
+ */
+ public static final String ATTR_LOG_FILECACHE_SIZE =
+ ConfigConstants.NAME_PREFIX_CFG + "db-log-filecache-size";
+
+
+ /**
+ * The name of the attribute which may specify any native JE properties.
+ */
+ public static final String ATTR_JE_PROPERTY =
+ ConfigConstants.NAME_PREFIX_CFG + "je-property";
+
+
+ /**
+ * A map of JE property names to the corresponding configuration attribute.
+ */
+ private static HashMap<String, String> attrMap =
+ new HashMap<String, String>();
+
+ /**
+ * A map of configuration attribute names to the corresponding configuration
+ * object getter method.
+ */
+ private static HashMap<String,Method> methodMap =
+ new HashMap<String, Method>();
+
+ /**
+ * A map of configuration attribute names to the corresponding configuration
+ * PropertyDefinition.
+ */
+ private static HashMap<String,PropertyDefinition> defnMap =
+ new HashMap<String, PropertyDefinition>();
+
+
+ // Pulled from resource/admin/ABBREVIATIONS.xsl. db is mose common.
+ private static final List<String> ABBREVIATIONS = Arrays.asList(new String[]
+ {"aci", "ip", "ssl", "dn", "rdn", "jmx", "smtp", "http",
+ "https", "ldap", "ldaps", "ldif", "jdbc", "tcp", "tls",
+ "pkcs11", "sasl", "gssapi", "md5", "je", "dse", "fifo",
+ "vlv", "uuid", "md5", "sha1", "sha256", "sha384", "sha512",
+ "tls", "db"});
+
+ /*
+ * e.g. db-cache-percent -> DBCachePercent
+ */
+ private static String propNametoCamlCase(String hyphenated)
+ {
+ String[] components = hyphenated.split("\\-");
+ StringBuilder buffer = new StringBuilder();
+ for (String component: components) {
+ if (ABBREVIATIONS.contains(component)) {
+ buffer.append(component.toUpperCase());
+ } else {
+ buffer.append(component.substring(0, 1).toUpperCase() +
+ component.substring(1));
+ }
+ }
+ return buffer.toString();
+ }
+
+
+ /**
+ * Register a JE property and its corresponding configuration attribute.
+ *
+ * @param propertyName The name of the JE property to be registered.
+ * @param attrName The name of the configuration attribute associated
+ * with the property.
+ * @throws Exception If there is an error in the attribute name.
+ */
+ private static void registerProp(String propertyName, String attrName)
+ throws Exception
+ {
+ // Strip off NAME_PREFIX_CFG.
+ String baseName = attrName.substring(7);
+
+ String methodBaseName = propNametoCamlCase(baseName);
+
+ Class<LocalDBBackendCfg> configClass = LocalDBBackendCfg.class;
+ LocalDBBackendCfgDefn defn = LocalDBBackendCfgDefn.getInstance();
+ Class<? extends LocalDBBackendCfgDefn> defClass = defn.getClass();
+
+ PropertyDefinition propDefn =
+ (PropertyDefinition)defClass.getMethod("get" + methodBaseName +
+ "PropertyDefinition").invoke(defn);
+
+ String methodName;
+ if (propDefn instanceof BooleanPropertyDefinition)
+ {
+ methodName = "is" + methodBaseName;
+ }
+ else
+ {
+ methodName = "get" + methodBaseName;
+ }
+
+ defnMap.put(attrName, propDefn);
+ methodMap.put(attrName, configClass.getMethod(methodName));
+ attrMap.put(propertyName, attrName);
+ }
+
+
+ /**
+ * Get the name of the configuration attribute associated with a JE property.
+ * @param jeProperty The name of the JE property.
+ * @return The name of the associated configuration attribute.
+ */
+ public static String getAttributeForProperty(String jeProperty)
+ {
+ return attrMap.get(jeProperty);
+ }
+
+ /**
+ * Get the value of a JE property that is mapped to a configuration attribute.
+ * @param cfg The configuration containing the property values.
+ * @param attrName The conriguration attribute type name.
+ * @return The string value of the JE property.
+ */
+ private static String getPropertyValue(LocalDBBackendCfg cfg, String attrName)
+ {
+ try
+ {
+ PropertyDefinition propDefn = defnMap.get(attrName);
+ Method method = methodMap.get(attrName);
+
+ if (propDefn instanceof DurationPropertyDefinition)
+ {
+ Long value = (Long)method.invoke(cfg);
+
+ // JE durations are in microseconds so we must convert.
+ DurationPropertyDefinition durationPropDefn =
+ (DurationPropertyDefinition)propDefn;
+ value = 1000*durationPropDefn.getBaseUnit().toMilliSeconds(value);
+
+ return String.valueOf(value);
+ }
+ else
+ {
+ Object value = method.invoke(cfg);
+
+ if (attrName.equals(ATTR_NUM_CLEANER_THREADS) && value == null)
+ {
+ // Automatically choose based on the number of processors. We will use
+ // similar heuristics to those used to define the default number of
+ // worker threads.
+ int cpus = Runtime.getRuntime().availableProcessors();
+ value = Integer.valueOf(Math.max(24, cpus * 2));
+
+ logger.debug(INFO_ERGONOMIC_SIZING_OF_JE_CLEANER_THREADS,
+ cfg.dn().rdn().getAttributeValue(0), (Number) value);
+ }
+ else if (attrName.equals(ATTR_NUM_LOCK_TABLES)
+ && value == null)
+ {
+ // Automatically choose based on the number of processors.
+ // We'll assume that the user has also allowed automatic
+ // configuration of cleaners and workers.
+ int cpus = Runtime.getRuntime().availableProcessors();
+ int cleaners = Math.max(24, cpus * 2);
+ int workers = Math.max(24, cpus * 2);
+ BigInteger tmp = BigInteger.valueOf((cleaners + workers) * 2);
+ value = tmp.nextProbablePrime();
+
+ logger.debug(INFO_ERGONOMIC_SIZING_OF_JE_LOCK_TABLES, cfg.dn().rdn().getAttributeValue(0), (Number) value);
+ }
+
+ return String.valueOf(value);
+ }
+ }
+ catch (Exception e)
+ {
+ logger.traceException(e);
+ return "";
+ }
+ }
+
+
+
+ static
+ {
+ // Register the parameters that have JE property names.
+ try
+ {
+ registerProp("je.maxMemoryPercent", ATTR_DATABASE_CACHE_PERCENT);
+ registerProp("je.maxMemory", ATTR_DATABASE_CACHE_SIZE);
+ registerProp("je.cleaner.minUtilization", ATTR_CLEANER_MIN_UTILIZATION);
+ registerProp("je.env.runCleaner", ATTR_DATABASE_RUN_CLEANER);
+ registerProp("je.evictor.lruOnly", ATTR_EVICTOR_LRU_ONLY);
+ registerProp("je.evictor.nodesPerScan", ATTR_EVICTOR_NODES_PER_SCAN);
+ registerProp("je.evictor.coreThreads", ATTR_EVICTOR_CORE_THREADS);
+ registerProp("je.evictor.maxThreads", ATTR_EVICTOR_MAX_THREADS);
+ registerProp("je.evictor.keepAlive", ATTR_EVICTOR_KEEP_ALIVE);
+ registerProp("je.log.fileMax", ATTR_DATABASE_LOG_FILE_MAX);
+ registerProp("je.checkpointer.bytesInterval",
+ ATTR_CHECKPOINTER_BYTES_INTERVAL);
+ registerProp("je.checkpointer.wakeupInterval",
+ ATTR_CHECKPOINTER_WAKEUP_INTERVAL);
+ registerProp("je.lock.nLockTables", ATTR_NUM_LOCK_TABLES);
+ registerProp("je.cleaner.threads", ATTR_NUM_CLEANER_THREADS);
+ registerProp("je.log.fileCacheSize", ATTR_LOG_FILECACHE_SIZE);
+ }
+ catch (Exception e)
+ {
+ logger.traceException(e);
+ }
+ }
+
+
+
+ /**
+ * Create a JE environment configuration with default values.
+ *
+ * @return A JE environment config containing default values.
+ */
+ public static EnvironmentConfig defaultConfig()
+ {
+ EnvironmentConfig envConfig = new EnvironmentConfig();
+
+ envConfig.setTransactional(true);
+ envConfig.setAllowCreate(true);
+
+ // "je.env.sharedLatches" is "true" by default since JE #12136 (3.3.62?)
+
+ // This parameter was set to false while diagnosing a Berkeley DB JE bug.
+ // Normally cleansed log files are deleted, but if this is set false
+ // they are instead renamed from .jdb to .del.
+ envConfig.setConfigParam(CLEANER_EXPUNGE, "true");
+
+ // Under heavy write load the check point can fall behind causing
+ // uncontrolled DB growth over time. This parameter makes the out of
+ // the box configuration more robust at the cost of a slight
+ // reduction in maximum write throughput. Experiments have shown
+ // that response time predictability is not impacted negatively.
+ envConfig.setConfigParam(CHECKPOINTER_HIGH_PRIORITY, "true");
+
+ // If the JVM is reasonably large then we can safely default to
+ // bigger read buffers. This will result in more scalable checkpointer
+ // and cleaner performance.
+ if (Runtime.getRuntime().maxMemory() > 256 * 1024 * 1024)
+ {
+ envConfig.setConfigParam(CLEANER_LOOK_AHEAD_CACHE_SIZE,
+ String.valueOf(2 * 1024 * 1024));
+ envConfig.setConfigParam(LOG_ITERATOR_READ_SIZE,
+ String.valueOf(2 * 1024 * 1024));
+ envConfig.setConfigParam(LOG_FAULT_READ_SIZE, String.valueOf(4 * 1024));
+ }
+
+ // Disable lock timeouts, meaning that no lock wait
+ // timelimit is enforced and a deadlocked operation
+ // will block indefinitely.
+ envConfig.setLockTimeout(0, TimeUnit.MICROSECONDS);
+
+ return envConfig;
+ }
+
+
+
+ /**
+ * Parse a configuration associated with a JE environment and create an
+ * environment config from it.
+ *
+ * @param cfg The configuration to be parsed.
+ * @return An environment config instance corresponding to the config entry.
+ * @throws ConfigException If there is an error in the provided configuration
+ * entry.
+ */
+ public static EnvironmentConfig parseConfigEntry(LocalDBBackendCfg cfg)
+ throws ConfigException
+ {
+ // See if the db cache size setting is valid.
+ if(cfg.getDBCacheSize() != 0)
+ {
+ if (MemoryBudget.getRuntimeMaxMemory() < cfg.getDBCacheSize()) {
+ throw new ConfigException(
+ ERR_CONFIG_JEB_CACHE_SIZE_GREATER_THAN_JVM_HEAP.get(
+ cfg.getDBCacheSize(), MemoryBudget.getRuntimeMaxMemory()));
+ }
+ if (cfg.getDBCacheSize() < MemoryBudget.MIN_MAX_MEMORY_SIZE) {
+ throw new ConfigException(
+ ERR_CONFIG_JEB_CACHE_SIZE_TOO_SMALL.get(
+ cfg.getDBCacheSize(), MemoryBudget.MIN_MAX_MEMORY_SIZE));
+ }
+ }
+
+ EnvironmentConfig envConfig = defaultConfig();
+
+ // Durability settings.
+ if (cfg.isDBTxnNoSync() && cfg.isDBTxnWriteNoSync())
+ {
+ throw new ConfigException(
+ ERR_CONFIG_JEB_DURABILITY_CONFLICT.get());
+ }
+ if (cfg.isDBTxnNoSync())
+ {
+ envConfig.setDurability(Durability.COMMIT_NO_SYNC);
+ }
+ if (cfg.isDBTxnWriteNoSync())
+ {
+ envConfig.setDurability(Durability.COMMIT_WRITE_NO_SYNC);
+ }
+
+ // Iterate through the config attributes associated with a JE property.
+ for (Map.Entry<String, String> mapEntry : attrMap.entrySet())
+ {
+ String jeProperty = mapEntry.getKey();
+ String attrName = mapEntry.getValue();
+
+ String value = getPropertyValue(cfg, attrName);
+ envConfig.setConfigParam(jeProperty, value);
+ }
+
+ // Set logging and file handler levels.
+ Logger parent = Logger.getLogger("com.sleepycat.je");
+ try
+ {
+ parent.setLevel(Level.parse(cfg.getDBLoggingLevel()));
+ }
+ catch (Exception e)
+ {
+ throw new ConfigException(ERR_JEB_INVALID_LOGGING_LEVEL.get(cfg.getDBLoggingLevel(), cfg.dn()));
+ }
+
+ final Level level = cfg.isDBLoggingFileHandlerOn() ? Level.ALL : Level.OFF;
+ envConfig.setConfigParam(FILE_LOGGING_LEVEL, level.getName());
+
+ // See if there are any native JE properties specified in the config
+ // and if so try to parse, evaluate and set them.
+ return setJEProperties(envConfig, cfg.getJEProperty(), attrMap);
+ }
+
+
+
+ /**
+ * Parse, validate and set native JE environment properties for
+ * a given environment config.
+ *
+ * @param envConfig The JE environment config for which to set
+ * the properties.
+ * @param jeProperties The JE environment properties to parse,
+ * validate and set.
+ * @param configAttrMap Component supported JE properties to
+ * their configuration attributes map.
+ * @return An environment config instance with given properties
+ * set.
+ * @throws ConfigException If there is an error while parsing,
+ * validating and setting any of the properties provided.
+ */
+ public static EnvironmentConfig setJEProperties(EnvironmentConfig envConfig,
+ SortedSet<String> jeProperties, HashMap<String, String> configAttrMap)
+ throws ConfigException
+ {
+ if (jeProperties.isEmpty()) {
+ // return default config.
+ return envConfig;
+ }
+
+ // Set to catch duplicate properties.
+ HashSet<String> uniqueJEProperties = new HashSet<String>();
+
+ // Iterate through the config values associated with a JE property.
+ for (String jeEntry : jeProperties)
+ {
+ StringTokenizer st = new StringTokenizer(jeEntry, "=");
+ if (st.countTokens() == 2) {
+ String jePropertyName = st.nextToken();
+ String jePropertyValue = st.nextToken();
+ // Check if it is a duplicate.
+ if (uniqueJEProperties.contains(jePropertyName)) {
+ LocalizableMessage message = ERR_CONFIG_JE_DUPLICATE_PROPERTY.get(
+ jePropertyName);
+ throw new ConfigException(message);
+ }
+ // Set JE property.
+ try {
+ envConfig.setConfigParam(jePropertyName, jePropertyValue);
+ // If this property shadows an existing config attribute.
+ if (configAttrMap.containsKey(jePropertyName)) {
+ LocalizableMessage message = ERR_CONFIG_JE_PROPERTY_SHADOWS_CONFIG.get(
+ jePropertyName, attrMap.get(jePropertyName));
+ throw new ConfigException(message);
+ }
+ // Add this property to unique set.
+ uniqueJEProperties.add(jePropertyName);
+ } catch(IllegalArgumentException e) {
+ logger.traceException(e);
+ LocalizableMessage message =
+ ERR_CONFIG_JE_PROPERTY_INVALID.get(
+ jeEntry, e.getMessage());
+ throw new ConfigException(message, e.getCause());
+ }
+ } else {
+ LocalizableMessage message =
+ ERR_CONFIG_JE_PROPERTY_INVALID_FORM.get(jeEntry);
+ throw new ConfigException(message);
+ }
+ }
+
+ return envConfig;
+ }
+
+
+
+}
diff --git a/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/DN2ID.java b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/DN2ID.java
new file mode 100644
index 0000000..0435930
--- /dev/null
+++ b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/DN2ID.java
@@ -0,0 +1,172 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License"). You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at legal-notices/CDDLv1_0.txt
+ * or http://forgerock.org/license/CDDLv1.0.html.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at legal-notices/CDDLv1_0.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information:
+ * Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ *
+ *
+ * Copyright 2006-2010 Sun Microsystems, Inc.
+ * Portions Copyright 2014 ForgeRock AS
+ */
+package org.opends.server.backends.pluggable;
+
+import org.forgerock.opendj.ldap.ByteSequence;
+import org.forgerock.opendj.ldap.ByteString;
+import org.opends.server.backends.pluggable.BackendImpl.ReadableStorage;
+import org.opends.server.backends.pluggable.BackendImpl.Storage;
+import org.opends.server.backends.pluggable.BackendImpl.StorageRuntimeException;
+import org.opends.server.backends.pluggable.BackendImpl.TreeName;
+import org.opends.server.backends.pluggable.BackendImpl.WriteableStorage;
+import org.opends.server.types.DN;
+
+import static org.opends.server.backends.pluggable.JebFormat.*;
+
+/**
+ * This class represents the DN database, or dn2id, which has one record
+ * for each entry. The key is the normalized entry DN and the value
+ * is the entry ID.
+ */
+public class DN2ID extends DatabaseContainer
+{
+ private final int prefixRDNComponents;
+
+ /**
+ * Create a DN2ID instance for the DN database in a given entryContainer.
+ *
+ * @param treeName The name of the DN database.
+ * @param env The JE environment.
+ * @param entryContainer The entryContainer of the DN database.
+ * @throws StorageRuntimeException If an error occurs in the JE database.
+ */
+ DN2ID(TreeName treeName, Storage env, EntryContainer entryContainer)
+ throws StorageRuntimeException
+ {
+ super(treeName, env, entryContainer);
+
+ prefixRDNComponents = entryContainer.getBaseDN().size();
+ }
+
+ /**
+ * Insert a new record into the DN database.
+ * @param txn A JE database transaction to be used for the database operation,
+ * or null if none.
+ * @param dn The entry DN, which is the key to the record.
+ * @param id The entry ID, which is the value of the record.
+ * @return true if the record was inserted, false if a record with that key
+ * already exists.
+ * @throws StorageRuntimeException If an error occurred while attempting to insert
+ * the new record.
+ */
+ public boolean insert(WriteableStorage txn, DN dn, EntryID id) throws StorageRuntimeException
+ {
+ ByteString key = dnToDNKey(dn, prefixRDNComponents);
+ ByteString value = id.toByteString();
+
+ return insert(txn, key, value);
+ }
+
+ /**
+ * Write a record to the DN database. If a record with the given key already
+ * exists, the record will be replaced, otherwise a new record will be
+ * inserted.
+ * @param txn A JE database transaction to be used for the database operation,
+ * or null if none.
+ * @param dn The entry DN, which is the key to the record.
+ * @param id The entry ID, which is the value of the record.
+ * @throws StorageRuntimeException If an error occurred while attempting to write
+ * the record.
+ */
+ public void put(WriteableStorage txn, DN dn, EntryID id) throws StorageRuntimeException
+ {
+ ByteString key = dnToDNKey(dn, prefixRDNComponents);
+ ByteString value = id.toByteString();
+
+ put(txn, key, value);
+ }
+
+ /**
+ * Write a record to the DN database, where the key and value are already
+ * formatted.
+ *
+ * @param txn
+ * A JE database transaction to be used for the database operation,
+ * or null if none.
+ * @param key
+ * A ByteString containing the record key.
+ * @param value
+ * A ByteString containing the record value.
+ * @throws StorageRuntimeException
+ * If an error occurred while attempting to write the record.
+ */
+ @Override
+ public void put(WriteableStorage txn, ByteSequence key, ByteSequence value) throws StorageRuntimeException
+ {
+ super.put(txn, key, value);
+ }
+
+ /**
+ * Remove a record from the DN database.
+ * @param txn A JE database transaction to be used for the database operation,
+ * or null if none.
+ * @param dn The entry DN, which is the key to the record.
+ * @return true if the record was removed, false if it was not removed.
+ * @throws StorageRuntimeException If an error occurred while attempting to remove
+ * the record.
+ */
+ public boolean remove(WriteableStorage txn, DN dn) throws StorageRuntimeException
+ {
+ ByteString key = dnToDNKey(dn, prefixRDNComponents);
+
+ return delete(txn, key);
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ protected boolean delete(WriteableStorage txn, ByteSequence key) throws StorageRuntimeException
+ {
+ return super.delete(txn, key);
+ }
+
+ /**
+ * Fetch the entry ID for a given DN.
+ * @param txn A JE database transaction to be used for the database read, or
+ * null if none is required.
+ * @param dn The DN for which the entry ID is desired.
+ * @param isRMW
+ * @return The entry ID, or null if the given DN is not in the DN database.
+ * @throws StorageRuntimeException If an error occurs in the JE database.
+ */
+ public EntryID get(ReadableStorage txn, DN dn, boolean isRMW) throws StorageRuntimeException
+ {
+ ByteString key = dnToDNKey(dn, prefixRDNComponents);
+ ByteString value = read(txn, key, isRMW);
+ if (value != null)
+ {
+ return new EntryID(value);
+ }
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public ByteString read(ReadableStorage txn, ByteSequence key, boolean isRMW)
+ {
+ return super.read(txn, key, isRMW);
+ }
+}
diff --git a/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/DN2URI.java b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/DN2URI.java
new file mode 100644
index 0000000..d4c4d94
--- /dev/null
+++ b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/DN2URI.java
@@ -0,0 +1,693 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License"). You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at legal-notices/CDDLv1_0.txt
+ * or http://forgerock.org/license/CDDLv1.0.html.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at legal-notices/CDDLv1_0.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information:
+ * Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ *
+ *
+ * Copyright 2006-2010 Sun Microsystems, Inc.
+ * Portions Copyright 2012-2014 ForgeRock AS
+ */
+package org.opends.server.backends.pluggable;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Set;
+
+import org.forgerock.i18n.LocalizableMessage;
+import org.forgerock.i18n.slf4j.LocalizedLogger;
+import org.forgerock.opendj.ldap.ByteSequence;
+import org.forgerock.opendj.ldap.ByteSequenceReader;
+import org.forgerock.opendj.ldap.ByteString;
+import org.forgerock.opendj.ldap.ByteStringBuilder;
+import org.forgerock.opendj.ldap.ConditionResult;
+import org.forgerock.opendj.ldap.ResultCode;
+import org.forgerock.opendj.ldap.SearchScope;
+import org.opends.server.backends.pluggable.BackendImpl.Cursor;
+import org.opends.server.backends.pluggable.BackendImpl.ReadableStorage;
+import org.opends.server.backends.pluggable.BackendImpl.Storage;
+import org.opends.server.backends.pluggable.BackendImpl.StorageRuntimeException;
+import org.opends.server.backends.pluggable.BackendImpl.TreeName;
+import org.opends.server.backends.pluggable.BackendImpl.WriteableStorage;
+import org.opends.server.core.DirectoryServer;
+import org.opends.server.core.SearchOperation;
+import org.opends.server.types.Attribute;
+import org.opends.server.types.AttributeType;
+import org.opends.server.types.DN;
+import org.opends.server.types.DirectoryException;
+import org.opends.server.types.Entry;
+import org.opends.server.types.LDAPURL;
+import org.opends.server.types.Modification;
+import org.opends.server.types.SearchResultReference;
+import org.opends.server.util.StaticUtils;
+
+import static org.opends.messages.JebMessages.*;
+import static org.opends.server.util.ServerConstants.*;
+
+/**
+ * This class represents the referral database which contains URIs from referral
+ * entries. The key is the DN of the referral entry and the value is that of a
+ * labeled URI in the ref attribute for that entry. Duplicate keys are permitted
+ * since a referral entry can contain multiple values of the ref attribute. Key
+ * order is the same as in the DN database so that all referrals in a subtree
+ * can be retrieved by cursoring through a range of the records.
+ */
+public class DN2URI extends DatabaseContainer
+{
+ private static final LocalizedLogger logger = LocalizedLogger.getLoggerForThisClass();
+
+ private final int prefixRDNComponents;
+
+ /**
+ * The standard attribute type that is used to specify the set of referral
+ * URLs in a referral entry.
+ */
+ private final AttributeType referralType =
+ DirectoryServer.getAttributeType(ATTR_REFERRAL_URL);
+
+ /**
+ * A flag that indicates whether there are any referrals contained in this
+ * database. It should only be set to {@code false} when it is known that
+ * there are no referrals.
+ */
+ private volatile ConditionResult containsReferrals =
+ ConditionResult.UNDEFINED;
+
+
+ /**
+ * Create a new object representing a referral database in a given
+ * entryContainer.
+ *
+ * @param treeName
+ * The name of the referral database.
+ * @param storage
+ * The JE environment.
+ * @param entryContainer
+ * The entryContainer of the DN database.
+ * @throws StorageRuntimeException
+ * If an error occurs in the JE database.
+ */
+ @SuppressWarnings("unchecked")
+ DN2URI(TreeName treeName, Storage storage, EntryContainer entryContainer)
+ throws StorageRuntimeException
+ {
+ super(treeName, storage, entryContainer);
+
+ prefixRDNComponents = entryContainer.getBaseDN().size();
+ }
+
+ private ByteSequence encode(Collection<String> col)
+ {
+ if (col != null)
+ {
+ ByteStringBuilder b = new ByteStringBuilder();
+ b.append(col.size());
+ for (String s : col)
+ {
+ byte[] bytes = StaticUtils.getBytes(s);
+ b.append(bytes.length);
+ b.append(bytes);
+ }
+ return b;
+ }
+ return ByteString.empty();
+ }
+
+ private Collection<String> decode(ByteSequence bs)
+ {
+ if (!bs.isEmpty())
+ {
+ ByteSequenceReader r = bs.asReader();
+ final int nbElems = r.getInt();
+ ArrayList<String> results = new ArrayList<String>(nbElems);
+ for (int i = 0; i < nbElems; i++)
+ {
+ final int stringLength = r.getInt();
+ results.add(r.getString(stringLength));
+ }
+ return results;
+ }
+ return new ArrayList<String>();
+ }
+
+ /**
+ * Insert a URI value in the referral database.
+ *
+ * @param txn A database transaction used for the update, or null if none is
+ * required.
+ * @param dn The DN of the referral entry.
+ * @param labeledURIs The labeled URI value of the ref attribute.
+ * @throws StorageRuntimeException If an error occurs in the JE database.
+ */
+ private void insert(WriteableStorage txn, DN dn, Collection<String> labeledURIs) throws StorageRuntimeException
+ {
+ ByteString key = toKey(dn);
+
+ ByteString oldValue = read(txn, key, true);
+ if (oldValue != null)
+ {
+ final Collection<String> newUris = decode(oldValue);
+ if (newUris.addAll(labeledURIs))
+ {
+ put(txn, key, encode(newUris));
+ }
+ }
+ else
+ {
+ txn.putIfAbsent(treeName, key, encode(labeledURIs));
+ }
+ containsReferrals = ConditionResult.TRUE;
+ }
+
+ /**
+ * Delete URI values for a given referral entry from the referral database.
+ *
+ * @param txn A database transaction used for the update, or null if none is
+ * required.
+ * @param dn The DN of the referral entry for which URI values are to be
+ * deleted.
+ * @return true if the values were deleted, false if not.
+ * @throws StorageRuntimeException If an error occurs in the JE database.
+ */
+ public boolean delete(WriteableStorage txn, DN dn) throws StorageRuntimeException
+ {
+ ByteString key = toKey(dn);
+
+ if (delete(txn, key))
+ {
+ containsReferrals = containsReferrals(txn);
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * Delete a single URI value from the referral database.
+ * @param txn A database transaction used for the update, or null if none is
+ * required.
+ * @param dn The DN of the referral entry.
+ * @param labeledURIs The URI value to be deleted.
+ * @return true if the value was deleted, false if not.
+ * @throws StorageRuntimeException If an error occurs in the JE database.
+ */
+ public boolean delete(WriteableStorage txn, DN dn, Collection<String> labeledURIs)
+ throws StorageRuntimeException
+ {
+ ByteString key = toKey(dn);
+
+ ByteString oldValue = read(txn, key, true);
+ if (oldValue != null)
+ {
+ final Collection<String> oldUris = decode(oldValue);
+ if (oldUris.removeAll(labeledURIs))
+ {
+ put(txn, key, encode(oldUris));
+ containsReferrals = containsReferrals(txn);
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Indicates whether the underlying database contains any referrals.
+ *
+ * @param txn The transaction to use when making the determination.
+ *
+ * @return {@code true} if it is believed that the underlying database may
+ * contain at least one referral, or {@code false} if it is certain
+ * that it doesn't.
+ */
+ private ConditionResult containsReferrals(ReadableStorage txn)
+ {
+ Cursor cursor = txn.openCursor(treeName);
+ try
+ {
+ return ConditionResult.valueOf(cursor.next());
+ }
+ catch (Exception e)
+ {
+ logger.traceException(e);
+
+ return ConditionResult.UNDEFINED;
+ }
+ finally
+ {
+ cursor.close();
+ }
+ }
+
+ /**
+ * Update the referral database for an entry that has been modified. Does
+ * not do anything unless the entry before the modification or the entry after
+ * the modification is a referral entry.
+ *
+ * @param txn A database transaction used for the update, or null if none is
+ * required.
+ * @param before The entry before the modifications have been applied.
+ * @param after The entry after the modifications have been applied.
+ * @param mods The sequence of modifications made to the entry.
+ * @throws StorageRuntimeException If an error occurs in the JE database.
+ */
+ public void modifyEntry(WriteableStorage txn, Entry before, Entry after,
+ List<Modification> mods)
+ throws StorageRuntimeException
+ {
+ DN entryDN = before.getName();
+ for (Modification mod : mods)
+ {
+ Attribute modAttr = mod.getAttribute();
+ AttributeType modAttrType = modAttr.getAttributeType();
+ if (modAttrType.equals(referralType))
+ {
+ Attribute a = mod.getAttribute();
+ switch (mod.getModificationType().asEnum())
+ {
+ case ADD:
+ if (a != null)
+ {
+ insert(txn, entryDN, toStrings(a));
+ }
+ break;
+
+ case DELETE:
+ if (a == null || a.isEmpty())
+ {
+ delete(txn, entryDN);
+ }
+ else
+ {
+ delete(txn, entryDN, toStrings(a));
+ }
+ break;
+
+ case INCREMENT:
+ // Nonsensical.
+ break;
+
+ case REPLACE:
+ delete(txn, entryDN);
+ if (a != null)
+ {
+ insert(txn, entryDN, toStrings(a));
+ }
+ break;
+ }
+ }
+ }
+ }
+
+ private List<String> toStrings(Attribute a)
+ {
+ List<String> results = new ArrayList<String>(a.size());
+ for (ByteString v : a)
+ {
+ results.add(v.toString());
+ }
+ return results;
+ }
+
+ /**
+ * Update the referral database for an entry that has been replaced. Does not
+ * do anything unless the entry before it was replaced or the entry after it
+ * was replaced is a referral entry.
+ *
+ * @param txn
+ * A database transaction used for the update, or null if none is
+ * required.
+ * @param before
+ * The entry before it was replaced.
+ * @param after
+ * The entry after it was replaced.
+ * @throws StorageRuntimeException
+ * If an error occurs in the JE database.
+ */
+ public void replaceEntry(WriteableStorage txn, Entry before, Entry after)
+ throws StorageRuntimeException
+ {
+ deleteEntry(txn, before);
+ addEntry(txn, after);
+ }
+
+ /**
+ * Update the referral database for a new entry. Does nothing if the entry
+ * is not a referral entry.
+ * @param txn A database transaction used for the update, or null if none is
+ * required.
+ * @param entry The entry to be added.
+ * @return True if the entry was added successfully or False otherwise.
+ * @throws StorageRuntimeException If an error occurs in the JE database.
+ */
+ public boolean addEntry(WriteableStorage txn, Entry entry)
+ throws StorageRuntimeException
+ {
+ Set<String> labeledURIs = entry.getReferralURLs();
+ if (labeledURIs != null)
+ {
+ insert(txn, entry.getName(), labeledURIs);
+ }
+ return true;
+ }
+
+ /**
+ * Update the referral database for a deleted entry. Does nothing if the entry
+ * was not a referral entry.
+ * @param txn A database transaction used for the update, or null if none is
+ * required.
+ * @param entry The entry to be deleted.
+ * @throws StorageRuntimeException If an error occurs in the JE database.
+ */
+ public void deleteEntry(WriteableStorage txn, Entry entry)
+ throws StorageRuntimeException
+ {
+ Set<String> labeledURIs = entry.getReferralURLs();
+ if (labeledURIs != null)
+ {
+ delete(txn, entry.getName());
+ }
+ }
+
+ /**
+ * Checks whether the target of an operation is a referral entry and throws
+ * a Directory referral exception if it is.
+ * @param entry The target entry of the operation, or the base entry of a
+ * search operation.
+ * @param searchScope The scope of the search operation, or null if the
+ * operation is not a search operation.
+ * @throws DirectoryException If a referral is found at or above the target
+ * DN. The referral URLs will be set appropriately for the references found
+ * in the referral entry.
+ */
+ public void checkTargetForReferral(Entry entry, SearchScope searchScope)
+ throws DirectoryException
+ {
+ Set<String> referralURLs = entry.getReferralURLs();
+ if (referralURLs != null)
+ {
+ throwReferralException(entry.getName(), entry.getName(), referralURLs,
+ searchScope);
+ }
+ }
+
+ /**
+ * Throws a Directory referral exception for the case where a referral entry
+ * exists at or above the target DN of an operation.
+ * @param targetDN The target DN of the operation, or the base object of a
+ * search operation.
+ * @param referralDN The DN of the referral entry.
+ * @param labeledURIs The set of labeled URIs in the referral entry.
+ * @param searchScope The scope of the search operation, or null if the
+ * operation is not a search operation.
+ * @throws DirectoryException If a referral is found at or above the target
+ * DN. The referral URLs will be set appropriately for the references found
+ * in the referral entry.
+ */
+ public void throwReferralException(DN targetDN, DN referralDN, Collection<String> labeledURIs, SearchScope searchScope)
+ throws DirectoryException
+ {
+ ArrayList<String> URIList = new ArrayList<String>(labeledURIs.size());
+ for (String labeledURI : labeledURIs)
+ {
+ // Remove the label part of the labeled URI if there is a label.
+ String uri = labeledURI;
+ int i = labeledURI.indexOf(' ');
+ if (i != -1)
+ {
+ uri = labeledURI.substring(0, i);
+ }
+
+ try
+ {
+ LDAPURL ldapurl = LDAPURL.decode(uri, false);
+
+ if ("ldap".equalsIgnoreCase(ldapurl.getScheme()))
+ {
+ DN urlBaseDN = targetDN;
+ if (!referralDN.equals(ldapurl.getBaseDN()))
+ {
+ urlBaseDN =
+ EntryContainer.modDN(targetDN,
+ referralDN.size(),
+ ldapurl.getBaseDN());
+ }
+ ldapurl.setBaseDN(urlBaseDN);
+ if (searchScope == null)
+ {
+ // RFC 3296, 5.2. Target Object Considerations:
+ // In cases where the URI to be returned is a LDAP URL, the server
+ // SHOULD trim any present scope, filter, or attribute list from the
+ // URI before returning it. Critical extensions MUST NOT be trimmed
+ // or modified.
+ StringBuilder builder = new StringBuilder(uri.length());
+ ldapurl.toString(builder, true);
+ uri = builder.toString();
+ }
+ else
+ {
+ // RFC 3296, 5.3. Base Object Considerations:
+ // In cases where the URI to be returned is a LDAP URL, the server
+ // MUST provide an explicit scope specifier from the LDAP URL prior
+ // to returning it.
+ ldapurl.getAttributes().clear();
+ ldapurl.setScope(searchScope);
+ ldapurl.setFilter(null);
+ uri = ldapurl.toString();
+ }
+ }
+ }
+ catch (DirectoryException e)
+ {
+ logger.traceException(e);
+ // Return the non-LDAP URI as is.
+ }
+
+ URIList.add(uri);
+ }
+
+ // Throw a directory referral exception containing the URIs.
+ LocalizableMessage msg = NOTE_JEB_REFERRAL_RESULT_MESSAGE.get(referralDN);
+ throw new DirectoryException(
+ ResultCode.REFERRAL, msg, referralDN, URIList, null);
+ }
+
+ /**
+ * Process referral entries that are above the target DN of an operation.
+ * @param targetDN The target DN of the operation, or the base object of a
+ * search operation.
+ * @param searchScope The scope of the search operation, or null if the
+ * operation is not a search operation.
+ * @throws DirectoryException If a referral is found at or above the target
+ * DN. The referral URLs will be set appropriately for the references found
+ * in the referral entry.
+ */
+ public void targetEntryReferrals(DN targetDN, SearchScope searchScope)
+ throws DirectoryException
+ {
+ if (containsReferrals == ConditionResult.UNDEFINED)
+ {
+ containsReferrals = containsReferrals(null);
+ }
+
+ if (containsReferrals == ConditionResult.FALSE)
+ {
+ return;
+ }
+
+ try
+ {
+ Cursor cursor = storage.openCursor(treeName);
+ try
+ {
+ // Go up through the DIT hierarchy until we find a referral.
+ for (DN dn = entryContainer.getParentWithinBase(targetDN); dn != null;
+ dn = entryContainer.getParentWithinBase(dn))
+ {
+ // Look for a record whose key matches the current DN.
+ if (cursor.positionToKey(toKey(dn)))
+ {
+ // Construct a set of all the labeled URIs in the referral.
+ Collection<String> labeledURIs = decode(cursor.getValue());
+ throwReferralException(targetDN, dn, labeledURIs, searchScope);
+ }
+ }
+ }
+ finally
+ {
+ cursor.close();
+ }
+ }
+ catch (StorageRuntimeException e)
+ {
+ logger.traceException(e);
+ }
+ }
+
+ /**
+ * Return search result references for a search operation using the referral
+ * database to find all referral entries within scope of the search.
+ * @param searchOp The search operation for which search result references
+ * should be returned.
+ * @return <CODE>true</CODE> if the caller should continue processing the
+ * search request and sending additional entries and references, or
+ * <CODE>false</CODE> if not for some reason (e.g., the size limit
+ * has been reached or the search has been abandoned).
+ * @throws DirectoryException If a Directory Server error occurs.
+ */
+ public boolean returnSearchReferences(SearchOperation searchOp)
+ throws DirectoryException
+ {
+ if (containsReferrals == ConditionResult.UNDEFINED)
+ {
+ containsReferrals = containsReferrals(null);
+ }
+
+ if (containsReferrals == ConditionResult.FALSE)
+ {
+ return true;
+ }
+
+ /*
+ * We will iterate forwards through a range of the keys to
+ * find subordinates of the base entry from the top of the tree downwards.
+ */
+ ByteString baseDN = toKey(searchOp.getBaseDN());
+ ByteStringBuilder suffix = new ByteStringBuilder(baseDN.length() + 1);
+ suffix.append(baseDN);
+ ByteStringBuilder end = new ByteStringBuilder(suffix);
+
+ /*
+ * Set the ending value to a value of equal length but slightly
+ * greater than the suffix. Since keys are compared in
+ * reverse order we must set the first byte (the comma).
+ * No possibility of overflow here.
+ */
+ suffix.append((byte) 0x00);
+ end.append((byte) 0x01);
+
+ ByteSequence startKey = suffix;
+ try
+ {
+ Cursor cursor = storage.openCursor(treeName);
+ try
+ {
+ // Initialize the cursor very close to the starting value then
+ // step forward until we pass the ending value.
+ boolean success = cursor.positionToKey(startKey);
+ while (success)
+ {
+ ByteString key = cursor.getKey();
+ int cmp = ByteSequence.COMPARATOR.compare(key, end);
+ if (cmp >= 0)
+ {
+ // We have gone past the ending value.
+ break;
+ }
+
+ // We have found a subordinate referral.
+ DN dn = JebFormat.dnFromDNKey(key, entryContainer.getBaseDN());
+
+ // Make sure the referral is within scope.
+ if (searchOp.getScope() == SearchScope.SINGLE_LEVEL
+ && JebFormat.findDNKeyParent(key) != baseDN.length())
+ {
+ continue;
+ }
+
+ // Construct a list of all the URIs in the referral.
+ Collection<String> labeledURIs = decode(cursor.getValue());
+ SearchResultReference reference = toSearchResultReference(dn, labeledURIs, searchOp.getScope());
+ if (!searchOp.returnReference(dn, reference))
+ {
+ return false;
+ }
+ success = cursor.next();
+ }
+ }
+ finally
+ {
+ cursor.close();
+ }
+ }
+ catch (StorageRuntimeException e)
+ {
+ logger.traceException(e);
+ }
+
+ return true;
+ }
+
+ private SearchResultReference toSearchResultReference(DN dn, Collection<String> labeledURIs, SearchScope scope)
+ {
+ ArrayList<String> URIList = new ArrayList<String>(labeledURIs.size());
+ for (String labeledURI : labeledURIs)
+ {
+ // Remove the label part of the labeled URI if there is a label.
+ String uri = labeledURI;
+ int i = labeledURI.indexOf(' ');
+ if (i != -1)
+ {
+ uri = labeledURI.substring(0, i);
+ }
+
+ // From RFC 3296 section 5.4:
+ // If the URI component is not a LDAP URL, it should be returned as
+ // is. If the LDAP URL's DN part is absent or empty, the DN part
+ // must be modified to contain the DN of the referral object. If
+ // the URI component is a LDAP URL, the URI SHOULD be modified to
+ // add an explicit scope specifier.
+ try
+ {
+ LDAPURL ldapurl = LDAPURL.decode(uri, false);
+
+ if ("ldap".equalsIgnoreCase(ldapurl.getScheme()))
+ {
+ if (ldapurl.getBaseDN().isRootDN())
+ {
+ ldapurl.setBaseDN(dn);
+ }
+ ldapurl.getAttributes().clear();
+ if (scope == SearchScope.SINGLE_LEVEL)
+ {
+ ldapurl.setScope(SearchScope.BASE_OBJECT);
+ }
+ else
+ {
+ ldapurl.setScope(SearchScope.WHOLE_SUBTREE);
+ }
+ ldapurl.setFilter(null);
+ uri = ldapurl.toString();
+ }
+ }
+ catch (DirectoryException e)
+ {
+ logger.traceException(e);
+ // Return the non-LDAP URI as is.
+ }
+
+ URIList.add(uri);
+ }
+ return new SearchResultReference(URIList);
+ }
+
+ private ByteString toKey(DN dn)
+ {
+ return JebFormat.dnToDNKey(dn, prefixRDNComponents);
+ }
+}
diff --git a/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/DataConfig.java b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/DataConfig.java
new file mode 100644
index 0000000..2ff604d
--- /dev/null
+++ b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/DataConfig.java
@@ -0,0 +1,133 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License"). You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at legal-notices/CDDLv1_0.txt
+ * or http://forgerock.org/license/CDDLv1.0.html.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at legal-notices/CDDLv1_0.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information:
+ * Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ *
+ *
+ * Copyright 2006-2008 Sun Microsystems, Inc.
+ * Portions Copyright 2014 ForgeRock AS
+ */
+package org.opends.server.backends.pluggable;
+
+import org.forgerock.util.Reject;
+import org.opends.server.api.CompressedSchema;
+import org.opends.server.types.EntryEncodeConfig;
+
+/**
+ * Configuration class to indicate desired compression and cryptographic options
+ * for the data stored in the database.
+ */
+public final class DataConfig
+{
+ /** Indicates whether data should be compressed before writing to the database. */
+ private boolean compressed;
+
+ /** The configuration to use when encoding entries in the database. */
+ private EntryEncodeConfig encodeConfig = new EntryEncodeConfig();
+
+ /**
+ * Construct a new DataConfig object with the specified settings.
+ *
+ * @param compressed true if data should be compressed, false if not.
+ * @param compactEncoding true if data should be encoded in compact form,
+ * false if not.
+ * @param compressedSchema the compressed schema manager to use. It must not
+ * be {@code null} if compactEncoding is {@code true}.
+ */
+ public DataConfig(boolean compressed, boolean compactEncoding, CompressedSchema compressedSchema)
+ {
+ this.compressed = compressed;
+ setCompactEncoding(compactEncoding, compressedSchema);
+ }
+
+ /**
+ * Determine whether data should be compressed before writing to the database.
+ * @return true if data should be compressed, false if not.
+ */
+ public boolean isCompressed()
+ {
+ return compressed;
+ }
+
+ /**
+ * Determine whether entries should be encoded with the compact form before
+ * writing to the database.
+ * @return true if data should be encoded in the compact form.
+ */
+ public boolean isCompactEncoding()
+ {
+ return encodeConfig.compressAttributeDescriptions();
+ }
+
+ /**
+ * Configure whether data should be compressed before writing to the database.
+ * @param compressed true if data should be compressed, false if not.
+ */
+ public void setCompressed(boolean compressed)
+ {
+ this.compressed = compressed;
+ }
+
+ /**
+ * Configure whether data should be encoded with the compact form before
+ * writing to the database.
+ * @param compactEncoding true if data should be encoded in compact form,
+ * false if not.
+ * @param compressedSchema The compressed schema manager to use. It must not
+ * be {@code null} if compactEncoding is {@code true}.
+ */
+ public void setCompactEncoding(boolean compactEncoding, CompressedSchema compressedSchema)
+ {
+ if (compressedSchema == null)
+ {
+ Reject.ifTrue(compactEncoding);
+ this.encodeConfig = new EntryEncodeConfig(false, compactEncoding, false);
+ }
+ else
+ {
+ this.encodeConfig = new EntryEncodeConfig(false, compactEncoding, compactEncoding, compressedSchema);
+ }
+ }
+
+ /**
+ * Get the EntryEncodeConfig object in use by this configuration.
+ * @return the EntryEncodeConfig object in use by this configuration.
+ */
+ public EntryEncodeConfig getEntryEncodeConfig()
+ {
+ return this.encodeConfig;
+ }
+
+ /**
+ * Get a string representation of this object.
+ * @return A string representation of this object.
+ */
+ @Override
+ public String toString()
+ {
+ final StringBuilder builder = new StringBuilder();
+ builder.append("DataConfig(compressed=");
+ builder.append(compressed);
+ builder.append(", ");
+ encodeConfig.toString(builder);
+ builder.append(")");
+ return builder.toString();
+ }
+}
diff --git a/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/DatabaseContainer.java b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/DatabaseContainer.java
new file mode 100644
index 0000000..f9832b6
--- /dev/null
+++ b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/DatabaseContainer.java
@@ -0,0 +1,339 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License"). You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at legal-notices/CDDLv1_0.txt
+ * or http://forgerock.org/license/CDDLv1.0.html.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at legal-notices/CDDLv1_0.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information:
+ * Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ *
+ *
+ * Copyright 2006-2008 Sun Microsystems, Inc.
+ * Portions Copyright 2011-2014 ForgeRock AS
+ */
+package org.opends.server.backends.pluggable;
+
+import java.io.Closeable;
+
+import org.forgerock.i18n.slf4j.LocalizedLogger;
+import org.forgerock.opendj.ldap.ByteSequence;
+import org.forgerock.opendj.ldap.ByteString;
+import org.opends.server.backends.pluggable.BackendImpl.Cursor;
+import org.opends.server.backends.pluggable.BackendImpl.ReadableStorage;
+import org.opends.server.backends.pluggable.BackendImpl.Storage;
+import org.opends.server.backends.pluggable.BackendImpl.StorageRuntimeException;
+import org.opends.server.backends.pluggable.BackendImpl.TreeName;
+import org.opends.server.backends.pluggable.BackendImpl.WriteableStorage;
+import org.opends.server.util.ServerConstants;
+import org.opends.server.util.StaticUtils;
+
+/**
+ * This class is a wrapper around the JE database object and provides basic
+ * read and write methods for entries.
+ */
+public abstract class DatabaseContainer implements Closeable
+{
+ private static final LocalizedLogger logger = LocalizedLogger.getLoggerForThisClass();
+
+ /** The database entryContainer. */
+ protected final EntryContainer entryContainer;
+ /** The name of the database within the entryContainer. */
+ protected TreeName treeName;
+
+ /** The reference to the JE Storage. */
+ protected final Storage storage;
+
+ /**
+ * Create a new DatabaseContainer object.
+ *
+ * @param treeName The name of the entry database.
+ * @param storage The JE Storage.
+ * @param entryContainer The entryContainer of the entry database.
+ */
+ protected DatabaseContainer(TreeName treeName, Storage storage, EntryContainer entryContainer)
+ {
+ this.storage = storage;
+ this.entryContainer = entryContainer;
+ this.treeName = treeName;
+ }
+
+ /**
+ * Opens a JE database in this database container. If the provided
+ * database configuration is transactional, a transaction will be
+ * created and used to perform the open.
+ *
+ * @throws StorageRuntimeException if a JE database error occurs while
+ * opening the index.
+ */
+ public void open() throws StorageRuntimeException
+ {
+ if (dbConfig.getTransactional())
+ {
+ // Open the database under a transaction.
+ Transaction txn = entryContainer.beginTransaction();
+ try
+ {
+ treeName = storage.openDatabase(txn, treeName, dbConfig);
+ if (logger.isTraceEnabled())
+ {
+ logger.trace("JE database %s opened. txnid=%d", treeName, txn.getId());
+ }
+ EntryContainer.transactionCommit(txn);
+ }
+ catch (StorageRuntimeException e)
+ {
+ EntryContainer.transactionAbort(txn);
+ throw e;
+ }
+ }
+ else
+ {
+ treeName = storage.openDatabase(null, treeName, dbConfig);
+ if (logger.isTraceEnabled())
+ {
+ logger.trace("JE database %s opened. txnid=none", treeName);
+ }
+ }
+ }
+
+ /**
+ * Flush any cached database information to disk and close the
+ * database container.
+ *
+ * The database container should not be closed while other processes
+ * acquired the container. The container should not be closed
+ * while cursors handles into the database remain open, or
+ * transactions that include operations on the database have not yet
+ * been committed or aborted.
+ *
+ * The container may not be accessed again after this method is
+ * called, regardless of the method's success or failure.
+ *
+ * @throws StorageRuntimeException if an error occurs.
+ */
+ @Override
+ public synchronized void close() throws StorageRuntimeException
+ {
+ if(dbConfig.getDeferredWrite())
+ {
+ treeName.sync();
+ }
+ storage.openTree(treeName)
+ treeName.close();
+ treeName = null;
+
+ if(logger.isTraceEnabled())
+ {
+ logger.trace("Closed tree %s", treeName);
+ }
+ }
+
+ /**
+ * Replace or insert a record into a JE database, with optional debug logging.
+ * This is a simple wrapper around the JE Database.put method.
+ * @param txn The JE transaction handle, or null if none.
+ * @param key The record key.
+ * @param value The record value.
+ * @throws StorageRuntimeException If an error occurs in the JE operation.
+ */
+ protected void put(WriteableStorage txn, ByteSequence key, ByteSequence value)
+ throws StorageRuntimeException
+ {
+ txn.put(treeName, key, value);
+ if (logger.isTraceEnabled())
+ {
+ logger.trace(messageToLog(true, treeName, txn, key, value));
+ }
+ }
+
+ /**
+ * Read a record from a JE database, with optional debug logging. This is a
+ * simple wrapper around the JE Database.get method.
+ * @param txn The JE transaction handle, or null if none.
+ * @param key The key of the record to be read.
+ * @return The operation status.
+ * @throws StorageRuntimeException If an error occurs in the JE operation.
+ */
+ protected ByteString read(ReadableStorage txn, ByteSequence key, boolean isRMW) throws StorageRuntimeException
+ {
+ ByteString value = txn.get(treeName, key);
+ if (logger.isTraceEnabled())
+ {
+ logger.trace(messageToLog(value != null, treeName, txn, key, value));
+ }
+ return value;
+ }
+
+ /**
+ * Insert a record into a JE database, with optional debug logging. This is a
+ * simple wrapper around the JE Database.putNoOverwrite method.
+ * @param txn The JE transaction handle, or null if none.
+ * @param key The record key.
+ * @param value The record value.
+ * @return <code>true</code> if the key-value mapping could be inserted, <code>false</code> if the key was already mapped to another value
+ * @throws StorageRuntimeException If an error occurs in the JE operation.
+ */
+ protected boolean insert(WriteableStorage txn, ByteString key, ByteString value) throws StorageRuntimeException
+ {
+ boolean result = txn.putIfAbsent(treeName, key, value);
+ if (logger.isTraceEnabled())
+ {
+ logger.trace(messageToLog(result, treeName, txn, key, value));
+ }
+ return result;
+ }
+
+ /**
+ * Delete a record from a JE database, with optional debug logging. This is a
+ * simple wrapper around the JE Database.delete method.
+ * @param txn The JE transaction handle, or null if none.
+ * @param key The key of the record to be read.
+ * @return <code>true</code> if the key mapping was removed, <code>false</code> otherwise
+ * @throws StorageRuntimeException If an error occurs in the JE operation.
+ */
+ protected boolean delete(WriteableStorage txn, ByteSequence key) throws StorageRuntimeException
+ {
+ boolean result = txn.remove(treeName, key);
+ if (logger.isTraceEnabled())
+ {
+ logger.trace(messageToLog(result, treeName, txn, key, null));
+ }
+ return result;
+ }
+
+ /**
+ * Open a JE cursor on the JE database. This is a simple wrapper around
+ * the JE Database.openCursor method.
+ * @param txn A JE database transaction to be used by the cursor,
+ * or null if none.
+ * @return A JE cursor.
+ * @throws StorageRuntimeException If an error occurs while attempting to open
+ * the cursor.
+ */
+ public Cursor openCursor(ReadableStorage txn) throws StorageRuntimeException
+ {
+ return txn.openCursor(treeName);
+ }
+
+ /**
+ * Get the count of key/data pairs in the database in a JE database.
+ * This is a simple wrapper around the JE Database.count method.
+ * @return The count of key/data pairs in the database.
+ * @throws StorageRuntimeException If an error occurs in the JE operation.
+ */
+ public long getRecordCount() throws StorageRuntimeException
+ {
+ long count = treeName.count();
+ if (logger.isTraceEnabled())
+ {
+ logger.trace(messageToLog(true, treeName, null, null, null));
+ }
+ return count;
+ }
+
+ /**
+ * Get a string representation of this object.
+ * @return return A string representation of this object.
+ */
+ @Override
+ public String toString()
+ {
+ return treeName.toString();
+ }
+
+ /**
+ * Get the JE database name for this database container.
+ *
+ * @return JE database name for this database container.
+ */
+ public TreeName getName()
+ {
+ return treeName;
+ }
+
+ /**
+ * Preload the database into cache.
+ *
+ * @param config The preload configuration.
+ * @return Statistics about the preload process.
+ * @throws StorageRuntimeException If an JE database error occurs
+ * during the preload.
+ */
+ public PreloadStats preload(PreloadConfig config)
+ throws StorageRuntimeException
+ {
+ return treeName.preload(config);
+ }
+
+ /**
+ * Set the JE database name to use for this container.
+ *
+ * @param name The database name to use for this container.
+ */
+ void setName(TreeName name)
+ {
+ this.treeName = name;
+ }
+
+ /** Returns the message to log given the provided information. */
+ private String messageToLog(boolean success, TreeName treeName, ReadableStorage txn, ByteSequence key,
+ ByteSequence value)
+ {
+ StringBuilder builder = new StringBuilder();
+ builder.append(" (");
+ builder.append(success ? "SUCCESS" : "ERROR");
+ builder.append(")");
+ builder.append(" db=");
+ builder.append(treeName);
+ if (txn != null)
+ {
+ builder.append(" txnid=");
+ try
+ {
+ builder.append(txn.getId());
+ }
+ catch (StorageRuntimeException de)
+ {
+ builder.append(de);
+ }
+ }
+ else
+ {
+ builder.append(" txnid=none");
+ }
+
+ builder.append(ServerConstants.EOL);
+ if (key != null)
+ {
+ builder.append("key:");
+ builder.append(ServerConstants.EOL);
+ StaticUtils.byteArrayToHexPlusAscii(builder, key.toByteArray(), 4);
+ }
+
+ // If the operation was successful we log the same common information
+ // plus the data
+ if (value != null)
+ {
+ builder.append("value(len=");
+ builder.append(value.length());
+ builder.append("):");
+ builder.append(ServerConstants.EOL);
+ StaticUtils.byteArrayToHexPlusAscii(builder, value.toByteArray(), 4);
+ }
+ return builder.toString();
+ }
+
+}
diff --git a/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/DbPreloadComparator.java b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/DbPreloadComparator.java
new file mode 100644
index 0000000..676472e
--- /dev/null
+++ b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/DbPreloadComparator.java
@@ -0,0 +1,80 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License"). You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at legal-notices/CDDLv1_0.txt
+ * or http://forgerock.org/license/CDDLv1.0.html.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at legal-notices/CDDLv1_0.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information:
+ * Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ *
+ *
+ * Copyright 2006-2008 Sun Microsystems, Inc.
+ * Portions Copyright 2014 ForgeRock AS
+ */
+package org.opends.server.backends.pluggable;
+
+import java.util.Comparator;
+
+import org.opends.server.backends.pluggable.BackendImpl.TreeName;
+
+/**
+ * This comparator is used to sort databases in order of priority
+ * for preloading into the cache.
+ */
+public class DbPreloadComparator
+ implements Comparator<DatabaseContainer>
+{
+
+ /**
+ * Calculate the relative priority of a database for preloading.
+ *
+ * @param database A handle to the database.
+ * @return 1 for id2entry database, 2 for dn2id database, 3 for all others.
+ */
+ static private int priority(DatabaseContainer database)
+ {
+ TreeName name = database.getName();
+ if (name.endsWith(EntryContainer.ID2ENTRY_DATABASE_NAME))
+ {
+ return 1;
+ }
+ else if (name.endsWith(EntryContainer.DN2ID_DATABASE_NAME))
+ {
+ return 2;
+ }
+ else
+ {
+ return 3;
+ }
+ }
+
+ /**
+ * Compares its two arguments for order. Returns a negative integer,
+ * zero, or a positive integer as the first argument is less than, equal
+ * to, or greater than the second.
+ *
+ * @param database1 the first object to be compared.
+ * @param database2 the second object to be compared.
+ * @return a negative integer, zero, or a positive integer as the
+ * first argument is less than, equal to, or greater than the
+ * second.
+ **/
+ @Override
+ public int compare(DatabaseContainer database1, DatabaseContainer database2)
+ {
+ return priority(database1) - priority(database2);
+ }
+}
diff --git a/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/EntryCachePreloader.java b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/EntryCachePreloader.java
new file mode 100644
index 0000000..a28e37b
--- /dev/null
+++ b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/EntryCachePreloader.java
@@ -0,0 +1,374 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License"). You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at legal-notices/CDDLv1_0.txt
+ * or http://forgerock.org/license/CDDLv1.0.html.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at legal-notices/CDDLv1_0.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information:
+ * Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ *
+ *
+ * Copyright 2008 Sun Microsystems, Inc.
+ * Portions Copyright 2014 ForgeRock AS
+ */
+package org.opends.server.backends.pluggable;
+
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Timer;
+import java.util.TimerTask;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.forgerock.i18n.slf4j.LocalizedLogger;
+import org.forgerock.opendj.ldap.ByteString;
+import org.opends.server.api.DirectoryThread;
+import org.opends.server.backends.pluggable.BackendImpl.Cursor;
+import org.opends.server.core.DirectoryServer;
+import org.opends.server.types.Entry;
+
+import static org.opends.messages.ExtensionMessages.*;
+import static org.opends.server.util.StaticUtils.*;
+
+/**
+ * This class defines a utility that will be used to pre-load the Directory
+ * Server entry cache. Pre-loader is multi-threaded and consist of the
+ * following threads:
+ *
+ * - The Arbiter thread which monitors overall pre-load progress and manages
+ * pre-load worker threads by adding or removing them as deemed necessary.
+ *
+ * - The Collector thread which collects all entries stored within the
+ * backend and places them to a blocking queue workers consume from.
+ *
+ * - Worker threads which are responsible for monitoring the collector feed
+ * and processing the actual entries for cache storage.
+ *
+ * This implementation is self-adjusting to any system workload and does not
+ * require any configuration parameters to optimize for initial system
+ * resources availability and/or any subsequent fluctuations.
+ */
+class EntryCachePreloader
+{
+ private static final LocalizedLogger logger = LocalizedLogger.getLoggerForThisClass();
+
+ /**
+ * BackendImpl object.
+ */
+ private BackendImpl backend;
+
+ /**
+ * Interrupt flag for the arbiter to terminate worker threads.
+ */
+ private AtomicBoolean interruptFlag = new AtomicBoolean(false);
+
+ /**
+ * Processed entries counter.
+ */
+ private AtomicLong processedEntries = new AtomicLong(0);
+
+ /**
+ * Progress report resolution.
+ */
+ private static final long progressInterval = 5000;
+
+ /**
+ * Default resolution time.
+ */
+ public static final long
+ PRELOAD_DEFAULT_SLEEP_TIME = 10000;
+
+ /**
+ * Effective synchronization time.
+ */
+ private static long syncSleepTime;
+
+ /**
+ * Default queue capacity.
+ */
+ public static final int
+ PRELOAD_DEFAULT_QUEUE_CAPACITY = 128;
+
+ /**
+ * Effective queue capacity.
+ */
+ private static int queueCapacity;
+
+ /**
+ * Worker threads.
+ */
+ private List<Thread> preloadThreads =
+ Collections.synchronizedList(
+ new LinkedList<Thread>());
+
+ /**
+ * Collector thread.
+ */
+ private EntryCacheCollector collector =
+ new EntryCacheCollector();
+
+ /**
+ * This queue is for workers to take from.
+ */
+ private LinkedBlockingQueue<PreloadEntry> entryQueue;
+
+ /**
+ * The number of bytes in a megabyte.
+ */
+ private static final int bytesPerMegabyte = 1024*1024;
+
+ /**
+ * Constructs the Entry Cache Pre-loader for
+ * a given JEB implementation instance.
+ *
+ * @param jeb The JEB instance to pre-load.
+ */
+ public EntryCachePreloader(BackendImpl jeb) {
+ // These should not be exposed as configuration
+ // parameters and are only useful for testing.
+ syncSleepTime = Long.getLong(
+ "org.opends.server.entrycache.preload.sleep",
+ PRELOAD_DEFAULT_SLEEP_TIME);
+ queueCapacity = Integer.getInteger(
+ "org.opends.server.entrycache.preload.queue",
+ PRELOAD_DEFAULT_QUEUE_CAPACITY);
+ entryQueue =
+ new LinkedBlockingQueue<PreloadEntry>(
+ queueCapacity);
+ this.backend = jeb;
+ }
+
+ /**
+ * The Arbiter thread.
+ */
+ protected void preload()
+ {
+ logger.info(NOTE_CACHE_PRELOAD_PROGRESS_START, backend.getBackendID());
+ // Start collector thread first.
+ collector.start();
+ // Kick off a single worker.
+ EntryCachePreloadWorker singleWorkerThread =
+ new EntryCachePreloadWorker();
+ singleWorkerThread.start();
+ preloadThreads.add(singleWorkerThread);
+ // Progress report timer task.
+ Timer timer = new Timer();
+ TimerTask progressTask = new TimerTask() {
+ // Persistent state restore progress report.
+ @Override
+ public void run() {
+ if (processedEntries.get() > 0) {
+ long freeMemory =
+ Runtime.getRuntime().freeMemory() / bytesPerMegabyte;
+ logger.info(NOTE_CACHE_PRELOAD_PROGRESS_REPORT, backend.getBackendID(), processedEntries.get(), freeMemory);
+ }
+ }
+ };
+ timer.scheduleAtFixedRate(progressTask, progressInterval,
+ progressInterval);
+ // Cycle to monitor progress and adjust workers.
+ long processedEntriesCycle = 0;
+ long processedEntriesDelta = 0;
+ long processedEntriesDeltaLow = 0;
+ long processedEntriesDeltaHigh = 0;
+ long lastKnownProcessedEntries = 0;
+ try {
+ while (!entryQueue.isEmpty() || collector.isAlive()) {
+
+ Thread.sleep(syncSleepTime);
+
+ processedEntriesCycle = processedEntries.get();
+ processedEntriesDelta =
+ processedEntriesCycle - lastKnownProcessedEntries;
+ lastKnownProcessedEntries = processedEntriesCycle;
+ // Spawn another worker if scaling up.
+ if (processedEntriesDelta > processedEntriesDeltaHigh) {
+ processedEntriesDeltaLow = processedEntriesDeltaHigh;
+ processedEntriesDeltaHigh = processedEntriesDelta;
+ EntryCachePreloadWorker workerThread =
+ new EntryCachePreloadWorker();
+ workerThread.start();
+ preloadThreads.add(workerThread);
+ }
+ // Interrupt random worker if scaling down.
+ if (processedEntriesDelta < processedEntriesDeltaLow) {
+ processedEntriesDeltaHigh = processedEntriesDeltaLow;
+ processedEntriesDeltaLow = processedEntriesDelta;
+ // Leave at least one worker to progress.
+ if (preloadThreads.size() > 1) {
+ interruptFlag.set(true);
+ }
+ }
+ }
+ // Join the collector.
+ if (collector.isAlive()) {
+ collector.join();
+ }
+ // Join all spawned workers.
+ for (Thread workerThread : preloadThreads) {
+ if (workerThread.isAlive()) {
+ workerThread.join();
+ }
+ }
+ // Cancel progress report task and report done.
+ timer.cancel();
+ logger.info(NOTE_CACHE_PRELOAD_PROGRESS_DONE, backend.getBackendID(), processedEntries.get());
+ } catch (InterruptedException ex) {
+ logger.traceException(ex);
+ // Interrupt the collector.
+ collector.interrupt();
+ // Interrupt all preload threads.
+ for (Thread thread : preloadThreads) {
+ thread.interrupt();
+ }
+ logger.warn(WARN_CACHE_PRELOAD_INTERRUPTED, backend.getBackendID());
+ } finally {
+ // Kill the timer task.
+ timer.cancel();
+ }
+ }
+
+ /**
+ * The worker thread.
+ */
+ private class EntryCachePreloadWorker extends DirectoryThread {
+ public EntryCachePreloadWorker() {
+ super("Entry Cache Preload Worker");
+ }
+ @Override
+ public void run() {
+ while (!entryQueue.isEmpty() || collector.isAlive()) {
+ // Check if interrupted.
+ if (Thread.interrupted()) {
+ return;
+ }
+ // Check for scaling down interruption.
+ if (interruptFlag.compareAndSet(true, false)) {
+ preloadThreads.remove(Thread.currentThread());
+ break;
+ }
+ // Dequeue the next entry.
+ try {
+ PreloadEntry preloadEntry = entryQueue.poll();
+ if (preloadEntry == null) {
+ continue;
+ }
+ long entryID = preloadEntry.entryIDBytes.toLong();
+ Entry entry =
+ ID2Entry.entryFromDatabase(preloadEntry.entryBytes,
+ backend.getRootContainer().getCompressedSchema());
+ try {
+ // Even if the entry does not end up in the cache its still
+ // treated as a processed entry anyways.
+ DirectoryServer.getEntryCache().putEntry(entry, backend, entryID);
+ processedEntries.getAndIncrement();
+ } catch (Exception ex) {
+ logger.traceException(ex);
+ logger.error(ERR_CACHE_PRELOAD_ENTRY_FAILED, entry.getName(),
+ (ex.getCause() != null ? ex.getCause().getMessage() :
+ stackTraceToSingleLineString(ex)));
+ }
+ } catch (Exception ex) {
+ break;
+ }
+ }
+ }
+ }
+
+ /**
+ * The Collector thread.
+ */
+ private class EntryCacheCollector extends DirectoryThread {
+ public EntryCacheCollector() {
+ super("Entry Cache Preload Collector");
+ }
+ @Override
+ public void run() {
+ Cursor cursor = null;
+ ID2Entry id2entry = null;
+ Collection<EntryContainer> entryContainers =
+ backend.getRootContainer().getEntryContainers();
+ Iterator<EntryContainer> ecIterator =
+ entryContainers.iterator();
+ boolean success = true;
+
+ try {
+ while (success) {
+ // Check if interrupted.
+ if (Thread.interrupted()) {
+ return;
+ }
+ try {
+ if (cursor == null) {
+ if (ecIterator.hasNext()) {
+ id2entry = ecIterator.next().getID2Entry();
+ } else {
+ break;
+ }
+ if (id2entry != null) {
+ cursor = id2entry.openCursor(null);
+ } else {
+ continue;
+ }
+ }
+ // BUG cursor might be null ? If not why testing below ?
+ success = cursor.next();
+ if (!success) {
+ // Reset cursor and continue.
+ close(cursor);
+ success = true;
+ cursor = null;
+ } else {
+ entryQueue.put(new PreloadEntry(cursor.getValue(), cursor.getKey()));
+ }
+ } catch (InterruptedException e) {
+ return;
+ } catch (Exception e) {
+ logger.traceException(e);
+ }
+ }
+ } finally {
+ close(cursor);
+ }
+ }
+ }
+
+ /**
+ * This inner class represents pre-load entry object.
+ */
+ private class PreloadEntry {
+
+ // Encoded Entry.
+ public ByteString entryBytes;
+
+ // Encoded EntryID.
+ public ByteString entryIDBytes;
+
+ /**
+ * Default constructor.
+ */
+ public PreloadEntry(ByteString entryBytes, ByteString entryIDBytes)
+ {
+ this.entryBytes = entryBytes;
+ this.entryIDBytes = entryIDBytes;
+ }
+ }
+}
diff --git a/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/EntryContainer.java b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/EntryContainer.java
new file mode 100644
index 0000000..a585eb3
--- /dev/null
+++ b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/EntryContainer.java
@@ -0,0 +1,3554 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License"). You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at legal-notices/CDDLv1_0.txt
+ * or http://forgerock.org/license/CDDLv1.0.html.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at legal-notices/CDDLv1_0.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information:
+ * Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ *
+ *
+ * Copyright 2006-2010 Sun Microsystems, Inc.
+ * Portions Copyright 2011-2014 ForgeRock AS
+ * Portions copyright 2013 Manuel Gaupp
+ */
+package org.opends.server.backends.pluggable;
+
+import java.util.*;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import org.forgerock.i18n.LocalizableMessage;
+import org.forgerock.i18n.LocalizableMessageBuilder;
+import org.forgerock.i18n.slf4j.LocalizedLogger;
+import org.forgerock.opendj.config.server.ConfigException;
+import org.forgerock.opendj.ldap.ByteSequence;
+import org.forgerock.opendj.ldap.ByteString;
+import org.forgerock.opendj.ldap.ByteStringBuilder;
+import org.forgerock.opendj.ldap.ResultCode;
+import org.forgerock.opendj.ldap.SearchScope;
+import org.forgerock.util.Utils;
+import org.opends.server.admin.server.ConfigurationAddListener;
+import org.opends.server.admin.server.ConfigurationChangeListener;
+import org.opends.server.admin.server.ConfigurationDeleteListener;
+import org.opends.server.admin.std.server.LocalDBBackendCfg;
+import org.opends.server.admin.std.server.LocalDBIndexCfg;
+import org.opends.server.admin.std.server.LocalDBVLVIndexCfg;
+import org.opends.server.api.Backend;
+import org.opends.server.api.ClientConnection;
+import org.opends.server.api.EntryCache;
+import org.opends.server.api.plugin.PluginResult.SubordinateDelete;
+import org.opends.server.api.plugin.PluginResult.SubordinateModifyDN;
+import org.opends.server.backends.pluggable.BackendImpl.Cursor;
+import org.opends.server.backends.pluggable.BackendImpl.ReadOperation;
+import org.opends.server.backends.pluggable.BackendImpl.ReadableStorage;
+import org.opends.server.backends.pluggable.BackendImpl.Storage;
+import org.opends.server.backends.pluggable.BackendImpl.StorageRuntimeException;
+import org.opends.server.backends.pluggable.BackendImpl.TreeName;
+import org.opends.server.backends.pluggable.BackendImpl.WriteOperation;
+import org.opends.server.backends.pluggable.BackendImpl.WriteableStorage;
+import org.opends.server.backends.pluggable.SuffixContainer;
+import org.opends.server.controls.*;
+import org.opends.server.core.*;
+import org.opends.server.types.*;
+import org.opends.server.util.ServerConstants;
+import org.opends.server.util.StaticUtils;
+
+import static org.opends.messages.JebMessages.*;
+import static org.opends.server.backends.pluggable.JebFormat.*;
+import static org.opends.server.core.DirectoryServer.*;
+import static org.opends.server.protocols.ldap.LDAPResultCode.*;
+import static org.opends.server.types.AdditionalLogItem.*;
+import static org.opends.server.util.StaticUtils.*;
+
+/**
+ * Storage container for LDAP entries. Each base DN of a JE backend is given
+ * its own entry container. The entry container is the object that implements
+ * the guts of the backend API methods for LDAP operations.
+ */
+public class EntryContainer
+ implements SuffixContainer, ConfigurationChangeListener<LocalDBBackendCfg>
+{
+ private static final LocalizedLogger logger = LocalizedLogger.getLoggerForThisClass();
+
+ /** The name of the entry database. */
+ public static final String ID2ENTRY_DATABASE_NAME = ID2ENTRY_INDEX_NAME;
+ /** The name of the DN database. */
+ public static final String DN2ID_DATABASE_NAME = DN2ID_INDEX_NAME;
+ /** The name of the children index database. */
+ private static final String ID2CHILDREN_DATABASE_NAME = ID2CHILDREN_INDEX_NAME;
+ /** The name of the subtree index database. */
+ private static final String ID2SUBTREE_DATABASE_NAME = ID2SUBTREE_INDEX_NAME;
+ /** The name of the referral database. */
+ private static final String REFERRAL_DATABASE_NAME = REFERRAL_INDEX_NAME;
+ /** The name of the state database. */
+ private static final String STATE_DATABASE_NAME = STATE_INDEX_NAME;
+
+ /** The attribute index configuration manager. */
+ private final AttributeJEIndexCfgManager attributeJEIndexCfgManager;
+ /** The vlv index configuration manager. */
+ private final VLVJEIndexCfgManager vlvJEIndexCfgManager;
+
+ /** The backend to which this entry container belongs. */
+ private final Backend<?> backend;
+
+ /** The root container in which this entryContainer belongs. */
+ private final RootContainer rootContainer;
+
+ /** The baseDN this entry container is responsible for. */
+ private final DN baseDN;
+
+ /** The backend configuration. */
+ private LocalDBBackendCfg config;
+
+ /** The JE database environment. */
+ private final Storage storage;
+
+ /** The DN database maps a normalized DN string to an entry ID (8 bytes). */
+ private DN2ID dn2id;
+ /** The entry database maps an entry ID (8 bytes) to a complete encoded entry. */
+ private ID2Entry id2entry;
+ /** Index maps entry ID to an entry ID list containing its children. */
+ private Index id2children;
+ /** Index maps entry ID to an entry ID list containing its subordinates. */
+ private Index id2subtree;
+ /** The referral database maps a normalized DN string to labeled URIs. */
+ private DN2URI dn2uri;
+ /** The state database maps a config DN to config entries. */
+ private State state;
+
+ /** The set of attribute indexes. */
+ private final HashMap<AttributeType, AttributeIndex> attrIndexMap = new HashMap<AttributeType, AttributeIndex>();
+
+ /** The set of VLV (Virtual List View) indexes. */
+ private final HashMap<String, VLVIndex> vlvIndexMap = new HashMap<String, VLVIndex>();
+
+ /**
+ * Prevents name clashes for common indexes (like id2entry) across multiple suffixes.
+ * For example when a root container contains multiple suffixes.
+ */
+ private TreeName databasePrefix;
+
+ /**
+ * This class is responsible for managing the configuration for attribute
+ * indexes used within this entry container.
+ */
+ private class AttributeJEIndexCfgManager implements
+ ConfigurationAddListener<LocalDBIndexCfg>,
+ ConfigurationDeleteListener<LocalDBIndexCfg>
+ {
+ /** {@inheritDoc} */
+ @Override
+ public boolean isConfigurationAddAcceptable(
+ LocalDBIndexCfg cfg,
+ List<LocalizableMessage> unacceptableReasons)
+ {
+ try
+ {
+ //Try creating all the indexes before confirming they are valid ones.
+ new AttributeIndex(cfg, EntryContainer.this);
+ return true;
+ }
+ catch(Exception e)
+ {
+ unacceptableReasons.add(LocalizableMessage.raw(e.getLocalizedMessage()));
+ return false;
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public ConfigChangeResult applyConfigurationAdd(LocalDBIndexCfg cfg)
+ {
+ boolean adminActionRequired = false;
+ List<LocalizableMessage> messages = new ArrayList<LocalizableMessage>();
+
+ try
+ {
+ AttributeIndex index = new AttributeIndex(cfg, EntryContainer.this);
+ index.open();
+ if(!index.isTrusted())
+ {
+ adminActionRequired = true;
+ messages.add(NOTE_JEB_INDEX_ADD_REQUIRES_REBUILD.get(
+ cfg.getAttribute().getNameOrOID()));
+ }
+ attrIndexMap.put(cfg.getAttribute(), index);
+ }
+ catch(Exception e)
+ {
+ messages.add(LocalizableMessage.raw(e.getLocalizedMessage()));
+ return new ConfigChangeResult(
+ DirectoryServer.getServerErrorResultCode(), adminActionRequired, messages);
+ }
+
+ return new ConfigChangeResult(ResultCode.SUCCESS, adminActionRequired, messages);
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public boolean isConfigurationDeleteAcceptable(
+ LocalDBIndexCfg cfg, List<LocalizableMessage> unacceptableReasons)
+ {
+ // TODO: validate more before returning true?
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public ConfigChangeResult applyConfigurationDelete(LocalDBIndexCfg cfg)
+ {
+ boolean adminActionRequired = false;
+ ArrayList<LocalizableMessage> messages = new ArrayList<LocalizableMessage>();
+
+ exclusiveLock.lock();
+ try
+ {
+ AttributeIndex index = attrIndexMap.get(cfg.getAttribute());
+ deleteAttributeIndex(index);
+ attrIndexMap.remove(cfg.getAttribute());
+ }
+ catch(StorageRuntimeException de)
+ {
+ messages.add(LocalizableMessage.raw(StaticUtils.stackTraceToSingleLineString(de)));
+ return new ConfigChangeResult(
+ DirectoryServer.getServerErrorResultCode(), adminActionRequired, messages);
+ }
+ finally
+ {
+ exclusiveLock.unlock();
+ }
+
+ return new ConfigChangeResult(ResultCode.SUCCESS, adminActionRequired, messages);
+ }
+ }
+
+ /**
+ * This class is responsible for managing the configuration for VLV indexes
+ * used within this entry container.
+ */
+ private class VLVJEIndexCfgManager implements
+ ConfigurationAddListener<LocalDBVLVIndexCfg>,
+ ConfigurationDeleteListener<LocalDBVLVIndexCfg>
+ {
+ /** {@inheritDoc} */
+ @Override
+ public boolean isConfigurationAddAcceptable(
+ LocalDBVLVIndexCfg cfg, List<LocalizableMessage> unacceptableReasons)
+ {
+ try
+ {
+ SearchFilter.createFilterFromString(cfg.getFilter());
+ }
+ catch(Exception e)
+ {
+ LocalizableMessage msg = ERR_JEB_CONFIG_VLV_INDEX_BAD_FILTER.get(
+ cfg.getFilter(), cfg.getName(),
+ e.getLocalizedMessage());
+ unacceptableReasons.add(msg);
+ return false;
+ }
+
+ String[] sortAttrs = cfg.getSortOrder().split(" ");
+ SortKey[] sortKeys = new SortKey[sortAttrs.length];
+ boolean[] ascending = new boolean[sortAttrs.length];
+ for(int i = 0; i < sortAttrs.length; i++)
+ {
+ try
+ {
+ if(sortAttrs[i].startsWith("-"))
+ {
+ ascending[i] = false;
+ sortAttrs[i] = sortAttrs[i].substring(1);
+ }
+ else
+ {
+ ascending[i] = true;
+ if(sortAttrs[i].startsWith("+"))
+ {
+ sortAttrs[i] = sortAttrs[i].substring(1);
+ }
+ }
+ }
+ catch(Exception e)
+ {
+ LocalizableMessage msg =
+ ERR_JEB_CONFIG_VLV_INDEX_UNDEFINED_ATTR.get(sortKeys[i], cfg.getName());
+ unacceptableReasons.add(msg);
+ return false;
+ }
+
+ AttributeType attrType =
+ DirectoryServer.getAttributeType(sortAttrs[i].toLowerCase());
+ if(attrType == null)
+ {
+ LocalizableMessage msg = ERR_JEB_CONFIG_VLV_INDEX_UNDEFINED_ATTR.get(
+ sortAttrs[i], cfg.getName());
+ unacceptableReasons.add(msg);
+ return false;
+ }
+ sortKeys[i] = new SortKey(attrType, ascending[i]);
+ }
+
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public ConfigChangeResult applyConfigurationAdd(LocalDBVLVIndexCfg cfg)
+ {
+ boolean adminActionRequired = false;
+ ArrayList<LocalizableMessage> messages = new ArrayList<LocalizableMessage>();
+
+ try
+ {
+ VLVIndex vlvIndex = new VLVIndex(cfg, state, storage, EntryContainer.this);
+ vlvIndex.open();
+ if(!vlvIndex.isTrusted())
+ {
+ adminActionRequired = true;
+ messages.add(NOTE_JEB_INDEX_ADD_REQUIRES_REBUILD.get(
+ cfg.getName()));
+ }
+ vlvIndexMap.put(cfg.getName().toLowerCase(), vlvIndex);
+ }
+ catch(Exception e)
+ {
+ messages.add(LocalizableMessage.raw(StaticUtils.stackTraceToSingleLineString(e)));
+ return new ConfigChangeResult(
+ DirectoryServer.getServerErrorResultCode(), adminActionRequired, messages);
+ }
+
+ return new ConfigChangeResult(ResultCode.SUCCESS, adminActionRequired, messages);
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public boolean isConfigurationDeleteAcceptable(
+ LocalDBVLVIndexCfg cfg,
+ List<LocalizableMessage> unacceptableReasons)
+ {
+ // TODO: validate more before returning true?
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public ConfigChangeResult applyConfigurationDelete(LocalDBVLVIndexCfg cfg)
+ {
+ boolean adminActionRequired = false;
+ List<LocalizableMessage> messages = new ArrayList<LocalizableMessage>();
+
+ exclusiveLock.lock();
+ try
+ {
+ VLVIndex vlvIndex =
+ vlvIndexMap.get(cfg.getName().toLowerCase());
+ deleteDatabase(vlvIndex);
+ vlvIndexMap.remove(cfg.getName());
+ }
+ catch(StorageRuntimeException de)
+ {
+ messages.add(LocalizableMessage.raw(StaticUtils.stackTraceToSingleLineString(de)));
+ return new ConfigChangeResult(
+ DirectoryServer.getServerErrorResultCode(), adminActionRequired, messages);
+ }
+ finally
+ {
+ exclusiveLock.unlock();
+ }
+
+ return new ConfigChangeResult(ResultCode.SUCCESS, adminActionRequired, messages);
+ }
+
+ }
+
+ /** A read write lock to handle schema changes and bulk changes. */
+ private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+ final Lock sharedLock = lock.readLock();
+ final Lock exclusiveLock = lock.writeLock();
+
+ /**
+ * Create a new entry container object.
+ *
+ * @param baseDN The baseDN this entry container will be responsible for
+ * storing on disk.
+ * @param databasePrefix The prefix to use in the database names used by
+ * this entry container.
+ * @param backend A reference to the JE backend that is creating this entry
+ * container. It is needed by the Directory Server entry cache
+ * methods.
+ * @param config The configuration of the JE backend.
+ * @param env The JE environment to create this entryContainer in.
+ * @param rootContainer The root container this entry container is in.
+ * @throws ConfigException if a configuration related error occurs.
+ */
+ public EntryContainer(DN baseDN, String databasePrefix, Backend<?> backend,
+ LocalDBBackendCfg config, Storage env, RootContainer rootContainer)
+ throws ConfigException
+ {
+ this.backend = backend;
+ this.baseDN = baseDN;
+ this.config = config;
+ this.storage = env;
+ this.rootContainer = rootContainer;
+
+ this.databasePrefix = preparePrefix(databasePrefix);
+
+ config.addLocalDBChangeListener(this);
+
+ attributeJEIndexCfgManager = new AttributeJEIndexCfgManager();
+ config.addLocalDBIndexAddListener(attributeJEIndexCfgManager);
+ config.addLocalDBIndexDeleteListener(attributeJEIndexCfgManager);
+
+ vlvJEIndexCfgManager = new VLVJEIndexCfgManager();
+ config.addLocalDBVLVIndexAddListener(vlvJEIndexCfgManager);
+ config.addLocalDBVLVIndexDeleteListener(vlvJEIndexCfgManager);
+ }
+
+ /**
+ * Opens the entryContainer for reading and writing.
+ *
+ * @throws StorageRuntimeException If an error occurs in the JE database.
+ * @throws ConfigException if a configuration related error occurs.
+ */
+ public void open()
+ throws StorageRuntimeException, ConfigException
+ {
+ try
+ {
+ DataConfig entryDataConfig =
+ new DataConfig(config.isEntriesCompressed(),
+ config.isCompactEncoding(),
+ rootContainer.getCompressedSchema());
+
+ id2entry = new ID2Entry(databasePrefix.child(ID2ENTRY_DATABASE_NAME),
+ entryDataConfig, storage, this);
+ id2entry.open();
+
+ dn2id = new DN2ID(databasePrefix.child(DN2ID_DATABASE_NAME), storage, this);
+ dn2id.open();
+
+ state = new State(databasePrefix.child(STATE_DATABASE_NAME), storage, this);
+ state.open();
+
+ if (config.isSubordinateIndexesEnabled())
+ {
+ openSubordinateIndexes();
+ }
+ else
+ {
+ // Use a null index and ensure that future attempts to use the real
+ // subordinate indexes will fail.
+ id2children = new NullIndex(databasePrefix.child(ID2CHILDREN_DATABASE_NAME),
+ new ID2CIndexer(), state, storage, this);
+ if (!storage.getConfig().getReadOnly())
+ {
+ state.putIndexTrustState(null, id2children, false);
+ }
+ id2children.open(); // No-op
+
+ id2subtree = new NullIndex(databasePrefix.child(ID2SUBTREE_DATABASE_NAME),
+ new ID2SIndexer(), state, storage, this);
+ if (!storage.getConfig().getReadOnly())
+ {
+ state.putIndexTrustState(null, id2subtree, false);
+ }
+ id2subtree.open(); // No-op
+
+ logger.info(NOTE_JEB_SUBORDINATE_INDEXES_DISABLED, backend.getBackendID());
+ }
+
+ dn2uri = new DN2URI(databasePrefix.child(REFERRAL_DATABASE_NAME), storage, this);
+ dn2uri.open();
+
+ for (String idx : config.listLocalDBIndexes())
+ {
+ LocalDBIndexCfg indexCfg = config.getLocalDBIndex(idx);
+
+ AttributeIndex index = new AttributeIndex(indexCfg, this);
+ index.open();
+ if(!index.isTrusted())
+ {
+ logger.info(NOTE_JEB_INDEX_ADD_REQUIRES_REBUILD, index.getName());
+ }
+ attrIndexMap.put(indexCfg.getAttribute(), index);
+ }
+
+ for(String idx : config.listLocalDBVLVIndexes())
+ {
+ LocalDBVLVIndexCfg vlvIndexCfg = config.getLocalDBVLVIndex(idx);
+
+ VLVIndex vlvIndex = new VLVIndex(vlvIndexCfg, state, storage, this);
+ vlvIndex.open();
+
+ if(!vlvIndex.isTrusted())
+ {
+ logger.info(NOTE_JEB_INDEX_ADD_REQUIRES_REBUILD, vlvIndex.getName());
+ }
+
+ vlvIndexMap.put(vlvIndexCfg.getName().toLowerCase(), vlvIndex);
+ }
+ }
+ catch (StorageRuntimeException de)
+ {
+ logger.traceException(de);
+ close();
+ throw de;
+ }
+ }
+
+ /**
+ * Closes the entry container.
+ *
+ * @throws StorageRuntimeException If an error occurs in the JE database.
+ */
+ @Override
+ public void close() throws StorageRuntimeException
+ {
+ // Close core indexes.
+ dn2id.close();
+ id2entry.close();
+ dn2uri.close();
+ id2children.close();
+ id2subtree.close();
+ state.close();
+
+ Utils.closeSilently(attrIndexMap.values());
+
+ for (VLVIndex vlvIndex : vlvIndexMap.values())
+ {
+ vlvIndex.close();
+ }
+
+ // Deregister any listeners.
+ config.removeLocalDBChangeListener(this);
+ config.removeLocalDBIndexAddListener(attributeJEIndexCfgManager);
+ config.removeLocalDBIndexDeleteListener(attributeJEIndexCfgManager);
+ config.removeLocalDBVLVIndexAddListener(vlvJEIndexCfgManager);
+ config.removeLocalDBVLVIndexDeleteListener(vlvJEIndexCfgManager);
+ }
+
+ /**
+ * Retrieves a reference to the root container in which this entry container
+ * exists.
+ *
+ * @return A reference to the root container in which this entry container
+ * exists.
+ */
+ public RootContainer getRootContainer()
+ {
+ return rootContainer;
+ }
+
+ public Storage getStorage()
+ {
+ return storage;
+ }
+
+ /**
+ * Get the DN database used by this entry container.
+ * The entryContainer must have been opened.
+ *
+ * @return The DN database.
+ */
+ public DN2ID getDN2ID()
+ {
+ return dn2id;
+ }
+
+ /**
+ * Get the entry database used by this entry container.
+ * The entryContainer must have been opened.
+ *
+ * @return The entry database.
+ */
+ public ID2Entry getID2Entry()
+ {
+ return id2entry;
+ }
+
+ /**
+ * Get the referral database used by this entry container.
+ * The entryContainer must have been opened.
+ *
+ * @return The referral database.
+ */
+ public DN2URI getDN2URI()
+ {
+ return dn2uri;
+ }
+
+ /**
+ * Get the children database used by this entry container.
+ * The entryContainer must have been opened.
+ *
+ * @return The children database.
+ */
+ public Index getID2Children()
+ {
+ return id2children;
+ }
+
+ /**
+ * Get the subtree database used by this entry container.
+ * The entryContainer must have been opened.
+ *
+ * @return The subtree database.
+ */
+ public Index getID2Subtree()
+ {
+ return id2subtree;
+ }
+
+ /**
+ * Get the state database used by this entry container.
+ * The entry container must have been opened.
+ *
+ * @return The state database.
+ */
+ public State getState()
+ {
+ return state;
+ }
+
+ /**
+ * Look for an attribute index for the given attribute type.
+ *
+ * @param attrType The attribute type for which an attribute index is needed.
+ * @return The attribute index or null if there is none for that type.
+ */
+ public AttributeIndex getAttributeIndex(AttributeType attrType)
+ {
+ return attrIndexMap.get(attrType);
+ }
+
+ /**
+ * Return attribute index map.
+ *
+ * @return The attribute index map.
+ */
+ public Map<AttributeType, AttributeIndex> getAttributeIndexMap() {
+ return attrIndexMap;
+ }
+
+ /**
+ * Look for an VLV index for the given index name.
+ *
+ * @param vlvIndexName The vlv index name for which an vlv index is needed.
+ * @return The VLV index or null if there is none with that name.
+ */
+ public VLVIndex getVLVIndex(String vlvIndexName)
+ {
+ return vlvIndexMap.get(vlvIndexName);
+ }
+
+ /**
+ * Retrieve all attribute indexes.
+ *
+ * @return All attribute indexes defined in this entry container.
+ */
+ public Collection<AttributeIndex> getAttributeIndexes()
+ {
+ return attrIndexMap.values();
+ }
+
+ /**
+ * Retrieve all VLV indexes.
+ *
+ * @return The collection of VLV indexes defined in this entry container.
+ */
+ public Collection<VLVIndex> getVLVIndexes()
+ {
+ return vlvIndexMap.values();
+ }
+
+ /**
+ * Determine the highest entryID in the entryContainer.
+ * The entryContainer must already be open.
+ *
+ * @return The highest entry ID.
+ * @throws StorageRuntimeException If an error occurs in the JE database.
+ */
+ public EntryID getHighestEntryID() throws StorageRuntimeException
+ {
+ Cursor cursor = storage.openCursor(id2entry.getName());
+ try
+ {
+ // Position a cursor on the last data item, and the key should give the highest ID.
+ if (cursor.positionToLastKey())
+ {
+ return new EntryID(cursor.getKey());
+ }
+ return new EntryID(0);
+ }
+ finally
+ {
+ cursor.close();
+ }
+ }
+
+ /**
+ * Determine the number of subordinate entries for a given entry.
+ *
+ * @param entryDN The distinguished name of the entry.
+ * @param subtree <code>true</code> will include all the entries under the
+ * given entries. <code>false</code> will only return the
+ * number of entries immediately under the given entry.
+ * @return The number of subordinate entries for the given entry or -1 if
+ * the entry does not exist.
+ * @throws StorageRuntimeException If an error occurs in the JE database.
+ */
+ public long getNumSubordinates(final DN entryDN, final boolean subtree)
+ throws StorageRuntimeException
+ {
+ try
+ {
+ return storage.read(new ReadOperation<Long>()
+ {
+ @Override
+ public Long run(ReadableStorage txn) throws Exception
+ {
+ EntryID entryID = dn2id.get(txn, entryDN, false);
+ if (entryID != null)
+ {
+ ByteString key = entryID.toByteString();
+ EntryIDSet entryIDSet;
+ if (subtree)
+ {
+ entryIDSet = id2subtree.readKey(key, txn);
+ }
+ else
+ {
+ entryIDSet = id2children.readKey(key, txn);
+ }
+ long count = entryIDSet.size();
+ if (count != Long.MAX_VALUE)
+ {
+ return count;
+ }
+ }
+ return -1L;
+ }
+ });
+ }
+ catch (Exception e)
+ {
+ throw new StorageRuntimeException(e);
+ }
+ }
+
+ /**
+ * Processes the specified search in this entryContainer.
+ * Matching entries should be provided back to the core server using the
+ * <CODE>SearchOperation.returnEntry</CODE> method.
+ *
+ * @param searchOperation The search operation to be processed.
+ * @throws DirectoryException
+ * If a problem occurs while processing the
+ * search.
+ * @throws StorageRuntimeException If an error occurs in the JE database.
+ * @throws CanceledOperationException if this operation should be cancelled.
+ */
+ public void search(final SearchOperation searchOperation)
+ throws DirectoryException, StorageRuntimeException, CanceledOperationException
+ {
+ try
+ {
+ storage.read(new ReadOperation<Void>()
+ {
+ @Override
+ public Void run(ReadableStorage txn) throws Exception
+ {
+ DN aBaseDN = searchOperation.getBaseDN();
+ SearchScope searchScope = searchOperation.getScope();
+
+ PagedResultsControl pageRequest = searchOperation.getRequestControl(PagedResultsControl.DECODER);
+ ServerSideSortRequestControl sortRequest =
+ searchOperation.getRequestControl(ServerSideSortRequestControl.DECODER);
+ if (sortRequest != null && !sortRequest.containsSortKeys() && sortRequest.isCritical())
+ {
+ /**
+ * If the control's criticality field is true then the server SHOULD
+ * do the following: return unavailableCriticalExtension as a return
+ * code in the searchResultDone message; include the
+ * sortKeyResponseControl in the searchResultDone message, and not
+ * send back any search result entries.
+ */
+ searchOperation.addResponseControl(new ServerSideSortResponseControl(NO_SUCH_ATTRIBUTE, null));
+ searchOperation.setResultCode(ResultCode.UNAVAILABLE_CRITICAL_EXTENSION);
+ return null;
+ }
+ VLVRequestControl vlvRequest = searchOperation.getRequestControl(VLVRequestControl.DECODER);
+
+ if (vlvRequest != null && pageRequest != null)
+ {
+ LocalizableMessage message = ERR_JEB_SEARCH_CANNOT_MIX_PAGEDRESULTS_AND_VLV.get();
+ throw new DirectoryException(ResultCode.CONSTRAINT_VIOLATION, message);
+ }
+
+ // Handle client abandon of paged results.
+ if (pageRequest != null)
+ {
+ if (pageRequest.getSize() == 0)
+ {
+ Control control = new PagedResultsControl(pageRequest.isCritical(), 0, null);
+ searchOperation.getResponseControls().add(control);
+ return null;
+ }
+ if (searchOperation.getSizeLimit() > 0 && pageRequest.getSize() >= searchOperation.getSizeLimit())
+ {
+ // The RFC says : "If the page size is greater than or equal to the
+ // sizeLimit value, the server should ignore the control as the
+ // request can be satisfied in a single page"
+ pageRequest = null;
+ }
+ }
+
+ // Handle base-object search first.
+ if (searchScope == SearchScope.BASE_OBJECT)
+ {
+ // Fetch the base entry.
+ Entry baseEntry = fetchBaseEntry(aBaseDN, searchScope);
+
+ if (!isManageDsaITOperation(searchOperation))
+ {
+ dn2uri.checkTargetForReferral(baseEntry, searchOperation.getScope());
+ }
+
+ if (searchOperation.getFilter().matchesEntry(baseEntry))
+ {
+ searchOperation.returnEntry(baseEntry, null);
+ }
+
+ if (pageRequest != null)
+ {
+ // Indicate no more pages.
+ Control control = new PagedResultsControl(pageRequest.isCritical(), 0, null);
+ searchOperation.getResponseControls().add(control);
+ }
+
+ return null;
+ }
+
+ // Check whether the client requested debug information about the
+ // contribution of the indexes to the search.
+ StringBuilder debugBuffer = null;
+ if (searchOperation.getAttributes().contains(ATTR_DEBUG_SEARCH_INDEX))
+ {
+ debugBuffer = new StringBuilder();
+ }
+
+ EntryIDSet entryIDList = null;
+ boolean candidatesAreInScope = false;
+ if (sortRequest != null)
+ {
+ for (VLVIndex vlvIndex : vlvIndexMap.values())
+ {
+ try
+ {
+ entryIDList = vlvIndex.evaluate(null, searchOperation, sortRequest, vlvRequest, debugBuffer);
+ if (entryIDList != null)
+ {
+ searchOperation.addResponseControl(new ServerSideSortResponseControl(SUCCESS, null));
+ candidatesAreInScope = true;
+ break;
+ }
+ }
+ catch (DirectoryException de)
+ {
+ searchOperation.addResponseControl(new ServerSideSortResponseControl(de.getResultCode().intValue(),
+ null));
+
+ if (sortRequest.isCritical())
+ {
+ throw de;
+ }
+ }
+ }
+ }
+
+ if (entryIDList == null)
+ {
+ // See if we could use a virtual attribute rule to process the
+ // search.
+ for (VirtualAttributeRule rule : DirectoryServer.getVirtualAttributes())
+ {
+ if (rule.getProvider().isSearchable(rule, searchOperation, true))
+ {
+ rule.getProvider().processSearch(rule, searchOperation);
+ return null;
+ }
+ }
+
+ // Create an index filter to get the search result candidate entries
+ IndexFilter indexFilter =
+ new IndexFilter(EntryContainer.this, searchOperation, debugBuffer, rootContainer.getMonitorProvider());
+
+ // Evaluate the filter against the attribute indexes.
+ entryIDList = indexFilter.evaluate();
+
+ // Evaluate the search scope against the id2children and id2subtree indexes
+ if (entryIDList.size() > IndexFilter.FILTER_CANDIDATE_THRESHOLD)
+ {
+ // Read the ID from dn2id.
+ EntryID baseID = dn2id.get(txn, aBaseDN, false);
+ if (baseID == null)
+ {
+ LocalizableMessage message = ERR_JEB_SEARCH_NO_SUCH_OBJECT.get(aBaseDN);
+ DN matchedDN = getMatchedDN(aBaseDN);
+ throw new DirectoryException(ResultCode.NO_SUCH_OBJECT, message, matchedDN, null);
+ }
+ ByteString baseIDData = baseID.toByteString();
+
+ EntryIDSet scopeList;
+ if (searchScope == SearchScope.SINGLE_LEVEL)
+ {
+ scopeList = id2children.readKey(baseIDData, txn);
+ }
+ else
+ {
+ scopeList = id2subtree.readKey(baseIDData, txn);
+ if (searchScope == SearchScope.WHOLE_SUBTREE)
+ {
+ // The id2subtree list does not include the base entry ID.
+ scopeList.add(baseID);
+ }
+ }
+ entryIDList.retainAll(scopeList);
+ if (debugBuffer != null)
+ {
+ debugBuffer.append(" scope=");
+ debugBuffer.append(searchScope);
+ scopeList.toString(debugBuffer);
+ }
+ if (scopeList.isDefined())
+ {
+ // In this case we know that every candidate is in scope.
+ candidatesAreInScope = true;
+ }
+ }
+
+ if (sortRequest != null)
+ {
+ try
+ {
+ // If the sort key is not present, the sorting will generate the
+ // default ordering. VLV search request goes through as if
+ // this sort key was not found in the user entry.
+ entryIDList =
+ EntryIDSetSorter.sort(EntryContainer.this, entryIDList, searchOperation,
+ sortRequest.getSortOrder(), vlvRequest);
+ if (sortRequest.containsSortKeys())
+ {
+ searchOperation.addResponseControl(new ServerSideSortResponseControl(SUCCESS, null));
+ }
+ else
+ {
+ /*
+ * There is no sort key associated with the sort control.
+ * Since it came here it means that the criticality is false
+ * so let the server return all search results unsorted and
+ * include the sortKeyResponseControl in the searchResultDone
+ * message.
+ */
+ searchOperation.addResponseControl(new ServerSideSortResponseControl(NO_SUCH_ATTRIBUTE, null));
+ }
+ }
+ catch (DirectoryException de)
+ {
+ searchOperation.addResponseControl(new ServerSideSortResponseControl(de.getResultCode().intValue(),
+ null));
+
+ if (sortRequest.isCritical())
+ {
+ throw de;
+ }
+ }
+ }
+ }
+
+ // If requested, construct and return a fictitious entry containing
+ // debug information, and no other entries.
+ if (debugBuffer != null)
+ {
+ debugBuffer.append(" final=");
+ entryIDList.toString(debugBuffer);
+
+ Attribute attr = Attributes.create(ATTR_DEBUG_SEARCH_INDEX, debugBuffer.toString());
+ Entry debugEntry = new Entry(DN.valueOf("cn=debugsearch"), null, null, null);
+ debugEntry.addAttribute(attr, new ArrayList<ByteString>());
+
+ searchOperation.returnEntry(debugEntry, null);
+ return null;
+ }
+
+ if (entryIDList.isDefined())
+ {
+ if (rootContainer.getMonitorProvider().isFilterUseEnabled())
+ {
+ rootContainer.getMonitorProvider().updateIndexedSearchCount();
+ }
+ searchIndexed(entryIDList, candidatesAreInScope, searchOperation, pageRequest);
+ }
+ else
+ {
+ if (rootContainer.getMonitorProvider().isFilterUseEnabled())
+ {
+ rootContainer.getMonitorProvider().updateUnindexedSearchCount();
+ }
+
+ searchOperation.addAdditionalLogItem(keyOnly(getClass(), "unindexed"));
+
+ // See if we could use a virtual attribute rule to process the
+ // search.
+ for (VirtualAttributeRule rule : DirectoryServer.getVirtualAttributes())
+ {
+ if (rule.getProvider().isSearchable(rule, searchOperation, false))
+ {
+ rule.getProvider().processSearch(rule, searchOperation);
+ return null;
+ }
+ }
+
+ ClientConnection clientConnection = searchOperation.getClientConnection();
+ if (!clientConnection.hasPrivilege(Privilege.UNINDEXED_SEARCH, searchOperation))
+ {
+ LocalizableMessage message = ERR_JEB_SEARCH_UNINDEXED_INSUFFICIENT_PRIVILEGES.get();
+ throw new DirectoryException(ResultCode.INSUFFICIENT_ACCESS_RIGHTS, message);
+ }
+
+ if (sortRequest != null)
+ {
+ // FIXME -- Add support for sorting unindexed searches using
+ // indexes
+ // like DSEE currently does.
+ searchOperation.addResponseControl(new ServerSideSortResponseControl(UNWILLING_TO_PERFORM, null));
+
+ if (sortRequest.isCritical())
+ {
+ LocalizableMessage message = ERR_JEB_SEARCH_CANNOT_SORT_UNINDEXED.get();
+ throw new DirectoryException(ResultCode.UNAVAILABLE_CRITICAL_EXTENSION, message);
+ }
+ }
+
+ searchNotIndexed(searchOperation, pageRequest);
+ }
+ return null;
+ }
+ });
+ }
+ catch (Exception e)
+ {
+ throw new StorageRuntimeException(e);
+ }
+ }
+
+ /**
+ * We were not able to obtain a set of candidate entry IDs for the
+ * search from the indexes.
+ * <p>
+ * Here we are relying on the DN key order to ensure children are
+ * returned after their parents.
+ * <ul>
+ * <li>iterate through a subtree range of the DN database
+ * <li>discard non-children DNs if the search scope is single level
+ * <li>fetch the entry by ID from the entry cache or the entry database
+ * <li>return the entry if it matches the filter
+ * </ul>
+ *
+ * @param searchOperation The search operation.
+ * @param pageRequest A Paged Results control, or null if none.
+ * @throws DirectoryException If an error prevented the search from being
+ * processed.
+ */
+ private void searchNotIndexed(SearchOperation searchOperation, PagedResultsControl pageRequest)
+ throws DirectoryException, CanceledOperationException
+ {
+ DN aBaseDN = searchOperation.getBaseDN();
+ SearchScope searchScope = searchOperation.getScope();
+ boolean manageDsaIT = isManageDsaITOperation(searchOperation);
+
+ // The base entry must already have been processed if this is
+ // a request for the next page in paged results. So we skip
+ // the base entry processing if the cookie is set.
+ if (pageRequest == null || pageRequest.getCookie().length() == 0)
+ {
+ // Fetch the base entry.
+ Entry baseEntry = fetchBaseEntry(aBaseDN, searchScope);
+
+ if (!manageDsaIT)
+ {
+ dn2uri.checkTargetForReferral(baseEntry, searchScope);
+ }
+
+ /*
+ * The base entry is only included for whole subtree search.
+ */
+ if (searchScope == SearchScope.WHOLE_SUBTREE
+ && searchOperation.getFilter().matchesEntry(baseEntry))
+ {
+ searchOperation.returnEntry(baseEntry, null);
+ }
+
+ if (!manageDsaIT
+ && !dn2uri.returnSearchReferences(searchOperation)
+ && pageRequest != null)
+ {
+ // Indicate no more pages.
+ Control control = new PagedResultsControl(pageRequest.isCritical(), 0, null);
+ searchOperation.getResponseControls().add(control);
+ }
+ }
+
+ /*
+ * We will iterate forwards through a range of the dn2id keys to
+ * find subordinates of the target entry from the top of the tree
+ * downwards. For example, any subordinates of "dc=example,dc=com" appear
+ * in dn2id with a key ending in ",dc=example,dc=com". The entry
+ * "cn=joe,ou=people,dc=example,dc=com" will appear after the entry
+ * "ou=people,dc=example,dc=com".
+ */
+ ByteString baseDNKey = dnToDNKey(aBaseDN, this.baseDN.size());
+ ByteStringBuilder suffix = copyOf(baseDNKey);
+ ByteStringBuilder end = copyOf(baseDNKey);
+
+ /*
+ * Set the ending value to a value of equal length but slightly
+ * greater than the suffix. Since keys are compared in
+ * reverse order we must set the first byte (the comma).
+ * No possibility of overflow here.
+ */
+ suffix.append((byte) 0x00);
+ end.append((byte) 0x01);
+
+ // Set the starting value.
+ ByteSequence begin;
+ if (pageRequest != null && pageRequest.getCookie().length() != 0)
+ {
+ // The cookie contains the DN of the next entry to be returned.
+ try
+ {
+ begin = ByteString.wrap(pageRequest.getCookie().toByteArray());
+ }
+ catch (Exception e)
+ {
+ logger.traceException(e);
+ String str = pageRequest.getCookie().toHexString();
+ LocalizableMessage msg = ERR_JEB_INVALID_PAGED_RESULTS_COOKIE.get(str);
+ throw new DirectoryException(ResultCode.UNWILLING_TO_PERFORM, msg, e);
+ }
+ }
+ else
+ {
+ // Set the starting value to the suffix.
+ begin = suffix;
+ }
+
+ ByteSequence startKey = begin;
+
+ int lookthroughCount = 0;
+ int lookthroughLimit = searchOperation.getClientConnection().getLookthroughLimit();
+
+ try
+ {
+ Cursor cursor = storage.openCursor(dn2id.getName());
+ try
+ {
+ // Initialize the cursor very close to the starting value.
+ boolean success = cursor.positionToKeyOrNext(startKey);
+
+ // Step forward until we pass the ending value.
+ while (success)
+ {
+ if(lookthroughLimit > 0 && lookthroughCount > lookthroughLimit)
+ {
+ //Lookthrough limit exceeded
+ searchOperation.setResultCode(ResultCode.ADMIN_LIMIT_EXCEEDED);
+ searchOperation.appendErrorMessage(
+ NOTE_JEB_LOOKTHROUGH_LIMIT_EXCEEDED.get(lookthroughLimit));
+ return;
+ }
+ int cmp = ByteSequence.COMPARATOR.compare(cursor.getKey(), end);
+ if (cmp >= 0)
+ {
+ // We have gone past the ending value.
+ break;
+ }
+
+ // We have found a subordinate entry.
+
+ EntryID entryID = new EntryID(cursor.getValue());
+
+ boolean isInScope =
+ searchScope != SearchScope.SINGLE_LEVEL
+ // Check if this entry is an immediate child.
+ || findDNKeyParent(cursor.getKey()) == baseDNKey.length();
+ if (isInScope)
+ {
+ // Process the candidate entry.
+ final Entry entry = getEntry(entryID);
+ if (entry != null)
+ {
+ lookthroughCount++;
+
+ if ((manageDsaIT || entry.getReferralURLs() == null)
+ && searchOperation.getFilter().matchesEntry(entry))
+ {
+ if (pageRequest != null
+ && searchOperation.getEntriesSent() == pageRequest.getSize())
+ {
+ // The current page is full.
+ // Set the cookie to remember where we were.
+ ByteString cookie = cursor.getKey();
+ Control control = new PagedResultsControl(pageRequest.isCritical(), 0, cookie);
+ searchOperation.getResponseControls().add(control);
+ return;
+ }
+
+ if (!searchOperation.returnEntry(entry, null))
+ {
+ // We have been told to discontinue processing of the
+ // search. This could be due to size limit exceeded or
+ // operation cancelled.
+ return;
+ }
+ }
+ }
+ }
+
+ searchOperation.checkIfCanceled(false);
+
+ // Move to the next record.
+ success = cursor.next();
+ }
+ }
+ finally
+ {
+ cursor.close();
+ }
+ }
+ catch (StorageRuntimeException e)
+ {
+ logger.traceException(e);
+ }
+
+ if (pageRequest != null)
+ {
+ // Indicate no more pages.
+ Control control = new PagedResultsControl(pageRequest.isCritical(), 0, null);
+ searchOperation.getResponseControls().add(control);
+ }
+ }
+
+ /**
+ * Returns the entry corresponding to the provided entryID.
+ *
+ * @param entryID
+ * the id of the entry to retrieve
+ * @return the entry corresponding to the provided entryID
+ * @throws DirectoryException
+ * If an error occurs retrieving the entry
+ */
+ public Entry getEntry(EntryID entryID) throws DirectoryException
+ {
+ // Try the entry cache first.
+ final EntryCache entryCache = getEntryCache();
+ final Entry cacheEntry = entryCache.getEntry(backend, entryID.longValue());
+ if (cacheEntry != null)
+ {
+ return cacheEntry;
+ }
+
+ final Entry entry = id2entry.get(null, entryID, false);
+ if (entry != null)
+ {
+ // Put the entry in the cache making sure not to overwrite a newer copy
+ // that may have been inserted since the time we read the cache.
+ entryCache.putEntryIfAbsent(entry, backend, entryID.longValue());
+ }
+ return entry;
+ }
+
+ /**
+ * We were able to obtain a set of candidate entry IDs for the
+ * search from the indexes.
+ * <p>
+ * Here we are relying on ID order to ensure children are returned
+ * after their parents.
+ * <ul>
+ * <li>Iterate through the candidate IDs
+ * <li>fetch entry by ID from cache or id2entry
+ * <li>put the entry in the cache if not present
+ * <li>discard entries that are not in scope
+ * <li>return entry if it matches the filter
+ * </ul>
+ *
+ * @param entryIDList The candidate entry IDs.
+ * @param candidatesAreInScope true if it is certain that every candidate
+ * entry is in the search scope.
+ * @param searchOperation The search operation.
+ * @param pageRequest A Paged Results control, or null if none.
+ * @throws DirectoryException If an error prevented the search from being
+ * processed.
+ */
+ private void searchIndexed(EntryIDSet entryIDList,
+ boolean candidatesAreInScope,
+ SearchOperation searchOperation,
+ PagedResultsControl pageRequest)
+ throws DirectoryException, CanceledOperationException
+ {
+ SearchScope searchScope = searchOperation.getScope();
+ DN aBaseDN = searchOperation.getBaseDN();
+ boolean manageDsaIT = isManageDsaITOperation(searchOperation);
+ boolean continueSearch = true;
+
+ // Set the starting value.
+ EntryID begin = null;
+ if (pageRequest != null && pageRequest.getCookie().length() != 0)
+ {
+ // The cookie contains the ID of the next entry to be returned.
+ try
+ {
+ begin = new EntryID(pageRequest.getCookie().toLong());
+ }
+ catch (Exception e)
+ {
+ logger.traceException(e);
+ String str = pageRequest.getCookie().toHexString();
+ LocalizableMessage msg = ERR_JEB_INVALID_PAGED_RESULTS_COOKIE.get(str);
+ throw new DirectoryException(ResultCode.UNWILLING_TO_PERFORM,
+ msg, e);
+ }
+ }
+ else if (!manageDsaIT)
+ {
+ // Return any search result references.
+ continueSearch = dn2uri.returnSearchReferences(searchOperation);
+ }
+
+ // Make sure the candidate list is smaller than the lookthrough limit
+ int lookthroughLimit =
+ searchOperation.getClientConnection().getLookthroughLimit();
+ if(lookthroughLimit > 0 && entryIDList.size() > lookthroughLimit)
+ {
+ //Lookthrough limit exceeded
+ searchOperation.setResultCode(ResultCode.ADMIN_LIMIT_EXCEEDED);
+ searchOperation.appendErrorMessage(
+ NOTE_JEB_LOOKTHROUGH_LIMIT_EXCEEDED.get(lookthroughLimit));
+ continueSearch = false;
+ }
+
+ // Iterate through the index candidates.
+ if (continueSearch)
+ {
+ for (Iterator<EntryID> it = entryIDList.iterator(begin); it.hasNext();)
+ {
+ final EntryID id = it.next();
+
+ Entry entry;
+ try
+ {
+ entry = getEntry(id);
+ }
+ catch (Exception e)
+ {
+ logger.traceException(e);
+ continue;
+ }
+
+ // Process the candidate entry.
+ if (entry != null)
+ {
+ // Filter the entry if it is in scope.
+ if (isInScope(candidatesAreInScope, searchScope, aBaseDN, entry)
+ && (manageDsaIT || entry.getReferralURLs() == null)
+ && searchOperation.getFilter().matchesEntry(entry))
+ {
+ if (pageRequest != null
+ && searchOperation.getEntriesSent() == pageRequest.getSize())
+ {
+ // The current page is full.
+ // Set the cookie to remember where we were.
+ ByteString cookie = id.toByteString();
+ Control control = new PagedResultsControl(pageRequest.isCritical(), 0, cookie);
+ searchOperation.getResponseControls().add(control);
+ return;
+ }
+
+ if (!searchOperation.returnEntry(entry, null))
+ {
+ // We have been told to discontinue processing of the
+ // search. This could be due to size limit exceeded or
+ // operation cancelled.
+ break;
+ }
+ }
+ }
+ }
+ searchOperation.checkIfCanceled(false);
+ }
+
+ // Before we return success from the search we must ensure the base entry
+ // exists. However, if we have returned at least one entry or subordinate
+ // reference it implies the base does exist, so we can omit the check.
+ if (searchOperation.getEntriesSent() == 0
+ && searchOperation.getReferencesSent() == 0)
+ {
+ // Fetch the base entry if it exists.
+ Entry baseEntry = fetchBaseEntry(aBaseDN, searchScope);
+
+ if (!manageDsaIT)
+ {
+ dn2uri.checkTargetForReferral(baseEntry, searchScope);
+ }
+ }
+
+ if (pageRequest != null)
+ {
+ // Indicate no more pages.
+ Control control = new PagedResultsControl(pageRequest.isCritical(), 0, null);
+ searchOperation.getResponseControls().add(control);
+ }
+ }
+
+ private boolean isInScope(boolean candidatesAreInScope, SearchScope searchScope, DN aBaseDN, Entry entry)
+ {
+ DN entryDN = entry.getName();
+
+ if (candidatesAreInScope)
+ {
+ return true;
+ }
+ else if (searchScope == SearchScope.SINGLE_LEVEL)
+ {
+ // Check if this entry is an immediate child.
+ if (entryDN.size() == aBaseDN.size() + 1
+ && entryDN.isDescendantOf(aBaseDN))
+ {
+ return true;
+ }
+ }
+ else if (searchScope == SearchScope.WHOLE_SUBTREE)
+ {
+ if (entryDN.isDescendantOf(aBaseDN))
+ {
+ return true;
+ }
+ }
+ else if (searchScope == SearchScope.SUBORDINATES
+ && entryDN.size() > aBaseDN.size()
+ && entryDN.isDescendantOf(aBaseDN))
+ {
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * Adds the provided entry to this database. This method must ensure that the
+ * entry is appropriate for the database and that no entry already exists with
+ * the same DN. The caller must hold a write lock on the DN of the provided
+ * entry.
+ *
+ * @param entry The entry to add to this database.
+ * @param addOperation The add operation with which the new entry is
+ * associated. This may be <CODE>null</CODE> for adds
+ * performed internally.
+ * @throws DirectoryException If a problem occurs while trying to add the
+ * entry.
+ * @throws StorageRuntimeException If an error occurs in the JE database.
+ * @throws CanceledOperationException if this operation should be cancelled.
+ */
+ public void addEntry(final Entry entry, final AddOperation addOperation)
+ throws StorageRuntimeException, DirectoryException, CanceledOperationException
+ {
+ try
+ {
+ storage.update(new WriteOperation()
+ {
+ @Override
+ public void run(WriteableStorage txn) throws Exception
+ {
+ DN parentDN = getParentWithinBase(entry.getName());
+
+ try
+ {
+ // Check whether the entry already exists.
+ if (dn2id.get(txn, entry.getName(), false) != null)
+ {
+ throw new DirectoryException(ResultCode.ENTRY_ALREADY_EXISTS, ERR_JEB_ADD_ENTRY_ALREADY_EXISTS.get(entry
+ .getName()));
+ }
+
+ // Check that the parent entry exists.
+ EntryID parentID = null;
+ if (parentDN != null)
+ {
+ // Check for referral entries above the target.
+ dn2uri.targetEntryReferrals(entry.getName(), null);
+
+ // Read the parent ID from dn2id.
+ parentID = dn2id.get(txn, parentDN, false);
+ if (parentID == null)
+ {
+ LocalizableMessage message = ERR_JEB_ADD_NO_SUCH_OBJECT.get(entry.getName());
+ DN matchedDN = getMatchedDN(baseDN);
+ throw new DirectoryException(ResultCode.NO_SUCH_OBJECT, message, matchedDN, null);
+ }
+ }
+
+ EntryID entryID = rootContainer.getNextEntryID();
+
+ // Insert into dn2id.
+ if (!dn2id.insert(txn, entry.getName(), entryID))
+ {
+ // Do not ever expect to come through here.
+ throw new DirectoryException(ResultCode.ENTRY_ALREADY_EXISTS, ERR_JEB_ADD_ENTRY_ALREADY_EXISTS.get(entry
+ .getName()));
+ }
+
+ // Update the referral database for referral entries.
+ if (!dn2uri.addEntry(txn, entry))
+ {
+ // Do not ever expect to come through here.
+ throw new DirectoryException(ResultCode.ENTRY_ALREADY_EXISTS, ERR_JEB_ADD_ENTRY_ALREADY_EXISTS.get(entry
+ .getName()));
+ }
+
+ // Insert into id2entry.
+ if (!id2entry.insert(txn, entryID, entry))
+ {
+ // Do not ever expect to come through here.
+ throw new DirectoryException(ResultCode.ENTRY_ALREADY_EXISTS, ERR_JEB_ADD_ENTRY_ALREADY_EXISTS.get(entry
+ .getName()));
+ }
+
+ // Insert into the indexes, in index configuration order.
+ final IndexBuffer indexBuffer = new IndexBuffer(EntryContainer.this);
+ indexInsertEntry(indexBuffer, entry, entryID);
+
+ // Insert into id2children and id2subtree.
+ // The database transaction locks on these records will be hotly
+ // contested so we do them last so as to hold the locks for the
+ // shortest duration.
+ if (parentDN != null)
+ {
+ final ByteString parentIDKeyBytes = parentID.toByteString();
+ id2children.insertID(indexBuffer, parentIDKeyBytes, entryID);
+ id2subtree.insertID(indexBuffer, parentIDKeyBytes, entryID);
+
+ // Iterate up through the superior entries, starting above the
+ // parent.
+ for (DN dn = getParentWithinBase(parentDN); dn != null; dn = getParentWithinBase(dn))
+ {
+ // Read the ID from dn2id.
+ EntryID nodeID = dn2id.get(txn, dn, false);
+ if (nodeID == null)
+ {
+ throw new JebException(ERR_JEB_MISSING_DN2ID_RECORD.get(dn));
+ }
+
+ // Insert into id2subtree for this node.
+ id2subtree.insertID(indexBuffer, nodeID.toByteString(), entryID);
+ }
+ }
+ indexBuffer.flush(txn);
+
+ if (addOperation != null)
+ {
+ // One last check before committing
+ addOperation.checkIfCanceled(true);
+ }
+
+ // Commit the transaction.
+ EntryContainer.transactionCommit(txn);
+
+ // Update the entry cache.
+ EntryCache<?> entryCache = DirectoryServer.getEntryCache();
+ if (entryCache != null)
+ {
+ entryCache.putEntry(entry, backend, entryID.longValue());
+ }
+ }
+ catch (StorageRuntimeException StorageRuntimeException)
+ {
+ EntryContainer.transactionAbort(txn);
+ throw StorageRuntimeException;
+ }
+ catch (DirectoryException directoryException)
+ {
+ EntryContainer.transactionAbort(txn);
+ throw directoryException;
+ }
+ catch (CanceledOperationException coe)
+ {
+ EntryContainer.transactionAbort(txn);
+ throw coe;
+ }
+ catch (Exception e)
+ {
+ EntryContainer.transactionAbort(txn);
+
+ String msg = e.getMessage();
+ if (msg == null)
+ {
+ msg = stackTraceToSingleLineString(e);
+ }
+ LocalizableMessage message = ERR_JEB_UNCHECKED_EXCEPTION.get(msg);
+ throw new DirectoryException(DirectoryServer.getServerErrorResultCode(), message, e);
+ }
+ }
+ });
+ }
+ catch (Exception e)
+ {
+ throw new StorageRuntimeException(e);
+ }
+ }
+
+ /**
+ * Removes the specified entry from this database. This method must ensure
+ * that the entry exists and that it does not have any subordinate entries
+ * (unless the database supports a subtree delete operation and the client
+ * included the appropriate information in the request). The caller must hold
+ * a write lock on the provided entry DN.
+ *
+ * @param entryDN The DN of the entry to remove from this database.
+ * @param deleteOperation The delete operation with which this action is
+ * associated. This may be <CODE>null</CODE> for
+ * deletes performed internally.
+ * @throws DirectoryException If a problem occurs while trying to remove the
+ * entry.
+ * @throws StorageRuntimeException If an error occurs in the JE database.
+ * @throws CanceledOperationException if this operation should be cancelled.
+ */
+ public void deleteEntry(final DN entryDN, final DeleteOperation deleteOperation)
+ throws DirectoryException, StorageRuntimeException, CanceledOperationException
+ {
+ try
+ {
+ storage.update(new WriteOperation()
+ {
+ @Override
+ public void run(WriteableStorage txn) throws Exception
+ {
+ final IndexBuffer indexBuffer = new IndexBuffer(EntryContainer.this);
+
+ try
+ {
+ // Check for referral entries above the target entry.
+ dn2uri.targetEntryReferrals(entryDN, null);
+
+ // Determine whether this is a subtree delete.
+ boolean isSubtreeDelete =
+ deleteOperation != null && deleteOperation.getRequestControl(SubtreeDeleteControl.DECODER) != null;
+
+ /*
+ * We will iterate forwards through a range of the dn2id keys to
+ * find subordinates of the target entry from the top of the tree
+ * downwards.
+ */
+ ByteString entryDNKey = dnToDNKey(entryDN, baseDN.size());
+ ByteStringBuilder suffix = copyOf(entryDNKey);
+ ByteStringBuilder end = copyOf(entryDNKey);
+
+ /*
+ * Set the ending value to a value of equal length but slightly
+ * greater than the suffix.
+ */
+ suffix.append((byte) 0x00);
+ end.append((byte) 0x01);
+
+ int subordinateEntriesDeleted = 0;
+
+ ByteSequence startKey = suffix;
+
+ CursorConfig cursorConfig = new CursorConfig();
+ cursorConfig.setReadCommitted(true);
+ Cursor cursor = dn2id.openCursor(txn);
+ try
+ {
+ // Initialize the cursor very close to the starting value.
+ boolean success = cursor.positionToKeyOrNext(startKey);
+
+ // Step forward until the key is greater than the starting value.
+ while (success && ByteSequence.COMPARATOR.compare(startKey, suffix) <= 0)
+ {
+ success = cursor.next();
+ }
+
+ // Step forward until we pass the ending value.
+ while (success)
+ {
+ int cmp = ByteSequence.COMPARATOR.compare(cursor.getKey(), end);
+ if (cmp >= 0)
+ {
+ // We have gone past the ending value.
+ break;
+ }
+
+ // We have found a subordinate entry.
+ if (!isSubtreeDelete)
+ {
+ // The subtree delete control was not specified and
+ // the target entry is not a leaf.
+ throw new DirectoryException(ResultCode.NOT_ALLOWED_ON_NONLEAF, ERR_JEB_DELETE_NOT_ALLOWED_ON_NONLEAF
+ .get(entryDN));
+ }
+
+ /*
+ * Delete this entry which by now must be a leaf because we have
+ * been deleting from the bottom of the tree upwards.
+ */
+ EntryID entryID = new EntryID(cursor.getValue());
+
+ // Invoke any subordinate delete plugins on the entry.
+ if (deleteOperation != null && !deleteOperation.isSynchronizationOperation())
+ {
+ Entry subordinateEntry = id2entry.get(txn, entryID, false);
+ SubordinateDelete pluginResult =
+ getPluginConfigManager().invokeSubordinateDeletePlugins(deleteOperation, subordinateEntry);
+
+ if (!pluginResult.continueProcessing())
+ {
+ LocalizableMessage message =
+ ERR_JEB_DELETE_ABORTED_BY_SUBORDINATE_PLUGIN.get(dnFromDNKey(cursor.getKey(), getBaseDN()));
+ throw new DirectoryException(DirectoryServer.getServerErrorResultCode(), message);
+ }
+ }
+
+ deleteEntry(txn, indexBuffer, true, entryDN, startKey, entryID);
+ subordinateEntriesDeleted++;
+
+ if (deleteOperation != null)
+ {
+ deleteOperation.checkIfCanceled(false);
+ }
+
+ // Get the next DN.
+ success = cursor.next();
+ }
+ }
+ finally
+ {
+ cursor.close();
+ }
+
+ // draft-armijo-ldap-treedelete, 4.1 Tree Delete Semantics:
+ // The server MUST NOT chase referrals stored in the tree. If
+ // information about referrals is stored in this section of the
+ // tree, this pointer will be deleted.
+ boolean manageDsaIT = isSubtreeDelete || isManageDsaITOperation(deleteOperation);
+ deleteEntry(txn, indexBuffer, manageDsaIT, entryDN, null, null);
+
+ indexBuffer.flush(txn);
+
+ if (deleteOperation != null)
+ {
+ // One last check before committing
+ deleteOperation.checkIfCanceled(true);
+ }
+
+ // Commit the transaction.
+ EntryContainer.transactionCommit(txn);
+
+ if (isSubtreeDelete)
+ {
+ deleteOperation.addAdditionalLogItem(unquotedKeyValue(getClass(), "deletedEntries",
+ subordinateEntriesDeleted + 1));
+ }
+ }
+ catch (StorageRuntimeException StorageRuntimeException)
+ {
+ EntryContainer.transactionAbort(txn);
+ throw StorageRuntimeException;
+ }
+ catch (DirectoryException directoryException)
+ {
+ EntryContainer.transactionAbort(txn);
+ throw directoryException;
+ }
+ catch (CanceledOperationException coe)
+ {
+ EntryContainer.transactionAbort(txn);
+ throw coe;
+ }
+ catch (Exception e)
+ {
+ EntryContainer.transactionAbort(txn);
+
+ String msg = e.getMessage();
+ if (msg == null)
+ {
+ msg = stackTraceToSingleLineString(e);
+ }
+ LocalizableMessage message = ERR_JEB_UNCHECKED_EXCEPTION.get(msg);
+ throw new DirectoryException(DirectoryServer.getServerErrorResultCode(), message, e);
+ }
+ }
+ });
+ }
+ catch (Exception e)
+ {
+ throw new StorageRuntimeException(e);
+ }
+ }
+
+ private ByteStringBuilder copyOf(ByteString bs)
+ {
+ ByteStringBuilder newBS = new ByteStringBuilder(bs.length() + 1);
+ newBS.append(bs);
+ return newBS;
+ }
+
+ private void deleteEntry(WriteableStorage txn,
+ IndexBuffer indexBuffer,
+ boolean manageDsaIT,
+ DN targetDN,
+ ByteSequence leafDNKey,
+ EntryID leafID)
+ throws StorageRuntimeException, DirectoryException, JebException
+ {
+ if(leafID == null || leafDNKey == null)
+ {
+ // Read the entry ID from dn2id.
+ if(leafDNKey == null)
+ {
+ leafDNKey = dnToDNKey(targetDN, baseDN.size());
+ }
+ ByteString value = dn2id.read(txn, leafDNKey, true);
+ if (value == null)
+ {
+ LocalizableMessage message = ERR_JEB_DELETE_NO_SUCH_OBJECT.get(leafDNKey);
+ DN matchedDN = getMatchedDN(baseDN);
+ throw new DirectoryException(ResultCode.NO_SUCH_OBJECT, message, matchedDN, null);
+ }
+ leafID = new EntryID(value);
+ }
+
+ // Remove from dn2id.
+ if (!dn2id.delete(txn, leafDNKey))
+ {
+ // Do not expect to ever come through here.
+ LocalizableMessage message = ERR_JEB_DELETE_NO_SUCH_OBJECT.get(leafDNKey);
+ DN matchedDN = getMatchedDN(baseDN);
+ throw new DirectoryException(ResultCode.NO_SUCH_OBJECT, message, matchedDN, null);
+ }
+
+ // Check that the entry exists in id2entry and read its contents.
+ Entry entry = id2entry.get(txn, leafID, true);
+ if (entry == null)
+ {
+ throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
+ ERR_JEB_MISSING_ID2ENTRY_RECORD.get(leafID));
+ }
+
+ if (!manageDsaIT)
+ {
+ dn2uri.checkTargetForReferral(entry, null);
+ }
+
+ // Update the referral database.
+ dn2uri.deleteEntry(txn, entry);
+
+ // Remove from id2entry.
+ if (!id2entry.remove(txn, leafID))
+ {
+ throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
+ ERR_JEB_MISSING_ID2ENTRY_RECORD.get(leafID));
+ }
+
+ // Remove from the indexes, in index config order.
+ indexRemoveEntry(indexBuffer, entry, leafID);
+
+ // Remove the id2c and id2s records for this entry.
+ final ByteString leafIDKeyBytes = ByteString.valueOf(leafID.longValue());
+ id2children.delete(indexBuffer, leafIDKeyBytes);
+ id2subtree.delete(indexBuffer, leafIDKeyBytes);
+
+ // Iterate up through the superior entries from the target entry.
+ boolean isParent = true;
+ for (DN parentDN = getParentWithinBase(targetDN); parentDN != null;
+ parentDN = getParentWithinBase(parentDN))
+ {
+ // Read the ID from dn2id.
+ EntryID parentID = dn2id.get(txn, parentDN, false);
+ if (parentID == null)
+ {
+ throw new JebException(ERR_JEB_MISSING_DN2ID_RECORD.get(parentDN));
+ }
+
+ ByteString parentIDBytes = ByteString.valueOf(parentID.longValue());
+ // Remove from id2children.
+ if (isParent)
+ {
+ id2children.removeID(indexBuffer, parentIDBytes, leafID);
+ isParent = false;
+ }
+ id2subtree.removeID(indexBuffer, parentIDBytes, leafID);
+ }
+
+ // Remove the entry from the entry cache.
+ EntryCache<?> entryCache = DirectoryServer.getEntryCache();
+ if (entryCache != null)
+ {
+ entryCache.removeEntry(entry.getName());
+ }
+ }
+
+ /**
+ * Indicates whether an entry with the specified DN exists.
+ *
+ * @param entryDN The DN of the entry for which to determine existence.
+ *
+ * @return <CODE>true</CODE> if the specified entry exists,
+ * or <CODE>false</CODE> if it does not.
+ *
+ * @throws DirectoryException If a problem occurs while trying to make the
+ * determination.
+ */
+ public boolean entryExists(final DN entryDN) throws DirectoryException
+ {
+ // Try the entry cache first.
+ EntryCache<?> entryCache = DirectoryServer.getEntryCache();
+ if (entryCache != null && entryCache.containsEntry(entryDN))
+ {
+ return true;
+ }
+
+ try
+ {
+ return storage.read(new ReadOperation<Boolean>()
+ {
+ @Override
+ public Boolean run(ReadableStorage txn) throws Exception
+ {
+ EntryID id = dn2id.get(null, entryDN, false);
+ return id != null;
+ }
+ });
+ }
+ catch (Exception e)
+ {
+ logger.traceException(e);
+ return false;
+ }
+ }
+
+ /**
+ * Fetch an entry by DN, trying the entry cache first, then the database.
+ * Retrieves the requested entry, trying the entry cache first,
+ * then the database. Note that the caller must hold a read or write lock
+ * on the specified DN.
+ *
+ * @param entryDN The distinguished name of the entry to retrieve.
+ * @return The requested entry, or <CODE>null</CODE> if the entry does not
+ * exist.
+ * @throws DirectoryException If a problem occurs while trying to retrieve
+ * the entry.
+ * @throws StorageRuntimeException An error occurred during a database operation.
+ */
+ public Entry getEntry(final DN entryDN)
+ throws StorageRuntimeException, DirectoryException
+ {
+ final EntryCache<?> entryCache = DirectoryServer.getEntryCache();
+ Entry entry = null;
+
+ // Try the entry cache first.
+ if (entryCache != null)
+ {
+ entry = entryCache.getEntry(entryDN);
+ }
+
+ if (entry == null)
+ {
+ try
+ {
+ return storage.read(new ReadOperation<Entry>()
+ {
+ @Override
+ public Entry run(ReadableStorage txn) throws Exception
+ {
+ // Read dn2id.
+ EntryID entryID = dn2id.get(txn, entryDN, false);
+ if (entryID == null)
+ {
+ // The entryDN does not exist.
+ // Check for referral entries above the target entry.
+ dn2uri.targetEntryReferrals(entryDN, null);
+ return null;
+ }
+
+ // Read id2entry.
+ Entry entry2 = id2entry.get(txn, entryID, false);
+ if (entry2 == null)
+ {
+ // The entryID does not exist.
+ throw new DirectoryException(DirectoryServer.getServerErrorResultCode(), ERR_JEB_MISSING_ID2ENTRY_RECORD
+ .get(entryID));
+ }
+
+ // Put the entry in the cache making sure not to overwrite
+ // a newer copy that may have been inserted since the time
+ // we read the cache.
+ if (entryCache != null)
+ {
+ entryCache.putEntryIfAbsent(entry2, backend, entryID.longValue());
+ }
+ return entry2;
+ }
+ });
+ }
+ catch (Exception e)
+ {
+ throw new StorageRuntimeException(e);
+ }
+ }
+
+ return entry;
+ }
+
+ /**
+ * The simplest case of replacing an entry in which the entry DN has
+ * not changed.
+ *
+ * @param oldEntry The old contents of the entry
+ * @param newEntry The new contents of the entry
+ * @param modifyOperation The modify operation with which this action is
+ * associated. This may be <CODE>null</CODE> for
+ * modifications performed internally.
+ * @throws StorageRuntimeException If an error occurs in the JE database.
+ * @throws DirectoryException If a Directory Server error occurs.
+ * @throws CanceledOperationException if this operation should be cancelled.
+ */
+ public void replaceEntry(final Entry oldEntry, final Entry newEntry, final ModifyOperation modifyOperation)
+ throws StorageRuntimeException, DirectoryException, CanceledOperationException
+ {
+ try
+ {
+ storage.update(new WriteOperation()
+ {
+ @Override
+ public void run(WriteableStorage txn) throws Exception
+ {
+ try
+ {
+ // Read dn2id.
+ EntryID entryID = dn2id.get(txn, newEntry.getName(), true);
+ if (entryID == null)
+ {
+ // The entry does not exist.
+ LocalizableMessage message =
+ ERR_JEB_MODIFY_NO_SUCH_OBJECT.get(newEntry.getName());
+ DN matchedDN = getMatchedDN(baseDN);
+ throw new DirectoryException(ResultCode.NO_SUCH_OBJECT,
+ message, matchedDN, null);
+ }
+
+ if (!isManageDsaITOperation(modifyOperation))
+ {
+ // Check if the entry is a referral entry.
+ dn2uri.checkTargetForReferral(oldEntry, null);
+ }
+
+ // Update the referral database.
+ if (modifyOperation != null)
+ {
+ // In this case we know from the operation what the modifications were.
+ List<Modification> mods = modifyOperation.getModifications();
+ dn2uri.modifyEntry(txn, oldEntry, newEntry, mods);
+ }
+ else
+ {
+ dn2uri.replaceEntry(txn, oldEntry, newEntry);
+ }
+
+ // Replace id2entry.
+ id2entry.put(txn, entryID, newEntry);
+
+ // Update the indexes.
+ final IndexBuffer indexBuffer = new IndexBuffer(EntryContainer.this);
+ if (modifyOperation != null)
+ {
+ // In this case we know from the operation what the modifications were.
+ List<Modification> mods = modifyOperation.getModifications();
+ indexModifications(indexBuffer, oldEntry, newEntry, entryID, mods);
+ }
+ else
+ {
+ // The most optimal would be to figure out what the modifications were.
+ indexRemoveEntry(indexBuffer, oldEntry, entryID);
+ indexInsertEntry(indexBuffer, newEntry, entryID);
+ }
+
+ indexBuffer.flush(txn);
+
+ if(modifyOperation != null)
+ {
+ // One last check before committing
+ modifyOperation.checkIfCanceled(true);
+ }
+
+ // Commit the transaction.
+ EntryContainer.transactionCommit(txn);
+
+ // Update the entry cache.
+ EntryCache<?> entryCache = DirectoryServer.getEntryCache();
+ if (entryCache != null)
+ {
+ entryCache.putEntry(newEntry, backend, entryID.longValue());
+ }
+ }
+ catch (StorageRuntimeException StorageRuntimeException)
+ {
+ EntryContainer.transactionAbort(txn);
+ throw StorageRuntimeException;
+ }
+ catch (DirectoryException directoryException)
+ {
+ EntryContainer.transactionAbort(txn);
+ throw directoryException;
+ }
+ catch (CanceledOperationException coe)
+ {
+ EntryContainer.transactionAbort(txn);
+ throw coe;
+ }
+ catch (Exception e)
+ {
+ EntryContainer.transactionAbort(txn);
+
+ String msg = e.getMessage();
+ if (msg == null)
+ {
+ msg = stackTraceToSingleLineString(e);
+ }
+ LocalizableMessage message = ERR_JEB_UNCHECKED_EXCEPTION.get(msg);
+ throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
+ message, e);
+ }
+ }
+ });
+ }
+ catch (Exception e)
+ {
+ throw new StorageRuntimeException(e);
+ }
+ }
+
+ /**
+ * Moves and/or renames the provided entry in this backend, altering any
+ * subordinate entries as necessary. This must ensure that an entry already
+ * exists with the provided current DN, and that no entry exists with the
+ * target DN of the provided entry. The caller must hold write locks on both
+ * the current DN and the new DN for the entry.
+ *
+ * @param currentDN The current DN of the entry to be replaced.
+ * @param entry The new content to use for the entry.
+ * @param modifyDNOperation The modify DN operation with which this action
+ * is associated. This may be <CODE>null</CODE>
+ * for modify DN operations performed internally.
+ * @throws DirectoryException
+ * If a problem occurs while trying to perform the rename.
+ * @throws CanceledOperationException
+ * If this backend noticed and reacted
+ * to a request to cancel or abandon the
+ * modify DN operation.
+ * @throws StorageRuntimeException If an error occurs in the JE database.
+ */
+ public void renameEntry(final DN currentDN, final Entry entry, final ModifyDNOperation modifyDNOperation)
+ throws StorageRuntimeException, DirectoryException, CanceledOperationException
+ {
+ try
+ {
+ storage.update(new WriteOperation()
+ {
+ @Override
+ public void run(WriteableStorage txn) throws Exception
+ {
+ DN oldSuperiorDN = getParentWithinBase(currentDN);
+ DN newSuperiorDN = getParentWithinBase(entry.getName());
+
+ final boolean isApexEntryMoved;
+ if (oldSuperiorDN != null)
+ {
+ isApexEntryMoved = !oldSuperiorDN.equals(newSuperiorDN);
+ }
+ else if (newSuperiorDN != null)
+ {
+ isApexEntryMoved = !newSuperiorDN.equals(oldSuperiorDN);
+ }
+ else
+ {
+ isApexEntryMoved = false;
+ }
+
+ IndexBuffer buffer = new IndexBuffer(EntryContainer.this);
+
+ try
+ {
+ // Check whether the renamed entry already exists.
+ if (!currentDN.equals(entry.getName()) && dn2id.get(txn, entry.getName(), false) != null)
+ {
+ LocalizableMessage message = ERR_JEB_MODIFYDN_ALREADY_EXISTS.get(entry.getName());
+ throw new DirectoryException(ResultCode.ENTRY_ALREADY_EXISTS, message);
+ }
+
+ EntryID oldApexID = dn2id.get(txn, currentDN, false);
+ if (oldApexID == null)
+ {
+ // Check for referral entries above the target entry.
+ dn2uri.targetEntryReferrals(currentDN, null);
+
+ LocalizableMessage message = ERR_JEB_MODIFYDN_NO_SUCH_OBJECT.get(currentDN);
+ DN matchedDN = getMatchedDN(baseDN);
+ throw new DirectoryException(ResultCode.NO_SUCH_OBJECT, message, matchedDN, null);
+ }
+
+ Entry oldApexEntry = id2entry.get(txn, oldApexID, false);
+ if (oldApexEntry == null)
+ {
+ throw new DirectoryException(DirectoryServer.getServerErrorResultCode(), ERR_JEB_MISSING_ID2ENTRY_RECORD
+ .get(oldApexID));
+ }
+
+ if (!isManageDsaITOperation(modifyDNOperation))
+ {
+ dn2uri.checkTargetForReferral(oldApexEntry, null);
+ }
+
+ EntryID newApexID = oldApexID;
+ if (newSuperiorDN != null && isApexEntryMoved)
+ {
+ /*
+ * We want to preserve the invariant that the ID of an entry is
+ * greater than its parent, since search results are returned in
+ * ID order.
+ */
+ EntryID newSuperiorID = dn2id.get(txn, newSuperiorDN, false);
+ if (newSuperiorID == null)
+ {
+ LocalizableMessage msg = ERR_JEB_NEW_SUPERIOR_NO_SUCH_OBJECT.get(newSuperiorDN);
+ DN matchedDN = getMatchedDN(baseDN);
+ throw new DirectoryException(ResultCode.NO_SUCH_OBJECT, msg, matchedDN, null);
+ }
+
+ if (newSuperiorID.compareTo(oldApexID) > 0)
+ {
+ // This move would break the above invariant so we must
+ // renumber every entry that moves. This is even more
+ // expensive since every entry has to be deleted from
+ // and added back into the attribute indexes.
+ newApexID = rootContainer.getNextEntryID();
+
+ if (logger.isTraceEnabled())
+ {
+ logger.trace("Move of target entry requires renumbering" + "all entries in the subtree. "
+ + "Old DN: %s " + "New DN: %s " + "Old entry ID: %d " + "New entry ID: %d "
+ + "New Superior ID: %d" + oldApexEntry.getName(), entry.getName(), oldApexID.longValue(),
+ newApexID.longValue(), newSuperiorID.longValue());
+ }
+ }
+ }
+
+ MovedEntry head = new MovedEntry(null, null, false);
+ MovedEntry current = head;
+ // Move or rename the apex entry.
+ removeApexEntry(txn, buffer, oldSuperiorDN, oldApexID, newApexID, oldApexEntry, entry, isApexEntryMoved,
+ modifyDNOperation, current);
+ current = current.next;
+
+ /*
+ * We will iterate forwards through a range of the dn2id keys to
+ * find subordinates of the target entry from the top of the tree
+ * downwards.
+ */
+ ByteString currentDNKey = dnToDNKey(currentDN, baseDN.size());
+ ByteStringBuilder suffix = copyOf(currentDNKey);
+ ByteStringBuilder end = copyOf(currentDNKey);
+
+ /*
+ * Set the ending value to a value of equal length but slightly
+ * greater than the suffix.
+ */
+ suffix.append((byte) 0x00);
+ end.append((byte) 0x01);
+
+ ByteSequence startKey = suffix;
+
+ Cursor cursor = txn.openCursor(dn2id.getName());
+ try
+ {
+ // Initialize the cursor very close to the starting value.
+ boolean success = cursor.positionToKeyOrNext(startKey);
+
+ // Step forward until the key is greater than the starting value.
+ while (success && ByteSequence.COMPARATOR.compare(cursor.getKey(), suffix) <= 0)
+ {
+ success = cursor.next();
+ }
+
+ // Step forward until we pass the ending value.
+ while (success)
+ {
+ int cmp = ByteSequence.COMPARATOR.compare(cursor.getKey(), end);
+ if (cmp >= 0)
+ {
+ // We have gone past the ending value.
+ break;
+ }
+
+ // We have found a subordinate entry.
+
+ EntryID oldID = new EntryID(cursor.getValue());
+ Entry oldEntry = id2entry.get(txn, oldID, false);
+
+ // Construct the new DN of the entry.
+ DN newDN = modDN(oldEntry.getName(), currentDN.size(), entry.getName());
+
+ // Assign a new entry ID if we are renumbering.
+ EntryID newID = oldID;
+ if (!newApexID.equals(oldApexID))
+ {
+ newID = rootContainer.getNextEntryID();
+
+ if (logger.isTraceEnabled())
+ {
+ logger.trace("Move of subordinate entry requires " + "renumbering. " + "Old DN: %s "
+ + "New DN: %s " + "Old entry ID: %d " + "New entry ID: %d", oldEntry.getName(), newDN, oldID
+ .longValue(), newID.longValue());
+ }
+ }
+
+ // Move this entry.
+ removeSubordinateEntry(txn, buffer, oldSuperiorDN, oldID, newID, oldEntry, newDN, isApexEntryMoved,
+ modifyDNOperation, current);
+ current = current.next;
+
+ if (modifyDNOperation != null)
+ {
+ modifyDNOperation.checkIfCanceled(false);
+ }
+
+ // Get the next DN.
+ success = cursor.next();
+ }
+ }
+ finally
+ {
+ cursor.close();
+ }
+
+ // Set current to the first moved entry and null out the head.
+ // This will allow processed moved entries to be GCed.
+ current = head.next;
+ head = null;
+ while (current != null)
+ {
+ addRenamedEntry(txn, buffer, current.entryID, current.entry, isApexEntryMoved, current.renumbered,
+ modifyDNOperation);
+ current = current.next;
+ }
+ buffer.flush(txn);
+
+ if (modifyDNOperation != null)
+ {
+ // One last check before committing
+ modifyDNOperation.checkIfCanceled(true);
+ }
+
+ // Commit the transaction.
+ EntryContainer.transactionCommit(txn);
+ }
+ catch (StorageRuntimeException StorageRuntimeException)
+ {
+ EntryContainer.transactionAbort(txn);
+ throw StorageRuntimeException;
+ }
+ catch (DirectoryException directoryException)
+ {
+ EntryContainer.transactionAbort(txn);
+ throw directoryException;
+ }
+ catch (CanceledOperationException coe)
+ {
+ EntryContainer.transactionAbort(txn);
+ throw coe;
+ }
+ catch (Exception e)
+ {
+ EntryContainer.transactionAbort(txn);
+
+ String msg = e.getMessage();
+ if (msg == null)
+ {
+ msg = stackTraceToSingleLineString(e);
+ }
+ LocalizableMessage message = ERR_JEB_UNCHECKED_EXCEPTION.get(msg);
+ throw new DirectoryException(DirectoryServer.getServerErrorResultCode(), message, e);
+ }
+ }
+ });
+ }
+ catch (Exception e)
+ {
+ throw new StorageRuntimeException(e);
+ }
+ }
+
+ /**
+ * Represents an renamed entry that was deleted from JE but yet to be added
+ * back.
+ */
+ private static class MovedEntry
+ {
+ EntryID entryID;
+ Entry entry;
+ MovedEntry next;
+ boolean renumbered;
+
+ private MovedEntry(EntryID entryID, Entry entry, boolean renumbered)
+ {
+ this.entryID = entryID;
+ this.entry = entry;
+ this.renumbered = renumbered;
+ }
+ }
+
+ private void addRenamedEntry(WriteableStorage txn, IndexBuffer buffer,
+ EntryID newID,
+ Entry newEntry,
+ boolean isApexEntryMoved,
+ boolean renumbered,
+ ModifyDNOperation modifyDNOperation)
+ throws DirectoryException, StorageRuntimeException
+ {
+ if (!dn2id.insert(txn, newEntry.getName(), newID))
+ {
+ LocalizableMessage message = ERR_JEB_MODIFYDN_ALREADY_EXISTS.get(newEntry.getName());
+ throw new DirectoryException(ResultCode.ENTRY_ALREADY_EXISTS, message);
+ }
+ id2entry.put(txn, newID, newEntry);
+ dn2uri.addEntry(txn, newEntry);
+
+ if (renumbered || modifyDNOperation == null)
+ {
+ // Reindex the entry with the new ID.
+ indexInsertEntry(buffer, newEntry, newID);
+ }
+
+ // Add the new ID to id2children and id2subtree of new apex parent entry.
+ if(isApexEntryMoved)
+ {
+ boolean isParent = true;
+ for (DN dn = getParentWithinBase(newEntry.getName()); dn != null;
+ dn = getParentWithinBase(dn))
+ {
+ EntryID parentID = dn2id.get(txn, dn, false);
+ ByteString parentIDKeyBytes = ByteString.valueOf(parentID.longValue());
+ if(isParent)
+ {
+ id2children.insertID(buffer, parentIDKeyBytes, newID);
+ isParent = false;
+ }
+ id2subtree.insertID(buffer, parentIDKeyBytes, newID);
+ }
+ }
+ }
+
+ private void removeApexEntry(WriteableStorage txn, IndexBuffer buffer,
+ DN oldSuperiorDN,
+ EntryID oldID, EntryID newID,
+ Entry oldEntry, Entry newEntry,
+ boolean isApexEntryMoved,
+ ModifyDNOperation modifyDNOperation,
+ MovedEntry tail)
+ throws DirectoryException, StorageRuntimeException
+ {
+ DN oldDN = oldEntry.getName();
+
+ // Remove the old DN from dn2id.
+ dn2id.remove(txn, oldDN);
+
+ // Remove old ID from id2entry and put the new entry
+ // (old entry with new DN) in id2entry.
+ if (!newID.equals(oldID))
+ {
+ id2entry.remove(txn, oldID);
+ }
+
+ // Update any referral records.
+ dn2uri.deleteEntry(txn, oldEntry);
+
+ tail.next = new MovedEntry(newID, newEntry, !newID.equals(oldID));
+
+ // Remove the old ID from id2children and id2subtree of
+ // the old apex parent entry.
+ if(oldSuperiorDN != null && isApexEntryMoved)
+ {
+ boolean isParent = true;
+ for (DN dn = oldSuperiorDN; dn != null; dn = getParentWithinBase(dn))
+ {
+ EntryID parentID = dn2id.get(txn, dn, false);
+ ByteString parentIDKeyBytes = ByteString.valueOf(parentID.longValue());
+ if(isParent)
+ {
+ id2children.removeID(buffer, parentIDKeyBytes, oldID);
+ isParent = false;
+ }
+ id2subtree.removeID(buffer, parentIDKeyBytes, oldID);
+ }
+ }
+
+ if (!newID.equals(oldID) || modifyDNOperation == null)
+ {
+ // All the subordinates will be renumbered so we have to rebuild
+ // id2c and id2s with the new ID.
+ ByteString oldIDKeyBytes = ByteString.valueOf(oldID.longValue());
+ id2children.delete(buffer, oldIDKeyBytes);
+ id2subtree.delete(buffer, oldIDKeyBytes);
+
+ // Reindex the entry with the new ID.
+ indexRemoveEntry(buffer, oldEntry, oldID);
+ }
+ else
+ {
+ // Update the indexes if needed.
+ indexModifications(buffer, oldEntry, newEntry, oldID,
+ modifyDNOperation.getModifications());
+ }
+
+ // Remove the entry from the entry cache.
+ EntryCache<?> entryCache = DirectoryServer.getEntryCache();
+ if (entryCache != null)
+ {
+ entryCache.removeEntry(oldDN);
+ }
+ }
+
+ private void removeSubordinateEntry(WriteableStorage txn, IndexBuffer buffer,
+ DN oldSuperiorDN,
+ EntryID oldID, EntryID newID,
+ Entry oldEntry, DN newDN,
+ boolean isApexEntryMoved,
+ ModifyDNOperation modifyDNOperation,
+ MovedEntry tail)
+ throws DirectoryException, StorageRuntimeException
+ {
+ DN oldDN = oldEntry.getName();
+ Entry newEntry = oldEntry.duplicate(false);
+ newEntry.setDN(newDN);
+ List<Modification> modifications =
+ Collections.unmodifiableList(new ArrayList<Modification>(0));
+
+ // Create a new entry that is a copy of the old entry but with the new DN.
+ // Also invoke any subordinate modify DN plugins on the entry.
+ // FIXME -- At the present time, we don't support subordinate modify DN
+ // plugins that make changes to subordinate entries and therefore
+ // provide an unmodifiable list for the modifications element.
+ // FIXME -- This will need to be updated appropriately if we decided that
+ // these plugins should be invoked for synchronization
+ // operations.
+ if (! modifyDNOperation.isSynchronizationOperation())
+ {
+ SubordinateModifyDN pluginResult =
+ getPluginConfigManager().invokeSubordinateModifyDNPlugins(
+ modifyDNOperation, oldEntry, newEntry, modifications);
+
+ if (!pluginResult.continueProcessing())
+ {
+ throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
+ ERR_JEB_MODIFYDN_ABORTED_BY_SUBORDINATE_PLUGIN.get(oldDN, newDN));
+ }
+
+ if (! modifications.isEmpty())
+ {
+ LocalizableMessageBuilder invalidReason = new LocalizableMessageBuilder();
+ if (! newEntry.conformsToSchema(null, false, false, false,
+ invalidReason))
+ {
+ LocalizableMessage message =
+ ERR_JEB_MODIFYDN_ABORTED_BY_SUBORDINATE_SCHEMA_ERROR.get(oldDN, newDN, invalidReason);
+ throw new DirectoryException(
+ DirectoryServer.getServerErrorResultCode(), message);
+ }
+ }
+ }
+
+ // Remove the old DN from dn2id.
+ dn2id.remove(txn, oldDN);
+
+ // Remove old ID from id2entry and put the new entry
+ // (old entry with new DN) in id2entry.
+ if (!newID.equals(oldID))
+ {
+ id2entry.remove(txn, oldID);
+ }
+
+ // Update any referral records.
+ dn2uri.deleteEntry(txn, oldEntry);
+
+ tail.next = new MovedEntry(newID, newEntry, !newID.equals(oldID));
+
+ if(isApexEntryMoved)
+ {
+ // Remove the old ID from id2subtree of old apex superior entries.
+ for (DN dn = oldSuperiorDN; dn != null; dn = getParentWithinBase(dn))
+ {
+ EntryID parentID = dn2id.get(txn, dn, false);
+ ByteString parentIDKeyBytes = ByteString.valueOf(parentID.longValue());
+ id2subtree.removeID(buffer, parentIDKeyBytes, oldID);
+ }
+ }
+
+ if (!newID.equals(oldID))
+ {
+ // All the subordinates will be renumbered so we have to rebuild
+ // id2c and id2s with the new ID.
+ ByteString oldIDKeyBytes = ByteString.valueOf(oldID.longValue());
+ id2children.delete(buffer, oldIDKeyBytes);
+ id2subtree.delete(buffer, oldIDKeyBytes);
+
+ // Reindex the entry with the new ID.
+ indexRemoveEntry(buffer, oldEntry, oldID);
+ }
+ else if (!modifications.isEmpty())
+ {
+ // Update the indexes.
+ indexModifications(buffer, oldEntry, newEntry, oldID, modifications);
+ }
+
+ // Remove the entry from the entry cache.
+ EntryCache<?> entryCache = DirectoryServer.getEntryCache();
+ if (entryCache != null)
+ {
+ entryCache.removeEntry(oldDN);
+ }
+ }
+
+ /**
+ * Make a new DN for a subordinate entry of a renamed or moved entry.
+ *
+ * @param oldDN The current DN of the subordinate entry.
+ * @param oldSuffixLen The current DN length of the renamed or moved entry.
+ * @param newSuffixDN The new DN of the renamed or moved entry.
+ * @return The new DN of the subordinate entry.
+ */
+ public static DN modDN(DN oldDN, int oldSuffixLen, DN newSuffixDN)
+ {
+ int oldDNNumComponents = oldDN.size();
+ int oldDNKeepComponents = oldDNNumComponents - oldSuffixLen;
+ int newSuffixDNComponents = newSuffixDN.size();
+
+ RDN[] newDNComponents = new RDN[oldDNKeepComponents+newSuffixDNComponents];
+ for (int i=0; i < oldDNKeepComponents; i++)
+ {
+ newDNComponents[i] = oldDN.getRDN(i);
+ }
+
+ for (int i=oldDNKeepComponents, j=0; j < newSuffixDNComponents; i++,j++)
+ {
+ newDNComponents[i] = newSuffixDN.getRDN(j);
+ }
+
+ return new DN(newDNComponents);
+ }
+
+ /**
+ * Insert a new entry into the attribute indexes.
+ *
+ * @param buffer The index buffer used to buffer up the index changes.
+ * @param entry The entry to be inserted into the indexes.
+ * @param entryID The ID of the entry to be inserted into the indexes.
+ * @throws StorageRuntimeException If an error occurs in the JE database.
+ * @throws DirectoryException If a Directory Server error occurs.
+ */
+ private void indexInsertEntry(IndexBuffer buffer, Entry entry, EntryID entryID)
+ throws StorageRuntimeException, DirectoryException
+ {
+ for (AttributeIndex index : attrIndexMap.values())
+ {
+ index.addEntry(buffer, entryID, entry);
+ }
+
+ for (VLVIndex vlvIndex : vlvIndexMap.values())
+ {
+ vlvIndex.addEntry(buffer, entryID, entry);
+ }
+ }
+
+ /**
+ * Remove an entry from the attribute indexes.
+ *
+ * @param buffer The index buffer used to buffer up the index changes.
+ * @param entry The entry to be removed from the indexes.
+ * @param entryID The ID of the entry to be removed from the indexes.
+ * @throws StorageRuntimeException If an error occurs in the JE database.
+ * @throws DirectoryException If a Directory Server error occurs.
+ */
+ private void indexRemoveEntry(IndexBuffer buffer, Entry entry, EntryID entryID)
+ throws StorageRuntimeException, DirectoryException
+ {
+ for (AttributeIndex index : attrIndexMap.values())
+ {
+ index.removeEntry(buffer, entryID, entry);
+ }
+
+ for (VLVIndex vlvIndex : vlvIndexMap.values())
+ {
+ vlvIndex.removeEntry(buffer, entryID, entry);
+ }
+ }
+
+ /**
+ * Update the attribute indexes to reflect the changes to the
+ * attributes of an entry resulting from a sequence of modifications.
+ *
+ * @param buffer The index buffer used to buffer up the index changes.
+ * @param oldEntry The contents of the entry before the change.
+ * @param newEntry The contents of the entry after the change.
+ * @param entryID The ID of the entry that was changed.
+ * @param mods The sequence of modifications made to the entry.
+ * @throws StorageRuntimeException If an error occurs in the JE database.
+ * @throws DirectoryException If a Directory Server error occurs.
+ */
+ private void indexModifications(IndexBuffer buffer, Entry oldEntry, Entry newEntry,
+ EntryID entryID, List<Modification> mods)
+ throws StorageRuntimeException, DirectoryException
+ {
+ // Process in index configuration order.
+ for (AttributeIndex index : attrIndexMap.values())
+ {
+ // Check whether any modifications apply to this indexed attribute.
+ if (isAttributeModified(index, mods))
+ {
+ index.modifyEntry(buffer, entryID, oldEntry, newEntry, mods);
+ }
+ }
+
+ for(VLVIndex vlvIndex : vlvIndexMap.values())
+ {
+ vlvIndex.modifyEntry(buffer, entryID, oldEntry, newEntry, mods);
+ }
+ }
+
+ /**
+ * Get a count of the number of entries stored in this entry container.
+ *
+ * @return The number of entries stored in this entry container.
+ * @throws StorageRuntimeException If an error occurs in the JE database.
+ */
+ @Override
+ public long getEntryCount() throws StorageRuntimeException
+ {
+ EntryID entryID = dn2id.get(null, baseDN, false);
+ if (entryID != null)
+ {
+ ByteString key = entryIDToDatabase(entryID.longValue());
+ EntryIDSet entryIDSet = id2subtree.readKey(key, null);
+
+ long count = entryIDSet.size();
+ if(count != Long.MAX_VALUE)
+ {
+ // Add the base entry itself
+ return ++count;
+ }
+ else
+ {
+ // The count is not maintained. Fall back to the slow method
+ return id2entry.getRecordCount();
+ }
+ }
+ else
+ {
+ // Base entry doesn't not exist so this entry container
+ // must not have any entries
+ return 0;
+ }
+ }
+
+ /**
+ * Get the number of values for which the entry limit has been exceeded
+ * since the entry container was opened.
+ * @return The number of values for which the entry limit has been exceeded.
+ */
+ public int getEntryLimitExceededCount()
+ {
+ int count = 0;
+ count += id2children.getEntryLimitExceededCount();
+ count += id2subtree.getEntryLimitExceededCount();
+ for (AttributeIndex index : attrIndexMap.values())
+ {
+ count += index.getEntryLimitExceededCount();
+ }
+ return count;
+ }
+
+
+ /**
+ * Get a list of the databases opened by the entryContainer.
+ * @param dbList A list of database containers.
+ */
+ public void listDatabases(List<DatabaseContainer> dbList)
+ {
+ dbList.add(dn2id);
+ dbList.add(id2entry);
+ dbList.add(dn2uri);
+ if (config.isSubordinateIndexesEnabled())
+ {
+ dbList.add(id2children);
+ dbList.add(id2subtree);
+ }
+ dbList.add(state);
+
+ for(AttributeIndex index : attrIndexMap.values())
+ {
+ index.listDatabases(dbList);
+ }
+
+ dbList.addAll(vlvIndexMap.values());
+ }
+
+ /**
+ * Determine whether the provided operation has the ManageDsaIT request
+ * control.
+ * @param operation The operation for which the determination is to be made.
+ * @return true if the operation has the ManageDsaIT request control, or false
+ * if not.
+ */
+ private static boolean isManageDsaITOperation(Operation operation)
+ {
+ if(operation != null)
+ {
+ List<Control> controls = operation.getRequestControls();
+ if (controls != null)
+ {
+ for (Control control : controls)
+ {
+ if (ServerConstants.OID_MANAGE_DSAIT_CONTROL.equals(control.getOID()))
+ {
+ return true;
+ }
+ }
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Begin a leaf transaction using the default configuration.
+ * Provides assertion debug logging.
+ * @return A JE transaction handle.
+ * @throws StorageRuntimeException If an error occurs while attempting to begin
+ * a new transaction.
+ */
+ public Transaction beginTransaction()
+ throws StorageRuntimeException
+ {
+ Transaction parentTxn = null;
+ TransactionConfig txnConfig = null;
+ Transaction txn = storage.beginTransaction(parentTxn, txnConfig);
+ if (logger.isTraceEnabled())
+ {
+ logger.trace("beginTransaction", "begin txnid=" + txn.getId());
+ }
+ return txn;
+ }
+
+ /**
+ * Commit a transaction.
+ * Provides assertion debug logging.
+ * @param txn The JE transaction handle.
+ * @throws StorageRuntimeException If an error occurs while attempting to commit
+ * the transaction.
+ */
+ public static void transactionCommit(WriteableStorage txn)
+ throws StorageRuntimeException
+ {
+ if (txn != null)
+ {
+ txn.commit();
+ if (logger.isTraceEnabled())
+ {
+ logger.trace("commit txnid=%d", txn.getId());
+ }
+ }
+ }
+
+ /**
+ * Abort a transaction.
+ * Provides assertion debug logging.
+ * @param txn The JE transaction handle.
+ * @throws StorageRuntimeException If an error occurs while attempting to abort the
+ * transaction.
+ */
+ public static void transactionAbort(WriteableStorage txn)
+ throws StorageRuntimeException
+ {
+ if (txn != null)
+ {
+ txn.abort();
+ if (logger.isTraceEnabled())
+ {
+ logger.trace("abort txnid=%d", txn.getId());
+ }
+ }
+ }
+
+ /**
+ * Delete this entry container from disk. The entry container should be
+ * closed before calling this method.
+ *
+ * @throws StorageRuntimeException If an error occurs while removing the entry
+ * container.
+ */
+ public void delete() throws StorageRuntimeException
+ {
+ List<DatabaseContainer> databases = new ArrayList<DatabaseContainer>();
+ listDatabases(databases);
+
+ if(storage.getConfig().getTransactional())
+ {
+ Transaction txn = beginTransaction();
+
+ try
+ {
+ for(DatabaseContainer db : databases)
+ {
+ storage.removeDatabase(txn, db.getName());
+ }
+
+ transactionCommit(txn);
+ }
+ catch(StorageRuntimeException de)
+ {
+ transactionAbort(txn);
+ throw de;
+ }
+ }
+ else
+ {
+ for(DatabaseContainer db : databases)
+ {
+ storage.removeDatabase(null, db.getName());
+ }
+ }
+ }
+
+ /**
+ * Remove a database from disk.
+ *
+ * @param database The database container to remove.
+ * @throws StorageRuntimeException If an error occurs while attempting to delete the
+ * database.
+ */
+ public void deleteDatabase(DatabaseContainer database)
+ throws StorageRuntimeException
+ {
+ if(database == state)
+ {
+ // The state database can not be removed individually.
+ return;
+ }
+
+ database.close();
+ if(storage.getConfig().getTransactional())
+ {
+ Transaction txn = beginTransaction();
+ try
+ {
+ storage.removeDatabase(txn, database.getName());
+ if(database instanceof Index)
+ {
+ state.removeIndexTrustState(txn, database);
+ }
+ transactionCommit(txn);
+ }
+ catch(StorageRuntimeException de)
+ {
+ transactionAbort(txn);
+ throw de;
+ }
+ }
+ else
+ {
+ storage.removeDatabase(null, database.getName());
+ if(database instanceof Index)
+ {
+ state.removeIndexTrustState(null, database);
+ }
+ }
+ }
+
+ /**
+ * Removes a attribute index from disk.
+ *
+ * @param attributeIndex The attribute index to remove.
+ * @throws StorageRuntimeException If an JE database error occurs while attempting
+ * to delete the index.
+ */
+ private void deleteAttributeIndex(AttributeIndex attributeIndex)
+ throws StorageRuntimeException
+ {
+ attributeIndex.close();
+ Transaction txn = storage.getConfig().getTransactional()
+ ? beginTransaction() : null;
+ try
+ {
+ for (Index index : attributeIndex.getAllIndexes())
+ {
+ storage.removeDatabase(txn, index.getName());
+ state.removeIndexTrustState(txn, index);
+ }
+ if (txn != null)
+ {
+ transactionCommit(txn);
+ }
+ }
+ catch(StorageRuntimeException de)
+ {
+ if (txn != null)
+ {
+ transactionAbort(txn);
+ }
+ throw de;
+ }
+ }
+
+ /**
+ * This method constructs a container name from a base DN. Only alphanumeric
+ * characters are preserved, all other characters are replaced with an
+ * underscore.
+ *
+ * @return The container name for the base DN.
+ */
+ public TreeName getDatabasePrefix()
+ {
+ return databasePrefix;
+ }
+
+ /**
+ * Sets a new database prefix for this entry container and rename all
+ * existing databases in use by this entry container.
+ *
+ * @param newDatabasePrefix The new database prefix to use.
+ * @throws StorageRuntimeException If an error occurs in the JE database.
+ * @throws JebException If an error occurs in the JE backend.
+ */
+ public void setDatabasePrefix(String newDatabasePrefix)
+ throws StorageRuntimeException, JebException
+
+ {
+ List<DatabaseContainer> databases = new ArrayList<DatabaseContainer>();
+ listDatabases(databases);
+
+ TreeName newDbPrefix = preparePrefix(newDatabasePrefix);
+
+ // close the containers.
+ for(DatabaseContainer db : databases)
+ {
+ db.close();
+ }
+
+ try
+ {
+ if(storage.getConfig().getTransactional())
+ {
+ //Rename under transaction
+ Transaction txn = beginTransaction();
+ try
+ {
+ for(DatabaseContainer db : databases)
+ {
+ TreeName oldName = db.getName();
+ String newName = oldName.replace(databasePrefix, newDbPrefix);
+ storage.renameDatabase(txn, oldName, newName);
+ }
+
+ transactionCommit(txn);
+
+ for(DatabaseContainer db : databases)
+ {
+ TreeName oldName = db.getName();
+ String newName = oldName.replace(databasePrefix, newDbPrefix);
+ db.setName(newName);
+ }
+
+ // Update the prefix.
+ this.databasePrefix = newDbPrefix;
+ }
+ catch(Exception e)
+ {
+ transactionAbort(txn);
+
+ String msg = e.getMessage();
+ if (msg == null)
+ {
+ msg = stackTraceToSingleLineString(e);
+ }
+ LocalizableMessage message = ERR_JEB_UNCHECKED_EXCEPTION.get(msg);
+ throw new JebException(message, e);
+ }
+ }
+ else
+ {
+ for(DatabaseContainer db : databases)
+ {
+ TreeName oldName = db.getName();
+ String newName = oldName.replace(databasePrefix, newDbPrefix);
+ storage.renameDatabase(null, oldName, newName);
+ db.setName(newName);
+ }
+
+ // Update the prefix.
+ this.databasePrefix = newDbPrefix;
+ }
+ }
+ finally
+ {
+ // Open the containers backup.
+ for(DatabaseContainer db : databases)
+ {
+ db.open();
+ }
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public DN getBaseDN()
+ {
+ return baseDN;
+ }
+
+ /**
+ * Get the parent of a DN in the scope of the base DN.
+ *
+ * @param dn A DN which is in the scope of the base DN.
+ * @return The parent DN, or null if the given DN is the base DN.
+ */
+ public DN getParentWithinBase(DN dn)
+ {
+ if (dn.equals(baseDN))
+ {
+ return null;
+ }
+ return dn.parent();
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public boolean isConfigurationChangeAcceptable(
+ LocalDBBackendCfg cfg, List<LocalizableMessage> unacceptableReasons)
+ {
+ // This is always true because only all config attributes used
+ // by the entry container should be validated by the admin framework.
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public ConfigChangeResult applyConfigurationChange(LocalDBBackendCfg cfg)
+ {
+ boolean adminActionRequired = false;
+ ArrayList<LocalizableMessage> messages = new ArrayList<LocalizableMessage>();
+
+ exclusiveLock.lock();
+ try
+ {
+ if (config.isSubordinateIndexesEnabled() != cfg.isSubordinateIndexesEnabled())
+ {
+ if (cfg.isSubordinateIndexesEnabled())
+ {
+ // Re-enabling subordinate indexes.
+ openSubordinateIndexes();
+ }
+ else
+ {
+ // Disabling subordinate indexes. Use a null index and ensure that
+ // future attempts to use the real indexes will fail.
+ id2children.close();
+ id2children = new NullIndex(databasePrefix.child(ID2CHILDREN_DATABASE_NAME),
+ new ID2CIndexer(), state, storage, this);
+ state.putIndexTrustState(null, id2children, false);
+ id2children.open(); // No-op
+
+ id2subtree.close();
+ id2subtree = new NullIndex(databasePrefix.child(ID2SUBTREE_DATABASE_NAME),
+ new ID2SIndexer(), state, storage, this);
+ state.putIndexTrustState(null, id2subtree, false);
+ id2subtree.open(); // No-op
+
+ logger.info(NOTE_JEB_SUBORDINATE_INDEXES_DISABLED, cfg.getBackendId());
+ }
+ }
+
+ if (config.getIndexEntryLimit() != cfg.getIndexEntryLimit())
+ {
+ if (id2children.setIndexEntryLimit(cfg.getIndexEntryLimit()))
+ {
+ adminActionRequired = true;
+ messages.add(NOTE_JEB_CONFIG_INDEX_ENTRY_LIMIT_REQUIRES_REBUILD.get(id2children.getName()));
+ }
+
+ if (id2subtree.setIndexEntryLimit(cfg.getIndexEntryLimit()))
+ {
+ adminActionRequired = true;
+ messages.add(NOTE_JEB_CONFIG_INDEX_ENTRY_LIMIT_REQUIRES_REBUILD.get(id2subtree.getName()));
+ }
+ }
+
+ DataConfig entryDataConfig = new DataConfig(cfg.isEntriesCompressed(),
+ cfg.isCompactEncoding(), rootContainer.getCompressedSchema());
+ id2entry.setDataConfig(entryDataConfig);
+
+ this.config = cfg;
+ }
+ catch (StorageRuntimeException e)
+ {
+ messages.add(LocalizableMessage.raw(stackTraceToSingleLineString(e)));
+ return new ConfigChangeResult(DirectoryServer.getServerErrorResultCode(),
+ false, messages);
+ }
+ finally
+ {
+ exclusiveLock.unlock();
+ }
+
+ return new ConfigChangeResult(ResultCode.SUCCESS, adminActionRequired, messages);
+ }
+
+ /**
+ * Get the environment config of the JE environment used in this entry
+ * container.
+ *
+ * @return The environment config of the JE environment.
+ * @throws StorageRuntimeException If an error occurs while retrieving the
+ * configuration object.
+ */
+ public EnvironmentConfig getEnvironmentConfig() throws StorageRuntimeException
+ {
+ return storage.getConfig();
+ }
+
+ /**
+ * Clear the contents of this entry container.
+ *
+ * @return The number of records deleted.
+ * @throws StorageRuntimeException If an error occurs while removing the entry
+ * container.
+ */
+ public long clear() throws StorageRuntimeException
+ {
+ List<DatabaseContainer> databases = new ArrayList<DatabaseContainer>();
+ listDatabases(databases);
+ long count = 0;
+
+ for(DatabaseContainer db : databases)
+ {
+ db.close();
+ }
+ try
+ {
+ if(storage.getConfig().getTransactional())
+ {
+ Transaction txn = beginTransaction();
+
+ try
+ {
+ for(DatabaseContainer db : databases)
+ {
+ count += storage.truncateDatabase(txn, db.getName(), true);
+ }
+
+ transactionCommit(txn);
+ }
+ catch(StorageRuntimeException de)
+ {
+ transactionAbort(txn);
+ throw de;
+ }
+ }
+ else
+ {
+ for(DatabaseContainer db : databases)
+ {
+ count += storage.truncateDatabase(null, db.getName(), true);
+ }
+ }
+ }
+ finally
+ {
+ for(DatabaseContainer db : databases)
+ {
+ db.open();
+ }
+
+ Transaction txn = null;
+ try
+ {
+ if(storage.getConfig().getTransactional()) {
+ txn = beginTransaction();
+ }
+ for(DatabaseContainer db : databases)
+ {
+ if (db instanceof Index)
+ {
+ Index index = (Index)db;
+ index.setTrusted(txn, true);
+ }
+ }
+ if(storage.getConfig().getTransactional()) {
+ transactionCommit(txn);
+ }
+ }
+ catch(Exception de)
+ {
+ logger.traceException(de);
+
+ // This is mainly used during the unit tests, so it's not essential.
+ try
+ {
+ if (txn != null)
+ {
+ transactionAbort(txn);
+ }
+ }
+ catch (Exception e)
+ {
+ logger.traceException(de);
+ }
+ }
+ }
+
+ return count;
+ }
+
+ /**
+ * Clear the contents for a database from disk.
+ *
+ * @param database The database to clear.
+ * @throws StorageRuntimeException if a JE database error occurs.
+ */
+ public void clearDatabase(DatabaseContainer database)
+ throws StorageRuntimeException
+ {
+ database.close();
+ try
+ {
+ if(storage.getConfig().getTransactional())
+ {
+ Transaction txn = beginTransaction();
+ try
+ {
+ storage.removeDatabase(txn, database.getName());
+ transactionCommit(txn);
+ }
+ catch(StorageRuntimeException de)
+ {
+ transactionAbort(txn);
+ throw de;
+ }
+ }
+ else
+ {
+ storage.removeDatabase(null, database.getName());
+ }
+ }
+ finally
+ {
+ database.open();
+ }
+ if(logger.isTraceEnabled())
+ {
+ logger.trace("Cleared the database %s", database.getName());
+ }
+ }
+
+
+ /**
+ * Finds an existing entry whose DN is the closest ancestor of a given baseDN.
+ *
+ * @param baseDN the DN for which we are searching a matched DN.
+ * @return the DN of the closest ancestor of the baseDN.
+ * @throws DirectoryException If an error prevented the check of an
+ * existing entry from being performed.
+ */
+ private DN getMatchedDN(DN baseDN) throws DirectoryException
+ {
+ DN parentDN = baseDN.getParentDNInSuffix();
+ while (parentDN != null && parentDN.isDescendantOf(getBaseDN()))
+ {
+ if (entryExists(parentDN))
+ {
+ return parentDN;
+ }
+ parentDN = parentDN.getParentDNInSuffix();
+ }
+ return null;
+ }
+
+ /**
+ * Opens the id2children and id2subtree indexes.
+ */
+ private void openSubordinateIndexes()
+ {
+ id2children = newIndex(ID2CHILDREN_DATABASE_NAME, new ID2CIndexer());
+ id2subtree = newIndex(ID2SUBTREE_DATABASE_NAME, new ID2SIndexer());
+ }
+
+ private Index newIndex(String name, Indexer indexer)
+ {
+ final Index index = new Index(databasePrefix.child(name),
+ indexer, state, config.getIndexEntryLimit(), 0, true, storage, this);
+ index.open();
+ if (!index.isTrusted())
+ {
+ logger.info(NOTE_JEB_INDEX_ADD_REQUIRES_REBUILD, index.getName());
+ }
+ return index;
+ }
+
+ /**
+ * Creates a new index for an attribute.
+ *
+ * @param indexName the name to give to the new index
+ * @param indexer the indexer to use when inserting data into the index
+ * @param indexEntryLimit the index entry limit
+ * @return a new index
+ */
+ Index newIndexForAttribute(TreeName indexName, Indexer indexer, int indexEntryLimit)
+ {
+ final int cursorEntryLimit = 100000;
+ return new Index(indexName, indexer, state, indexEntryLimit, cursorEntryLimit, false, storage, this);
+ }
+
+
+ /**
+ * Checks if any modifications apply to this indexed attribute.
+ * @param index the indexed attributes.
+ * @param mods the modifications to check for.
+ * @return true if any apply, false otherwise.
+ */
+ private boolean isAttributeModified(AttributeIndex index,
+ List<Modification> mods)
+ {
+ boolean attributeModified = false;
+ AttributeType indexAttributeType = index.getAttributeType();
+ Iterable<AttributeType> subTypes =
+ DirectoryServer.getSchema().getSubTypes(indexAttributeType);
+
+ for (Modification mod : mods)
+ {
+ Attribute modAttr = mod.getAttribute();
+ AttributeType modAttrType = modAttr.getAttributeType();
+ if (modAttrType.equals(indexAttributeType))
+ {
+ attributeModified = true;
+ break;
+ }
+ for(AttributeType subType : subTypes)
+ {
+ if(modAttrType.equals(subType))
+ {
+ attributeModified = true;
+ break;
+ }
+ }
+ }
+ return attributeModified;
+ }
+
+
+ /**
+ * Fetch the base Entry of the EntryContainer.
+ * @param baseDN the DN for the base entry
+ * @param searchScope the scope under which this is fetched.
+ * Scope is used for referral processing.
+ * @return the Entry matching the baseDN.
+ * @throws DirectoryException if the baseDN doesn't exist.
+ */
+ private Entry fetchBaseEntry(DN baseDN, SearchScope searchScope)
+ throws DirectoryException
+ {
+ // Fetch the base entry.
+ Entry baseEntry = null;
+ try
+ {
+ baseEntry = getEntry(baseDN);
+ }
+ catch (Exception e)
+ {
+ logger.traceException(e);
+ }
+
+ // The base entry must exist for a successful result.
+ if (baseEntry == null)
+ {
+ // Check for referral entries above the base entry.
+ dn2uri.targetEntryReferrals(baseDN, searchScope);
+
+ LocalizableMessage message = ERR_JEB_SEARCH_NO_SUCH_OBJECT.get(baseDN);
+ DN matchedDN = getMatchedDN(baseDN);
+ throw new DirectoryException(ResultCode.NO_SUCH_OBJECT,
+ message, matchedDN, null);
+ }
+
+ return baseEntry;
+ }
+
+
+ /**
+ * Transform a database prefix string to one usable by the DB.
+ * @param databasePrefix the database prefix
+ * @return a new string when non letter or digit characters
+ * have been replaced with underscore
+ */
+ private TreeName preparePrefix(String databasePrefix)
+ {
+ StringBuilder builder = new StringBuilder(databasePrefix.length());
+ for (int i = 0; i < databasePrefix.length(); i++)
+ {
+ char ch = databasePrefix.charAt(i);
+ if (Character.isLetterOrDigit(ch))
+ {
+ builder.append(ch);
+ }
+ else
+ {
+ builder.append('_');
+ }
+ }
+ return TreeName.of(builder.toString());
+ }
+
+ /** Get the exclusive lock. */
+ public void lock() {
+ exclusiveLock.lock();
+ }
+
+ /** Unlock the exclusive lock. */
+ public void unlock() {
+ exclusiveLock.unlock();
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public String toString() {
+ return databasePrefix.toString();
+ }
+}
diff --git a/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/EntryID.java b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/EntryID.java
new file mode 100644
index 0000000..1c0a354
--- /dev/null
+++ b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/EntryID.java
@@ -0,0 +1,159 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License"). You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at legal-notices/CDDLv1_0.txt
+ * or http://forgerock.org/license/CDDLv1.0.html.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at legal-notices/CDDLv1_0.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information:
+ * Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ *
+ *
+ * Copyright 2006-2008 Sun Microsystems, Inc.
+ * Portions Copyright 2014 ForgeRock AS
+ */
+package org.opends.server.backends.pluggable;
+
+import org.forgerock.opendj.ldap.ByteString;
+
+/**
+ * An integer identifier assigned to each entry in the JE backend.
+ * An entry ID is implemented by this class as a long.
+ * There are static methods to assign monotonically increasing entry IDs,
+ * starting from 1.
+ */
+public class EntryID implements Comparable<EntryID>
+{
+ /** The identifier integer value. */
+ private final long id;
+ /** The value in database format, created when necessary. */
+ private ByteString value;
+
+ /**
+ * Create a new entry ID object from a given long value.
+ * @param id The long value of the ID.
+ */
+ public EntryID(long id)
+ {
+ this.id = id;
+ }
+
+ /**
+ * Create a new entry ID object from a value in database format.
+ * @param value The database value of the ID.
+ */
+ public EntryID(ByteString value)
+ {
+ this.value = value;
+ id = value.toLong();
+ }
+
+ /**
+ * Get the value of the entry ID as a long.
+ * @return The entry ID.
+ */
+ public long longValue()
+ {
+ return id;
+ }
+
+ /**
+ * Get the value of the ID in database format.
+ * @return The value of the ID in database format.
+ */
+ public ByteString toByteString()
+ {
+ if (value == null)
+ {
+ value = ByteString.valueOf(id);
+ }
+ return value;
+ }
+
+ /**
+ * Compares this object with the specified object for order. Returns a
+ * negative integer, zero, or a positive integer as this object is less
+ * than, equal to, or greater than the specified object.<p>
+ * <p/>
+ *
+ * @param that the Object to be compared.
+ * @return a negative integer, zero, or a positive integer as this object
+ * is less than, equal to, or greater than the specified object.
+ * @throws ClassCastException if the specified object's type prevents it
+ * from being compared to this Object.
+ */
+ @Override
+ public int compareTo(EntryID that) throws ClassCastException
+ {
+ final long result = this.id - that.id;
+ if (result < 0)
+ {
+ return -1;
+ }
+ else if (result > 0)
+ {
+ return 1;
+ }
+ return 0;
+ }
+
+ /**
+ * Indicates whether some other object is "equal to" this one.
+ *
+ * @param that the reference object with which to compare.
+ * @return <code>true</code> if this object is the same as the obj
+ * argument; <code>false</code> otherwise.
+ * @see #hashCode()
+ * @see java.util.Hashtable
+ */
+ @Override
+ public boolean equals(Object that)
+ {
+ if (this == that)
+ {
+ return true;
+ }
+ if (!(that instanceof EntryID))
+ {
+ return false;
+ }
+ return this.id == ((EntryID) that).id;
+ }
+
+ /**
+ * Returns a hash code value for the object. This method is
+ * supported for the benefit of hashtables such as those provided by
+ * <code>java.util.Hashtable</code>.
+ *
+ * @return a hash code value for this object.
+ * @see java.lang.Object#equals(java.lang.Object)
+ * @see java.util.Hashtable
+ */
+ @Override
+ public int hashCode()
+ {
+ return (int) id;
+ }
+
+ /**
+ * Get a string representation of this object.
+ * @return A string representation of this object.
+ */
+ @Override
+ public String toString()
+ {
+ return Long.toString(id);
+ }
+}
diff --git a/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/EntryIDSet.java b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/EntryIDSet.java
new file mode 100644
index 0000000..6f9881a
--- /dev/null
+++ b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/EntryIDSet.java
@@ -0,0 +1,676 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License"). You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at legal-notices/CDDLv1_0.txt
+ * or http://forgerock.org/license/CDDLv1.0.html.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at legal-notices/CDDLv1_0.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information:
+ * Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ *
+ *
+ * Copyright 2006-2008 Sun Microsystems, Inc.
+ * Portions Copyright 2014 ForgeRock AS
+ */
+package org.opends.server.backends.pluggable;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Iterator;
+
+import org.forgerock.opendj.ldap.ByteSequence;
+import org.forgerock.opendj.ldap.ByteString;
+
+/**
+ * Represents a set of Entry IDs. It can represent a set where the IDs are
+ * not defined, for example when the index entry limit has been exceeded.
+ */
+public class EntryIDSet implements Iterable<EntryID>
+{
+
+ /**
+ * The IDs are stored here in an array in ascending order.
+ * A null array implies not defined, rather than zero IDs.
+ */
+ private long[] values;
+
+ /**
+ * The size of the set when it is not defined. This value is only maintained
+ * when the set is undefined.
+ */
+ private long undefinedSize = Long.MAX_VALUE;
+
+ /**
+ * The database key containing this set, if the set was constructed
+ * directly from the database.
+ */
+ private final ByteSequence key;
+
+ /** Create a new undefined set. */
+ public EntryIDSet()
+ {
+ this.key = null;
+ this.undefinedSize = Long.MAX_VALUE;
+ }
+
+ /**
+ * Create a new undefined set with a initial size.
+ *
+ * @param size The undefined size for this set.
+ */
+ public EntryIDSet(long size)
+ {
+ this.key = null;
+ this.undefinedSize = size;
+ }
+
+ /**
+ * Create a new entry ID set from the raw database value.
+ *
+ * @param keyBytes The database key that contains this value.
+ * @param bytes The database value, or null if there are no entry IDs.
+ */
+ public EntryIDSet(byte[] keyBytes, byte[] bytes)
+ {
+ this(keyBytes != null ? ByteString.wrap(keyBytes) : null,
+ bytes != null ? ByteString.wrap(bytes) : null);
+ }
+
+ /**
+ * Create a new entry ID set from the raw database value.
+ *
+ * @param key
+ * The database key that contains this value.
+ * @param bytes
+ * The database value, or null if there are no entry IDs.
+ */
+ public EntryIDSet(ByteSequence key, ByteString bytes)
+ {
+ this.key = key;
+
+ if (bytes == null)
+ {
+ values = new long[0];
+ return;
+ }
+
+ if (bytes.length() == 0)
+ {
+ // Entry limit has exceeded and there is no encoded undefined set size.
+ undefinedSize = Long.MAX_VALUE;
+ }
+ else if ((bytes.byteAt(0) & 0x80) == 0x80)
+ {
+ // Entry limit has exceeded and there is an encoded undefined set size.
+ undefinedSize =
+ JebFormat.entryIDUndefinedSizeFromDatabase(bytes.toByteArray());
+ }
+ else
+ {
+ // Seems like entry limit has not been exceeded and the bytes is a
+ // list of entry IDs.
+ values = JebFormat.entryIDListFromDatabase(bytes);
+ }
+ }
+
+ /**
+ * Construct an EntryIDSet from an array of longs.
+ *
+ * @param values The array of IDs represented as longs.
+ * @param pos The position of the first ID to take from the array.
+ * @param len the number of IDs to take from the array.
+ */
+ EntryIDSet(long[] values, int pos, int len)
+ {
+ this.key = null;
+ this.values = new long[len];
+ System.arraycopy(values, pos, this.values, 0, len);
+ }
+
+ /**
+ * Create a new set of entry IDs that is the union of several entry ID sets.
+ *
+ * @param sets A list of entry ID sets.
+ * @param allowDuplicates true if duplicate IDs are allowed in the resulting
+ * set, or if the provided sets are sure not to overlap; false if
+ * duplicates should be eliminated.
+ * @return The union of the provided entry ID sets.
+ */
+ public static EntryIDSet unionOfSets(ArrayList<EntryIDSet> sets,
+ boolean allowDuplicates)
+ {
+ int count = 0;
+
+ boolean undefined = false;
+ for (EntryIDSet l : sets)
+ {
+ if (!l.isDefined())
+ {
+ if(l.undefinedSize == Long.MAX_VALUE)
+ {
+ return new EntryIDSet();
+ }
+ undefined = true;
+ }
+ count += l.size();
+ }
+
+ if(undefined)
+ {
+ return new EntryIDSet(count);
+ }
+
+ boolean needSort = false;
+ long[] n = new long[count];
+ int pos = 0;
+ for (EntryIDSet l : sets)
+ {
+ if (l.values.length != 0)
+ {
+ if (!needSort && pos > 0 && l.values[0] < n[pos-1])
+ {
+ needSort = true;
+ }
+ System.arraycopy(l.values, 0, n, pos, l.values.length);
+ pos += l.values.length;
+ }
+ }
+ if (needSort)
+ {
+ Arrays.sort(n);
+ }
+ if (allowDuplicates)
+ {
+ EntryIDSet ret = new EntryIDSet();
+ ret.values = n;
+ return ret;
+ }
+ long[] n1 = new long[n.length];
+ long last = -1;
+ int j = 0;
+ for (long l : n)
+ {
+ if (l != last)
+ {
+ last = n1[j++] = l;
+ }
+ }
+ if (j == n1.length)
+ {
+ EntryIDSet ret = new EntryIDSet();
+ ret.values = n1;
+ return ret;
+ }
+ else
+ {
+ return new EntryIDSet(n1, 0, j);
+ }
+ }
+
+ /**
+ * Get the size of this entry ID set.
+ *
+ * @return The number of IDs in the set.
+ */
+ public long size()
+ {
+ if (values != null)
+ {
+ return values.length;
+ }
+ return undefinedSize;
+ }
+
+ /**
+ * Get a string representation of this object.
+ * @return A string representation of this object.
+ */
+ @Override
+ public String toString()
+ {
+ StringBuilder buffer = new StringBuilder(16);
+ toString(buffer);
+ return buffer.toString();
+ }
+
+ /**
+ * Convert to a short string to aid with debugging.
+ *
+ * @param buffer The string is appended to this string builder.
+ */
+ public void toString(StringBuilder buffer)
+ {
+ if (!isDefined())
+ {
+ if (key != null)
+ {
+ // The index entry limit was exceeded
+ if(undefinedSize == Long.MAX_VALUE)
+ {
+ buffer.append("[LIMIT-EXCEEDED]");
+ }
+ else
+ {
+ buffer.append("[LIMIT-EXCEEDED:");
+ buffer.append(undefinedSize);
+ buffer.append("]");
+ }
+ }
+ else
+ {
+ // Not indexed
+ buffer.append("[NOT-INDEXED]");
+ }
+ }
+ else
+ {
+ buffer.append("[COUNT:");
+ buffer.append(size());
+ buffer.append("]");
+ }
+ }
+
+ /**
+ * Determine whether this set of IDs is defined.
+ *
+ * @return true if the set of IDs is defined.
+ */
+ public boolean isDefined()
+ {
+ return values != null;
+ }
+
+ /**
+ * Get a database representation of this object.
+ * @return A database representation of this object as a byte array.
+ */
+ public ByteString toByteString()
+ {
+ if(isDefined())
+ {
+ return ByteString.wrap(JebFormat.entryIDListToDatabase(values));
+ }
+ else
+ {
+ return ByteString.wrap(JebFormat.entryIDUndefinedSizeToDatabase(undefinedSize));
+ }
+ }
+
+ /**
+ * Insert an ID into this set.
+ *
+ * @param entryID The ID to be inserted.
+ * @return true if the set was changed, false if it was not changed,
+ * for example if the set is undefined or the ID was already present.
+ */
+ public boolean add(EntryID entryID)
+ {
+ if (values == null)
+ {
+ if(undefinedSize != Long.MAX_VALUE)
+ {
+ undefinedSize++;
+ }
+ return true;
+ }
+
+ long id = entryID.longValue();
+ if (values.length == 0)
+ {
+ values = new long[] { id };
+ return true;
+ }
+
+ if (id > values[values.length-1])
+ {
+ long[] updatedValues = Arrays.copyOf(values, values.length + 1);
+ updatedValues[values.length] = id;
+ values = updatedValues;
+ }
+ else
+ {
+ int pos = Arrays.binarySearch(values, id);
+ if (pos >= 0)
+ {
+ // The ID is already present.
+ return false;
+ }
+
+ // For a negative return value r, the index -(r+1) gives the array
+ // index at which the specified value can be inserted to maintain
+ // the sorted order of the array.
+ pos = -(pos+1);
+
+ long[] updatedValues = new long[values.length+1];
+ System.arraycopy(values, 0, updatedValues, 0, pos);
+ System.arraycopy(values, pos, updatedValues, pos+1, values.length-pos);
+ updatedValues[pos] = id;
+ values = updatedValues;
+ }
+
+ return true;
+ }
+
+ /**
+ * Remove an ID from this set.
+ *
+ * @param entryID The ID to be removed
+ * @return true if the set was changed, false if it was not changed,
+ * for example if the set was undefined or the ID was not present.
+ */
+ public boolean remove(EntryID entryID)
+ {
+ if (values == null)
+ {
+ if(undefinedSize != Long.MAX_VALUE)
+ {
+ undefinedSize--;
+ }
+ return true;
+ }
+
+ if (values.length == 0)
+ {
+ return false;
+ }
+
+ // Binary search to locate the ID.
+ long id = entryID.longValue();
+ int pos = Arrays.binarySearch(values, id);
+ if (pos >= 0)
+ {
+ // Found it.
+ long[] updatedValues = new long[values.length-1];
+ System.arraycopy(values, 0, updatedValues, 0, pos);
+ System.arraycopy(values, pos+1, updatedValues, pos, values.length-pos-1);
+ values = updatedValues;
+ return true;
+ }
+ // Not found.
+ return false;
+ }
+
+ /**
+ * Check whether this set of entry IDs contains a given ID.
+ *
+ * @param entryID The ID to be checked.
+ * @return true if this set contains the given ID,
+ * or if the set is undefined.
+ */
+ public boolean contains(EntryID entryID)
+ {
+ if (values == null)
+ {
+ return true;
+ }
+
+ final long id = entryID.longValue();
+ return values.length != 0
+ && id <= values[values.length - 1]
+ && Arrays.binarySearch(values, id) >= 0;
+ }
+
+ /**
+ * Takes the intersection of this set with another.
+ * Retain those IDs that appear in the given set.
+ *
+ * @param that The set of IDs that are to be retained from this object.
+ */
+ public void retainAll(EntryIDSet that)
+ {
+ if (!isDefined())
+ {
+ this.values = that.values;
+ this.undefinedSize = that.undefinedSize;
+ return;
+ }
+
+ if (!that.isDefined())
+ {
+ return;
+ }
+
+ // TODO Perhaps Arrays.asList and retainAll list method are more efficient?
+
+ long[] a = this.values;
+ long[] b = that.values;
+
+ int ai = 0, bi = 0, ci = 0;
+ long[] c = new long[Math.min(a.length,b.length)];
+ while (ai < a.length && bi < b.length)
+ {
+ if (a[ai] == b[bi])
+ {
+ c[ci] = a[ai];
+ ai++;
+ bi++;
+ ci++;
+ }
+ else if (a[ai] > b[bi])
+ {
+ bi++;
+ }
+ else
+ {
+ ai++;
+ }
+ }
+ if (ci < c.length)
+ {
+ values = Arrays.copyOf(c, ci);
+ }
+ else
+ {
+ values = c;
+ }
+ }
+
+ /**
+ * Add all the IDs from a given set that are not already present.
+ *
+ * @param that The set of IDs to be added. It MUST be defined
+ */
+ public void addAll(EntryIDSet that)
+ {
+ if(!that.isDefined())
+ {
+ return;
+ }
+
+ if (!isDefined())
+ {
+ // Assume there are no overlap between IDs in that set with this set
+ if(undefinedSize != Long.MAX_VALUE)
+ {
+ undefinedSize += that.size();
+ }
+ return;
+ }
+
+ long[] a = this.values;
+ long[] b = that.values;
+
+ if (a.length == 0)
+ {
+ values = b;
+ return;
+ }
+
+ if (b.length == 0)
+ {
+ return;
+ }
+
+ // Optimize for case where the two sets are sure to have no overlap.
+ if (b[0] > a[a.length-1])
+ {
+ // All IDs in 'b' are greater than those in 'a'.
+ long[] n = new long[a.length + b.length];
+ System.arraycopy(a, 0, n, 0, a.length);
+ System.arraycopy(b, 0, n, a.length, b.length);
+ values = n;
+ return;
+ }
+
+ if (a[0] > b[b.length-1])
+ {
+ // All IDs in 'a' are greater than those in 'b'.
+ long[] n = new long[a.length + b.length];
+ System.arraycopy(b, 0, n, 0, b.length);
+ System.arraycopy(a, 0, n, b.length, a.length);
+ values = n;
+ return;
+ }
+
+ long[] n;
+ if ( b.length < a.length ) {
+ n = a;
+ a = b;
+ b = n;
+ }
+
+ n = new long[a.length + b.length];
+
+ int ai, bi, ni;
+ for ( ni = 0, ai = 0, bi = 0; ai < a.length && bi < b.length; ) {
+ if ( a[ai] < b[bi] ) {
+ n[ni++] = a[ai++];
+ } else if ( b[bi] < a[ai] ) {
+ n[ni++] = b[bi++];
+ } else {
+ n[ni++] = a[ai];
+ ai++;
+ bi++;
+ }
+ }
+
+ // Copy any remainder from the first array.
+ int aRemain = a.length - ai;
+ if (aRemain > 0)
+ {
+ System.arraycopy(a, ai, n, ni, aRemain);
+ ni += aRemain;
+ }
+
+ // Copy any remainder from the second array.
+ int bRemain = b.length - bi;
+ if (bRemain > 0)
+ {
+ System.arraycopy(b, bi, n, ni, bRemain);
+ ni += bRemain;
+ }
+
+ if (ni < n.length)
+ {
+ values = Arrays.copyOf(n, ni);
+ }
+ else
+ {
+ values = n;
+ }
+ }
+
+ /**
+ * Delete all IDs in this set that are in a given set.
+ *
+ * @param that The set of IDs to be deleted. It MUST be defined.
+ */
+ public void deleteAll(EntryIDSet that)
+ {
+ if(!that.isDefined())
+ {
+ return;
+ }
+
+ if (!isDefined())
+ {
+ // Assume all IDs in the given set exists in this set.
+ if(undefinedSize != Long.MAX_VALUE)
+ {
+ undefinedSize -= that.size();
+ }
+ return;
+ }
+
+ long[] a = this.values;
+ long[] b = that.values;
+
+ if (a.length == 0 || b.length == 0
+ // Optimize for cases where the two sets are sure to have no overlap.
+ || b[0] > a[a.length-1]
+ || a[0] > b[b.length-1])
+ {
+ return;
+ }
+
+ long[] n = new long[a.length];
+
+ int ai, bi, ni;
+ for ( ni = 0, ai = 0, bi = 0; ai < a.length && bi < b.length; ) {
+ if ( a[ai] < b[bi] ) {
+ n[ni++] = a[ai++];
+ } else if ( b[bi] < a[ai] ) {
+ bi++;
+ } else {
+ ai++;
+ bi++;
+ }
+ }
+
+ System.arraycopy(a, ai, n, ni, a.length - ai);
+ ni += a.length - ai;
+
+ if (ni < a.length)
+ {
+ values = Arrays.copyOf(n, ni);
+ }
+ else
+ {
+ values = n;
+ }
+ }
+
+ /**
+ * Create an iterator over the set or an empty iterator
+ * if the set is not defined.
+ *
+ * @return An EntryID iterator.
+ */
+ @Override
+ public Iterator<EntryID> iterator()
+ {
+ return iterator(null);
+ }
+
+ /**
+ * Create an iterator over the set or an empty iterator
+ * if the set is not defined.
+ *
+ * @param begin The entry ID of the first entry to return in the list.
+ *
+ * @return An EntryID iterator.
+ */
+ public Iterator<EntryID> iterator(EntryID begin)
+ {
+ if (values != null)
+ {
+ // The set is defined.
+ return new IDSetIterator(values, begin);
+ }
+ // The set is not defined.
+ return new IDSetIterator(new long[0]);
+ }
+
+}
diff --git a/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/EntryIDSetSorter.java b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/EntryIDSetSorter.java
new file mode 100644
index 0000000..b77d737
--- /dev/null
+++ b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/EntryIDSetSorter.java
@@ -0,0 +1,271 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License"). You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at legal-notices/CDDLv1_0.txt
+ * or http://forgerock.org/license/CDDLv1.0.html.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at legal-notices/CDDLv1_0.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information:
+ * Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ *
+ *
+ * Copyright 2008 Sun Microsystems, Inc.
+ * Portions Copyright 2011-2014 ForgeRock AS
+ */
+package org.opends.server.backends.pluggable;
+
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.Map;
+import java.util.TreeMap;
+
+import org.forgerock.i18n.LocalizableMessage;
+import org.forgerock.opendj.ldap.ByteString;
+import org.forgerock.opendj.ldap.ResultCode;
+import org.forgerock.opendj.ldap.SearchScope;
+import org.opends.server.backends.pluggable.SuffixContainer;
+import org.opends.server.controls.VLVRequestControl;
+import org.opends.server.controls.VLVResponseControl;
+import org.opends.server.core.DirectoryServer;
+import org.opends.server.core.SearchOperation;
+import org.opends.server.protocols.ldap.LDAPResultCode;
+import org.opends.server.types.*;
+
+import static org.opends.messages.JebMessages.*;
+import static org.opends.server.util.StaticUtils.*;
+
+/**
+ * This class provides a mechanism for sorting the contents of an entry ID set
+ * based on a given sort order.
+ */
+public class EntryIDSetSorter
+{
+ /**
+ * Creates a new entry ID set which is a sorted representation of the provided
+ * set using the given sort order.
+ *
+ * @param suffixContainer The suffix container with which the ID list is associated.
+ * @param entryIDSet The entry ID set to be sorted.
+ * @param searchOperation The search operation being processed.
+ * @param sortOrder The sort order to use for the entry ID set.
+ * @param vlvRequest The VLV request control included in the search
+ * request, or {@code null} if there was none.
+ *
+ * @return A new entry ID set which is a sorted representation of the
+ * provided set using the given sort order.
+ *
+ * @throws DirectoryException If an error occurs while performing the sort.
+ */
+ public static EntryIDSet sort(SuffixContainer suffixContainer,
+ EntryIDSet entryIDSet,
+ SearchOperation searchOperation,
+ SortOrder sortOrder,
+ VLVRequestControl vlvRequest)
+ throws DirectoryException
+ {
+ if (! entryIDSet.isDefined())
+ {
+ return new EntryIDSet();
+ }
+
+ DN baseDN = searchOperation.getBaseDN();
+ SearchScope scope = searchOperation.getScope();
+ SearchFilter filter = searchOperation.getFilter();
+
+ TreeMap<SortValues,EntryID> sortMap = new TreeMap<SortValues,EntryID>();
+ for (EntryID id : entryIDSet)
+ {
+ try
+ {
+ Entry e = suffixContainer.getEntry(id);
+ if (e.matchesBaseAndScope(baseDN, scope) && filter.matchesEntry(e))
+ {
+ sortMap.put(new SortValues(id, e, sortOrder), id);
+ }
+ }
+ catch (Exception e)
+ {
+ LocalizableMessage message = ERR_ENTRYIDSORTER_CANNOT_EXAMINE_ENTRY.get(id, getExceptionMessage(e));
+ throw new DirectoryException(DirectoryServer.getServerErrorResultCode(), message, e);
+ }
+ }
+
+
+ // See if there is a VLV request to further pare down the set of results,
+ // and if there is where it should be processed by offset or assertion value.
+ long[] sortedIDs;
+ if (vlvRequest != null)
+ {
+ int beforeCount = vlvRequest.getBeforeCount();
+ int afterCount = vlvRequest.getAfterCount();
+
+ if (vlvRequest.getTargetType() == VLVRequestControl.TYPE_TARGET_BYOFFSET)
+ {
+ int targetOffset = vlvRequest.getOffset();
+ if (targetOffset < 0)
+ {
+ // The client specified a negative target offset. This should never be allowed.
+ searchOperation.addResponseControl(
+ new VLVResponseControl(targetOffset, sortMap.size(),
+ LDAPResultCode.OFFSET_RANGE_ERROR));
+
+ LocalizableMessage message = ERR_ENTRYIDSORTER_NEGATIVE_START_POS.get();
+ throw new DirectoryException(ResultCode.VIRTUAL_LIST_VIEW_ERROR,
+ message);
+ }
+ else if (targetOffset == 0)
+ {
+ // This is an easy mistake to make, since VLV offsets start at 1
+ // instead of 0. We'll assume the client meant to use 1.
+ targetOffset = 1;
+ }
+
+ int listOffset = targetOffset - 1; // VLV offsets start at 1, not 0.
+ int startPos = listOffset - beforeCount;
+ if (startPos < 0)
+ {
+ // This can happen if beforeCount >= offset, and in this case we'll
+ // just adjust the start position to ignore the range of beforeCount
+ // that doesn't exist.
+ startPos = 0;
+ beforeCount = listOffset;
+ }
+ else if (startPos >= sortMap.size())
+ {
+ // The start position is beyond the end of the list. In this case,
+ // we'll assume that the start position was one greater than the
+ // size of the list and will only return the beforeCount entries.
+ targetOffset = sortMap.size() + 1;
+ listOffset = sortMap.size();
+ startPos = listOffset - beforeCount;
+ afterCount = 0;
+ }
+
+ int count = 1 + beforeCount + afterCount;
+ sortedIDs = new long[count];
+
+ int treePos = 0;
+ int arrayPos = 0;
+ for (EntryID id : sortMap.values())
+ {
+ if (treePos++ < startPos)
+ {
+ continue;
+ }
+
+ sortedIDs[arrayPos++] = id.longValue();
+ if (arrayPos >= count)
+ {
+ break;
+ }
+ }
+
+ if (arrayPos < count)
+ {
+ // We don't have enough entries in the set to meet the requested
+ // page size, so we'll need to shorten the array.
+ long[] newIDArray = new long[arrayPos];
+ System.arraycopy(sortedIDs, 0, newIDArray, 0, arrayPos);
+ sortedIDs = newIDArray;
+ }
+
+ searchOperation.addResponseControl(
+ new VLVResponseControl(targetOffset, sortMap.size(),
+ LDAPResultCode.SUCCESS));
+ }
+ else
+ {
+ ByteString assertionValue = vlvRequest.getGreaterThanOrEqualAssertion();
+
+ boolean targetFound = false;
+ int targetOffset = 0;
+ int includedBeforeCount = 0;
+ int includedAfterCount = 0;
+ int listSize = 0;
+ LinkedList<EntryID> idList = new LinkedList<EntryID>();
+ for (Map.Entry<SortValues, EntryID> entry : sortMap.entrySet())
+ {
+ SortValues sortValues = entry.getKey();
+ EntryID id = entry.getValue();
+
+ if (targetFound)
+ {
+ idList.add(id);
+ listSize++;
+ includedAfterCount++;
+ if (includedAfterCount >= afterCount)
+ {
+ break;
+ }
+ }
+ else
+ {
+ targetFound = sortValues.compareTo(assertionValue) >= 0;
+ targetOffset++;
+
+ if (targetFound)
+ {
+ idList.add(id);
+ listSize++;
+ }
+ else if (beforeCount > 0)
+ {
+ idList.add(id);
+ includedBeforeCount++;
+ if (includedBeforeCount > beforeCount)
+ {
+ idList.removeFirst();
+ includedBeforeCount--;
+ }
+ else
+ {
+ listSize++;
+ }
+ }
+ }
+ }
+
+ if (! targetFound)
+ {
+ // No entry was found to be greater than or equal to the sort key, so
+ // the target offset will be one greater than the content count.
+ targetOffset = sortMap.size() + 1;
+ }
+
+ sortedIDs = new long[listSize];
+ Iterator<EntryID> idIterator = idList.iterator();
+ for (int i=0; i < listSize; i++)
+ {
+ sortedIDs[i] = idIterator.next().longValue();
+ }
+
+ searchOperation.addResponseControl(
+ new VLVResponseControl(targetOffset, sortMap.size(),
+ LDAPResultCode.SUCCESS));
+ }
+ }
+ else
+ {
+ sortedIDs = new long[sortMap.size()];
+ int i=0;
+ for (EntryID id : sortMap.values())
+ {
+ sortedIDs[i++] = id.longValue();
+ }
+ }
+
+ return new EntryIDSet(sortedIDs, 0, sortedIDs.length);
+ }
+}
diff --git a/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/EnvManager.java b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/EnvManager.java
new file mode 100644
index 0000000..2df136f
--- /dev/null
+++ b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/EnvManager.java
@@ -0,0 +1,141 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License"). You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at legal-notices/CDDLv1_0.txt
+ * or http://forgerock.org/license/CDDLv1.0.html.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at legal-notices/CDDLv1_0.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information:
+ * Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ *
+ *
+ * Copyright 2006-2009 Sun Microsystems, Inc.
+ * Portions Copyright 2014 ForgeRock AS
+ */
+package org.opends.server.backends.pluggable;
+import org.forgerock.i18n.LocalizableMessage;
+
+import org.forgerock.i18n.slf4j.LocalizedLogger;
+import static org.opends.messages.JebMessages.*;
+
+import java.io.File;
+import java.io.FilenameFilter;
+
+/**
+ * A singleton class to manage the life-cycle of a JE database environment.
+ */
+public class EnvManager
+{
+ private static final LocalizedLogger logger = LocalizedLogger.getLoggerForThisClass();
+
+
+ /**
+ * A filename filter to match all kinds of JE files.
+ */
+ private static final FilenameFilter jeAllFilesFilter;
+
+ static
+ {
+ // A filename filter to match all kinds of JE files.
+ // JE has a com.sleepycat.je.log.JEFileFilter that would be useful
+ // here but is not public.
+ jeAllFilesFilter = new FilenameFilter()
+ {
+ public boolean accept(File d, String name)
+ {
+ return name.endsWith(".jdb") ||
+ name.endsWith(".del") ||
+ name.startsWith("je.");
+ }
+ };
+ }
+
+ /**
+ * Creates the environment home directory, deleting any existing data files
+ * if the directory already exists.
+ * The environment must not be open.
+ *
+ * @param homeDir The backend home directory.
+ * @throws JebException If an error occurs in the JE backend.
+ */
+ public static void createHomeDir(String homeDir)
+ throws JebException
+ {
+ File dir = new File(homeDir);
+
+ if (dir.exists())
+ {
+ if (!dir.isDirectory())
+ {
+ LocalizableMessage message = ERR_JEB_DIRECTORY_INVALID.get(homeDir);
+ throw new JebException(message);
+ }
+ removeFiles(homeDir);
+ }
+ else
+ {
+ try
+ {
+ dir.mkdir();
+ }
+ catch (Exception e)
+ {
+ logger.traceException(e);
+ LocalizableMessage message = ERR_JEB_CREATE_FAIL.get(e.getMessage());
+ throw new JebException(message, e);
+ }
+ }
+ }
+
+ /**
+ * Deletes all the data files associated with the environment.
+ * The environment must not be open.
+ *
+ * @param homeDir The backend home directory
+ * @throws JebException If an error occurs in the JE backend or if the
+ * specified home directory does not exist.
+ */
+ public static void removeFiles(String homeDir)
+ throws JebException
+ {
+ File dir = new File(homeDir);
+ if (!dir.exists())
+ {
+ LocalizableMessage message = ERR_JEB_DIRECTORY_DOES_NOT_EXIST.get(homeDir);
+ throw new JebException(message);
+ }
+ if (!dir.isDirectory())
+ {
+ LocalizableMessage message = ERR_JEB_DIRECTORY_INVALID.get(homeDir);
+ throw new JebException(message);
+ }
+
+ try
+ {
+ File[] jdbFiles = dir.listFiles(jeAllFilesFilter);
+ for (File f : jdbFiles)
+ {
+ f.delete();
+ }
+ }
+ catch (Exception e)
+ {
+ logger.traceException(e);
+ LocalizableMessage message = ERR_JEB_REMOVE_FAIL.get(e.getMessage());
+ throw new JebException(message, e);
+ }
+ }
+
+}
diff --git a/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/EqualityIndexer.java b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/EqualityIndexer.java
new file mode 100644
index 0000000..c6ab6f2
--- /dev/null
+++ b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/EqualityIndexer.java
@@ -0,0 +1,78 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License"). You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at legal-notices/CDDLv1_0.txt
+ * or http://forgerock.org/license/CDDLv1.0.html.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at legal-notices/CDDLv1_0.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information:
+ * Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ *
+ *
+ * Copyright 2006-2010 Sun Microsystems, Inc.
+ * Portions Copyright 2014 ForgeRock AS
+ */
+package org.opends.server.backends.pluggable;
+
+import java.util.Collection;
+
+import org.forgerock.opendj.ldap.ByteSequence;
+import org.forgerock.opendj.ldap.ByteString;
+import org.forgerock.opendj.ldap.DecodeException;
+import org.forgerock.opendj.ldap.schema.MatchingRule;
+import org.forgerock.opendj.ldap.schema.Schema;
+import org.forgerock.opendj.ldap.spi.Indexer;
+import org.forgerock.opendj.ldap.spi.IndexingOptions;
+import org.opends.server.types.AttributeType;
+
+/**
+ * An implementation of an Indexer for attribute equality.
+ */
+public class EqualityIndexer implements Indexer
+{
+
+ /**
+ * The attribute type equality matching rule which is also the
+ * comparator for the index keys generated by this class.
+ */
+ private final MatchingRule equalityRule;
+
+ /**
+ * Create a new attribute equality indexer for the given index configuration.
+ * @param attributeType The attribute type for which an indexer is
+ * required.
+ */
+ public EqualityIndexer(AttributeType attributeType)
+ {
+ this.equalityRule = attributeType.getEqualityMatchingRule();
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public String getIndexID()
+ {
+ return "equality";
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void createKeys(Schema schema, ByteSequence value,
+ IndexingOptions options, Collection<ByteString> keys)
+ throws DecodeException
+ {
+ keys.add(equalityRule.normalizeAttributeValue(value));
+ }
+
+}
diff --git a/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/ExportJob.java b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/ExportJob.java
new file mode 100644
index 0000000..993506c
--- /dev/null
+++ b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/ExportJob.java
@@ -0,0 +1,311 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License"). You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at legal-notices/CDDLv1_0.txt
+ * or http://forgerock.org/license/CDDLv1.0.html.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at legal-notices/CDDLv1_0.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information:
+ * Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ *
+ *
+ * Copyright 2006-2008 Sun Microsystems, Inc.
+ * Portions Copyright 2012-2014 ForgeRock AS.
+ */
+package org.opends.server.backends.pluggable;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Timer;
+import java.util.TimerTask;
+
+import org.forgerock.i18n.slf4j.LocalizedLogger;
+import org.forgerock.opendj.ldap.ByteString;
+import org.opends.server.backends.pluggable.BackendImpl.Cursor;
+import org.opends.server.backends.pluggable.BackendImpl.Storage;
+import org.opends.server.backends.pluggable.BackendImpl.StorageRuntimeException;
+import org.opends.server.types.DN;
+import org.opends.server.types.Entry;
+import org.opends.server.types.LDIFExportConfig;
+import org.opends.server.util.LDIFException;
+import org.opends.server.util.StaticUtils;
+
+import static org.opends.messages.JebMessages.*;
+
+/**
+ * Export a JE backend to LDIF.
+ */
+public class ExportJob
+{
+ private static final LocalizedLogger logger = LocalizedLogger.getLoggerForThisClass();
+
+
+ /**
+ * The requested LDIF export configuration.
+ */
+ private LDIFExportConfig exportConfig;
+
+ /**
+ * The number of milliseconds between job progress reports.
+ */
+ private long progressInterval = 10000;
+
+ /**
+ * The current number of entries exported.
+ */
+ private long exportedCount = 0;
+
+ /**
+ * The current number of entries skipped.
+ */
+ private long skippedCount = 0;
+
+ /**
+ * Create a new export job.
+ *
+ * @param exportConfig The requested LDIF export configuration.
+ */
+ public ExportJob(LDIFExportConfig exportConfig)
+ {
+ this.exportConfig = exportConfig;
+ }
+
+ /**
+ * Export entries from the backend to an LDIF file.
+ * @param rootContainer The root container to export.
+ * @throws StorageRuntimeException If an error occurs in the JE database.
+ * @throws IOException If an I/O error occurs while writing an entry.
+ * @throws JebException If an error occurs in the JE backend.
+ * @throws LDIFException If an error occurs while trying to determine whether
+ * to write an entry.
+ */
+ public void exportLDIF(RootContainer rootContainer)
+ throws IOException, LDIFException, StorageRuntimeException, JebException
+ {
+ List<DN> includeBranches = exportConfig.getIncludeBranches();
+ DN baseDN;
+ ArrayList<EntryContainer> exportContainers =
+ new ArrayList<EntryContainer>();
+
+ for (EntryContainer entryContainer : rootContainer.getEntryContainers())
+ {
+ // Skip containers that are not covered by the include branches.
+ baseDN = entryContainer.getBaseDN();
+
+ if (includeBranches == null || includeBranches.isEmpty())
+ {
+ exportContainers.add(entryContainer);
+ }
+ else
+ {
+ for (DN includeBranch : includeBranches)
+ {
+ if (includeBranch.isDescendantOf(baseDN) ||
+ includeBranch.isAncestorOf(baseDN))
+ {
+ exportContainers.add(entryContainer);
+ break;
+ }
+ }
+ }
+ }
+
+ // Make a note of the time we started.
+ long startTime = System.currentTimeMillis();
+
+ // Start a timer for the progress report.
+ Timer timer = new Timer();
+ TimerTask progressTask = new ProgressTask();
+ timer.scheduleAtFixedRate(progressTask, progressInterval,
+ progressInterval);
+
+ // Iterate through the containers.
+ try
+ {
+ for (EntryContainer exportContainer : exportContainers)
+ {
+ if (exportConfig.isCancelled())
+ {
+ break;
+ }
+
+ exportContainer.sharedLock.lock();
+ try
+ {
+ exportContainer(exportContainer);
+ }
+ finally
+ {
+ exportContainer.sharedLock.unlock();
+ }
+ }
+ }
+ finally
+ {
+ timer.cancel();
+ }
+
+
+ long finishTime = System.currentTimeMillis();
+ long totalTime = (finishTime - startTime);
+
+ float rate = 0;
+ if (totalTime > 0)
+ {
+ rate = 1000f*exportedCount / totalTime;
+ }
+
+ logger.info(NOTE_JEB_EXPORT_FINAL_STATUS, exportedCount, skippedCount, totalTime/1000, rate);
+
+ }
+
+ /**
+ * Export the entries in a single entry entryContainer, in other words from
+ * one of the base DNs.
+ * @param entryContainer The entry container that holds the entries to be
+ * exported.
+ * @throws StorageRuntimeException If an error occurs in the JE database.
+ * @throws IOException If an error occurs while writing an entry.
+ * @throws LDIFException If an error occurs while trying to determine
+ * whether to write an entry.
+ */
+ private void exportContainer(EntryContainer entryContainer)
+ throws StorageRuntimeException, IOException, LDIFException
+ {
+ Storage storage = entryContainer.getStorage();
+
+ Cursor cursor = storage.openCursor(entryContainer.getID2Entry().getName());
+ try
+ {
+ while (cursor.next())
+ {
+ if (exportConfig.isCancelled())
+ {
+ break;
+ }
+
+ ByteString key=cursor.getKey();
+ EntryID entryID = null;
+ try
+ {
+ entryID = new EntryID(key);
+ }
+ catch (Exception e)
+ {
+ if (logger.isTraceEnabled())
+ {
+ logger.traceException(e);
+
+ logger.trace("Malformed id2entry ID %s.%n",
+ StaticUtils.bytesToHex(key.toByteArray()));
+ }
+ skippedCount++;
+ continue;
+ }
+
+ if (entryID.longValue() == 0)
+ {
+ // This is the stored entry count.
+ continue;
+ }
+
+ ByteString value = cursor.getValue();
+ Entry entry = null;
+ try
+ {
+ entry = ID2Entry.entryFromDatabase(value,
+ entryContainer.getRootContainer().getCompressedSchema());
+ }
+ catch (Exception e)
+ {
+ if (logger.isTraceEnabled())
+ {
+ logger.traceException(e);
+
+ logger.trace("Malformed id2entry record for ID %d:%n%s%n",
+ entryID.longValue(),
+ StaticUtils.bytesToHex(value.toByteArray()));
+ }
+ skippedCount++;
+ continue;
+ }
+
+ if (entry.toLDIF(exportConfig))
+ {
+ exportedCount++;
+ }
+ else
+ {
+ skippedCount++;
+ }
+ }
+ }
+ finally
+ {
+ cursor.close();
+ }
+ }
+
+ /**
+ * This class reports progress of the export job at fixed intervals.
+ */
+ class ProgressTask extends TimerTask
+ {
+ /**
+ * The number of entries that had been exported at the time of the
+ * previous progress report.
+ */
+ private long previousCount = 0;
+
+ /**
+ * The time in milliseconds of the previous progress report.
+ */
+ private long previousTime;
+
+ /**
+ * Create a new export progress task.
+ */
+ public ProgressTask()
+ {
+ previousTime = System.currentTimeMillis();
+ }
+
+ /**
+ * The action to be performed by this timer task.
+ */
+ @Override
+ public void run()
+ {
+ long latestCount = exportedCount;
+ long deltaCount = (latestCount - previousCount);
+ long latestTime = System.currentTimeMillis();
+ long deltaTime = latestTime - previousTime;
+
+ if (deltaTime == 0)
+ {
+ return;
+ }
+
+ float rate = 1000f*deltaCount / deltaTime;
+
+ logger.info(NOTE_JEB_EXPORT_PROGRESS_REPORT, latestCount, skippedCount, rate);
+
+ previousCount = latestCount;
+ previousTime = latestTime;
+ }
+ }
+
+}
diff --git a/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/ID2CIndexer.java b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/ID2CIndexer.java
new file mode 100644
index 0000000..7618085
--- /dev/null
+++ b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/ID2CIndexer.java
@@ -0,0 +1,94 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License"). You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at legal-notices/CDDLv1_0.txt
+ * or http://forgerock.org/license/CDDLv1.0.html.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at legal-notices/CDDLv1_0.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information:
+ * Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ *
+ *
+ * Copyright 2006-2008 Sun Microsystems, Inc.
+ * Portions Copyright 2014 ForgeRock AS
+ */
+package org.opends.server.backends.pluggable;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.forgerock.opendj.ldap.ByteString;
+import org.forgerock.opendj.ldap.spi.IndexingOptions;
+import org.opends.server.types.Entry;
+import org.opends.server.types.Modification;
+
+/**
+ * Implementation of an Indexer for the children index.
+ */
+public class ID2CIndexer extends Indexer
+{
+ /**
+ * Create a new indexer for a children index.
+ */
+ public ID2CIndexer()
+ {
+ // No implementation required.
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public String toString()
+ {
+ return "id2children";
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void indexEntry(Entry entry, Set<ByteString> addKeys, IndexingOptions options)
+ {
+ // The superior entry IDs are in the entry attachment.
+ ArrayList<EntryID> ids = (ArrayList<EntryID>) entry.getAttachment();
+
+ // Skip the entry's own ID.
+ Iterator<EntryID> iter = ids.iterator();
+ iter.next();
+
+ // Get the parent ID.
+ if (iter.hasNext())
+ {
+ addKeys.add(iter.next().toByteString());
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void replaceEntry(Entry oldEntry, Entry newEntry,
+ Map<ByteString, Boolean> modifiedKeys, IndexingOptions options)
+ {
+ // Nothing to do.
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void modifyEntry(Entry oldEntry, Entry newEntry,
+ List<Modification> mods,
+ Map<ByteString, Boolean> modifiedKeys, IndexingOptions options)
+ {
+ // Nothing to do.
+ }
+}
diff --git a/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/ID2Entry.java b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/ID2Entry.java
new file mode 100644
index 0000000..5171e77
--- /dev/null
+++ b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/ID2Entry.java
@@ -0,0 +1,442 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License"). You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at legal-notices/CDDLv1_0.txt
+ * or http://forgerock.org/license/CDDLv1.0.html.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at legal-notices/CDDLv1_0.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information:
+ * Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ *
+ *
+ * Copyright 2006-2010 Sun Microsystems, Inc.
+ * Portions Copyright 2012-2014 ForgeRock AS.
+ */
+package org.opends.server.backends.pluggable;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.zip.DataFormatException;
+import java.util.zip.DeflaterOutputStream;
+import java.util.zip.InflaterOutputStream;
+
+import org.forgerock.i18n.slf4j.LocalizedLogger;
+import org.forgerock.opendj.io.ASN1;
+import org.forgerock.opendj.io.ASN1Reader;
+import org.forgerock.opendj.io.ASN1Writer;
+import org.forgerock.opendj.ldap.ByteSequence;
+import org.forgerock.opendj.ldap.ByteString;
+import org.forgerock.opendj.ldap.ByteStringBuilder;
+import org.forgerock.opendj.ldap.DecodeException;
+import org.opends.server.api.CompressedSchema;
+import org.opends.server.backends.pluggable.BackendImpl.ReadableStorage;
+import org.opends.server.backends.pluggable.BackendImpl.Storage;
+import org.opends.server.backends.pluggable.BackendImpl.StorageRuntimeException;
+import org.opends.server.backends.pluggable.BackendImpl.TreeName;
+import org.opends.server.backends.pluggable.BackendImpl.WriteableStorage;
+import org.opends.server.core.DirectoryServer;
+import org.opends.server.types.DirectoryException;
+import org.opends.server.types.Entry;
+import org.opends.server.types.LDAPException;
+
+import static org.forgerock.util.Utils.*;
+import static org.opends.messages.JebMessages.*;
+import static org.opends.server.core.DirectoryServer.*;
+
+/**
+ * Represents the database containing the LDAP entries. The database key is
+ * the entry ID and the value is the entry contents.
+ */
+public class ID2Entry extends DatabaseContainer
+{
+ private static final LocalizedLogger logger = LocalizedLogger.getLoggerForThisClass();
+
+ /** Parameters for compression and encryption. */
+ private DataConfig dataConfig;
+
+ /** Cached encoding buffers. */
+ private static final ThreadLocal<EntryCodec> ENTRY_CODEC_CACHE = new ThreadLocal<EntryCodec>()
+ {
+ @Override
+ protected EntryCodec initialValue()
+ {
+ return new EntryCodec();
+ }
+ };
+
+ private static EntryCodec acquireEntryCodec()
+ {
+ EntryCodec codec = ENTRY_CODEC_CACHE.get();
+ if (codec.maxBufferSize != getMaxInternalBufferSize())
+ {
+ // Setting has changed, so recreate the codec.
+ codec = new EntryCodec();
+ ENTRY_CODEC_CACHE.set(codec);
+ }
+ return codec;
+ }
+
+ /**
+ * A cached set of ByteStringBuilder buffers and ASN1Writer used to encode
+ * entries.
+ */
+ private static class EntryCodec
+ {
+ private static final int BUFFER_INIT_SIZE = 512;
+
+ private final ByteStringBuilder encodedBuffer = new ByteStringBuilder();
+ private final ByteStringBuilder entryBuffer = new ByteStringBuilder();
+ private final ByteStringBuilder compressedEntryBuffer = new ByteStringBuilder();
+ private final ASN1Writer writer;
+ private final int maxBufferSize;
+
+ private EntryCodec()
+ {
+ this.maxBufferSize = getMaxInternalBufferSize();
+ this.writer = ASN1.getWriter(encodedBuffer, maxBufferSize);
+ }
+
+ private void release()
+ {
+ closeSilently(writer);
+ encodedBuffer.clearAndTruncate(maxBufferSize, BUFFER_INIT_SIZE);
+ entryBuffer.clearAndTruncate(maxBufferSize, BUFFER_INIT_SIZE);
+ compressedEntryBuffer.clearAndTruncate(maxBufferSize, BUFFER_INIT_SIZE);
+ }
+
+ private Entry decode(ByteString bytes, CompressedSchema compressedSchema)
+ throws DirectoryException, DecodeException, LDAPException,
+ DataFormatException, IOException
+ {
+ // Get the format version.
+ byte formatVersion = bytes.byteAt(0);
+ if(formatVersion != JebFormat.FORMAT_VERSION)
+ {
+ throw DecodeException.error(ERR_JEB_INCOMPATIBLE_ENTRY_VERSION.get(formatVersion));
+ }
+
+ // Read the ASN1 sequence.
+ ASN1Reader reader = ASN1.getReader(bytes.subSequence(1, bytes.length()));
+ reader.readStartSequence();
+
+ // See if it was compressed.
+ int uncompressedSize = (int)reader.readInteger();
+ if(uncompressedSize > 0)
+ {
+ // It was compressed.
+ reader.readOctetString(compressedEntryBuffer);
+
+ OutputStream decompressor = null;
+ try
+ {
+ // TODO: Should handle the case where uncompress fails
+ decompressor = new InflaterOutputStream(entryBuffer.asOutputStream());
+ compressedEntryBuffer.copyTo(decompressor);
+ }
+ finally {
+ closeSilently(decompressor);
+ }
+
+ // Since we are used the cached buffers (ByteStringBuilders),
+ // the decoded attribute values will not refer back to the
+ // original buffer.
+ return Entry.decode(entryBuffer.asReader(), compressedSchema);
+ }
+ else
+ {
+ // Since we don't have to do any decompression, we can just decode
+ // the entry directly.
+ ByteString encodedEntry = reader.readOctetString();
+ return Entry.decode(encodedEntry.asReader(), compressedSchema);
+ }
+ }
+
+ private ByteString encodeCopy(Entry entry, DataConfig dataConfig)
+ throws DirectoryException
+ {
+ encodeVolatile(entry, dataConfig);
+ return encodedBuffer.toByteString();
+ }
+
+ private ByteString encodeInternal(Entry entry, DataConfig dataConfig)
+ throws DirectoryException
+ {
+ encodeVolatile(entry, dataConfig);
+ return encodedBuffer.toByteString();
+ }
+
+ private void encodeVolatile(Entry entry, DataConfig dataConfig) throws DirectoryException
+ {
+ // Encode the entry for later use.
+ entry.encode(entryBuffer, dataConfig.getEntryEncodeConfig());
+
+ // First write the DB format version byte.
+ encodedBuffer.append(JebFormat.FORMAT_VERSION);
+
+ try
+ {
+ // Then start the ASN1 sequence.
+ writer.writeStartSequence(JebFormat.TAG_DATABASE_ENTRY);
+
+ if (dataConfig.isCompressed())
+ {
+ OutputStream compressor = null;
+ try {
+ compressor = new DeflaterOutputStream(compressedEntryBuffer.asOutputStream());
+ entryBuffer.copyTo(compressor);
+ }
+ finally {
+ closeSilently(compressor);
+ }
+
+ // Compression needed and successful.
+ writer.writeInteger(entryBuffer.length());
+ writer.writeOctetString(compressedEntryBuffer);
+ }
+ else
+ {
+ writer.writeInteger(0);
+ writer.writeOctetString(entryBuffer);
+ }
+
+ writer.writeEndSequence();
+ }
+ catch(IOException ioe)
+ {
+ // TODO: This should never happen with byte buffer.
+ logger.traceException(ioe);
+ }
+ }
+ }
+
+ /**
+ * Create a new ID2Entry object.
+ *
+ * @param name The name of the entry database.
+ * @param dataConfig The desired compression and encryption options for data
+ * stored in the entry database.
+ * @param storage The JE Storage.
+ * @param entryContainer The entryContainer of the entry database.
+ * @throws StorageRuntimeException If an error occurs in the JE database.
+ *
+ */
+ ID2Entry(TreeName name, DataConfig dataConfig, Storage storage, EntryContainer entryContainer)
+ throws StorageRuntimeException
+ {
+ super(name, storage, entryContainer);
+ this.dataConfig = dataConfig;
+ }
+
+ /**
+ * Decodes an entry from its database representation.
+ * <p>
+ * An entry on disk is ASN1 encoded in this format:
+ *
+ * <pre>
+ * ByteString ::= [APPLICATION 0] IMPLICIT SEQUENCE {
+ * uncompressedSize INTEGER, -- A zero value means not compressed.
+ * dataBytes OCTET STRING -- Optionally compressed encoding of
+ * the data bytes.
+ * }
+ *
+ * ID2EntryValue ::= ByteString
+ * -- Where dataBytes contains an encoding of DirectoryServerEntry.
+ *
+ * DirectoryServerEntry ::= [APPLICATION 1] IMPLICIT SEQUENCE {
+ * dn LDAPDN,
+ * objectClasses SET OF LDAPString,
+ * userAttributes AttributeList,
+ * operationalAttributes AttributeList
+ * }
+ * </pre>
+ *
+ * @param bytes A byte array containing the encoded database value.
+ * @param compressedSchema The compressed schema manager to use when decoding.
+ * @return The decoded entry.
+ * @throws DecodeException If the data is not in the expected ASN.1 encoding
+ * format.
+ * @throws LDAPException If the data is not in the expected ASN.1 encoding
+ * format.
+ * @throws DataFormatException If an error occurs while trying to decompress
+ * compressed data.
+ * @throws DirectoryException If a Directory Server error occurs.
+ * @throws IOException if an error occurs while reading the ASN1 sequence.
+ */
+ public static Entry entryFromDatabase(ByteString bytes,
+ CompressedSchema compressedSchema) throws DirectoryException,
+ DecodeException, LDAPException, DataFormatException, IOException
+ {
+ EntryCodec codec = acquireEntryCodec();
+ try
+ {
+ return codec.decode(bytes, compressedSchema);
+ }
+ finally
+ {
+ codec.release();
+ }
+ }
+
+ /**
+ * Encodes an entry to the raw database format, with optional compression.
+ *
+ * @param entry The entry to encode.
+ * @param dataConfig Compression and cryptographic options.
+ * @return A ByteSTring containing the encoded database value.
+ *
+ * @throws DirectoryException If a problem occurs while attempting to encode
+ * the entry.
+ */
+ public static ByteString entryToDatabase(Entry entry, DataConfig dataConfig)
+ throws DirectoryException
+ {
+ EntryCodec codec = acquireEntryCodec();
+ try
+ {
+ return codec.encodeCopy(entry, dataConfig);
+ }
+ finally
+ {
+ codec.release();
+ }
+ }
+
+
+
+ /**
+ * Insert a record into the entry database.
+ *
+ * @param txn The database transaction or null if none.
+ * @param id The entry ID which forms the key.
+ * @param entry The LDAP entry.
+ * @return true if the entry was inserted, false if a record with that
+ * ID already existed.
+ * @throws StorageRuntimeException If an error occurs in the JE database.
+ * @throws DirectoryException If a problem occurs while attempting to encode
+ * the entry.
+ */
+ public boolean insert(WriteableStorage txn, EntryID id, Entry entry)
+ throws StorageRuntimeException, DirectoryException
+ {
+ ByteString key = id.toByteString();
+ EntryCodec codec = acquireEntryCodec();
+ try
+ {
+ ByteString value = codec.encodeInternal(entry, dataConfig);
+ return insert(txn, key, value);
+ }
+ finally
+ {
+ codec.release();
+ }
+ }
+
+ /**
+ * Write a record in the entry database.
+ *
+ * @param txn The database transaction or null if none.
+ * @param id The entry ID which forms the key.
+ * @param entry The LDAP entry.
+ * @throws StorageRuntimeException If an error occurs in the JE database.
+ * @throws DirectoryException If a problem occurs while attempting to encode
+ * the entry.
+ */
+ public void put(WriteableStorage txn, EntryID id, Entry entry)
+ throws StorageRuntimeException, DirectoryException
+ {
+ ByteString key = id.toByteString();
+ EntryCodec codec = acquireEntryCodec();
+ try
+ {
+ ByteString value = codec.encodeInternal(entry, dataConfig);
+ put(txn, key, value);
+ }
+ finally
+ {
+ codec.release();
+ }
+ }
+
+ /**
+ * Write a pre-formatted record into the entry database.
+ *
+ * @param txn The database transaction or null if none.
+ * @param key The key containing a pre-formatted entry ID.
+ * @param value The data value containing a pre-formatted LDAP entry.
+ * @throws StorageRuntimeException If an error occurs in the JE database.
+ */
+ @Override
+ public void put(WriteableStorage txn, ByteSequence key, ByteSequence value) throws StorageRuntimeException
+ {
+ super.put(txn, key, value);
+ }
+
+ /**
+ * Remove a record from the entry database.
+ *
+ * @param txn The database transaction or null if none.
+ * @param id The entry ID which forms the key.
+ * @return true if the entry was removed, false if it was not.
+ * @throws StorageRuntimeException If an error occurs in the JE database.
+ */
+ public boolean remove(WriteableStorage txn, EntryID id) throws StorageRuntimeException
+ {
+ return delete(txn, id.toByteString());
+ }
+
+ /**
+ * Fetch a record from the entry database.
+ *
+ * @param txn The database transaction or null if none.
+ * @param id The desired entry ID which forms the key.
+ * @return The requested entry, or null if there is no such record.
+ * @throws DirectoryException If a problem occurs while getting the entry.
+ * @throws StorageRuntimeException If an error occurs in the JE database.
+ */
+ public Entry get(ReadableStorage txn, EntryID id, boolean isRMW)
+ throws DirectoryException, StorageRuntimeException
+ {
+ ByteString value = read(txn, id.toByteString(), isRMW);
+ if (value == null)
+ {
+ return null;
+ }
+
+ try
+ {
+ Entry entry = entryFromDatabase(value,
+ entryContainer.getRootContainer().getCompressedSchema());
+ entry.processVirtualAttributes();
+ return entry;
+ }
+ catch (Exception e)
+ {
+ throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
+ ERR_JEB_ENTRY_DATABASE_CORRUPT.get(id));
+ }
+ }
+
+ /**
+ * Set the desired compression and encryption options for data
+ * stored in the entry database.
+ *
+ * @param dataConfig The desired compression and encryption options for data
+ * stored in the entry database.
+ */
+ public void setDataConfig(DataConfig dataConfig)
+ {
+ this.dataConfig = dataConfig;
+ }
+}
diff --git a/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/ID2SIndexer.java b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/ID2SIndexer.java
new file mode 100644
index 0000000..d4e8da9
--- /dev/null
+++ b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/ID2SIndexer.java
@@ -0,0 +1,92 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License"). You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at legal-notices/CDDLv1_0.txt
+ * or http://forgerock.org/license/CDDLv1.0.html.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at legal-notices/CDDLv1_0.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information:
+ * Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ *
+ *
+ * Copyright 2006-2008 Sun Microsystems, Inc.
+ * Portions Copyright 2014 ForgeRock AS
+ */
+package org.opends.server.backends.pluggable;
+
+import java.util.*;
+
+import org.forgerock.opendj.ldap.ByteString;
+import org.forgerock.opendj.ldap.spi.IndexingOptions;
+import org.opends.server.types.Entry;
+import org.opends.server.types.Modification;
+
+
+
+/**
+ * Implementation of an Indexer for the subtree index.
+ */
+public class ID2SIndexer extends Indexer
+{
+ /**
+ * Create a new indexer for a subtree index.
+ */
+ public ID2SIndexer()
+ {
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public String toString()
+ {
+ return "id2subtree";
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void indexEntry(Entry entry, Set<ByteString> addKeys, IndexingOptions options)
+ {
+ // The superior entry IDs are in the entry attachment.
+ ArrayList<EntryID> ids = (ArrayList<EntryID>) entry.getAttachment();
+
+ // Skip the entry's own ID.
+ Iterator<EntryID> iter = ids.iterator();
+ iter.next();
+
+ // Iterate through the superior IDs.
+ while (iter.hasNext())
+ {
+ ByteString nodeIDData = iter.next().toByteString();
+ addKeys.add(nodeIDData);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void replaceEntry(Entry oldEntry, Entry newEntry,
+ Map<ByteString, Boolean> modifiedKeys, IndexingOptions options)
+ {
+ // Nothing to do.
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void modifyEntry(Entry oldEntry, Entry newEntry,
+ List<Modification> mods,
+ Map<ByteString, Boolean> modifiedKeys, IndexingOptions options)
+ {
+ // Nothing to do.
+ }
+}
diff --git a/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/IDSetIterator.java b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/IDSetIterator.java
new file mode 100644
index 0000000..12e462f
--- /dev/null
+++ b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/IDSetIterator.java
@@ -0,0 +1,133 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License"). You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at legal-notices/CDDLv1_0.txt
+ * or http://forgerock.org/license/CDDLv1.0.html.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at legal-notices/CDDLv1_0.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information:
+ * Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ *
+ *
+ * Copyright 2006-2008 Sun Microsystems, Inc.
+ * Portions Copyright 2014 ForgeRock AS
+ */
+package org.opends.server.backends.pluggable;
+
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+
+/**
+ * Iterator for a set of Entry IDs. It must return values in order of ID.
+ */
+public class IDSetIterator implements Iterator<EntryID>
+{
+ /**
+ * An array of ID values in order of ID.
+ */
+ private long[] entryIDList;
+
+ /**
+ * Current position of the iterator as an index into the array of IDs.
+ */
+ private int i;
+
+ /**
+ * Create a new iterator for a given array of entry IDs.
+ * @param entryIDList An array of IDs in order or ID.
+ */
+ public IDSetIterator(long[] entryIDList)
+ {
+ this.entryIDList = entryIDList;
+ }
+
+ /**
+ * Create a new iterator for a given array of entry IDs.
+ * @param entryIDList An array of IDs in order or ID.
+ * @param begin The entry ID of the first entry that should be returned, or
+ * {@code null} if it should start at the beginning of the list.
+ */
+ public IDSetIterator(long[] entryIDList, EntryID begin)
+ {
+ this.entryIDList = entryIDList;
+
+ if (begin == null)
+ {
+ i = 0;
+ }
+ else
+ {
+ for (i=0; i < entryIDList.length; i++)
+ {
+ if (entryIDList[i] == begin.longValue())
+ {
+ break;
+ }
+ }
+
+ if (i >= entryIDList.length)
+ {
+ i = 0;
+ }
+ }
+ }
+
+ /**
+ * Returns <tt>true</tt> if the iteration has more elements. (In other
+ * words, returns <tt>true</tt> if <tt>next</tt> would return an element
+ * rather than throwing an exception.)
+ *
+ * @return <tt>true</tt> if the iterator has more elements.
+ */
+ public boolean hasNext()
+ {
+ return i < entryIDList.length;
+ }
+
+ /**
+ * Returns the next element in the iteration. Calling this method
+ * repeatedly until the {@link #hasNext()} method returns false will
+ * return each element in the underlying collection exactly once.
+ *
+ * @return the next element in the iteration.
+ * @throws java.util.NoSuchElementException
+ * iteration has no more elements.
+ */
+ public EntryID next()
+ throws NoSuchElementException
+ {
+ if (i < entryIDList.length)
+ {
+ return new EntryID(entryIDList[i++]);
+ }
+ throw new NoSuchElementException();
+ }
+
+ /**
+ *
+ * Removes from the underlying collection the last element returned by the
+ * iterator (optional operation). This method can be called only once per
+ * call to <tt>next</tt>. The behavior of an iterator is unspecified if
+ * the underlying collection is modified while the iteration is in
+ * progress in any way other than by calling this method.
+ *
+ * @exception UnsupportedOperationException if the <tt>remove</tt>
+ * operation is not supported by this Iterator.
+ */
+ public void remove() throws UnsupportedOperationException
+ {
+ throw new UnsupportedOperationException();
+ }
+}
diff --git a/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/Index.java b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/Index.java
new file mode 100644
index 0000000..18c9221
--- /dev/null
+++ b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/Index.java
@@ -0,0 +1,785 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License"). You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at legal-notices/CDDLv1_0.txt
+ * or http://forgerock.org/license/CDDLv1.0.html.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at legal-notices/CDDLv1_0.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information:
+ * Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ *
+ *
+ * Copyright 2006-2010 Sun Microsystems, Inc.
+ * Portions Copyright 2012-2014 ForgeRock AS
+ */
+package org.opends.server.backends.pluggable;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
+import org.forgerock.i18n.slf4j.LocalizedLogger;
+import org.forgerock.opendj.ldap.ByteSequence;
+import org.forgerock.opendj.ldap.ByteString;
+import org.forgerock.opendj.ldap.ConditionResult;
+import org.forgerock.opendj.ldap.spi.IndexingOptions;
+import org.opends.server.backends.pluggable.IndexBuffer.BufferedIndexValues;
+import org.opends.server.backends.pluggable.BackendImpl.Cursor;
+import org.opends.server.backends.pluggable.BackendImpl.ReadableStorage;
+import org.opends.server.backends.pluggable.BackendImpl.Storage;
+import org.opends.server.backends.pluggable.BackendImpl.StorageRuntimeException;
+import org.opends.server.backends.pluggable.BackendImpl.TreeName;
+import org.opends.server.backends.pluggable.BackendImpl.WriteableStorage;
+import org.opends.server.types.DirectoryException;
+import org.opends.server.types.Entry;
+import org.opends.server.types.Modification;
+import org.opends.server.util.StaticUtils;
+
+import static org.opends.messages.JebMessages.*;
+
+/**
+ * Represents an index implemented by a JE database in which each key maps to
+ * a set of entry IDs. The key is a byte array, and is constructed from some
+ * normalized form of an attribute value (or fragment of a value) appearing
+ * in the entry.
+ */
+public class Index extends DatabaseContainer
+{
+ private static final LocalizedLogger logger = LocalizedLogger.getLoggerForThisClass();
+
+ /** The indexer object to construct index keys from LDAP attribute values. */
+ public Indexer indexer;
+
+ /** The limit on the number of entry IDs that may be indexed by one key. */
+ private int indexEntryLimit;
+ /**
+ * Limit on the number of entry IDs that may be retrieved by cursoring
+ * through an index.
+ */
+ private final int cursorEntryLimit;
+ /**
+ * Number of keys that have exceeded the entry limit since this
+ * object was created.
+ */
+ private int entryLimitExceededCount;
+
+ /** The max number of tries to rewrite phantom records. */
+ private final int phantomWriteRetries = 3;
+
+ /**
+ * Whether to maintain a count of IDs for a key once the entry limit
+ * has exceeded.
+ */
+ private final boolean maintainCount;
+
+ private final State state;
+
+ /**
+ * A flag to indicate if this index should be trusted to be consistent
+ * with the entries database. If not trusted, we assume that existing
+ * entryIDSets for a key is still accurate. However, keys that do not
+ * exist are undefined instead of an empty entryIDSet. The following
+ * rules will be observed when the index is not trusted:
+ *
+ * - no entryIDs will be added to a non-existing key.
+ * - undefined entryIdSet will be returned whenever a key is not found.
+ */
+ private boolean trusted;
+
+ /**
+ * A flag to indicate if a rebuild process is running on this index.
+ * During the rebuild process, we assume that no entryIDSets are
+ * accurate and return an undefined set on all read operations.
+ * However all write operations will succeed. The rebuildRunning
+ * flag overrides all behaviors of the trusted flag.
+ */
+ private boolean rebuildRunning;
+
+ /** Thread local area to store per thread cursors. */
+ private final ThreadLocal<Cursor> curLocal = new ThreadLocal<Cursor>();
+
+ /**
+ * Create a new index object.
+ * @param name The name of the index database within the entryContainer.
+ * @param indexer The indexer object to construct index keys from LDAP
+ * attribute values.
+ * @param state The state database to persist index state info.
+ * @param indexEntryLimit The configured limit on the number of entry IDs
+ * that may be indexed by one key.
+ * @param cursorEntryLimit The configured limit on the number of entry IDs
+ * @param maintainCount Whether to maintain a count of IDs for a key once
+ * the entry limit has exceeded.
+ * @param storage The JE Storage
+ * @param entryContainer The database entryContainer holding this index.
+ * @throws StorageRuntimeException If an error occurs in the JE database.
+ */
+ public Index(TreeName name, Indexer indexer, State state,
+ int indexEntryLimit, int cursorEntryLimit, boolean maintainCount,
+ Storage storage, EntryContainer entryContainer)
+ throws StorageRuntimeException
+ {
+ super(name, storage, entryContainer);
+ this.indexer = indexer;
+ this.indexEntryLimit = indexEntryLimit;
+ this.cursorEntryLimit = cursorEntryLimit;
+ this.maintainCount = maintainCount;
+
+ this.state = state;
+ this.trusted = state.getIndexTrustState(null, this);
+ if (!trusted && entryContainer.getHighestEntryID().longValue() == 0)
+ {
+ // If there are no entries in the entry container then there
+ // is no reason why this index can't be upgraded to trusted.
+ setTrusted(null, true);
+ }
+ }
+
+ /**
+ * Add an add entry ID operation into a index buffer.
+ *
+ * @param buffer The index buffer to insert the ID into.
+ * @param keyBytes The index key bytes.
+ * @param entryID The entry ID.
+ */
+ public void insertID(IndexBuffer buffer, ByteString keyBytes, EntryID entryID)
+ {
+ getBufferedIndexValues(buffer, keyBytes).addEntryID(keyBytes, entryID);
+ }
+
+ /**
+ * Update the set of entry IDs for a given key.
+ *
+ * @param txn A database transaction, or null if none is required.
+ * @param key The database key.
+ * @param deletedIDs The IDs to remove for the key.
+ * @param addedIDs the IDs to add for the key.
+ * @throws StorageRuntimeException If a database error occurs.
+ */
+ void updateKey(WriteableStorage txn, ByteString key, EntryIDSet deletedIDs, EntryIDSet addedIDs)
+ throws StorageRuntimeException
+ {
+ if(deletedIDs == null && addedIDs == null)
+ {
+ boolean success = delete(txn, key);
+ if (success && logger.isTraceEnabled())
+ {
+ StringBuilder builder = new StringBuilder();
+ StaticUtils.byteArrayToHexPlusAscii(builder, key.toByteArray(), 4);
+ logger.trace("The expected key does not exist in the index %s.\nKey:%s ", treeName, builder);
+ }
+ return;
+ }
+
+ // Handle cases where nothing is changed early to avoid DB access.
+ if (isNullOrEmpty(deletedIDs) && isNullOrEmpty(addedIDs))
+ {
+ return;
+ }
+
+ if(maintainCount)
+ {
+ updateKeyWithRMW(txn, key, deletedIDs, addedIDs);
+ }
+ else
+ {
+ ByteString value = read(txn, key, false);
+ if(value != null)
+ {
+ EntryIDSet entryIDList = new EntryIDSet(key, value);
+ if (entryIDList.isDefined())
+ {
+ updateKeyWithRMW(txn, key, deletedIDs, addedIDs);
+ }
+ }
+ else
+ {
+ if (deletedIDs != null && trusted && !rebuildRunning)
+ {
+ logIndexCorruptError(txn, key);
+ }
+
+ if ((rebuildRunning || trusted) && isNotNullOrEmpty(addedIDs))
+ {
+ if(!insert(txn, key, addedIDs.toByteString()))
+ {
+ updateKeyWithRMW(txn, key, deletedIDs, addedIDs);
+ }
+ }
+ }
+ }
+ }
+
+ private boolean isNullOrEmpty(EntryIDSet entryIDSet)
+ {
+ return entryIDSet == null || entryIDSet.size() == 0;
+ }
+
+ private boolean isNotNullOrEmpty(EntryIDSet entryIDSet)
+ {
+ return entryIDSet != null && entryIDSet.size() > 0;
+ }
+
+ private void updateKeyWithRMW(WriteableStorage txn,
+ ByteString key,
+ EntryIDSet deletedIDs,
+ EntryIDSet addedIDs)
+ throws StorageRuntimeException
+ {
+ final ByteString value = read(txn, key, false);
+ if(value != null)
+ {
+ EntryIDSet entryIDList = computeEntryIDList(key, value, deletedIDs, addedIDs);
+ ByteString after = entryIDList.toByteString();
+ if (after != null)
+ {
+ put(txn, key, after);
+ }
+ else
+ {
+ // No more IDs, so remove the key. If index is not
+ // trusted then this will cause all subsequent reads
+ // for this key to return undefined set.
+ delete(txn, key);
+ }
+ }
+ else
+ {
+ if (deletedIDs != null && trusted && !rebuildRunning)
+ {
+ logIndexCorruptError(txn, key);
+ }
+
+ if ((rebuildRunning || trusted) && isNotNullOrEmpty(addedIDs))
+ {
+ insert(txn, key, addedIDs.toByteString());
+ }
+ }
+ }
+
+ private EntryIDSet computeEntryIDList(ByteString key, ByteString value, EntryIDSet deletedIDs,
+ EntryIDSet addedIDs)
+ {
+ EntryIDSet entryIDList = new EntryIDSet(key, value);
+ if(addedIDs != null)
+ {
+ if(entryIDList.isDefined() && indexEntryLimit > 0)
+ {
+ long idCountDelta = addedIDs.size();
+ if(deletedIDs != null)
+ {
+ idCountDelta -= deletedIDs.size();
+ }
+ if(idCountDelta + entryIDList.size() >= indexEntryLimit)
+ {
+ if(maintainCount)
+ {
+ entryIDList = new EntryIDSet(entryIDList.size() + idCountDelta);
+ }
+ else
+ {
+ entryIDList = new EntryIDSet();
+ }
+ entryLimitExceededCount++;
+
+ if(logger.isTraceEnabled())
+ {
+ StringBuilder builder = new StringBuilder();
+ StaticUtils.byteArrayToHexPlusAscii(builder, key.toByteArray(), 4);
+ logger.trace("Index entry exceeded in index %s. " +
+ "Limit: %d. ID list size: %d.\nKey:%s",
+ treeName, indexEntryLimit, idCountDelta + addedIDs.size(), builder);
+
+ }
+ }
+ else
+ {
+ entryIDList.addAll(addedIDs);
+ if(deletedIDs != null)
+ {
+ entryIDList.deleteAll(deletedIDs);
+ }
+ }
+ }
+ else
+ {
+ entryIDList.addAll(addedIDs);
+ if(deletedIDs != null)
+ {
+ entryIDList.deleteAll(deletedIDs);
+ }
+ }
+ }
+ else if(deletedIDs != null)
+ {
+ entryIDList.deleteAll(deletedIDs);
+ }
+ return entryIDList;
+ }
+
+ /**
+ * Add an remove entry ID operation into a index buffer.
+ *
+ * @param buffer The index buffer to insert the ID into.
+ * @param keyBytes The index key bytes.
+ * @param entryID The entry ID.
+ */
+ public void removeID(IndexBuffer buffer, ByteString keyBytes, EntryID entryID)
+ {
+ getBufferedIndexValues(buffer, keyBytes).deleteEntryID(keyBytes, entryID);
+ }
+
+ private void logIndexCorruptError(WriteableStorage txn, ByteString key)
+ {
+ if (logger.isTraceEnabled())
+ {
+ StringBuilder builder = new StringBuilder();
+ StaticUtils.byteArrayToHexPlusAscii(builder, key.toByteArray(), 4);
+ logger.trace("The expected key does not exist in the index %s.\nKey:%s", treeName, builder);
+ }
+
+ setTrusted(txn, false);
+ logger.error(ERR_JEB_INDEX_CORRUPT_REQUIRES_REBUILD, treeName);
+ }
+
+ /**
+ * Buffered delete of a key from the JE database.
+ * @param buffer The index buffer to use to store the deleted keys
+ * @param keyBytes The index key bytes.
+ */
+ public void delete(IndexBuffer buffer, ByteString keyBytes)
+ {
+ getBufferedIndexValues(buffer, keyBytes);
+ }
+
+ private BufferedIndexValues getBufferedIndexValues(IndexBuffer buffer, ByteString keyBytes)
+ {
+ return buffer.getBufferedIndexValues(this, keyBytes);
+ }
+
+ /**
+ * Check if an entry ID is in the set of IDs indexed by a given key.
+ *
+ * @param txn A database transaction, or null if none is required.
+ * @param key The index key.
+ * @param entryID The entry ID.
+ * @return true if the entry ID is indexed by the given key,
+ * false if it is not indexed by the given key,
+ * undefined if the key has exceeded the entry limit.
+ * @throws StorageRuntimeException If an error occurs in the JE database.
+ */
+ public ConditionResult containsID(ReadableStorage txn, ByteString key, EntryID entryID)
+ throws StorageRuntimeException
+ {
+ if(rebuildRunning)
+ {
+ return ConditionResult.UNDEFINED;
+ }
+
+ ByteString value = read(txn, key, false);
+ if (value != null)
+ {
+ EntryIDSet entryIDList = new EntryIDSet(key, value);
+ if (!entryIDList.isDefined())
+ {
+ return ConditionResult.UNDEFINED;
+ }
+ return ConditionResult.valueOf(entryIDList.contains(entryID));
+ }
+ else if (trusted)
+ {
+ return ConditionResult.FALSE;
+ }
+ else
+ {
+ return ConditionResult.UNDEFINED;
+ }
+ }
+
+ /**
+ * Reads the set of entry IDs for a given key.
+ *
+ * @param key The database key.
+ * @param txn A database transaction, or null if none is required.
+ * @return The entry IDs indexed by this key.
+ */
+ public EntryIDSet readKey(ByteSequence key, ReadableStorage txn)
+ {
+ if(rebuildRunning)
+ {
+ return new EntryIDSet();
+ }
+
+ try
+ {
+ ByteString value = read(txn, key, false);
+ if (value == null)
+ {
+ if(trusted)
+ {
+ return new EntryIDSet(key, null);
+ }
+ else
+ {
+ return new EntryIDSet();
+ }
+ }
+ return new EntryIDSet(key, value);
+ }
+ catch (StorageRuntimeException e)
+ {
+ logger.traceException(e);
+ return new EntryIDSet();
+ }
+ }
+
+ /**
+ * Writes the set of entry IDs for a given key.
+ *
+ * @param key The database key.
+ * @param entryIDList The entry IDs indexed by this key.
+ * @param txn A database transaction, or null if none is required.
+ * @throws StorageRuntimeException If an error occurs in the JE database.
+ */
+ public void writeKey(WriteableStorage txn, ByteString key, EntryIDSet entryIDList)
+ throws StorageRuntimeException
+ {
+ ByteString value = entryIDList.toByteString();
+ if (value != null)
+ {
+ if (!entryIDList.isDefined())
+ {
+ entryLimitExceededCount++;
+ }
+ put(txn, key, value);
+ }
+ else
+ {
+ // No more IDs, so remove the key.
+ delete(txn, key);
+ }
+ }
+
+ /**
+ * Reads a range of keys and collects all their entry IDs into a
+ * single set.
+ *
+ * @param lower The lower bound of the range. A 0 length byte array indicates
+ * no lower bound and the range will start from the
+ * smallest key.
+ * @param upper The upper bound of the range. A 0 length byte array indicates
+ * no upper bound and the range will end at the largest
+ * key.
+ * @param lowerIncluded true if a key exactly matching the lower bound
+ * is included in the range, false if only keys
+ * strictly greater than the lower bound are included.
+ * This value is ignored if the lower bound is not
+ * specified.
+ * @param upperIncluded true if a key exactly matching the upper bound
+ * is included in the range, false if only keys
+ * strictly less than the upper bound are included.
+ * This value is ignored if the upper bound is not
+ * specified.
+ * @return The set of entry IDs.
+ */
+ public EntryIDSet readRange(ByteSequence lower, ByteSequence upper,
+ boolean lowerIncluded, boolean upperIncluded)
+ {
+ // If this index is not trusted, then just return an undefined id set.
+ if(rebuildRunning || !trusted)
+ {
+ return new EntryIDSet();
+ }
+
+ try
+ {
+ // Total number of IDs found so far.
+ int totalIDCount = 0;
+
+ ArrayList<EntryIDSet> lists = new ArrayList<EntryIDSet>();
+
+ Cursor cursor = storage.openCursor(treeName);
+ try
+ {
+ ByteSequence key = ByteString.empty();
+ boolean success;
+ // Set the lower bound if necessary.
+ if (lower.length() > 0)
+ {
+ // Initialize the cursor to the lower bound.
+ key = lower;
+ success = cursor.positionToKeyOrNext(key);
+
+ // Advance past the lower bound if necessary.
+ if (success
+ && !lowerIncluded
+ && ByteSequence.COMPARATOR.compare(key, lower) == 0)
+ {
+ // Do not include the lower value.
+ success = cursor.next();
+ if (success)
+ {
+ key = cursor.getKey();
+ }
+ }
+ }
+ else
+ {
+ success = cursor.next();
+ if (success)
+ {
+ key = cursor.getKey();
+ }
+ }
+
+ if (!success)
+ {
+ // There are no values.
+ return new EntryIDSet(key, null);
+ }
+
+ // Step through the keys until we hit the upper bound or the last key.
+ while (success)
+ {
+ // Check against the upper bound if necessary
+ if (upper.length() > 0)
+ {
+ int cmp = ByteSequence.COMPARATOR.compare(cursor.getKey(), upper);
+ if (cmp > 0 || (cmp == 0 && !upperIncluded))
+ {
+ break;
+ }
+ }
+ EntryIDSet list = new EntryIDSet(key, cursor.getValue());
+ if (!list.isDefined())
+ {
+ // There is no point continuing.
+ return list;
+ }
+ totalIDCount += list.size();
+ if (cursorEntryLimit > 0 && totalIDCount > cursorEntryLimit)
+ {
+ // There are too many. Give up and return an undefined list.
+ return new EntryIDSet();
+ }
+ lists.add(list);
+ success = cursor.next();
+ }
+
+ return EntryIDSet.unionOfSets(lists, false);
+ }
+ finally
+ {
+ cursor.close();
+ }
+ }
+ catch (StorageRuntimeException e)
+ {
+ logger.traceException(e);
+ return new EntryIDSet();
+ }
+ }
+
+ /**
+ * Get the number of keys that have exceeded the entry limit since this
+ * object was created.
+ * @return The number of keys that have exceeded the entry limit since this
+ * object was created.
+ */
+ public int getEntryLimitExceededCount()
+ {
+ return entryLimitExceededCount;
+ }
+
+ /**
+ * Close any cursors open against this index.
+ *
+ * @throws StorageRuntimeException If a database error occurs.
+ */
+ public void closeCursor() throws StorageRuntimeException {
+ Cursor cursor = curLocal.get();
+ if(cursor != null) {
+ cursor.close();
+ curLocal.remove();
+ }
+ }
+
+ /**
+ * Update the index buffer for a deleted entry.
+ *
+ * @param buffer The index buffer to use to store the deleted keys
+ * @param entryID The entry ID.
+ * @param entry The entry to be indexed.
+ * @param options The indexing options to use
+ * @throws StorageRuntimeException If an error occurs in the JE database.
+ * @throws DirectoryException If a Directory Server error occurs.
+ */
+ public void addEntry(IndexBuffer buffer, EntryID entryID, Entry entry,
+ IndexingOptions options) throws StorageRuntimeException, DirectoryException
+ {
+ HashSet<ByteString> addKeys = new HashSet<ByteString>();
+ indexer.indexEntry(entry, addKeys, options);
+
+ for (ByteString keyBytes : addKeys)
+ {
+ insertID(buffer, keyBytes, entryID);
+ }
+ }
+
+ /**
+ * Update the index buffer for a deleted entry.
+ *
+ * @param buffer The index buffer to use to store the deleted keys
+ * @param entryID The entry ID
+ * @param entry The contents of the deleted entry.
+ * @param options The indexing options to use
+ * @throws StorageRuntimeException If an error occurs in the JE database.
+ * @throws DirectoryException If a Directory Server error occurs.
+ */
+ public void removeEntry(IndexBuffer buffer, EntryID entryID, Entry entry,
+ IndexingOptions options) throws StorageRuntimeException, DirectoryException
+ {
+ HashSet<ByteString> delKeys = new HashSet<ByteString>();
+ indexer.indexEntry(entry, delKeys, options);
+
+ for (ByteString keyBytes : delKeys)
+ {
+ removeID(buffer, keyBytes, entryID);
+ }
+ }
+
+ /**
+ * Update the index to reflect a sequence of modifications in a Modify
+ * operation.
+ *
+ * @param buffer The index buffer to use to store the deleted keys
+ * @param entryID The ID of the entry that was modified.
+ * @param oldEntry The entry before the modifications were applied.
+ * @param newEntry The entry after the modifications were applied.
+ * @param mods The sequence of modifications in the Modify operation.
+ * @param options The indexing options to use
+ * @throws StorageRuntimeException If an error occurs in the JE database.
+ */
+ public void modifyEntry(IndexBuffer buffer,
+ EntryID entryID,
+ Entry oldEntry,
+ Entry newEntry,
+ List<Modification> mods, IndexingOptions options)
+ throws StorageRuntimeException
+ {
+ TreeMap<ByteString, Boolean> modifiedKeys =
+ new TreeMap<ByteString, Boolean>(ByteSequence.COMPARATOR);
+ indexer.modifyEntry(oldEntry, newEntry, mods, modifiedKeys, options);
+
+ for (Map.Entry<ByteString, Boolean> modifiedKey : modifiedKeys.entrySet())
+ {
+ if(modifiedKey.getValue())
+ {
+ insertID(buffer, modifiedKey.getKey(), entryID);
+ }
+ else
+ {
+ removeID(buffer, modifiedKey.getKey(), entryID);
+ }
+ }
+ }
+
+ /**
+ * Set the index entry limit.
+ *
+ * @param indexEntryLimit The index entry limit to set.
+ * @return True if a rebuild is required or false otherwise.
+ */
+ public boolean setIndexEntryLimit(int indexEntryLimit)
+ {
+ final boolean rebuildRequired =
+ this.indexEntryLimit < indexEntryLimit && entryLimitExceededCount > 0;
+ this.indexEntryLimit = indexEntryLimit;
+ return rebuildRequired;
+ }
+
+ /**
+ * Set the indexer.
+ *
+ * @param indexer The indexer to set
+ */
+ public void setIndexer(Indexer indexer)
+ {
+ this.indexer = indexer;
+ }
+
+ /**
+ * Return entry limit.
+ *
+ * @return The entry limit.
+ */
+ public int getIndexEntryLimit() {
+ return this.indexEntryLimit;
+ }
+
+ /**
+ * Set the index trust state.
+ * @param txn A database transaction, or null if none is required.
+ * @param trusted True if this index should be trusted or false
+ * otherwise.
+ * @throws StorageRuntimeException If an error occurs in the JE database.
+ */
+ public synchronized void setTrusted(WriteableStorage txn, boolean trusted)
+ throws StorageRuntimeException
+ {
+ this.trusted = trusted;
+ state.putIndexTrustState(txn, this, trusted);
+ }
+
+ /**
+ * Return true iff this index is trusted.
+ * @return the trusted state of this index
+ */
+ public synchronized boolean isTrusted()
+ {
+ return trusted;
+ }
+
+ /**
+ * Return <code>true</code> iff this index is being rebuilt.
+ * @return The rebuild state of this index
+ */
+ public synchronized boolean isRebuildRunning()
+ {
+ return rebuildRunning;
+ }
+
+ /**
+ * Set the rebuild status of this index.
+ * @param rebuildRunning True if a rebuild process on this index
+ * is running or False otherwise.
+ */
+ public synchronized void setRebuildStatus(boolean rebuildRunning)
+ {
+ this.rebuildRunning = rebuildRunning;
+ }
+
+ /**
+ * Whether this index maintains a count of IDs for keys once the
+ * entry limit has exceeded.
+ * @return <code>true</code> if this index maintains court of IDs
+ * or <code>false</code> otherwise
+ */
+ public boolean getMaintainCount()
+ {
+ return maintainCount;
+ }
+}
diff --git a/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/IndexBuffer.java b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/IndexBuffer.java
new file mode 100644
index 0000000..86ad3c2
--- /dev/null
+++ b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/IndexBuffer.java
@@ -0,0 +1,289 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License"). You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at legal-notices/CDDLv1_0.txt
+ * or http://forgerock.org/license/CDDLv1.0.html.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at legal-notices/CDDLv1_0.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information:
+ * Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ *
+ *
+ * Copyright 2006-2008 Sun Microsystems, Inc.
+ * Portions Copyright 2014 ForgeRock AS
+ */
+package org.opends.server.backends.pluggable;
+
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.TreeSet;
+
+import org.forgerock.opendj.ldap.ByteSequence;
+import org.forgerock.opendj.ldap.ByteString;
+import org.opends.server.backends.pluggable.BackendImpl.StorageRuntimeException;
+import org.opends.server.backends.pluggable.BackendImpl.WriteableStorage;
+import org.opends.server.types.DirectoryException;
+
+/**
+ * A buffered index is used to buffer multiple reads or writes to the
+ * same index key into a single read or write.
+ * It can only be used to buffer multiple reads and writes under
+ * the same transaction. The transaction may be null if it is known
+ * that there are no other concurrent updates to the index.
+ */
+public class IndexBuffer
+{
+ private final EntryContainer entryContainer;
+
+ /**
+ * The buffered records stored as a map from the record key to the
+ * buffered value for that key for each index.
+ */
+ private final LinkedHashMap<Index, TreeMap<ByteString, BufferedIndexValues>> bufferedIndexes =
+ new LinkedHashMap<Index, TreeMap<ByteString, BufferedIndexValues>>();
+
+ /** The buffered records stored as a set of buffered VLV values for each index. */
+ private final LinkedHashMap<VLVIndex, BufferedVLVValues> bufferedVLVIndexes =
+ new LinkedHashMap<VLVIndex, BufferedVLVValues>();
+
+ /** A simple class representing a pair of added and deleted indexed IDs. */
+ static class BufferedIndexValues
+ {
+ private EntryIDSet addedIDs;
+ private EntryIDSet deletedIDs;
+
+ /**
+ * Adds the provided entryID to this object associating it with the provided keyBytes.
+ *
+ * @param keyBytes the keyBytes mapping for this entryID
+ * @param entryID the entryID to add
+ */
+ void addEntryID(ByteString keyBytes, EntryID entryID)
+ {
+ if (!remove(deletedIDs, entryID))
+ {
+ if (this.addedIDs == null)
+ {
+ this.addedIDs = new EntryIDSet(keyBytes, null);
+ }
+ this.addedIDs.add(entryID);
+ }
+ }
+
+ /**
+ * Deletes the provided entryID from this object.
+ *
+ * @param keyBytes the keyBytes mapping for this entryID
+ * @param entryID the entryID to delete
+ */
+ void deleteEntryID(ByteString keyBytes, EntryID entryID)
+ {
+ if (!remove(addedIDs, entryID))
+ {
+ if (this.deletedIDs == null)
+ {
+ this.deletedIDs = new EntryIDSet(keyBytes, null);
+ }
+ this.deletedIDs.add(entryID);
+ }
+ }
+
+ private boolean remove(EntryIDSet ids, EntryID entryID)
+ {
+ if (ids != null && ids.contains(entryID))
+ {
+ ids.remove(entryID);
+ return true;
+ }
+ return false;
+ }
+ }
+
+ /** A simple class representing a pair of added and deleted VLV values. */
+ static class BufferedVLVValues
+ {
+ private TreeSet<SortValues> addedValues;
+ private TreeSet<SortValues> deletedValues;
+
+ /**
+ * Adds the provided values to this object.
+ *
+ * @param sortValues the values to add
+ */
+ void addValues(SortValues sortValues)
+ {
+ if (!remove(deletedValues, sortValues))
+ {
+ if (this.addedValues == null)
+ {
+ this.addedValues = new TreeSet<SortValues>();
+ }
+ this.addedValues.add(sortValues);
+ }
+ }
+
+ /**
+ * Deletes the provided values from this object.
+ *
+ * @param sortValues the values to delete
+ */
+ void deleteValues(SortValues sortValues)
+ {
+ if (!remove(addedValues, sortValues))
+ {
+ if (this.deletedValues == null)
+ {
+ this.deletedValues = new TreeSet<SortValues>();
+ }
+ this.deletedValues.add(sortValues);
+ }
+ }
+
+ private boolean remove(TreeSet<SortValues> values, SortValues sortValues)
+ {
+ if (values != null && values.contains(sortValues))
+ {
+ values.remove(sortValues);
+ return true;
+ }
+ return false;
+ }
+ }
+
+ /**
+ * Construct a new empty index buffer object.
+ *
+ * @param entryContainer The database entryContainer using this
+ * index buffer.
+ */
+ public IndexBuffer(EntryContainer entryContainer)
+ {
+ this.entryContainer = entryContainer;
+ }
+
+ /**
+ * Get the buffered VLV values for the given VLV index.
+ *
+ * @param vlvIndex The VLV index with the buffered values to retrieve.
+ * @return The buffered VLV values or <code>null</code> if there are
+ * no buffered VLV values for the specified VLV index.
+ */
+ public BufferedVLVValues getVLVIndex(VLVIndex vlvIndex)
+ {
+ BufferedVLVValues bufferedValues = bufferedVLVIndexes.get(vlvIndex);
+ if (bufferedValues == null)
+ {
+ bufferedValues = new BufferedVLVValues();
+ bufferedVLVIndexes.put(vlvIndex, bufferedValues);
+ }
+ return bufferedValues;
+ }
+
+ /**
+ * Get the buffered index values for the given index and keyBytes.
+ *
+ * @param index
+ * The index for which to retrieve the buffered index values
+ * @param keyBytes
+ * The keyBytes for which to retrieve the buffered index values
+ * @return The buffered index values, it can never be null
+ */
+ BufferedIndexValues getBufferedIndexValues(Index index, ByteString keyBytes)
+ {
+ BufferedIndexValues values = null;
+
+ TreeMap<ByteString, BufferedIndexValues> bufferedOperations = bufferedIndexes.get(index);
+ if (bufferedOperations == null)
+ {
+ bufferedOperations = new TreeMap<ByteString, BufferedIndexValues>(ByteSequence.COMPARATOR);
+ bufferedIndexes.put(index, bufferedOperations);
+ }
+ else
+ {
+ values = bufferedOperations.get(keyBytes);
+ }
+
+ if (values == null)
+ {
+ values = new BufferedIndexValues();
+ bufferedOperations.put(keyBytes, values);
+ }
+ return values;
+ }
+
+ /**
+ * Flush the buffered index changes until the given transaction to
+ * the database.
+ *
+ * @param txn The database transaction to be used for the updates.
+ * @throws StorageRuntimeException If an error occurs in the JE database.
+ * @throws DirectoryException If a Directory Server error occurs.
+ */
+ public void flush(WriteableStorage txn) throws StorageRuntimeException, DirectoryException
+ {
+ for (AttributeIndex attributeIndex : entryContainer.getAttributeIndexes())
+ {
+ for (Index index : attributeIndex.getAllIndexes())
+ {
+ updateKeys(index, txn, bufferedIndexes.remove(index));
+ }
+ }
+
+ for (VLVIndex vlvIndex : entryContainer.getVLVIndexes())
+ {
+ BufferedVLVValues bufferedVLVValues = bufferedVLVIndexes.remove(vlvIndex);
+ if (bufferedVLVValues != null)
+ {
+ vlvIndex.updateIndex(txn, bufferedVLVValues.addedValues, bufferedVLVValues.deletedValues);
+ }
+ }
+
+ final Index id2children = entryContainer.getID2Children();
+ updateKeys(id2children, txn, bufferedIndexes.remove(id2children));
+
+ final Index id2subtree = entryContainer.getID2Subtree();
+ final TreeMap<ByteString, BufferedIndexValues> bufferedValues = bufferedIndexes.remove(id2subtree);
+ if (bufferedValues != null)
+ {
+ /*
+ * OPENDJ-1375: add keys in reverse order to be consistent with single
+ * entry processing in add/delete processing. This is necessary in order
+ * to avoid deadlocks.
+ */
+ updateKeys(id2subtree, txn, bufferedValues.descendingMap());
+ }
+ }
+
+ private void updateKeys(Index index, WriteableStorage txn,
+ Map<ByteString, BufferedIndexValues> bufferedValues)
+ {
+ if (bufferedValues != null)
+ {
+ final Iterator<Map.Entry<ByteString, BufferedIndexValues>> it = bufferedValues.entrySet().iterator();
+ while (it.hasNext())
+ {
+ final Map.Entry<ByteString, BufferedIndexValues> entry = it.next();
+ final ByteString key = entry.getKey();
+ final BufferedIndexValues values = entry.getValue();
+
+ index.updateKey(txn, key, values.deletedIDs, values.addedIDs);
+
+ it.remove();
+ }
+ }
+ }
+}
diff --git a/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/IndexFilter.java b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/IndexFilter.java
new file mode 100644
index 0000000..d192e15
--- /dev/null
+++ b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/IndexFilter.java
@@ -0,0 +1,394 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License"). You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at legal-notices/CDDLv1_0.txt
+ * or http://forgerock.org/license/CDDLv1.0.html.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at legal-notices/CDDLv1_0.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information:
+ * Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ *
+ *
+ * Copyright 2006-2010 Sun Microsystems, Inc.
+ * Portions copyright 2011-2014 ForgeRock AS
+ *
+ */
+package org.opends.server.backends.pluggable;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.opends.server.backends.pluggable.AttributeIndex.IndexFilterType;
+import org.opends.server.core.SearchOperation;
+import org.opends.server.monitors.DatabaseEnvironmentMonitor;
+import org.opends.server.types.AttributeType;
+import org.opends.server.types.FilterType;
+import org.opends.server.types.SearchFilter;
+
+import static org.opends.messages.JebMessages.*;
+
+/**
+ * An index filter is used to apply a search operation to a set of indexes
+ * to generate a set of candidate entries.
+ */
+public class IndexFilter
+{
+ /**
+ * Stop processing the filter against the indexes when the
+ * number of candidates is smaller than this value.
+ */
+ public static final int FILTER_CANDIDATE_THRESHOLD = 10;
+
+ /**
+ * The entry entryContainer holding the attribute indexes.
+ */
+ private final EntryContainer entryContainer;
+
+ /**
+ * The search operation provides the search base, scope and filter.
+ * It can also be checked periodically for cancellation.
+ */
+ private final SearchOperation searchOp;
+
+ /**
+ * A string builder to hold a diagnostic string which helps determine
+ * how the indexed contributed to the search operation.
+ */
+ private final StringBuilder buffer;
+
+ private final DatabaseEnvironmentMonitor monitor;
+
+ /**
+ * Construct an index filter for a search operation.
+ *
+ * @param entryContainer The entry entryContainer.
+ * @param searchOp The search operation to be evaluated.
+ * @param monitor The monitor to gather filter usage stats.
+ *
+ * @param debugBuilder If not null, a diagnostic string will be written
+ * which will help determine how the indexes contributed
+ * to this search.
+ */
+ public IndexFilter(EntryContainer entryContainer,
+ SearchOperation searchOp,
+ StringBuilder debugBuilder,
+ DatabaseEnvironmentMonitor monitor)
+ {
+ this.entryContainer = entryContainer;
+ this.searchOp = searchOp;
+ this.buffer = debugBuilder;
+ this.monitor = monitor;
+ }
+
+ /**
+ * Evaluate the search operation against the indexes.
+ *
+ * @return A set of entry IDs representing candidate entries.
+ */
+ public EntryIDSet evaluate()
+ {
+ if (buffer != null)
+ {
+ buffer.append("filter=");
+ }
+ return evaluateFilter(searchOp.getFilter());
+ }
+
+ /**
+ * Evaluate a search filter against the indexes.
+ *
+ * @param filter The search filter to be evaluated.
+ * @return A set of entry IDs representing candidate entries.
+ */
+ private EntryIDSet evaluateFilter(SearchFilter filter)
+ {
+ EntryIDSet candidates = evaluate(filter);
+ if (buffer != null)
+ {
+ candidates.toString(buffer);
+ }
+ return candidates;
+ }
+
+ private EntryIDSet evaluate(SearchFilter filter)
+ {
+ switch (filter.getFilterType())
+ {
+ case AND:
+ if (buffer != null)
+ {
+ buffer.append("(&");
+ }
+ final EntryIDSet res1 = evaluateLogicalAndFilter(filter);
+ if (buffer != null)
+ {
+ buffer.append(")");
+ }
+ return res1;
+
+ case OR:
+ if (buffer != null)
+ {
+ buffer.append("(|");
+ }
+ final EntryIDSet res2 = evaluateLogicalOrFilter(filter);
+ if (buffer != null)
+ {
+ buffer.append(")");
+ }
+ return res2;
+
+ case EQUALITY:
+ return evaluateFilterWithDiagnostic(IndexFilterType.EQUALITY, filter);
+
+ case GREATER_OR_EQUAL:
+ return evaluateFilterWithDiagnostic(IndexFilterType.GREATER_OR_EQUAL, filter);
+
+ case SUBSTRING:
+ return evaluateFilterWithDiagnostic(IndexFilterType.SUBSTRING, filter);
+
+ case LESS_OR_EQUAL:
+ return evaluateFilterWithDiagnostic(IndexFilterType.LESS_OR_EQUAL, filter);
+
+ case PRESENT:
+ return evaluateFilterWithDiagnostic(IndexFilterType.PRESENCE, filter);
+
+ case APPROXIMATE_MATCH:
+ return evaluateFilterWithDiagnostic(IndexFilterType.APPROXIMATE, filter);
+
+ case EXTENSIBLE_MATCH:
+ if (buffer!= null)
+ {
+ filter.toString(buffer);
+ }
+ return evaluateExtensibleFilter(filter);
+
+ case NOT:
+ default:
+ if (buffer != null)
+ {
+ filter.toString(buffer);
+ }
+ //NYI
+ return new EntryIDSet();
+ }
+ }
+
+ /**
+ * Evaluate a logical AND search filter against the indexes.
+ *
+ * @param andFilter The AND search filter to be evaluated.
+ * @return A set of entry IDs representing candidate entries.
+ */
+ private EntryIDSet evaluateLogicalAndFilter(SearchFilter andFilter)
+ {
+ // Start off with an undefined set.
+ EntryIDSet results = new EntryIDSet();
+
+ // Put the slow range filters (greater-or-equal, less-or-equal)
+ // into a hash map, the faster components (equality, presence, approx)
+ // into one list and the remainder into another list.
+
+ ArrayList<SearchFilter> fastComps = new ArrayList<SearchFilter>();
+ ArrayList<SearchFilter> otherComps = new ArrayList<SearchFilter>();
+ HashMap<AttributeType, ArrayList<SearchFilter>> rangeComps =
+ new HashMap<AttributeType, ArrayList<SearchFilter>>();
+
+ for (SearchFilter filter : andFilter.getFilterComponents())
+ {
+ FilterType filterType = filter.getFilterType();
+ if (filterType == FilterType.GREATER_OR_EQUAL ||
+ filterType == FilterType.LESS_OR_EQUAL)
+ {
+ ArrayList<SearchFilter> rangeList;
+ rangeList = rangeComps.get(filter.getAttributeType());
+ if (rangeList == null)
+ {
+ rangeList = new ArrayList<SearchFilter>();
+ rangeComps.put(filter.getAttributeType(), rangeList);
+ }
+ rangeList.add(filter);
+ }
+ else if (filterType == FilterType.EQUALITY ||
+ filterType == FilterType.PRESENT ||
+ filterType == FilterType.APPROXIMATE_MATCH)
+ {
+ fastComps.add(filter);
+ }
+ else
+ {
+ otherComps.add(filter);
+ }
+ }
+
+ // First, process the fast components.
+ if (evaluateFilters(results, fastComps)
+ // Next, process the other (non-range) components.
+ || evaluateFilters(results, otherComps)
+ // Are there any range component pairs like (cn>=A)(cn<=B) ?
+ || rangeComps.isEmpty())
+ {
+ return results;
+ }
+
+ // Next, process range component pairs like (cn>=A)(cn<=B).
+ ArrayList<SearchFilter> remainComps = new ArrayList<SearchFilter>();
+ for (Map.Entry<AttributeType, ArrayList<SearchFilter>> rangeEntry : rangeComps.entrySet())
+ {
+ ArrayList<SearchFilter> rangeList = rangeEntry.getValue();
+ if (rangeList.size() == 2)
+ {
+ SearchFilter filter1 = rangeList.get(0);
+ SearchFilter filter2 = rangeList.get(1);
+
+ AttributeIndex attributeIndex = entryContainer.getAttributeIndex(rangeEntry.getKey());
+ if (attributeIndex == null)
+ {
+ if(monitor.isFilterUseEnabled())
+ {
+ monitor.updateStats(SearchFilter.createANDFilter(rangeList),
+ INFO_JEB_INDEX_FILTER_INDEX_TYPE_DISABLED.get("ordering",
+ rangeEntry.getKey().getNameOrOID()));
+ }
+ continue;
+ }
+
+ EntryIDSet set = attributeIndex.evaluateBoundedRange(filter1, filter2, buffer, monitor);
+ if(monitor.isFilterUseEnabled() && set.isDefined())
+ {
+ monitor.updateStats(SearchFilter.createANDFilter(rangeList), set.size());
+ }
+ if (retainAll(results, set))
+ {
+ return results;
+ }
+ }
+ else
+ {
+ // Add to the remaining range components to be processed.
+ remainComps.addAll(rangeList);
+ }
+ }
+
+ // Finally, process the remaining slow range components.
+ evaluateFilters(results, remainComps);
+
+ return results;
+ }
+
+ private boolean evaluateFilters(EntryIDSet results, ArrayList<SearchFilter> filters)
+ {
+ for (SearchFilter filter : filters)
+ {
+ final EntryIDSet filteredSet = evaluateFilter(filter);
+ if (retainAll(results, filteredSet))
+ {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Retain all IDs in a given set that appear in a second set.
+ *
+ * @param a The set of entry IDs to be updated.
+ * @param b Only those IDs that are in this set are retained.
+ * @return true if the number of IDs in the updated set is now below
+ * the filter candidate threshold.
+ */
+ private boolean retainAll(EntryIDSet a, EntryIDSet b)
+ {
+ a.retainAll(b);
+
+ // We may have reached the point of diminishing returns where
+ // it is quicker to stop now and process the current small number of candidates.
+ return a.isDefined() && a.size() <= FILTER_CANDIDATE_THRESHOLD;
+ }
+
+ /**
+ * Evaluate a logical OR search filter against the indexes.
+ *
+ * @param orFilter The OR search filter to be evaluated.
+ * @return A set of entry IDs representing candidate entries.
+ */
+ private EntryIDSet evaluateLogicalOrFilter(SearchFilter orFilter)
+ {
+ ArrayList<EntryIDSet> candidateSets = new ArrayList<EntryIDSet>(
+ orFilter.getFilterComponents().size());
+
+ for (SearchFilter filter : orFilter.getFilterComponents())
+ {
+ EntryIDSet set = evaluateFilter(filter);
+ if (!set.isDefined())
+ {
+ // There is no point continuing.
+ return set;
+ }
+ candidateSets.add(set);
+ }
+ return EntryIDSet.unionOfSets(candidateSets, false);
+ }
+
+ private EntryIDSet evaluateFilterWithDiagnostic(IndexFilterType indexFilterType, SearchFilter filter)
+ {
+ if (buffer != null)
+ {
+ filter.toString(buffer);
+ }
+ return evaluateFilter(indexFilterType, filter);
+ }
+
+ private EntryIDSet evaluateFilter(IndexFilterType indexFilterType, SearchFilter filter)
+ {
+ AttributeIndex attributeIndex = entryContainer.getAttributeIndex(filter.getAttributeType());
+ if (attributeIndex != null)
+ {
+ return attributeIndex.evaluateFilter(indexFilterType, filter, buffer, monitor);
+ }
+
+ if (monitor.isFilterUseEnabled())
+ {
+ monitor.updateStats(filter, INFO_JEB_INDEX_FILTER_INDEX_TYPE_DISABLED.get(
+ indexFilterType.toString(), filter.getAttributeType().getNameOrOID()));
+ }
+ return new EntryIDSet();
+ }
+
+ /**
+ * Evaluate an extensible filter against the indexes.
+ *
+ * @param extensibleFilter The extensible filter to be evaluated.
+ * @return A set of entry IDs representing candidate entries.
+ */
+ private EntryIDSet evaluateExtensibleFilter(SearchFilter extensibleFilter)
+ {
+ if (extensibleFilter.getDNAttributes())
+ {
+ // This will always be unindexed since the filter potentially matches
+ // entries containing the specified attribute type as well as any entry
+ // containing the attribute in its DN as part of a superior RDN.
+ return IndexQuery.createNullIndexQuery().evaluate(null);
+ }
+
+ AttributeIndex attributeIndex = entryContainer.getAttributeIndex(extensibleFilter.getAttributeType());
+ if (attributeIndex != null)
+ {
+ return attributeIndex.evaluateExtensibleFilter(extensibleFilter, buffer, monitor);
+ }
+ return IndexQuery.createNullIndexQuery().evaluate(null);
+ }
+}
diff --git a/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/IndexQuery.java b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/IndexQuery.java
new file mode 100644
index 0000000..1bd337b
--- /dev/null
+++ b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/IndexQuery.java
@@ -0,0 +1,213 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License"). You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at legal-notices/CDDLv1_0.txt
+ * or http://forgerock.org/license/CDDLv1.0.html.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at legal-notices/CDDLv1_0.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information:
+ * Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ *
+ *
+ * Copyright 2009-2010 Sun Microsystems, Inc.
+ * Portions Copyright 2014 ForgeRock AS
+ */
+package org.opends.server.backends.pluggable;
+
+import java.util.Collection;
+
+import org.forgerock.i18n.LocalizableMessageBuilder;
+
+import static org.opends.server.backends.jeb.IndexFilter.*;
+
+/**
+ * This class represents a JE Backend Query.
+ */
+@org.opends.server.types.PublicAPI(
+ stability = org.opends.server.types.StabilityLevel.VOLATILE,
+ mayInstantiate = false,
+ mayExtend = true,
+ mayInvoke = false)
+public abstract class IndexQuery
+{
+ /**
+ * Evaluates the index query and returns the EntryIDSet.
+ *
+ * @param debugMessage If not null, diagnostic message will be written
+ * which will help to determine why the returned
+ * EntryIDSet is not defined.
+ * @return The EntryIDSet as a result of evaluation of this query.
+ */
+ public abstract EntryIDSet evaluate(LocalizableMessageBuilder debugMessage);
+
+
+
+ /**
+ * Creates an IntersectionIndexQuery object from a collection of
+ * IndexQuery objects.
+ *
+ * @param subIndexQueries
+ * A collection of IndexQuery objects.
+ * @return An IntersectionIndexQuery object.
+ */
+ public static IndexQuery createIntersectionIndexQuery(
+ Collection<IndexQuery> subIndexQueries)
+ {
+ return new IntersectionIndexQuery(subIndexQueries);
+ }
+
+
+
+ /**
+ * Creates a union IndexQuery object from a collection of IndexQuery
+ * objects.
+ *
+ * @param subIndexQueries
+ * Collection of IndexQuery objects.
+ * @return A UnionIndexQuery object.
+ */
+ public static IndexQuery createUnionIndexQuery(
+ Collection<IndexQuery> subIndexQueries)
+ {
+ return new UnionIndexQuery(subIndexQueries);
+ }
+
+
+
+ /**
+ * Creates an empty IndexQuery object.
+ *
+ * @return A NullIndexQuery object.
+ */
+ public static IndexQuery createNullIndexQuery()
+ {
+ return new NullIndexQuery();
+ }
+
+
+
+ /**
+ * This class creates a Null IndexQuery. It is used when there is no
+ * record in the index. It may also be used when the index contains
+ * all the records but an empty EntryIDSet should be returned as part
+ * of the optimization.
+ */
+ private static final class NullIndexQuery extends IndexQuery
+ {
+ /** {@inheritDoc} */
+ @Override
+ public EntryIDSet evaluate(LocalizableMessageBuilder debugMessage)
+ {
+ return new EntryIDSet();
+ }
+ }
+
+ /**
+ * This class creates an intersection IndexQuery from a collection of
+ * IndexQuery objects.
+ */
+ private static final class IntersectionIndexQuery extends IndexQuery
+ {
+ /**
+ * Collection of IndexQuery objects.
+ */
+ private final Collection<IndexQuery> subIndexQueries;
+
+
+
+ /**
+ * Creates an instance of IntersectionIndexQuery.
+ *
+ * @param subIndexQueries
+ * Collection of IndexQuery objects.
+ */
+ private IntersectionIndexQuery(Collection<IndexQuery> subIndexQueries)
+ {
+ this.subIndexQueries = subIndexQueries;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public EntryIDSet evaluate(LocalizableMessageBuilder debugMessage)
+ {
+ EntryIDSet entryIDs = null;
+ for (IndexQuery query : subIndexQueries)
+ {
+ if (entryIDs == null)
+ {
+ entryIDs = query.evaluate(debugMessage);
+ }
+ else
+ {
+ entryIDs.retainAll(query.evaluate(debugMessage));
+ }
+ if (entryIDs.isDefined()
+ && entryIDs.size() <= FILTER_CANDIDATE_THRESHOLD)
+ {
+ break;
+ }
+ }
+ return entryIDs;
+ }
+ }
+
+ /**
+ * This class creates a union of IndexQuery objects.
+ */
+ private static final class UnionIndexQuery extends IndexQuery
+ {
+ /**
+ * Collection containing IndexQuery objects.
+ */
+ private final Collection<IndexQuery> subIndexQueries;
+
+
+
+ /**
+ * Creates an instance of UnionIndexQuery.
+ *
+ * @param subIndexQueries
+ * The Collection of IndexQuery objects.
+ */
+ private UnionIndexQuery(Collection<IndexQuery> subIndexQueries)
+ {
+ this.subIndexQueries = subIndexQueries;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public EntryIDSet evaluate(LocalizableMessageBuilder debugMessage)
+ {
+ EntryIDSet entryIDs = null;
+ for (IndexQuery query : subIndexQueries)
+ {
+ if (entryIDs == null)
+ {
+ entryIDs = query.evaluate(debugMessage);
+ }
+ else
+ {
+ entryIDs.addAll(query.evaluate(debugMessage));
+ }
+ if (entryIDs.isDefined()
+ && entryIDs.size() <= FILTER_CANDIDATE_THRESHOLD)
+ {
+ break;
+ }
+ }
+ return entryIDs;
+ }
+ }
+}
diff --git a/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/IndexQueryFactoryImpl.java b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/IndexQueryFactoryImpl.java
new file mode 100644
index 0000000..dce3c39
--- /dev/null
+++ b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/IndexQueryFactoryImpl.java
@@ -0,0 +1,214 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License"). You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at legal-notices/CDDLv1_0.txt
+ * or http://forgerock.org/license/CDDLv1.0.html.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at legal-notices/CDDLv1_0.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information:
+ * Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ *
+ *
+ * Copyright 2009-2010 Sun Microsystems, Inc.
+ * Portions Copyright 2014 ForgeRock AS
+ */
+package org.opends.server.backends.pluggable;
+
+import java.util.Collection;
+import java.util.Map;
+
+import org.forgerock.i18n.LocalizableMessageBuilder;
+import org.forgerock.opendj.ldap.ByteSequence;
+import org.forgerock.opendj.ldap.spi.IndexQueryFactory;
+import org.forgerock.opendj.ldap.spi.IndexingOptions;
+
+import static org.opends.messages.JebMessages.*;
+
+/**
+ * This class is an implementation of IndexQueryFactory which creates
+ * IndexQuery objects as part of the query of the JEB index.
+ */
+public final class IndexQueryFactoryImpl implements
+ IndexQueryFactory<IndexQuery>
+{
+
+ private static final String PRESENCE_INDEX_KEY = "presence";
+
+ /**
+ * The Map containing the string type identifier and the corresponding index.
+ */
+ private final Map<String, Index> indexMap;
+ private final IndexingOptions indexingOptions;
+
+ /**
+ * Creates a new IndexQueryFactoryImpl object.
+ *
+ * @param indexMap
+ * A map containing the index id and the corresponding index.
+ * @param indexingOptions
+ * The options to use for indexing
+ */
+ public IndexQueryFactoryImpl(Map<String, Index> indexMap, IndexingOptions indexingOptions)
+ {
+ this.indexMap = indexMap;
+ this.indexingOptions = indexingOptions;
+ }
+
+
+
+ /** {@inheritDoc} */
+ @Override
+ public IndexQuery createExactMatchQuery(final String indexID, final ByteSequence key)
+ {
+ return new IndexQuery()
+ {
+
+ @Override
+ public EntryIDSet evaluate(LocalizableMessageBuilder debugMessage)
+ {
+ // Read the database and get Record for the key.
+ // Select the right index to be used.
+ Index index = indexMap.get(indexID);
+ if (index == null)
+ {
+ if(debugMessage != null)
+ {
+ debugMessage.append(INFO_JEB_INDEX_FILTER_INDEX_TYPE_DISABLED.get(indexID, ""));
+ }
+ return createMatchAllQuery().evaluate(debugMessage);
+ }
+ EntryIDSet entrySet = index.readKey(key, null);
+ if(debugMessage != null && !entrySet.isDefined())
+ {
+ updateStatsUndefinedResults(debugMessage, index);
+ }
+ return entrySet;
+ }
+ };
+ }
+
+
+
+ /** {@inheritDoc} */
+ @Override
+ public IndexQuery createRangeMatchQuery(final String indexID,
+ final ByteSequence lowerBound, final ByteSequence upperBound,
+ final boolean includeLowerBound, final boolean includeUpperBound)
+ {
+ return new IndexQuery()
+ {
+
+ @Override
+ public EntryIDSet evaluate(LocalizableMessageBuilder debugMessage)
+ {
+ // Find the right index.
+ Index index = indexMap.get(indexID);
+ if (index == null)
+ {
+ if(debugMessage != null)
+ {
+ debugMessage.append(INFO_JEB_INDEX_FILTER_INDEX_TYPE_DISABLED.get(indexID, ""));
+ }
+ return createMatchAllQuery().evaluate(debugMessage);
+ }
+ EntryIDSet entrySet = index.readRange(lowerBound, upperBound,
+ includeLowerBound, includeUpperBound);
+ if(debugMessage != null && !entrySet.isDefined())
+ {
+ updateStatsUndefinedResults(debugMessage, index);
+ }
+ return entrySet;
+ }
+ };
+ }
+
+
+
+ /** {@inheritDoc} */
+ @Override
+ public IndexQuery createIntersectionQuery(Collection<IndexQuery> subqueries)
+ {
+ return IndexQuery.createIntersectionIndexQuery(subqueries);
+ }
+
+
+
+ /** {@inheritDoc} */
+ @Override
+ public IndexQuery createUnionQuery(Collection<IndexQuery> subqueries)
+ {
+ return IndexQuery.createUnionIndexQuery(subqueries);
+ }
+
+
+
+ /**
+ * {@inheritDoc}
+ * <p>
+ * It returns an empty EntryIDSet object when either all or no record
+ * sets are requested.
+ */
+ @Override
+ public IndexQuery createMatchAllQuery()
+ {
+ return new IndexQuery()
+ {
+ @Override
+ public EntryIDSet evaluate(LocalizableMessageBuilder debugMessage)
+ {
+ final String indexID = PRESENCE_INDEX_KEY;
+ final Index index = indexMap.get(indexID);
+ if (index == null)
+ {
+ if(debugMessage != null)
+ {
+ debugMessage.append(INFO_JEB_INDEX_FILTER_INDEX_TYPE_DISABLED.get(indexID, ""));
+ }
+ return new EntryIDSet();
+ }
+
+ EntryIDSet entrySet = index.readKey(PresenceIndexer.presenceKey, null);
+ if (debugMessage != null && !entrySet.isDefined())
+ {
+ updateStatsUndefinedResults(debugMessage, index);
+ }
+ return entrySet;
+ }
+ };
+ }
+
+ private static void updateStatsUndefinedResults(LocalizableMessageBuilder debugMessage, Index index)
+ {
+ if (!index.isTrusted())
+ {
+ debugMessage.append(INFO_JEB_INDEX_FILTER_INDEX_NOT_TRUSTED.get(index.getName()));
+ }
+ else if (index.isRebuildRunning())
+ {
+ debugMessage.append(INFO_JEB_INDEX_FILTER_INDEX_REBUILD_IN_PROGRESS.get(index.getName()));
+ }
+ else
+ {
+ debugMessage.append(INFO_JEB_INDEX_FILTER_INDEX_LIMIT_EXCEEDED.get(index.getName()));
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public IndexingOptions getIndexingOptions()
+ {
+ return indexingOptions;
+ }
+}
diff --git a/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/Indexer.java b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/Indexer.java
new file mode 100644
index 0000000..c7f339b
--- /dev/null
+++ b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/Indexer.java
@@ -0,0 +1,86 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License"). You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at legal-notices/CDDLv1_0.txt
+ * or http://forgerock.org/license/CDDLv1.0.html.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at legal-notices/CDDLv1_0.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information:
+ * Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ *
+ *
+ * Copyright 2006-2008 Sun Microsystems, Inc.
+ * Portions copyright 2012-2014 ForgeRock AS.
+ */
+package org.opends.server.backends.pluggable;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.forgerock.opendj.ldap.ByteString;
+import org.forgerock.opendj.ldap.spi.IndexingOptions;
+import org.opends.server.types.Entry;
+import org.opends.server.types.Modification;
+
+/**
+ * This class attempts to abstract the generation and comparison of keys
+ * for an index. It is subclassed for the specific type of indexing.
+ */
+public abstract class Indexer
+{
+ /**
+ * Generate the set of index keys for an entry.
+ *
+ * @param entry The entry.
+ * @param keys The set into which the generated keys will be inserted.
+ * @param options The indexing options to use
+ */
+ public abstract void indexEntry(Entry entry, Set<ByteString> keys, IndexingOptions options);
+
+ /**
+ * Generate the set of index keys to be added and the set of index keys
+ * to be deleted for an entry that has been replaced.
+ *
+ * @param oldEntry The original entry contents.
+ * @param newEntry The new entry contents.
+ * @param modifiedKeys The map into which the modified keys will be inserted.
+ * @param options The indexing options to use
+ */
+ public abstract void replaceEntry(Entry oldEntry, Entry newEntry,
+ Map<ByteString, Boolean> modifiedKeys, IndexingOptions options);
+
+ /**
+ * Generate the set of index keys to be added and the set of index keys
+ * to be deleted for an entry that was modified.
+ *
+ * @param oldEntry The original entry contents.
+ * @param newEntry The new entry contents.
+ * @param mods The set of modifications that were applied to the entry.
+ * @param modifiedKeys The map into which the modified keys will be inserted.
+ * @param options The indexing options to use
+ */
+ public abstract void modifyEntry(Entry oldEntry, Entry newEntry,
+ List<Modification> mods, Map<ByteString, Boolean> modifiedKeys,
+ IndexingOptions options);
+
+ /**
+ * Get a string representation of this object. The returned value is
+ * used to name an index created using this object.
+ * @return A string representation of this object.
+ */
+ @Override
+ public abstract String toString();
+}
diff --git a/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/JECompressedSchema.java b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/JECompressedSchema.java
new file mode 100644
index 0000000..0292ce9
--- /dev/null
+++ b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/JECompressedSchema.java
@@ -0,0 +1,311 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License"). You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at legal-notices/CDDLv1_0.txt
+ * or http://forgerock.org/license/CDDLv1.0.html.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at legal-notices/CDDLv1_0.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information:
+ * Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ *
+ *
+ * Copyright 2008-2009 Sun Microsystems, Inc.
+ * Portions Copyright 2013-2014 ForgeRock AS.
+ */
+package org.opends.server.backends.pluggable;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.forgerock.i18n.LocalizableMessage;
+import org.forgerock.i18n.slf4j.LocalizedLogger;
+import org.forgerock.opendj.io.ASN1;
+import org.forgerock.opendj.io.ASN1Reader;
+import org.forgerock.opendj.io.ASN1Writer;
+import org.forgerock.opendj.ldap.ByteStringBuilder;
+import org.opends.server.api.CompressedSchema;
+import org.opends.server.backends.pluggable.BackendImpl.Cursor;
+import org.opends.server.backends.pluggable.BackendImpl.Storage;
+import org.opends.server.backends.pluggable.BackendImpl.StorageRuntimeException;
+import org.opends.server.core.DirectoryServer;
+import org.opends.server.types.DirectoryException;
+import org.opends.server.types.InitializationException;
+import org.opends.server.util.StaticUtils;
+
+
+
+
+
+
+
+
+
+
+import static com.sleepycat.je.LockMode.*;
+import static com.sleepycat.je.OperationStatus.*;
+
+import static org.opends.messages.JebMessages.*;
+
+/**
+ * This class provides a compressed schema implementation whose definitions are
+ * stored in a Berkeley DB JE database.
+ */
+public final class JECompressedSchema extends CompressedSchema
+{
+ private static final LocalizedLogger logger = LocalizedLogger.getLoggerForThisClass();
+
+ /** The name of the database used to store compressed attribute description definitions. */
+ private static final String DB_NAME_AD = "compressed_attributes";
+ /** The name of the database used to store compressed object class set definitions. */
+ private static final String DB_NAME_OC = "compressed_object_classes";
+
+ /** The compressed attribute description schema database. */
+ private Database adDatabase;
+ /** The environment in which the databases are held. */
+ private Storage environment;
+ /** The compressed object class set schema database. */
+ private Database ocDatabase;
+
+ private final ByteStringBuilder storeAttributeWriterBuffer = new ByteStringBuilder();
+ private final ASN1Writer storeAttributeWriter = ASN1.getWriter(storeAttributeWriterBuffer);
+ private final ByteStringBuilder storeObjectClassesWriterBuffer = new ByteStringBuilder();
+ private final ASN1Writer storeObjectClassesWriter = ASN1.getWriter(storeObjectClassesWriterBuffer);
+
+
+
+ /**
+ * Creates a new instance of this JE compressed schema manager.
+ *
+ * @param environment
+ * A reference to the database environment in which the databases
+ * will be held.
+ * @throws StorageRuntimeException
+ * If a database problem occurs while loading the compressed schema
+ * definitions from the database.
+ * @throws InitializationException
+ * If an error occurs while loading and processing the compressed
+ * schema definitions.
+ */
+ public JECompressedSchema(final Storage environment)
+ throws StorageRuntimeException, InitializationException
+ {
+ this.environment = environment;
+ load();
+ }
+
+
+
+ /**
+ * Closes the databases and releases any resources held by this compressed
+ * schema manager.
+ */
+ public void close()
+ {
+ close0(adDatabase);
+ close0(ocDatabase);
+
+ adDatabase = null;
+ ocDatabase = null;
+ environment = null;
+ }
+
+ private void close0(Database database)
+ {
+ try
+ {
+ database.sync();
+ }
+ catch (final Exception e)
+ {
+ // Ignore.
+ }
+ StaticUtils.close(database);
+ }
+
+
+
+ /** {@inheritDoc} */
+ @Override
+ protected void storeAttribute(final byte[] encodedAttribute,
+ final String attributeName, final Collection<String> attributeOptions)
+ throws DirectoryException
+ {
+ try
+ {
+ storeAttributeWriterBuffer.clear();
+ storeAttributeWriter.writeStartSequence();
+ storeAttributeWriter.writeOctetString(attributeName);
+ for (final String option : attributeOptions)
+ {
+ storeAttributeWriter.writeOctetString(option);
+ }
+ storeAttributeWriter.writeEndSequence();
+ store(adDatabase, encodedAttribute, storeAttributeWriterBuffer);
+ }
+ catch (final IOException e)
+ {
+ // TODO: Shouldn't happen but should log a message
+ }
+ }
+
+
+
+ /** {@inheritDoc} */
+ @Override
+ protected void storeObjectClasses(final byte[] encodedObjectClasses,
+ final Collection<String> objectClassNames) throws DirectoryException
+ {
+ try
+ {
+ storeObjectClassesWriterBuffer.clear();
+ storeObjectClassesWriter.writeStartSequence();
+ for (final String ocName : objectClassNames)
+ {
+ storeObjectClassesWriter.writeOctetString(ocName);
+ }
+ storeObjectClassesWriter.writeEndSequence();
+ store(ocDatabase, encodedObjectClasses, storeObjectClassesWriterBuffer);
+ }
+ catch (final IOException e)
+ {
+ // TODO: Shouldn't happen but should log a message
+ }
+ }
+
+
+
+ /**
+ * Loads the compressed schema information from the database.
+ *
+ * @throws StorageRuntimeException
+ * If a database error occurs while loading the definitions from the
+ * database.
+ * @throws InitializationException
+ * If an error occurs while loading and processing the definitions.
+ */
+ private void load() throws StorageRuntimeException, InitializationException
+ {
+ final DatabaseConfig dbConfig = JEBUtils.toDatabaseConfigNoDuplicates(environment);
+
+ adDatabase = environment.openDatabase(null, DB_NAME_AD, dbConfig);
+ ocDatabase = environment.openDatabase(null, DB_NAME_OC, dbConfig);
+
+ // Cursor through the object class database and load the object class set
+ // definitions. At the same time, figure out the highest token value and
+ // initialize the object class counter to one greater than that.
+ final Cursor ocCursor = ocDatabase.openCursor(null);
+ try
+ {
+ while (ocCursor.next())
+ {
+ final byte[] encodedObjectClasses = ocCursor.getKey().toByteArray();
+ final ASN1Reader reader = ASN1.getReader(ocCursor.getValue());
+ reader.readStartSequence();
+ final List<String> objectClassNames = new LinkedList<String>();
+ while (reader.hasNextElement())
+ {
+ objectClassNames.add(reader.readOctetStringAsString());
+ }
+ reader.readEndSequence();
+ loadObjectClasses(encodedObjectClasses, objectClassNames);
+ }
+ }
+ catch (final IOException e)
+ {
+ logger.traceException(e);
+ throw new InitializationException(
+ ERR_JEB_COMPSCHEMA_CANNOT_DECODE_OC_TOKEN.get(e.getMessage()), e);
+ }
+ finally
+ {
+ ocCursor.close();
+ }
+
+ // Cursor through the attribute description database and load the attribute
+ // set definitions.
+ final Cursor adCursor = adDatabase.openCursor(null);
+ try
+ {
+ while (adCursor.next())
+ {
+ final byte[] encodedAttribute = adCursor.getKey().toByteArray();
+ final ASN1Reader reader = ASN1.getReader(adCursor.getValue());
+ reader.readStartSequence();
+ final String attributeName = reader.readOctetStringAsString();
+ final List<String> attributeOptions = new LinkedList<String>();
+ while (reader.hasNextElement())
+ {
+ attributeOptions.add(reader.readOctetStringAsString());
+ }
+ reader.readEndSequence();
+ loadAttribute(encodedAttribute, attributeName, attributeOptions);
+ }
+ }
+ catch (final IOException e)
+ {
+ logger.traceException(e);
+ throw new InitializationException(
+ ERR_JEB_COMPSCHEMA_CANNOT_DECODE_AD_TOKEN.get(e.getMessage()), e);
+ }
+ finally
+ {
+ adCursor.close();
+ }
+ }
+
+
+
+ private void store(final Database database, final byte[] key, final ByteStringBuilder value) throws DirectoryException
+ {
+ if (!putNoOverwrite(database, key, value))
+ {
+ final LocalizableMessage m = ERR_JEB_COMPSCHEMA_CANNOT_STORE_MULTIPLE_FAILURES.get();
+ throw new DirectoryException(DirectoryServer.getServerErrorResultCode(), m);
+ }
+ }
+
+ private boolean putNoOverwrite(final Database database, final byte[] key, final ByteStringBuilder value)
+ throws DirectoryException
+ {
+ final ByteString keyEntry = new ByteString(key);
+ final ByteString valueEntry = new ByteString(value.getBackingArray(), 0, value.length());
+ for (int i = 0; i < 3; i++)
+ {
+ try
+ {
+ final OperationStatus status = database.putNoOverwrite(null, keyEntry, valueEntry);
+ if (status != SUCCESS)
+ {
+ final LocalizableMessage m = ERR_JEB_COMPSCHEMA_CANNOT_STORE_STATUS.get(status);
+ throw new DirectoryException(DirectoryServer.getServerErrorResultCode(), m);
+ }
+ return true;
+ }
+ catch (final LockConflictException ce)
+ {
+ continue;
+ }
+ catch (final StorageRuntimeException de)
+ {
+ final LocalizableMessage m = ERR_JEB_COMPSCHEMA_CANNOT_STORE_EX.get(de.getMessage());
+ throw new DirectoryException(DirectoryServer.getServerErrorResultCode(), m, de);
+ }
+ }
+ return false;
+ }
+
+}
diff --git a/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/JebException.java b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/JebException.java
new file mode 100644
index 0000000..1329cfa
--- /dev/null
+++ b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/JebException.java
@@ -0,0 +1,90 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License"). You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at legal-notices/CDDLv1_0.txt
+ * or http://forgerock.org/license/CDDLv1.0.html.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at legal-notices/CDDLv1_0.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information:
+ * Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ *
+ *
+ * Copyright 2006-2009 Sun Microsystems, Inc.
+ * Portions Copyright 2014 ForgeRock AS
+ */
+package org.opends.server.backends.pluggable;
+
+
+
+import org.opends.server.types.IdentifiedException;
+import org.forgerock.i18n.LocalizableMessage;
+
+
+/**
+ * This class defines an exception that may be thrown if a problem occurs in the
+ * JE backend database.
+ */
+public class JebException
+ extends IdentifiedException
+{
+ /**
+ * The serial version identifier required to satisfy the compiler because this
+ * class extends <CODE>java.lang.Exception</CODE>, which implements the
+ * <CODE>java.io.Serializable</CODE> interface. This value was generated
+ * using the <CODE>serialver</CODE> command-line utility included with the
+ * Java SDK.
+ */
+ static final long serialVersionUID = 3110979454298870834L;
+
+
+
+ /**
+ * Creates a new JE backend exception.
+ */
+ public JebException()
+ {
+ super();
+ }
+
+
+
+ /**
+ * Creates a new JE backend exception with the provided message.
+ *
+ * @param message The message that explains the problem that occurred.
+ */
+ public JebException(LocalizableMessage message)
+ {
+ super(message);
+ }
+
+
+
+ /**
+ * Creates a new JE backend exception with the provided message and root
+ * cause.
+ *
+ * @param message The message that explains the problem that occurred.
+ * @param cause The exception that was caught to trigger this exception.
+ */
+ public JebException(LocalizableMessage message, Throwable cause)
+ {
+ super(message, cause);
+ }
+
+
+
+}
+
diff --git a/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/JebFormat.java b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/JebFormat.java
new file mode 100644
index 0000000..9096baa
--- /dev/null
+++ b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/JebFormat.java
@@ -0,0 +1,385 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License"). You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at legal-notices/CDDLv1_0.txt
+ * or http://forgerock.org/license/CDDLv1.0.html.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at legal-notices/CDDLv1_0.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information:
+ * Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ *
+ *
+ * Copyright 2006-2010 Sun Microsystems, Inc.
+ * Portions Copyright 2014 ForgeRock AS
+ */
+package org.opends.server.backends.pluggable;
+
+import java.util.Iterator;
+import java.util.TreeSet;
+
+import org.forgerock.opendj.ldap.ByteSequence;
+import org.forgerock.opendj.ldap.ByteString;
+import org.forgerock.opendj.ldap.ByteStringBuilder;
+import org.opends.server.types.DN;
+import org.opends.server.types.DirectoryException;
+import org.opends.server.types.RDN;
+import org.opends.server.util.StaticUtils;
+
+/**
+ * Handles the disk representation of LDAP data.
+ */
+public class JebFormat
+{
+
+ /**
+ * The format version used by this class to encode and decode a ByteString.
+ */
+ public static final byte FORMAT_VERSION = 0x01;
+
+ /**
+ * The ASN1 tag for the ByteString type.
+ */
+ public static final byte TAG_DATABASE_ENTRY = 0x60;
+
+ /**
+ * The ASN1 tag for the DirectoryServerEntry type.
+ */
+ public static final byte TAG_DIRECTORY_SERVER_ENTRY = 0x61;
+
+ /**
+ * Decode a long from a byte array, starting at start index and ending at end
+ * index.
+ *
+ * @param bytes
+ * The bytes value of the long.
+ * @param start
+ * the array index where to start computing the long
+ * @param end
+ * the array index exclusive where to end computing the long
+ * @return the long representation of the read bytes.
+ * @throws ArrayIndexOutOfBoundsException
+ * if the bytes array length is less than end.
+ */
+ public static long toLong(byte[] bytes, int start, int end)
+ throws ArrayIndexOutOfBoundsException
+ {
+ long v = 0;
+ for (int i = start; i < end; i++)
+ {
+ v <<= 8;
+ v |= (bytes[i] & 0xFF);
+ }
+ return v;
+ }
+
+ /**
+ * Decode an entry ID count from its database representation.
+ *
+ * @param bytes The database value of the entry ID count.
+ * @return The entry ID count.
+ * Cannot be negative if encoded with #entryIDUndefinedSizeToDatabase(long)
+ * @see #entryIDUndefinedSizeToDatabase(long)
+ */
+ public static long entryIDUndefinedSizeFromDatabase(byte[] bytes)
+ {
+ if(bytes == null)
+ {
+ return 0;
+ }
+
+ if(bytes.length == 8)
+ {
+ long v = 0;
+ v |= (bytes[0] & 0x7F);
+ for (int i = 1; i < 8; i++)
+ {
+ v <<= 8;
+ v |= (bytes[i] & 0xFF);
+ }
+ return v;
+ }
+ return Long.MAX_VALUE;
+ }
+
+ /**
+ * Decode an array of entry ID values from its database representation.
+ *
+ * @param bytes The raw database value, null if there is no value and
+ * hence no entry IDs. Note that this method will throw an
+ * ArrayIndexOutOfBoundsException if the bytes array length is
+ * not a multiple of 8.
+ * @return An array of entry ID values.
+ * @see #entryIDListToDatabase(long[])
+ */
+ public static long[] entryIDListFromDatabase(ByteSequence bytes)
+ {
+ int count = bytes.length() / 8;
+ long[] entryIDList = new long[count];
+ for (int pos = 0, i = 0; i < count; i++)
+ {
+ long v = 0;
+ v |= (bytes.byteAt(pos++) & 0xFFL) << 56;
+ v |= (bytes.byteAt(pos++) & 0xFFL) << 48;
+ v |= (bytes.byteAt(pos++) & 0xFFL) << 40;
+ v |= (bytes.byteAt(pos++) & 0xFFL) << 32;
+ v |= (bytes.byteAt(pos++) & 0xFFL) << 24;
+ v |= (bytes.byteAt(pos++) & 0xFFL) << 16;
+ v |= (bytes.byteAt(pos++) & 0xFFL) << 8;
+ v |= (bytes.byteAt(pos++) & 0xFFL);
+ entryIDList[i] = v;
+ }
+
+ return entryIDList;
+ }
+
+ /**
+ * Decode a integer array using the specified byte array read from DB.
+ *
+ * @param bytes The byte array.
+ * @return An integer array.
+ */
+ public static int[] intArrayFromDatabaseBytes(byte[] bytes) {
+ byte[] decodedBytes = bytes;
+
+ int count = decodedBytes.length / 8;
+ int[] entryIDList = new int[count];
+ for (int pos = 0, i = 0; i < count; i++) {
+ int v = 0;
+ pos +=4;
+ v |= (decodedBytes[pos++] & 0xFFL) << 24;
+ v |= (decodedBytes[pos++] & 0xFFL) << 16;
+ v |= (decodedBytes[pos++] & 0xFFL) << 8;
+ v |= (decodedBytes[pos++] & 0xFFL);
+ entryIDList[i] = v;
+ }
+
+ return entryIDList;
+ }
+
+ /**
+ * Encode an entry ID value to its database representation.
+ *
+ * @param id The entry ID value to be encoded.
+ * @return The encoded database value of the entry ID.
+ * @see #entryIDFromDatabase(byte[])
+ */
+ public static ByteString entryIDToDatabase(long id)
+ {
+ return ByteString.valueOf(id);
+ }
+
+ /**
+ * Encode an entry ID set count to its database representation.
+ *
+ * @param count The entry ID set count to be encoded.
+ * @return The encoded database value of the entry ID set count.
+ * @see #entryIDUndefinedSizeFromDatabase(byte[])
+ */
+ public static byte[] entryIDUndefinedSizeToDatabase(long count)
+ {
+ byte[] bytes = new byte[8];
+ long v = count;
+ for (int i = 7; i >= 1; i--)
+ {
+ bytes[i] = (byte) (v & 0xFF);
+ v >>>= 8;
+ }
+ bytes[0] = (byte) ((v | 0x80) & 0xFF);
+ return bytes;
+ }
+
+ /**
+ * Encode an array of entry ID values to its database representation.
+ *
+ * @param entryIDArray An array of entry ID values.
+ * @return The encoded database value.
+ * @see #entryIDListFromDatabase(byte[])
+ */
+ public static byte[] entryIDListToDatabase(long[] entryIDArray)
+ {
+ if (entryIDArray.length == 0)
+ {
+ // Zero values
+ return null;
+ }
+
+ byte[] bytes = new byte[8*entryIDArray.length];
+ for (int pos = 0, i = 0; i < entryIDArray.length; i++)
+ {
+ long v = entryIDArray[i];
+ bytes[pos++] = (byte) ((v >>> 56) & 0xFF);
+ bytes[pos++] = (byte) ((v >>> 48) & 0xFF);
+ bytes[pos++] = (byte) ((v >>> 40) & 0xFF);
+ bytes[pos++] = (byte) ((v >>> 32) & 0xFF);
+ bytes[pos++] = (byte) ((v >>> 24) & 0xFF);
+ bytes[pos++] = (byte) ((v >>> 16) & 0xFF);
+ bytes[pos++] = (byte) ((v >>> 8) & 0xFF);
+ bytes[pos++] = (byte) (v & 0xFF);
+ }
+
+ return bytes;
+ }
+
+ /**
+ * Decode a DN value from its database key representation.
+ *
+ * @param dnKey The database key value of the DN.
+ * @param prefix The DN to prefix the decoded DN value.
+ * @return The decoded DN value.
+ * @throws DirectoryException if an error occurs while decoding the DN value.
+ * @see #dnToDNKey(DN, int)
+ */
+ public static DN dnFromDNKey(ByteSequence dnKey, DN prefix) throws DirectoryException
+ {
+ DN dn = prefix;
+ boolean escaped = false;
+ ByteStringBuilder buffer = new ByteStringBuilder();
+ for(int i = 0; i < dnKey.length(); i++)
+ {
+ if(dnKey.byteAt(i) == 0x5C)
+ {
+ escaped = true;
+ continue;
+ }
+ else if(!escaped && dnKey.byteAt(i) == 0x01)
+ {
+ buffer.append(0x01);
+ escaped = false;
+ continue;
+ }
+ else if(!escaped && dnKey.byteAt(i) == 0x00)
+ {
+ if(buffer.length() > 0)
+ {
+ dn = dn.child(RDN.decode(buffer.toString()));
+ buffer.clear();
+ }
+ }
+ else
+ {
+ if(escaped)
+ {
+ buffer.append(0x5C);
+ escaped = false;
+ }
+ buffer.append(dnKey.byteAt(i));
+ }
+ }
+
+ if(buffer.length() > 0)
+ {
+ dn = dn.child(RDN.decode(buffer.toString()));
+ }
+
+ return dn;
+ }
+
+ /**
+ * Find the length of bytes that represents the superior DN of the given
+ * DN key. The superior DN is represented by the initial bytes of the DN key.
+ *
+ * @param dnKey The database key value of the DN.
+ * @param offset Starting position in the database key data.
+ * @param length The length of the database key data.
+ * @return The length of the superior DN or -1 if the given dn is the
+ * root DN or 0 if the superior DN is removed.
+ */
+ public static int findDNKeyParent(byte[] dnKey, int offset, int length)
+ {
+ if(length == 0)
+ {
+ // This is the root or base DN
+ return -1;
+ }
+
+ // We will walk backwords through the buffer and find the first
+ // unescaped comma
+ for(int i = offset+length - 1; i >= offset; i--)
+ {
+ if(dnKey[i] == 0x00 && i-1 >= offset && dnKey[i-1] != 0x5C)
+ {
+ return i;
+ }
+ }
+ return offset;
+ }
+
+ public static int findDNKeyParent(ByteSequence dnKey)
+ {
+ if (dnKey.length() == 0)
+ {
+ // This is the root or base DN
+ return -1;
+ }
+
+ // We will walk backwords through the buffer and find the first
+ // unescaped comma
+ for (int i = dnKey.length() - 1; i >= 0; i--)
+ {
+ if (dnKey.byteAt(i) == 0x00 && i - 1 >= 0 && dnKey.byteAt(i - 1) != 0x5C)
+ {
+ return i;
+ }
+ }
+ return 0;
+ }
+
+ /**
+ * Create a DN database key from an entry DN.
+ * @param dn The entry DN.
+ * @param prefixRDNs The number of prefix RDNs to remove from the encoded
+ * representation.
+ * @return A ByteString containing the key.
+ * @see #dnFromDNKey(byte[], int, int, DN)
+ */
+ public static ByteString dnToDNKey(DN dn, int prefixRDNs)
+ {
+ StringBuilder buffer = new StringBuilder();
+ for (int i = dn.size() - prefixRDNs - 1; i >= 0; i--)
+ {
+ buffer.append('\u0000');
+ formatRDNKey(dn.getRDN(i), buffer);
+ }
+
+ return ByteString.wrap(StaticUtils.getBytes(buffer.toString()));
+ }
+
+ private static void formatRDNKey(RDN rdn, StringBuilder buffer)
+ {
+ if (!rdn.isMultiValued())
+ {
+ rdn.toNormalizedString(buffer);
+ }
+ else
+ {
+ TreeSet<String> rdnElementStrings = new TreeSet<String>();
+
+ for (int i=0; i < rdn.getNumValues(); i++)
+ {
+ StringBuilder b2 = new StringBuilder();
+ rdn.getNormalizedAVAString(i, b2);
+ rdnElementStrings.add(b2.toString());
+ }
+
+ Iterator<String> iterator = rdnElementStrings.iterator();
+ buffer.append(iterator.next().replace("\u0001", "\\\u0001"));
+ while (iterator.hasNext())
+ {
+ buffer.append('\u0001');
+ buffer.append(iterator.next().replace("\u0001", "\\\u0001"));
+ }
+ }
+ }
+}
diff --git a/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/NullIndex.java b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/NullIndex.java
new file mode 100644
index 0000000..f10b99b
--- /dev/null
+++ b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/NullIndex.java
@@ -0,0 +1,266 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License"). You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at legal-notices/CDDLv1_0.txt
+ * or http://forgerock.org/license/CDDLv1.0.html.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at legal-notices/CDDLv1_0.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information:
+ * Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ *
+ * Copyright 2011-2014 ForgeRock AS
+ */
+package org.opends.server.backends.pluggable;
+
+import java.util.List;
+
+import org.forgerock.opendj.ldap.ByteSequence;
+import org.forgerock.opendj.ldap.ByteString;
+import org.forgerock.opendj.ldap.ConditionResult;
+import org.forgerock.opendj.ldap.spi.IndexingOptions;
+import org.opends.server.backends.pluggable.BackendImpl.ReadableStorage;
+import org.opends.server.backends.pluggable.BackendImpl.Storage;
+import org.opends.server.backends.pluggable.BackendImpl.StorageRuntimeException;
+import org.opends.server.backends.pluggable.BackendImpl.TreeName;
+import org.opends.server.backends.pluggable.BackendImpl.WriteableStorage;
+import org.opends.server.types.DirectoryException;
+import org.opends.server.types.Entry;
+import org.opends.server.types.Modification;
+
+import com.sleepycat.je.PreloadConfig;
+import com.sleepycat.je.PreloadStats;
+
+/**
+ * A null index which replaces id2children and id2subtree when they have been
+ * disabled.
+ */
+final class NullIndex extends Index
+{
+
+ /**
+ * Create a new null index object.
+ *
+ * @param name
+ * The name of the index database within the entryContainer.
+ * @param indexer
+ * The indexer object to construct index keys from LDAP attribute
+ * values.
+ * @param state
+ * The state database to persist index state info.
+ * @param storage
+ * The JE Storage
+ * @param entryContainer
+ * The database entryContainer holding this index.
+ * @throws StorageRuntimeException
+ * If an error occurs in the JE database.
+ */
+ public NullIndex(TreeName name, Indexer indexer, State state, Storage storage,
+ EntryContainer entryContainer) throws StorageRuntimeException
+ {
+ super(name, indexer, state, 0, 0, false, storage, entryContainer);
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ void updateKey(WriteableStorage txn, ByteString key, EntryIDSet deletedIDs,
+ EntryIDSet addedIDs) throws StorageRuntimeException
+ {
+ // Do nothing.
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void delete(IndexBuffer buffer, ByteString keyBytes)
+ {
+ // Do nothing.
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public ConditionResult containsID(ReadableStorage txn, ByteString key,
+ EntryID entryID) throws StorageRuntimeException
+ {
+ return ConditionResult.UNDEFINED;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public EntryIDSet readKey(ByteSequence key, ReadableStorage txn)
+ {
+ return new EntryIDSet();
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void writeKey(WriteableStorage txn, ByteString key,
+ EntryIDSet entryIDList) throws StorageRuntimeException
+ {
+ // Do nothing.
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public EntryIDSet readRange(ByteSequence lower, ByteSequence upper,
+ boolean lowerIncluded, boolean upperIncluded)
+ {
+ return new EntryIDSet();
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public int getEntryLimitExceededCount()
+ {
+ return 0;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void closeCursor() throws StorageRuntimeException
+ {
+ // Do nothing.
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void addEntry(IndexBuffer buffer, EntryID entryID, Entry entry, IndexingOptions options)
+ throws StorageRuntimeException, DirectoryException
+ {
+ // Do nothing.
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void removeEntry(IndexBuffer buffer, EntryID entryID, Entry entry, IndexingOptions options)
+ throws StorageRuntimeException, DirectoryException
+ {
+ // Do nothing.
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void modifyEntry(IndexBuffer buffer, EntryID entryID, Entry oldEntry,
+ Entry newEntry, List<Modification> mods, IndexingOptions options) throws StorageRuntimeException
+ {
+ // Do nothing.
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public boolean setIndexEntryLimit(int indexEntryLimit)
+ {
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public int getIndexEntryLimit()
+ {
+ return 0;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void setTrusted(WriteableStorage txn, boolean trusted)
+ throws StorageRuntimeException
+ {
+ // Do nothing.
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public boolean isTrusted()
+ {
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public boolean isRebuildRunning()
+ {
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void setRebuildStatus(boolean rebuildRunning)
+ {
+ // Do nothing.
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public boolean getMaintainCount()
+ {
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void open() throws StorageRuntimeException
+ {
+ // Do nothing.
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void close() throws StorageRuntimeException
+ {
+ // Do nothing.
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ protected void put(WriteableStorage txn, ByteSequence key, ByteSequence value) throws StorageRuntimeException
+ {
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ protected ByteString read(ReadableStorage txn, ByteSequence key, boolean isRMW) throws StorageRuntimeException
+ {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ protected boolean insert(WriteableStorage txn, ByteString key,
+ ByteString value) throws StorageRuntimeException
+ {
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ protected boolean delete(WriteableStorage txn, ByteSequence key)
+ throws StorageRuntimeException
+ {
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public long getRecordCount() throws StorageRuntimeException
+ {
+ return 0;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public PreloadStats preload(PreloadConfig config) throws StorageRuntimeException
+ {
+ return new PreloadStats();
+ }
+
+}
diff --git a/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/PresenceIndexer.java b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/PresenceIndexer.java
new file mode 100644
index 0000000..88e372e
--- /dev/null
+++ b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/PresenceIndexer.java
@@ -0,0 +1,117 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License"). You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at legal-notices/CDDLv1_0.txt
+ * or http://forgerock.org/license/CDDLv1.0.html.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at legal-notices/CDDLv1_0.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information:
+ * Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ *
+ *
+ * Copyright 2006-2008 Sun Microsystems, Inc.
+ * Portions Copyright 2014 ForgeRock AS
+ */
+package org.opends.server.backends.pluggable;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.forgerock.opendj.ldap.ByteString;
+import org.forgerock.opendj.ldap.spi.IndexingOptions;
+import org.opends.server.types.Attribute;
+import org.opends.server.types.AttributeType;
+import org.opends.server.types.Entry;
+import org.opends.server.types.Modification;
+
+/**
+ * An implementation of an Indexer for attribute presence.
+ */
+public class PresenceIndexer extends Indexer
+{
+ /** The key bytes used for the presence index. */
+ static final byte[] presenceKeyBytes = "+".getBytes();
+
+ /** The key bytes used for the presence index as a {@link ByteString}. */
+ static final ByteString presenceKey = ByteString.wrap(presenceKeyBytes);
+
+ /** The attribute type for which this instance will generate index keys. */
+ private AttributeType attributeType;
+
+ /**
+ * Create a new attribute presence indexer.
+ * @param attributeType The attribute type for which the indexer
+ * is required.
+ */
+ public PresenceIndexer(AttributeType attributeType)
+ {
+ this.attributeType = attributeType;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public String toString()
+ {
+ return attributeType.getNameOrOID() + ".presence";
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void indexEntry(Entry entry, Set<ByteString> keys, IndexingOptions options)
+ {
+ List<Attribute> attrList = entry.getAttribute(attributeType);
+ if (attrList != null)
+ {
+ if (!attrList.isEmpty())
+ {
+ keys.add(presenceKey);
+ }
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void replaceEntry(Entry oldEntry, Entry newEntry,
+ Map<ByteString, Boolean> modifiedKeys, IndexingOptions options)
+ {
+ modifyEntry(oldEntry, newEntry, Collections.<Modification>emptyList(), modifiedKeys, options);
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void modifyEntry(Entry oldEntry, Entry newEntry,
+ List<Modification> mods,
+ Map<ByteString, Boolean> modifiedKeys, IndexingOptions options)
+ {
+ List<Attribute> newAttributes = newEntry.getAttribute(attributeType, true);
+ List<Attribute> oldAttributes = oldEntry.getAttribute(attributeType, true);
+ if(oldAttributes == null)
+ {
+ if(newAttributes != null)
+ {
+ modifiedKeys.put(presenceKey, true);
+ }
+ }
+ else
+ {
+ if(newAttributes == null)
+ {
+ modifiedKeys.put(presenceKey, false);
+ }
+ }
+ }
+}
diff --git a/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/RebuildConfig.java b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/RebuildConfig.java
new file mode 100644
index 0000000..4b08862
--- /dev/null
+++ b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/RebuildConfig.java
@@ -0,0 +1,295 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License"). You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at legal-notices/CDDLv1_0.txt
+ * or http://forgerock.org/license/CDDLv1.0.html.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at legal-notices/CDDLv1_0.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information:
+ * Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ *
+ *
+ * Copyright 2006-2009 Sun Microsystems, Inc.
+ * Portions Copyright 2011-2014 ForgeRock AS
+ */
+package org.opends.server.backends.pluggable;
+
+import org.opends.server.types.DN;
+
+import java.util.ArrayList;
+
+/**
+ * Configuration for the indexType rebuild process.
+ */
+public class RebuildConfig
+{
+ /**
+ * Identifies how indexes will be selected for rebuild.
+ */
+ public static enum RebuildMode
+ {
+ /**
+ * Rebuild all indexes, including system indexes.
+ */
+ ALL,
+
+ /**
+ * Rebuild all degraded indexes, including system indexes.
+ */
+ DEGRADED,
+
+ /**
+ * Rebuild used defined list of indexes.
+ */
+ USER_DEFINED;
+ }
+
+ /**
+ * The base DN to rebuild.
+ */
+ private DN baseDN;
+
+ /**
+ * The names of indexes to rebuild.
+ */
+ private ArrayList<String> rebuildList;
+
+ private RebuildMode rebuildMode = RebuildMode.USER_DEFINED;
+
+ private String tmpDirectory;
+
+ private boolean isClearDegradedState;
+
+ /**
+ * Create a new rebuild configuration.
+ */
+ public RebuildConfig()
+ {
+ rebuildList = new ArrayList<String>();
+ }
+
+ /**
+ * Get the base DN to rebuild.
+ *
+ * @return The base DN to rebuild.
+ */
+ public DN getBaseDN()
+ {
+ return baseDN;
+ }
+
+ /**
+ * Set the base DN to rebuild.
+ *
+ * @param baseDN
+ * The base DN to rebuild.
+ */
+ public void setBaseDN(DN baseDN)
+ {
+ this.baseDN = baseDN;
+ }
+
+ /**
+ * Get the list of indexes to rebuild in this configuration.
+ *
+ * @return The list of indexes to rebuild.
+ */
+ public ArrayList<String> getRebuildList()
+ {
+ return rebuildList;
+ }
+
+ /**
+ * Add an index to be rebuilt into the configuration. Duplicate index names
+ * will be ignored. Adding an index that causes a mix of complete and partial
+ * rebuild for the same attribute index in the configuration will remove the
+ * partial and just keep the complete attribute index name. (ie. uid and
+ * uid.presence).
+ *
+ * @param index
+ * The index to add.
+ */
+ public void addRebuildIndex(String index)
+ {
+ String[] newIndexParts = index.split("\\.");
+
+ for (String s : new ArrayList<String>(rebuildList))
+ {
+ String[] existingIndexParts = s.split("\\.");
+ if (existingIndexParts[0].equalsIgnoreCase(newIndexParts[0]))
+ {
+ if (newIndexParts.length == 1 && existingIndexParts.length == 1)
+ {
+ return;
+ }
+ else if (newIndexParts.length > 1 && existingIndexParts.length == 1)
+ {
+ return;
+ }
+ else if (newIndexParts.length == 1 && existingIndexParts.length > 1)
+ {
+ rebuildList.remove(s);
+ }
+ else if (newIndexParts[1].equalsIgnoreCase(existingIndexParts[1]))
+ {
+ return;
+ }
+ }
+ }
+
+ this.rebuildList.add(index);
+ }
+
+ /**
+ * Check the given config for conflicts with this config. A conflict is
+ * detected if both configs specify the same indexType/database to be rebuilt.
+ *
+ * @param config
+ * The rebuild config to check against.
+ * @return the name of the indexType causing the conflict or null if no
+ * conflict is detected.
+ */
+ public String checkConflicts(RebuildConfig config)
+ {
+ //If they specify different base DNs, no conflicts can occur.
+ if (this.baseDN.equals(config.baseDN))
+ {
+ for (String thisIndex : this.rebuildList)
+ {
+ for (String thatIndex : config.rebuildList)
+ {
+ String[] existingIndexParts = thisIndex.split("\\.");
+ String[] newIndexParts = thatIndex.split("\\.");
+ if (existingIndexParts[0].equalsIgnoreCase(newIndexParts[0]))
+ {
+ if (newIndexParts.length == 1 && existingIndexParts.length == 1)
+ {
+ return thatIndex;
+ }
+ else if (newIndexParts.length > 1 && existingIndexParts.length == 1)
+ {
+ return thatIndex;
+ }
+ else if (newIndexParts.length == 1 && existingIndexParts.length > 1)
+ {
+ return thatIndex;
+ }
+ else if (newIndexParts[1].equalsIgnoreCase(existingIndexParts[1]))
+ {
+ return thatIndex;
+ }
+ }
+ }
+ }
+ }
+
+ return null;
+ }
+
+ /**
+ * Test if this rebuild config includes any system indexes to rebuild.
+ *
+ * @return True if rebuilding of system indexes are included. False otherwise.
+ * @throws InitializationException
+ */
+ public boolean includesSystemIndex()
+ {
+ for (String index : rebuildList)
+ {
+ // Removed because the id2entry is not A system indexes is THE
+ // primary system index. It cannot be rebuilt.
+ /*if (index.equalsIgnoreCase("id2entry"))
+ {
+ return true;
+ }*/
+ if (index.equalsIgnoreCase("dn2id"))
+ {
+ return true;
+ }
+ if (index.equalsIgnoreCase("dn2uri"))
+ {
+ return true;
+ }
+ }
+
+ return false;
+ }
+
+ /**
+ * Set the temporary directory to the specified path.
+ *
+ * @param path
+ * The path to set the temporary directory to.
+ */
+ public void setTmpDirectory(String path)
+ {
+ tmpDirectory = path;
+ }
+
+ /**
+ * Return the temporary directory path.
+ *
+ * @return The temporary directory string.
+ */
+ public String getTmpDirectory()
+ {
+ return tmpDirectory;
+ }
+
+ /**
+ * Sets the rebuild mode.
+ *
+ * @param mode
+ * The new rebuild mode.
+ */
+ public void setRebuildMode(RebuildMode mode)
+ {
+ rebuildMode = mode;
+ }
+
+ /**
+ * Returns the rebuild mode.
+ *
+ * @return The rebuild mode.
+ */
+ public RebuildMode getRebuildMode()
+ {
+ return rebuildMode;
+ }
+
+ /**
+ * Returns {@code true} if indexes should be forcefully marked as valid even
+ * if they are currently degraded.
+ *
+ * @return {@code true} if index should be forcefully marked as valid.
+ */
+ public boolean isClearDegradedState()
+ {
+ return isClearDegradedState;
+ }
+
+ /**
+ * Sets the 'clear degraded index' status.
+ *
+ * @param isClearDegradedState
+ * {@code true} if indexes should be forcefully marked as valid even
+ * if they are currently degraded.
+ */
+ public void isClearDegradedState(boolean isClearDegradedState)
+ {
+ this.isClearDegradedState = isClearDegradedState;
+ }
+
+}
diff --git a/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/RootContainer.java b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/RootContainer.java
new file mode 100644
index 0000000..0fe1368
--- /dev/null
+++ b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/RootContainer.java
@@ -0,0 +1,890 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License"). You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at legal-notices/CDDLv1_0.txt
+ * or http://forgerock.org/license/CDDLv1.0.html.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at legal-notices/CDDLv1_0.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information:
+ * Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ *
+ *
+ * Copyright 2006-2010 Sun Microsystems, Inc.
+ * Portions Copyright 2011-2014 ForgeRock AS
+ */
+package org.opends.server.backends.pluggable;
+
+import java.io.File;
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.forgerock.i18n.LocalizableMessage;
+import org.forgerock.i18n.slf4j.LocalizedLogger;
+import org.forgerock.opendj.config.server.ConfigException;
+import org.forgerock.opendj.ldap.ResultCode;
+import org.opends.server.admin.server.ConfigurationChangeListener;
+import org.opends.server.admin.std.server.LocalDBBackendCfg;
+import org.opends.server.api.Backend;
+import org.opends.server.backends.pluggable.BackendImpl.Storage;
+import org.opends.server.backends.pluggable.BackendImpl.StorageRuntimeException;
+import org.opends.server.core.DirectoryServer;
+import org.opends.server.monitors.DatabaseEnvironmentMonitor;
+import org.opends.server.types.ConfigChangeResult;
+import org.opends.server.types.DN;
+import org.opends.server.types.FilePermission;
+import org.opends.server.types.InitializationException;
+
+
+
+
+
+import static org.opends.messages.ConfigMessages.*;
+import static org.opends.messages.JebMessages.*;
+import static org.opends.server.util.StaticUtils.*;
+
+/**
+ * Wrapper class for the JE environment. Root container holds all the entry
+ * containers for each base DN. It also maintains all the openings and closings
+ * of the entry containers.
+ */
+public class RootContainer
+ implements ConfigurationChangeListener<LocalDBBackendCfg>
+{
+ private static final LocalizedLogger logger = LocalizedLogger.getLoggerForThisClass();
+
+ /** The JE database environment. */
+ private Storage storage;
+
+ /** Used to force a checkpoint during import. */
+ private final CheckpointConfig importForceCheckPoint = new CheckpointConfig();
+
+ /** The backend configuration. */
+ private LocalDBBackendCfg config;
+
+ /** The backend to which this entry root container belongs. */
+ private final Backend<?> backend;
+
+ /** The database environment monitor for this JE environment. */
+ private DatabaseEnvironmentMonitor monitor;
+
+ /** The base DNs contained in this root container. */
+ private final ConcurrentHashMap<DN, EntryContainer> entryContainers = new ConcurrentHashMap<DN, EntryContainer>();
+
+ /** The cached value of the next entry identifier to be assigned. */
+ private AtomicLong nextid = new AtomicLong(1);
+
+ /** The compressed schema manager for this backend. */
+ private JECompressedSchema compressedSchema;
+
+
+
+ /**
+ * Creates a new RootContainer object. Each root container represents a JE
+ * environment.
+ *
+ * @param config The configuration of the JE backend.
+ * @param backend A reference to the JE back end that is creating this
+ * root container.
+ */
+ public RootContainer(Backend<?> backend, LocalDBBackendCfg config)
+ {
+ this.backend = backend;
+ this.config = config;
+
+ getMonitorProvider().enableFilterUseStats(config.isIndexFilterAnalyzerEnabled());
+ getMonitorProvider().setMaxEntries(config.getIndexFilterAnalyzerMaxFilters());
+
+ config.addLocalDBChangeListener(this);
+ importForceCheckPoint.setForce(true);
+ }
+
+ /**
+ * Opens the root container using the JE configuration object provided.
+ *
+ * @param envConfig The JE environment configuration.
+ * @throws StorageRuntimeException If a database error occurs when creating
+ * the environment.
+ * @throws InitializationException If an initialization error occurs while
+ * creating the environment.
+ * @throws ConfigException If an configuration error occurs while
+ * creating the environment.
+ */
+ public void open(EnvironmentConfig envConfig)
+ throws StorageRuntimeException, InitializationException, ConfigException
+ {
+ // Determine the backend database directory.
+ File parentDirectory = getFileForPath(config.getDBDirectory());
+ File backendDirectory = new File(parentDirectory, config.getBackendId());
+
+ // Create the directory if it doesn't exist.
+ if (!backendDirectory.exists())
+ {
+ if(!backendDirectory.mkdirs())
+ {
+ LocalizableMessage message =
+ ERR_JEB_CREATE_FAIL.get(backendDirectory.getPath());
+ throw new ConfigException(message);
+ }
+ }
+ //Make sure the directory is valid.
+ else if (!backendDirectory.isDirectory())
+ {
+ throw new ConfigException(ERR_JEB_DIRECTORY_INVALID.get(backendDirectory.getPath()));
+ }
+
+ FilePermission backendPermission;
+ try
+ {
+ backendPermission =
+ FilePermission.decodeUNIXMode(config.getDBDirectoryPermissions());
+ }
+ catch(Exception e)
+ {
+ throw new ConfigException(ERR_CONFIG_BACKEND_MODE_INVALID.get(config.dn()));
+ }
+
+ //Make sure the mode will allow the server itself access to
+ //the database
+ if(!backendPermission.isOwnerWritable() ||
+ !backendPermission.isOwnerReadable() ||
+ !backendPermission.isOwnerExecutable())
+ {
+ LocalizableMessage message = ERR_CONFIG_BACKEND_INSANE_MODE.get(
+ config.getDBDirectoryPermissions());
+ throw new ConfigException(message);
+ }
+
+ // Get the backend database backendDirectory permissions and apply
+ if(FilePermission.canSetPermissions())
+ {
+ try
+ {
+ if(!FilePermission.setPermissions(backendDirectory, backendPermission))
+ {
+ logger.warn(WARN_JEB_UNABLE_SET_PERMISSIONS, backendPermission, backendDirectory);
+ }
+ }
+ catch(Exception e)
+ {
+ // Log an warning that the permissions were not set.
+ logger.warn(WARN_JEB_SET_PERMISSIONS_FAILED, backendDirectory, e);
+ }
+ }
+
+ // Open the database environment
+ storage = new Storage(backendDirectory, envConfig);
+
+ if (logger.isTraceEnabled())
+ {
+ logger.trace("JE (%s) environment opened with the following config: %n%s",
+ JEVersion.CURRENT_VERSION, storage.getConfig());
+
+ // Get current size of heap in bytes
+ long heapSize = Runtime.getRuntime().totalMemory();
+
+ // Get maximum size of heap in bytes. The heap cannot grow beyond this size.
+ // Any attempt will result in an OutOfMemoryException.
+ long heapMaxSize = Runtime.getRuntime().maxMemory();
+
+ // Get amount of free memory within the heap in bytes. This size will increase
+ // after garbage collection and decrease as new objects are created.
+ long heapFreeSize = Runtime.getRuntime().freeMemory();
+
+ logger.trace("Current size of heap: %d bytes", heapSize);
+ logger.trace("Max size of heap: %d bytes", heapMaxSize);
+ logger.trace("Free memory in heap: %d bytes", heapFreeSize);
+ }
+
+ compressedSchema = new JECompressedSchema(storage);
+ openAndRegisterEntryContainers(config.getBaseDN());
+ }
+
+ /**
+ * Opens the entry container for a base DN. If the entry container does not
+ * exist for the base DN, it will be created. The entry container will be
+ * opened with the same mode as the root container. Any entry containers
+ * opened in a read only root container will also be read only. Any entry
+ * containers opened in a non transactional root container will also be non
+ * transactional.
+ *
+ * @param baseDN The base DN of the entry container to open.
+ * @param name The name of the entry container or <CODE>NULL</CODE> to open
+ * the default entry container for the given base DN.
+ * @return The opened entry container.
+ * @throws StorageRuntimeException If an error occurs while opening the entry
+ * container.
+ * @throws ConfigException If an configuration error occurs while opening
+ * the entry container.
+ */
+ public EntryContainer openEntryContainer(DN baseDN, String name)
+ throws StorageRuntimeException, ConfigException
+ {
+ String databasePrefix;
+ if(name == null || name.equals(""))
+ {
+ databasePrefix = baseDN.toNormalizedString();
+ }
+ else
+ {
+ databasePrefix = name;
+ }
+
+ EntryContainer ec = new EntryContainer(baseDN, databasePrefix,
+ backend, config, storage, this);
+ ec.open();
+ return ec;
+ }
+
+ /**
+ * Registers the entry container for a base DN.
+ *
+ * @param baseDN The base DN of the entry container to close.
+ * @param entryContainer The entry container to register for the baseDN.
+ * @throws InitializationException If an error occurs while opening the
+ * entry container.
+ */
+ public void registerEntryContainer(DN baseDN, EntryContainer entryContainer)
+ throws InitializationException
+ {
+ EntryContainer ec1 = this.entryContainers.get(baseDN);
+
+ // If an entry container for this baseDN is already open we don't allow
+ // another to be opened.
+ if (ec1 != null)
+ {
+ throw new InitializationException(ERR_JEB_ENTRY_CONTAINER_ALREADY_REGISTERED.get(
+ ec1.getDatabasePrefix(), baseDN));
+ }
+
+ this.entryContainers.put(baseDN, entryContainer);
+ }
+
+ /**
+ * Opens the entry containers for multiple base DNs.
+ *
+ * @param baseDNs The base DNs of the entry containers to open.
+ * @throws StorageRuntimeException If a database error occurs while opening
+ * the entry container.
+ * @throws InitializationException If an initialization error occurs while
+ * opening the entry container.
+ * @throws ConfigException If a configuration error occurs while
+ * opening the entry container.
+ */
+ private void openAndRegisterEntryContainers(Set<DN> baseDNs)
+ throws StorageRuntimeException, InitializationException, ConfigException
+ {
+ EntryID id;
+ EntryID highestID = null;
+ for(DN baseDN : baseDNs)
+ {
+ EntryContainer ec = openEntryContainer(baseDN, null);
+ id = ec.getHighestEntryID();
+ registerEntryContainer(baseDN, ec);
+ if(highestID == null || id.compareTo(highestID) > 0)
+ {
+ highestID = id;
+ }
+ }
+
+ nextid = new AtomicLong(highestID.longValue() + 1);
+ }
+
+ /**
+ * Unregisters the entry container for a base DN.
+ *
+ * @param baseDN The base DN of the entry container to close.
+ * @return The entry container that was unregistered or NULL if a entry
+ * container for the base DN was not registered.
+ */
+ public EntryContainer unregisterEntryContainer(DN baseDN)
+ {
+ return entryContainers.remove(baseDN);
+ }
+
+ /**
+ * Retrieves the compressed schema manager for this backend.
+ *
+ * @return The compressed schema manager for this backend.
+ */
+ public JECompressedSchema getCompressedSchema()
+ {
+ return compressedSchema;
+ }
+
+ /**
+ * Get the DatabaseEnvironmentMonitor object for JE environment used by this
+ * root container.
+ *
+ * @return The DatabaseEnvironmentMonito object.
+ */
+ public DatabaseEnvironmentMonitor getMonitorProvider()
+ {
+ if(monitor == null)
+ {
+ String monitorName = backend.getBackendID() + " Database Storage";
+ monitor = new DatabaseEnvironmentMonitor(monitorName, this);
+ }
+
+ return monitor;
+ }
+
+ /**
+ * Preload the database cache. There is no preload if the configured preload
+ * time limit is zero.
+ *
+ * @param timeLimit The time limit for the preload process.
+ */
+ public void preload(long timeLimit)
+ {
+ if (timeLimit > 0)
+ {
+ // Get a list of all the databases used by the backend.
+ ArrayList<DatabaseContainer> dbList = new ArrayList<DatabaseContainer>();
+ for (EntryContainer ec : entryContainers.values())
+ {
+ ec.sharedLock.lock();
+ try
+ {
+ ec.listDatabases(dbList);
+ }
+ finally
+ {
+ ec.sharedLock.unlock();
+ }
+ }
+
+ // Sort the list in order of priority.
+ Collections.sort(dbList, new DbPreloadComparator());
+
+ // Preload each database until we reach the time limit or the cache
+ // is filled.
+ try
+ {
+ // Configure preload of Leaf Nodes (LNs) containing the data values.
+ PreloadConfig preloadConfig = new PreloadConfig();
+ preloadConfig.setLoadLNs(true);
+
+ logger.info(NOTE_JEB_CACHE_PRELOAD_STARTED, backend.getBackendID());
+
+ boolean isInterrupted = false;
+
+ long timeEnd = System.currentTimeMillis() + timeLimit;
+
+ for (DatabaseContainer db : dbList)
+ {
+ // Calculate the remaining time.
+ long timeRemaining = timeEnd - System.currentTimeMillis();
+ if (timeRemaining <= 0)
+ {
+ break;
+ }
+
+ preloadConfig.setMaxMillisecs(timeRemaining);
+ PreloadStats preloadStats = db.preload(preloadConfig);
+
+ if(logger.isTraceEnabled())
+ {
+ logger.trace("file=" + db.getName() + " LNs=" + preloadStats.getNLNsLoaded());
+ }
+
+ // Stop if the cache is full or the time limit has been exceeded.
+ PreloadStatus preloadStatus = preloadStats.getStatus();
+ if (preloadStatus != PreloadStatus.SUCCESS)
+ {
+ if (preloadStatus == PreloadStatus.EXCEEDED_TIME) {
+ logger.info(NOTE_JEB_CACHE_PRELOAD_INTERRUPTED_BY_TIME, backend.getBackendID(), db.getName());
+ } else if (preloadStatus == PreloadStatus.FILLED_CACHE) {
+ logger.info(NOTE_JEB_CACHE_PRELOAD_INTERRUPTED_BY_SIZE, backend.getBackendID(), db.getName());
+ } else {
+ logger.info(NOTE_JEB_CACHE_PRELOAD_INTERRUPTED_UNKNOWN, backend.getBackendID(), db.getName());
+ }
+
+ isInterrupted = true;
+ break;
+ }
+
+ logger.info(NOTE_JEB_CACHE_DB_PRELOADED, db.getName());
+ }
+
+ if (!isInterrupted) {
+ logger.info(NOTE_JEB_CACHE_PRELOAD_DONE, backend.getBackendID());
+ }
+
+ // Log an informational message about the size of the cache.
+ EnvironmentStats stats = storage.getStats(new StatsConfig());
+ long total = stats.getCacheTotalBytes();
+
+ logger.info(NOTE_JEB_CACHE_SIZE_AFTER_PRELOAD, total / (1024 * 1024));
+ }
+ catch (StorageRuntimeException e)
+ {
+ logger.traceException(e);
+
+ logger.error(ERR_JEB_CACHE_PRELOAD, backend.getBackendID(),
+ stackTraceToSingleLineString(e.getCause() != null ? e.getCause() : e));
+ }
+ }
+ }
+
+ /**
+ * Closes this root container.
+ *
+ * @throws StorageRuntimeException If an error occurs while attempting to close
+ * the root container.
+ */
+ public void close() throws StorageRuntimeException
+ {
+ for(DN baseDN : entryContainers.keySet())
+ {
+ EntryContainer ec = unregisterEntryContainer(baseDN);
+ ec.exclusiveLock.lock();
+ try
+ {
+ ec.close();
+ }
+ finally
+ {
+ ec.exclusiveLock.unlock();
+ }
+ }
+
+ compressedSchema.close();
+ config.removeLocalDBChangeListener(this);
+
+ if (storage != null)
+ {
+ storage.close();
+ storage = null;
+ }
+ }
+
+ /**
+ * Return all the entry containers in this root container.
+ *
+ * @return The entry containers in this root container.
+ */
+ public Collection<EntryContainer> getEntryContainers()
+ {
+ return entryContainers.values();
+ }
+
+ /**
+ * Returns all the baseDNs this root container stores.
+ *
+ * @return The set of DNs this root container stores.
+ */
+ public Set<DN> getBaseDNs()
+ {
+ return entryContainers.keySet();
+ }
+
+ /**
+ * Return the entry container for a specific base DN.
+ *
+ * @param baseDN The base DN of the entry container to retrieve.
+ * @return The entry container for the base DN.
+ */
+ public EntryContainer getEntryContainer(DN baseDN)
+ {
+ EntryContainer ec = null;
+ DN nodeDN = baseDN;
+
+ while (ec == null && nodeDN != null)
+ {
+ ec = entryContainers.get(nodeDN);
+ if (ec == null)
+ {
+ nodeDN = nodeDN.getParentDNInSuffix();
+ }
+ }
+
+ return ec;
+ }
+
+ /**
+ * Get the environment stats of the JE environment used in this root
+ * container.
+ *
+ * @param statsConfig The configuration to use for the EnvironmentStats
+ * object.
+ * @return The environment status of the JE environment.
+ * @throws StorageRuntimeException If an error occurs while retrieving the stats
+ * object.
+ */
+ public EnvironmentStats getEnvironmentStats(StatsConfig statsConfig)
+ throws StorageRuntimeException
+ {
+ return storage.getStats(statsConfig);
+ }
+
+ /**
+ * Get the environment transaction stats of the JE environment used
+ * in this root container.
+ *
+ * @param statsConfig The configuration to use for the EnvironmentStats
+ * object.
+ * @return The environment status of the JE environment.
+ * @throws StorageRuntimeException If an error occurs while retrieving the stats
+ * object.
+ */
+ public TransactionStats getEnvironmentTransactionStats(
+ StatsConfig statsConfig) throws StorageRuntimeException
+ {
+ return storage.getTransactionStats(statsConfig);
+ }
+
+ /**
+ * Get the environment config of the JE environment used in this root
+ * container.
+ *
+ * @return The environment config of the JE environment.
+ * @throws StorageRuntimeException If an error occurs while retrieving the
+ * configuration object.
+ */
+ public EnvironmentConfig getEnvironmentConfig() throws StorageRuntimeException
+ {
+ return storage.getConfig();
+ }
+
+ /**
+ * Get the backend configuration used by this root container.
+ *
+ * @return The JE backend configuration used by this root container.
+ */
+ public LocalDBBackendCfg getConfiguration()
+ {
+ return config;
+ }
+
+ /**
+ * Get the total number of entries in this root container.
+ *
+ * @return The number of entries in this root container
+ * @throws StorageRuntimeException If an error occurs while retrieving the entry
+ * count.
+ */
+ public long getEntryCount() throws StorageRuntimeException
+ {
+ long entryCount = 0;
+ for(EntryContainer ec : this.entryContainers.values())
+ {
+ ec.sharedLock.lock();
+ try
+ {
+ entryCount += ec.getEntryCount();
+ }
+ finally
+ {
+ ec.sharedLock.unlock();
+ }
+ }
+
+ return entryCount;
+ }
+
+ /**
+ * Assign the next entry ID.
+ *
+ * @return The assigned entry ID.
+ */
+ public EntryID getNextEntryID()
+ {
+ return new EntryID(nextid.getAndIncrement());
+ }
+
+ /**
+ * Return the lowest entry ID assigned.
+ *
+ * @return The lowest entry ID assigned.
+ */
+ public Long getLowestEntryID()
+ {
+ return 1L;
+ }
+
+ /**
+ * Resets the next entry ID counter to zero. This should only be used after
+ * clearing all databases.
+ */
+ public void resetNextEntryID()
+ {
+ nextid.set(1);
+ }
+
+
+
+ /** {@inheritDoc} */
+ @Override
+ public boolean isConfigurationChangeAcceptable(
+ LocalDBBackendCfg cfg,
+ List<LocalizableMessage> unacceptableReasons)
+ {
+ boolean acceptable = true;
+
+ File parentDirectory = getFileForPath(config.getDBDirectory());
+ File backendDirectory = new File(parentDirectory, config.getBackendId());
+
+ //Make sure the directory either already exists or is able to create.
+ if (!backendDirectory.exists())
+ {
+ if(!backendDirectory.mkdirs())
+ {
+ unacceptableReasons.add(ERR_JEB_CREATE_FAIL.get(backendDirectory.getPath()));
+ acceptable = false;
+ }
+ else
+ {
+ backendDirectory.delete();
+ }
+ }
+ //Make sure the directory is valid.
+ else if (!backendDirectory.isDirectory())
+ {
+ unacceptableReasons.add(ERR_JEB_DIRECTORY_INVALID.get(backendDirectory.getPath()));
+ acceptable = false;
+ }
+
+ try
+ {
+ FilePermission newBackendPermission =
+ FilePermission.decodeUNIXMode(cfg.getDBDirectoryPermissions());
+
+ //Make sure the mode will allow the server itself access to
+ //the database
+ if(!newBackendPermission.isOwnerWritable() ||
+ !newBackendPermission.isOwnerReadable() ||
+ !newBackendPermission.isOwnerExecutable())
+ {
+ LocalizableMessage message = ERR_CONFIG_BACKEND_INSANE_MODE.get(
+ cfg.getDBDirectoryPermissions());
+ unacceptableReasons.add(message);
+ acceptable = false;
+ }
+ }
+ catch(Exception e)
+ {
+ unacceptableReasons.add(ERR_CONFIG_BACKEND_MODE_INVALID.get(cfg.dn()));
+ acceptable = false;
+ }
+
+ try
+ {
+ ConfigurableEnvironment.parseConfigEntry(cfg);
+ }
+ catch (Exception e)
+ {
+ unacceptableReasons.add(LocalizableMessage.raw(e.getLocalizedMessage()));
+ acceptable = false;
+ }
+
+ return acceptable;
+ }
+
+
+
+ /** {@inheritDoc} */
+ @Override
+ public ConfigChangeResult applyConfigurationChange(LocalDBBackendCfg cfg)
+ {
+ boolean adminActionRequired = false;
+ ArrayList<LocalizableMessage> messages = new ArrayList<LocalizableMessage>();
+
+ try
+ {
+ if(storage != null)
+ {
+ // Check if any JE non-mutable properties were changed.
+ EnvironmentConfig oldEnvConfig = storage.getConfig();
+ EnvironmentConfig newEnvConfig =
+ ConfigurableEnvironment.parseConfigEntry(cfg);
+ Map<?,?> paramsMap = EnvironmentParams.SUPPORTED_PARAMS;
+
+ // Iterate through native JE properties.
+ SortedSet<String> jeProperties = cfg.getJEProperty();
+ for (String jeEntry : jeProperties) {
+ // There is no need to validate properties yet again.
+ StringTokenizer st = new StringTokenizer(jeEntry, "=");
+ if (st.countTokens() == 2) {
+ String jePropertyName = st.nextToken();
+ String jePropertyValue = st.nextToken();
+ ConfigParam param = (ConfigParam) paramsMap.get(jePropertyName);
+ if (!param.isMutable()) {
+ String oldValue = oldEnvConfig.getConfigParam(param.getName());
+ if (!oldValue.equalsIgnoreCase(jePropertyValue)) {
+ adminActionRequired = true;
+ messages.add(INFO_CONFIG_JE_PROPERTY_REQUIRES_RESTART.get(jePropertyName));
+ if(logger.isTraceEnabled()) {
+ logger.trace("The change to the following property " +
+ "will take effect when the component is restarted: " +
+ jePropertyName);
+ }
+ }
+ }
+ }
+ }
+
+ // Iterate through JE configuration attributes.
+ for (Object o : paramsMap.values())
+ {
+ ConfigParam param = (ConfigParam) o;
+ if (!param.isMutable())
+ {
+ String oldValue = oldEnvConfig.getConfigParam(param.getName());
+ String newValue = newEnvConfig.getConfigParam(param.getName());
+ if (!oldValue.equalsIgnoreCase(newValue))
+ {
+ adminActionRequired = true;
+ String configAttr = ConfigurableEnvironment.
+ getAttributeForProperty(param.getName());
+ if (configAttr != null)
+ {
+ messages.add(NOTE_JEB_CONFIG_ATTR_REQUIRES_RESTART.get(configAttr));
+ }
+ else
+ {
+ messages.add(NOTE_JEB_CONFIG_ATTR_REQUIRES_RESTART.get(param.getName()));
+ }
+ if(logger.isTraceEnabled())
+ {
+ logger.trace("The change to the following property will " +
+ "take effect when the backend is restarted: " +
+ param.getName());
+ }
+ }
+ }
+ }
+
+ // This takes care of changes to the JE environment for those
+ // properties that are mutable at runtime.
+ storage.setMutableConfig(newEnvConfig);
+
+ logger.trace("JE database configuration: %s", storage.getConfig());
+ }
+
+ // Create the directory if it doesn't exist.
+ if(!cfg.getDBDirectory().equals(this.config.getDBDirectory()))
+ {
+ File parentDirectory = getFileForPath(cfg.getDBDirectory());
+ File backendDirectory =
+ new File(parentDirectory, cfg.getBackendId());
+
+ if (!backendDirectory.exists())
+ {
+ if(!backendDirectory.mkdirs())
+ {
+ messages.add(ERR_JEB_CREATE_FAIL.get(backendDirectory.getPath()));
+ return new ConfigChangeResult(
+ DirectoryServer.getServerErrorResultCode(),
+ adminActionRequired,
+ messages);
+ }
+ }
+ //Make sure the directory is valid.
+ else if (!backendDirectory.isDirectory())
+ {
+ messages.add(ERR_JEB_DIRECTORY_INVALID.get(backendDirectory.getPath()));
+ return new ConfigChangeResult(
+ DirectoryServer.getServerErrorResultCode(),
+ adminActionRequired,
+ messages);
+ }
+
+ adminActionRequired = true;
+ messages.add(NOTE_JEB_CONFIG_DB_DIR_REQUIRES_RESTART.get(
+ this.config.getDBDirectory(), cfg.getDBDirectory()));
+ }
+
+ if(!cfg.getDBDirectoryPermissions().equalsIgnoreCase(
+ config.getDBDirectoryPermissions()) ||
+ !cfg.getDBDirectory().equals(this.config.getDBDirectory()))
+ {
+ FilePermission backendPermission;
+ try
+ {
+ backendPermission =
+ FilePermission.decodeUNIXMode(cfg.getDBDirectoryPermissions());
+ }
+ catch(Exception e)
+ {
+ messages.add(ERR_CONFIG_BACKEND_MODE_INVALID.get(config.dn()));
+ return new ConfigChangeResult(
+ DirectoryServer.getServerErrorResultCode(),
+ adminActionRequired,
+ messages);
+ }
+
+ //Make sure the mode will allow the server itself access to
+ //the database
+ if(!backendPermission.isOwnerWritable() ||
+ !backendPermission.isOwnerReadable() ||
+ !backendPermission.isOwnerExecutable())
+ {
+ messages.add(ERR_CONFIG_BACKEND_INSANE_MODE.get(
+ cfg.getDBDirectoryPermissions()));
+ return new ConfigChangeResult(
+ DirectoryServer.getServerErrorResultCode(),
+ adminActionRequired,
+ messages);
+ }
+
+ // Get the backend database backendDirectory permissions and apply
+ if(FilePermission.canSetPermissions())
+ {
+ File parentDirectory = getFileForPath(config.getDBDirectory());
+ File backendDirectory = new File(parentDirectory, config.getBackendId());
+ try
+ {
+ if (!FilePermission.setPermissions(backendDirectory, backendPermission))
+ {
+ logger.warn(WARN_JEB_UNABLE_SET_PERMISSIONS, backendPermission, backendDirectory);
+ }
+ }
+ catch(Exception e)
+ {
+ // Log an warning that the permissions were not set.
+ logger.warn(WARN_JEB_SET_PERMISSIONS_FAILED, backendDirectory, e);
+ }
+ }
+ }
+
+ getMonitorProvider().enableFilterUseStats(
+ cfg.isIndexFilterAnalyzerEnabled());
+ getMonitorProvider()
+ .setMaxEntries(cfg.getIndexFilterAnalyzerMaxFilters());
+
+ this.config = cfg;
+ }
+ catch (Exception e)
+ {
+ messages.add(LocalizableMessage.raw(stackTraceToSingleLineString(e)));
+ return new ConfigChangeResult(DirectoryServer.getServerErrorResultCode(),
+ adminActionRequired,
+ messages);
+ }
+
+ return new ConfigChangeResult(ResultCode.SUCCESS, adminActionRequired, messages);
+ }
+
+ /**
+ * Returns whether this container JE database environment is
+ * open, valid and can be used.
+ *
+ * @return {@code true} if valid, or {@code false} otherwise.
+ */
+ public boolean isValid() {
+ return storage.isValid();
+ }
+}
diff --git a/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/SortValues.java b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/SortValues.java
new file mode 100644
index 0000000..b2cae8e
--- /dev/null
+++ b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/SortValues.java
@@ -0,0 +1,275 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License"). You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at legal-notices/CDDLv1_0.txt
+ * or http://forgerock.org/license/CDDLv1.0.html.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at legal-notices/CDDLv1_0.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information:
+ * Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ *
+ *
+ * Copyright 2008 Sun Microsystems, Inc.
+ * Portions Copyright 2014 ForgeRock AS
+ */
+package org.opends.server.backends.pluggable;
+
+import java.util.List;
+
+import org.forgerock.opendj.ldap.ByteString;
+import org.opends.server.types.Attribute;
+import org.opends.server.types.AttributeType;
+import org.opends.server.types.Entry;
+import org.opends.server.types.SortKey;
+import org.opends.server.types.SortOrder;
+
+/**
+ * This class defines a data structure that holds a set of attribute values that
+ * are associated with a sort order for a given entry. Any or all of the
+ * attribute values may be {@code null} if the entry does not include any values
+ * for the attribute type targeted by the corresponding sort key.
+ * <BR><BR>
+ * This class implements the {@code Comparable} interface and may therefore be
+ * used to order the elements in components like {@code TreeMap} and
+ * {@code TreeSet}.
+ * <p>
+ * FIXME: replace with the SDK's SortKey?
+ */
+public class SortValues
+ implements Comparable<SortValues>
+{
+ /** The set of sort keys (attribute values) in this sort order. */
+ private ByteString[] values;
+ /**
+ * The types of sort keys.
+ *
+ * @see #values
+ */
+ private AttributeType[] types;
+
+ /** The entry ID for the entry associated with this sort values. */
+ private EntryID entryID;
+
+ /** The sort order for this set of sort values. */
+ private SortOrder sortOrder;
+
+
+
+ /**
+ * Creates a new sort values object with the provided information.
+ *
+ * @param entryID The entry ID for the entry associated with this set of
+ * values.
+ * @param values The attribute values for this sort values.
+ * @param sortOrder The sort order to use to obtain the necessary values.
+ */
+ public SortValues(EntryID entryID, ByteString[] values,
+ SortOrder sortOrder)
+ {
+ this.entryID = entryID;
+ this.sortOrder = sortOrder;
+ this.values = values;
+
+ final SortKey[] sortKeys = sortOrder.getSortKeys();
+ this.types = new AttributeType[sortKeys.length];
+ for (int i = 0; i < sortKeys.length; i++)
+ {
+ types[i] = sortKeys[i].getAttributeType();
+ }
+ }
+
+ /**
+ * Creates a new sort values object with the provided information.
+ *
+ * @param entryID The entry ID for the entry associated with this set of
+ * values.
+ * @param entry The entry containing the values to extract and use when
+ * sorting.
+ * @param sortOrder The sort order to use to obtain the necessary values.
+ */
+ public SortValues(EntryID entryID, Entry entry, SortOrder sortOrder)
+ {
+ this.entryID = entryID;
+ this.sortOrder = sortOrder;
+
+ SortKey[] sortKeys = sortOrder.getSortKeys();
+ this.values = new ByteString[sortKeys.length];
+ this.types = new AttributeType[sortKeys.length];
+ for (int i=0; i < sortKeys.length; i++)
+ {
+ SortKey sortKey = sortKeys[i];
+ types[i] = sortKey.getAttributeType();
+ List<Attribute> attrList = entry.getAttribute(types[i]);
+ if (attrList != null)
+ {
+ values[i] = findBestMatchingValue(sortKey, attrList);
+ }
+ }
+ }
+
+ /**
+ * Finds the best matching attribute value for the provided sort key in the
+ * provided attribute list.
+ * <p>
+ * There may be multiple versions of this attribute in the target entry (e.g.,
+ * with different sets of options), and it may also be a multivalued
+ * attribute. In that case, we need to find the value that is the best match
+ * for the corresponding sort key (i.e., for sorting in ascending order, we
+ * want to find the lowest value; for sorting in descending order, we want to
+ * find the highest value). This is handled by the SortKey.compareValues
+ * method.
+ */
+ private ByteString findBestMatchingValue(SortKey sortKey, List<Attribute> attrList)
+ {
+ ByteString sortValue = null;
+ for (Attribute a : attrList)
+ {
+ for (ByteString v : a)
+ {
+ if (sortValue == null || sortKey.compareValues(v, sortValue) < 0)
+ {
+ sortValue = v;
+ }
+ }
+ }
+ return sortValue;
+ }
+
+ /**
+ * Compares this set of sort values with the provided set of values to
+ * determine their relative order in a sorted list.
+ *
+ * @param sortValues The set of values to compare against this sort values.
+ * It must also have the same sort order as this set of
+ * values.
+ *
+ * @return A negative value if this sort values object should come before the
+ * provided values in a sorted list, a positive value if this sort
+ * values object should come after the provided values in a sorted
+ * list, or zero if there is no significant difference in their
+ * relative order.
+ */
+ @Override
+ public int compareTo(SortValues sortValues)
+ {
+ SortKey[] sortKeys = sortOrder.getSortKeys();
+
+ for (int i=0; i < values.length; i++)
+ {
+ int compareValue = sortKeys[i].compareValues(values[i], sortValues.values[i]);
+ if (compareValue != 0)
+ {
+ return compareValue;
+ }
+ }
+
+ // If we've gotten here, then we can't tell a difference between the sets of
+ // sort values, so sort based on entry ID.
+ return entryID.compareTo(sortValues.entryID);
+ }
+
+ /**
+ * Compares the first element in this set of sort values with the provided
+ * assertion value to determine whether the assertion value is greater than or
+ * equal to the initial sort value. This is used during VLV processing to
+ * find the offset by assertion value.
+ *
+ * @param assertionValue The assertion value to compare against the first
+ * sort value.
+ *
+ * @return A negative value if the provided assertion value should come
+ * before the first sort value, zero if the provided assertion value
+ * is equal to the first sort value, or a positive value if the
+ * provided assertion value should come after the first sort value.
+ */
+ public int compareTo(ByteString assertionValue)
+ {
+ SortKey sortKey = sortOrder.getSortKeys()[0];
+ return sortKey.compareValues(values[0], assertionValue);
+ }
+
+ /**
+ * Retrieves a string representation of this sort values object.
+ *
+ * @return A string representation of this sort values object.
+ */
+ @Override
+ public String toString()
+ {
+ StringBuilder buffer = new StringBuilder();
+ toString(buffer);
+ return buffer.toString();
+ }
+
+ /**
+ * Appends a string representation of this sort values object to the provided
+ * buffer.
+ *
+ * @param buffer The buffer to which the information should be appended.
+ */
+ public void toString(StringBuilder buffer)
+ {
+ buffer.append("SortValues(");
+
+ SortKey[] sortKeys = sortOrder.getSortKeys();
+ for (int i=0; i < sortKeys.length; i++)
+ {
+ if (i > 0)
+ {
+ buffer.append(",");
+ }
+
+ buffer.append(sortKeys[i].ascending() ? "+" : "-");
+
+ buffer.append(sortKeys[i].getAttributeType().getNameOrOID());
+ buffer.append("=");
+ buffer.append(values[i]);
+ }
+
+ buffer.append(", id=");
+ buffer.append(entryID);
+ buffer.append(")");
+ }
+
+ /**
+ * Retrieve the attribute values in this sort values.
+ *
+ * @return The array of attribute values for this sort values.
+ */
+ public ByteString[] getValues()
+ {
+ return values;
+ }
+
+ /**
+ * Retrieve the type of the attribute values in this sort values.
+ *
+ * @return The array of type of the attribute values for this sort values.
+ */
+ public AttributeType[] getTypes()
+ {
+ return types;
+ }
+
+ /**
+ * Retrieve the entry ID in this sort values.
+ *
+ * @return The entry ID for this sort values.
+ */
+ public long getEntryID()
+ {
+ return entryID.longValue();
+ }
+}
diff --git a/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/SortValuesSet.java b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/SortValuesSet.java
new file mode 100644
index 0000000..824f32a
--- /dev/null
+++ b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/SortValuesSet.java
@@ -0,0 +1,699 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License"). You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at legal-notices/CDDLv1_0.txt
+ * or http://forgerock.org/license/CDDLv1.0.html.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at legal-notices/CDDLv1_0.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information:
+ * Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ *
+ *
+ * Copyright 2006-2008 Sun Microsystems, Inc.
+ * Portions Copyright 2014 ForgeRock AS
+ */
+package org.opends.server.backends.pluggable;
+
+import org.forgerock.opendj.ldap.ByteString;
+import org.forgerock.opendj.ldap.ByteStringBuilder;
+import org.forgerock.opendj.ldap.DecodeException;
+import org.forgerock.opendj.ldap.ResultCode;
+import org.forgerock.opendj.ldap.schema.MatchingRule;
+import org.opends.server.backends.pluggable.BackendImpl.StorageRuntimeException;
+import org.opends.server.types.AttributeType;
+import org.opends.server.types.DirectoryException;
+import org.opends.server.types.SortKey;
+
+import static org.opends.server.backends.pluggable.JebFormat.*;
+
+
+
+/**
+ * This class represents a partial sorted set of sorted entries in a VLV
+ * index.
+ */
+public class SortValuesSet
+{
+ private long[] entryIDs;
+
+ private int[] valuesBytesOffsets;
+ private byte[] valuesBytes;
+
+ private ByteString key;
+
+ private VLVIndex vlvIndex;
+
+ /**
+ * Construct an empty sort values set with the given information.
+ *
+ * @param vlvIndex The VLV index using this set.
+ */
+ public SortValuesSet(VLVIndex vlvIndex)
+ {
+ this.key = ByteString.empty();
+ this.entryIDs = null;
+ this.valuesBytes = null;
+ this.valuesBytesOffsets = null;
+ this.vlvIndex = vlvIndex;
+ }
+
+ /**
+ * Construct a sort values set from the database.
+ *
+ * @param key The database key used to locate this set.
+ * @param value The bytes to decode and construct this set.
+ * @param vlvIndex The VLV index using this set.
+ */
+ public SortValuesSet(ByteString key, ByteString value, VLVIndex vlvIndex)
+ {
+ this.key = key;
+ this.vlvIndex = vlvIndex;
+ if(value == null)
+ {
+ entryIDs = new long[0];
+ return;
+ }
+
+ entryIDs = getEncodedIDs(value, 0);
+ int valuesBytesOffset = entryIDs.length * 8 + 4;
+ int valuesBytesLength = value.length() - valuesBytesOffset;
+ valuesBytes = new byte[valuesBytesLength];
+ System.arraycopy(value, valuesBytesOffset, valuesBytes, 0,
+ valuesBytesLength);
+ this.valuesBytesOffsets = null;
+ }
+
+ private SortValuesSet()
+ {}
+
+ /**
+ * Add the given entryID and values from this VLV index.
+ *
+ * @param entryID The entry ID to add.
+ * @param values The values to add.
+ * @param types The types of the values to add.
+ * @return True if the information was successfully added or False
+ * otherwise.
+ * @throws DirectoryException If a Directory Server error occurs.
+ * @throws StorageRuntimeException If an error occurs in the JE database.
+ */
+ public boolean add(long entryID, ByteString[] values, AttributeType[] types)
+ throws StorageRuntimeException, DirectoryException
+ {
+ if(values == null)
+ {
+ return false;
+ }
+
+ if(entryIDs == null || entryIDs.length == 0)
+ {
+ entryIDs = new long[] { entryID };
+ valuesBytes = attributeValuesToDatabase(values, types);
+ if(valuesBytesOffsets != null)
+ {
+ valuesBytesOffsets = new int[] { 0 };
+ }
+ return true;
+ }
+ if (vlvIndex.comparator.compare(
+ this, entryIDs.length - 1, entryID, values) < 0)
+ {
+ long[] updatedEntryIDs = new long[entryIDs.length + 1];
+ System.arraycopy(entryIDs, 0, updatedEntryIDs, 0, entryIDs.length);
+ updatedEntryIDs[entryIDs.length] = entryID;
+
+ byte[] newValuesBytes = attributeValuesToDatabase(values, types);
+ byte[] updatedValuesBytes = new byte[valuesBytes.length +
+ newValuesBytes.length];
+ System.arraycopy(valuesBytes, 0, updatedValuesBytes, 0,
+ valuesBytes.length);
+ System.arraycopy(newValuesBytes, 0, updatedValuesBytes,
+ valuesBytes.length,
+ newValuesBytes.length);
+
+ if(valuesBytesOffsets != null)
+ {
+ int[] updatedValuesBytesOffsets =
+ new int[valuesBytesOffsets.length + 1];
+ System.arraycopy(valuesBytesOffsets, 0, updatedValuesBytesOffsets,
+ 0, valuesBytesOffsets.length);
+ updatedValuesBytesOffsets[valuesBytesOffsets.length] =
+ updatedValuesBytes.length - newValuesBytes.length;
+ valuesBytesOffsets = updatedValuesBytesOffsets;
+ }
+
+ entryIDs = updatedEntryIDs;
+ valuesBytes = updatedValuesBytes;
+ return true;
+ }
+ else
+ {
+ int pos = binarySearch(entryID, values);
+ if(pos >= 0)
+ {
+ if(entryIDs[pos] == entryID)
+ {
+ // The entry ID is alreadly present.
+ return false;
+ }
+ }
+ else
+ {
+ // For a negative return value r, the vlvIndex -(r+1) gives the array
+ // ndex at which the specified value can be inserted to maintain
+ // the sorted order of the array.
+ pos = -(pos+1);
+ }
+
+ long[] updatedEntryIDs = new long[entryIDs.length + 1];
+ System.arraycopy(entryIDs, 0, updatedEntryIDs, 0, pos);
+ System.arraycopy(entryIDs, pos, updatedEntryIDs, pos+1,
+ entryIDs.length-pos);
+ updatedEntryIDs[pos] = entryID;
+
+ byte[] newValuesBytes = attributeValuesToDatabase(values, types);
+ // BUG valuesBytesOffsets might be null ? If not why testing below ?
+ int valuesPos = valuesBytesOffsets[pos];
+ byte[] updatedValuesBytes = new byte[valuesBytes.length +
+ newValuesBytes.length];
+ System.arraycopy(valuesBytes, 0, updatedValuesBytes, 0, valuesPos);
+ System.arraycopy(valuesBytes, valuesPos, updatedValuesBytes,
+ valuesPos + newValuesBytes.length,
+ valuesBytes.length - valuesPos);
+ System.arraycopy(newValuesBytes, 0, updatedValuesBytes, valuesPos,
+ newValuesBytes.length);
+
+ if(valuesBytesOffsets != null)
+ {
+ int[] updatedValuesBytesOffsets =
+ new int[valuesBytesOffsets.length + 1];
+ System.arraycopy(valuesBytesOffsets, 0, updatedValuesBytesOffsets,
+ 0, pos);
+ // Update the rest of the offsets one by one - Expensive!
+ for(int i = pos; i < valuesBytesOffsets.length; i++)
+ {
+ updatedValuesBytesOffsets[i+1] =
+ valuesBytesOffsets[i] + newValuesBytes.length;
+ }
+ updatedValuesBytesOffsets[pos] = valuesBytesOffsets[pos];
+ valuesBytesOffsets = updatedValuesBytesOffsets;
+ }
+
+ entryIDs = updatedEntryIDs;
+ valuesBytes = updatedValuesBytes;
+ }
+
+ return true;
+ }
+
+ /**
+ * Remove the given entryID and values from this VLV index.
+ *
+ * @param entryID The entry ID to remove.
+ * @param values The values to remove.
+ * @return True if the information was successfully removed or False
+ * otherwise.
+ * @throws DirectoryException If a Directory Server error occurs.
+ * @throws StorageRuntimeException If an error occurs in the JE database.
+ */
+ public boolean remove(long entryID, ByteString[] values)
+ throws StorageRuntimeException, DirectoryException
+ {
+ if(entryIDs == null || entryIDs.length == 0)
+ {
+ return false;
+ }
+
+ if(valuesBytesOffsets == null)
+ {
+ updateValuesBytesOffsets();
+ }
+
+ int pos = binarySearch(entryID, values);
+ if(pos < 0)
+ {
+ // Not found.
+ return false;
+ }
+ else
+ {
+ // Found it.
+ long[] updatedEntryIDs = new long[entryIDs.length - 1];
+ System.arraycopy(entryIDs, 0, updatedEntryIDs, 0, pos);
+ System.arraycopy(entryIDs, pos+1, updatedEntryIDs, pos,
+ entryIDs.length-pos-1);
+ int valuesLength;
+ int valuesPos = valuesBytesOffsets[pos];
+ if(pos < valuesBytesOffsets.length - 1)
+ {
+ valuesLength = valuesBytesOffsets[pos+1] - valuesPos;
+ }
+ else
+ {
+ valuesLength = valuesBytes.length - valuesPos;
+ }
+ byte[] updatedValuesBytes = new byte[valuesBytes.length - valuesLength];
+ System.arraycopy(valuesBytes, 0, updatedValuesBytes, 0, valuesPos);
+ System.arraycopy(valuesBytes, valuesPos + valuesLength,
+ updatedValuesBytes, valuesPos,
+ valuesBytes.length - valuesPos - valuesLength);
+
+ int[] updatedValuesBytesOffsets = new int[valuesBytesOffsets.length - 1];
+ System.arraycopy(valuesBytesOffsets, 0, updatedValuesBytesOffsets,
+ 0, pos);
+ // Update the rest of the offsets one by one - Expensive!
+ for(int i = pos + 1; i < valuesBytesOffsets.length; i++)
+ {
+ updatedValuesBytesOffsets[i-1] =
+ valuesBytesOffsets[i] - valuesLength;
+ }
+
+ entryIDs = updatedEntryIDs;
+ valuesBytes = updatedValuesBytes;
+ valuesBytesOffsets = updatedValuesBytesOffsets;
+ return true;
+ }
+ }
+
+ /**
+ * Split portions of this set into another set. The values of the new set is
+ * from the end of this set.
+ *
+ * @param splitLength The size of the new set.
+ * @return The split set.
+ */
+ public SortValuesSet split(int splitLength)
+ {
+ if(valuesBytesOffsets == null)
+ {
+ updateValuesBytesOffsets();
+ }
+
+ long[] splitEntryIDs = new long[splitLength];
+ byte[] splitValuesBytes = new byte[valuesBytes.length -
+ valuesBytesOffsets[valuesBytesOffsets.length - splitLength]];
+ int[] splitValuesBytesOffsets = new int[splitLength];
+
+ long[] updatedEntryIDs = new long[entryIDs.length - splitEntryIDs.length];
+ System.arraycopy(entryIDs, 0, updatedEntryIDs, 0, updatedEntryIDs.length);
+ System.arraycopy(entryIDs, updatedEntryIDs.length, splitEntryIDs, 0,
+ splitEntryIDs.length);
+
+ byte[] updatedValuesBytes =
+ new byte[valuesBytesOffsets[valuesBytesOffsets.length - splitLength]];
+ System.arraycopy(valuesBytes, 0, updatedValuesBytes, 0,
+ updatedValuesBytes.length);
+ System.arraycopy(valuesBytes, updatedValuesBytes.length, splitValuesBytes,
+ 0, splitValuesBytes.length);
+
+ int[] updatedValuesBytesOffsets =
+ new int[valuesBytesOffsets.length - splitValuesBytesOffsets.length];
+ System.arraycopy(valuesBytesOffsets, 0, updatedValuesBytesOffsets,
+ 0, updatedValuesBytesOffsets.length);
+ for(int i = updatedValuesBytesOffsets.length;
+ i < valuesBytesOffsets.length; i++)
+ {
+ splitValuesBytesOffsets[i - updatedValuesBytesOffsets.length] =
+ valuesBytesOffsets[i] -
+ valuesBytesOffsets[updatedValuesBytesOffsets.length];
+ }
+
+ SortValuesSet splitValuesSet = new SortValuesSet();
+
+ splitValuesSet.entryIDs = splitEntryIDs;
+ splitValuesSet.key = this.key;
+ splitValuesSet.valuesBytes = splitValuesBytes;
+ splitValuesSet.valuesBytesOffsets = splitValuesBytesOffsets;
+ splitValuesSet.vlvIndex = this.vlvIndex;
+
+ entryIDs = updatedEntryIDs;
+ valuesBytes = updatedValuesBytes;
+ valuesBytesOffsets = updatedValuesBytesOffsets;
+ key = null;
+
+ return splitValuesSet;
+ }
+
+ /**
+ * Encode this set to its database format.
+ *
+ * @return The encoded bytes representing this set or null if
+ * this set is empty.
+ */
+ public ByteString toByteString()
+ {
+ if(size() == 0)
+ {
+ return null;
+ }
+
+ byte[] entryIDBytes = JebFormat.entryIDListToDatabase(entryIDs);
+ byte[] concatBytes = new byte[entryIDBytes.length + valuesBytes.length + 4];
+ int v = entryIDs.length;
+
+ for (int j = 3; j >= 0; j--)
+ {
+ concatBytes[j] = (byte) (v & 0xFF);
+ v >>>= 8;
+ }
+
+ System.arraycopy(entryIDBytes, 0, concatBytes, 4, entryIDBytes.length);
+ System.arraycopy(valuesBytes, 0, concatBytes, entryIDBytes.length+4,
+ valuesBytes.length);
+
+ return ByteString.valueOf(concatBytes);
+ }
+
+ /**
+ * Get the size of the provided encoded set.
+ *
+ * @param bytes The encoded bytes of a SortValuesSet to decode the size from.
+ * @param offset The byte offset to start decoding.
+ * @return The size of the provided encoded set.
+ */
+ public static int getEncodedSize(ByteString bytes, int offset)
+ {
+ int v = 0;
+ for (int i = offset; i < offset + 4; i++)
+ {
+ v <<= 8;
+ v |= (bytes.byteAt(i) & 0xFF);
+ }
+ return v;
+ }
+
+ /**
+ * Get the IDs from the provided encoded set.
+ *
+ * @param bytes The encoded bytes of a SortValuesSet to decode the IDs from.
+ * @param offset The byte offset to start decoding.
+ * @return The decoded IDs in the provided encoded set.
+ */
+ public static long[] getEncodedIDs(ByteString bytes, int offset)
+ {
+ int length = getEncodedSize(bytes, offset) * 8;
+ int offset2 = offset + 4;
+ ByteString entryIDBytes = bytes.subSequence(offset2, offset2 + length);
+ return JebFormat.entryIDListFromDatabase(entryIDBytes);
+ }
+
+ /**
+ * Searches this set for the specified values and entry ID using the binary
+ * search algorithm.
+ *
+ * @param entryID The entry ID to match or -1 if not matching on entry ID.
+ * @param values The values to match.
+ * @return Index of the entry matching the values and optionally the entry ID
+ * if it is found or a negative index if its not found.
+ * @throws DirectoryException If a Directory Server error occurs.
+ * @throws StorageRuntimeException If an error occurs in the JE database.
+ */
+ int binarySearch(long entryID, ByteString... values)
+ throws StorageRuntimeException, DirectoryException
+ {
+ if(entryIDs == null || entryIDs.length == 0)
+ {
+ return -1;
+ }
+
+ int i = 0;
+ for(int j = entryIDs.length - 1; i <= j;)
+ {
+ int k = i + j >> 1;
+ int l = vlvIndex.comparator.compare(this, k, entryID, values);
+ if (l < 0)
+ {
+ i = k + 1;
+ }
+ else if (l > 0)
+ {
+ j = k - 1;
+ }
+ else
+ {
+ return k;
+ }
+ }
+
+ return -(i + 1);
+ }
+
+ /**
+ * Retrieve the size of this set.
+ *
+ * @return The size of this set.
+ */
+ public int size()
+ {
+ if(entryIDs == null)
+ {
+ return 0;
+ }
+
+ return entryIDs.length;
+ }
+
+ /**
+ * Retrieve the entry IDs in this set.
+ *
+ * @return The entry IDs in this set.
+ */
+ public long[] getEntryIDs()
+ {
+ return entryIDs;
+ }
+
+ private byte[] attributeValuesToDatabase(ByteString[] values,
+ AttributeType[] types) throws DirectoryException
+ {
+ try
+ {
+ final ByteStringBuilder builder = new ByteStringBuilder();
+
+ for (int i = 0; i < values.length; i++)
+ {
+ final ByteString v = values[i];
+ if (v == null)
+ {
+ builder.appendBERLength(0);
+ }
+ else
+ {
+ final MatchingRule eqRule = types[i].getEqualityMatchingRule();
+ final ByteString nv = eqRule.normalizeAttributeValue(v);
+ builder.appendBERLength(nv.length());
+ builder.append(nv);
+ }
+ }
+ builder.trimToSize();
+
+ return builder.getBackingArray();
+ }
+ catch (DecodeException e)
+ {
+ throw new DirectoryException(
+ ResultCode.INVALID_ATTRIBUTE_SYNTAX, e.getMessageObject(), e);
+ }
+ }
+
+ /**
+ * Returns the key to use for this set of sort values in the database.
+ *
+ * @return The key as an array of bytes that should be used for this set in
+ * the database or NULL if this set is empty.
+ * @throws DirectoryException If a Directory Server error occurs.
+ * @throws StorageRuntimeException If an error occurs in the JE database.
+ */
+ public ByteString getKeyBytes()
+ throws StorageRuntimeException, DirectoryException
+ {
+ if(entryIDs == null || entryIDs.length == 0)
+ {
+ return null;
+ }
+
+ if(key != null)
+ {
+ return key;
+ }
+
+ if(valuesBytesOffsets == null)
+ {
+ updateValuesBytesOffsets();
+ }
+
+ int vBytesPos = valuesBytesOffsets[valuesBytesOffsets.length - 1];
+ int vBytesLength = valuesBytes.length - vBytesPos;
+
+ ByteString idBytes = entryIDToDatabase(entryIDs[entryIDs.length - 1]);
+ ByteStringBuilder keyBytes = new ByteStringBuilder(vBytesLength + idBytes.length());
+ keyBytes.append(valuesBytes, vBytesPos, vBytesLength);
+ keyBytes.append(idBytes);
+
+ key = keyBytes.toByteString();
+ return key;
+ }
+
+ /**
+ * Returns the key to use for this set of sort values in the database.
+ *
+ * @return The key as a sort values object that should be used for this set in
+ * the database or NULL if this set is empty or unbounded.
+ * @throws DirectoryException If a Directory Server error occurs.
+ * @throws StorageRuntimeException If an error occurs in the JE database.
+ */
+ public SortValues getKeySortValues()
+ throws StorageRuntimeException, DirectoryException
+ {
+ if(entryIDs == null || entryIDs.length == 0)
+ {
+ return null;
+ }
+
+ if(key != null && key.length() == 0)
+ {
+ return null;
+ }
+
+ EntryID id = new EntryID(entryIDs[entryIDs.length - 1]);
+ SortKey[] sortKeys = vlvIndex.sortOrder.getSortKeys();
+ int numValues = sortKeys.length;
+ ByteString[] values = new ByteString[numValues];
+ for (int i = (entryIDs.length - 1) * numValues, j = 0;
+ i < entryIDs.length * numValues;
+ i++, j++)
+ {
+ values[j] = getValue(i);
+ }
+
+ return new SortValues(id, values, vlvIndex.sortOrder);
+ }
+
+ /**
+ * Returns the sort values at the index in this set.
+ *
+ * @param index The index of the sort values to get.
+ * @return The sort values object at the specified index.
+ * @throws DirectoryException If a Directory Server error occurs.
+ * @throws StorageRuntimeException If an error occurs in the JE database.
+ * @throws JebException If an error occurs in the JE database.
+ **/
+ public SortValues getSortValues(int index)
+ throws JebException, StorageRuntimeException, DirectoryException
+ {
+ if(entryIDs == null || entryIDs.length == 0)
+ {
+ return null;
+ }
+
+ EntryID id = new EntryID(entryIDs[index]);
+ SortKey[] sortKeys = vlvIndex.sortOrder.getSortKeys();
+ int numValues = sortKeys.length;
+ ByteString[] values = new ByteString[numValues];
+ for (int i = index * numValues, j = 0;
+ i < (index + 1) * numValues;
+ i++, j++)
+ {
+ values[j] = getValue(i);
+ }
+
+ return new SortValues(id, values, vlvIndex.sortOrder);
+ }
+
+ private void updateValuesBytesOffsets()
+ {
+ valuesBytesOffsets = new int[entryIDs.length];
+ int vBytesPos = 0;
+ int numAttributes = vlvIndex.sortOrder.getSortKeys().length;
+
+ for(int pos = 0; pos < entryIDs.length; pos++)
+ {
+ valuesBytesOffsets[pos] = vBytesPos;
+
+ for(int i = 0; i < numAttributes; i++)
+ {
+ int valueLength = valuesBytes[vBytesPos] & 0x7F;
+ if (valueLength != valuesBytes[vBytesPos++])
+ {
+ int valueLengthBytes = valueLength;
+ valueLength = 0;
+ for (int j=0; j < valueLengthBytes; j++, vBytesPos++)
+ {
+ valueLength = (valueLength << 8) | (valuesBytes[vBytesPos] & 0xFF);
+ }
+ }
+
+ vBytesPos += valueLength;
+ }
+ }
+ }
+
+ /**
+ * Retrieve an attribute value from this values set. The index is the
+ * absolute index. (ie. for a sort on 3 attributes per entry, an vlvIndex of 6
+ * will be the 1st attribute value of the 3rd entry).
+ *
+ * @param index The vlvIndex of the attribute value to retrieve.
+ * @return The byte array representation of the attribute value.
+ * @throws DirectoryException If a Directory Server error occurs.
+ * @throws StorageRuntimeException If an error occurs in the JE database.
+ */
+ public ByteString getValue(int index)
+ throws StorageRuntimeException, DirectoryException
+ {
+ if(valuesBytesOffsets == null)
+ {
+ updateValuesBytesOffsets();
+ }
+ int numAttributes = vlvIndex.sortOrder.getSortKeys().length;
+ int vIndex = index / numAttributes;
+ int vOffset = index % numAttributes;
+ int vBytesPos = valuesBytesOffsets[vIndex];
+
+ // Find the desired value in the sort order set.
+ for(int i = 0; i <= vOffset; i++)
+ {
+ int valueLength = valuesBytes[vBytesPos] & 0x7F;
+ if (valueLength != valuesBytes[vBytesPos++])
+ {
+ int valueLengthBytes = valueLength;
+ valueLength = 0;
+ for (int j=0; j < valueLengthBytes; j++, vBytesPos++)
+ {
+ valueLength = (valueLength << 8) | (valuesBytes[vBytesPos] & 0xFF);
+ }
+ }
+
+ if(i == vOffset)
+ {
+ if(valueLength == 0)
+ {
+ return null;
+ }
+ else
+ {
+ byte[] valueBytes = new byte[valueLength];
+ System.arraycopy(valuesBytes, vBytesPos, valueBytes, 0, valueLength);
+ return ByteString.wrap(valueBytes);
+ }
+ }
+ else
+ {
+ vBytesPos += valueLength;
+ }
+ }
+ return ByteString.empty();
+ }
+}
diff --git a/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/State.java b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/State.java
new file mode 100644
index 0000000..bfd1e0e
--- /dev/null
+++ b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/State.java
@@ -0,0 +1,122 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License"). You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at legal-notices/CDDLv1_0.txt
+ * or http://forgerock.org/license/CDDLv1.0.html.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at legal-notices/CDDLv1_0.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information:
+ * Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ *
+ *
+ * Copyright 2006-2008 Sun Microsystems, Inc.
+ * Portions copyright 2011-2014 ForgeRock AS
+ */
+package org.opends.server.backends.pluggable;
+
+import org.forgerock.opendj.ldap.ByteString;
+import org.opends.server.backends.pluggable.BackendImpl.ReadableStorage;
+import org.opends.server.backends.pluggable.BackendImpl.Storage;
+import org.opends.server.backends.pluggable.BackendImpl.StorageRuntimeException;
+import org.opends.server.backends.pluggable.BackendImpl.TreeName;
+import org.opends.server.backends.pluggable.BackendImpl.WriteableStorage;
+import org.opends.server.util.StaticUtils;
+
+/**
+ * This class is responsible for storing the configuration state of
+ * the JE backend for a particular suffix.
+ */
+public class State extends DatabaseContainer
+{
+ private static final ByteString falseBytes = ByteString.wrap(new byte[] { 0x00 });
+ private static final ByteString trueBytes = ByteString.wrap(new byte[] { 0x01 });
+
+ /**
+ * Create a new State object.
+ *
+ * @param name The name of the entry database.
+ * @param env The JE Storage.
+ * @param entryContainer The entryContainer of the entry database.
+ */
+ State(TreeName name, Storage env, EntryContainer entryContainer)
+ {
+ super(name, env, entryContainer);
+ }
+
+ /**
+ * Return the key associated with the index in the state database.
+ *
+ * @param index The index we need the key for.
+ * @return the key
+ * @throws StorageRuntimeException If an error occurs in the JE database.
+ */
+ private ByteString keyForIndex(DatabaseContainer index)
+ throws StorageRuntimeException
+ {
+ String shortName = index.getName().toString();
+ return ByteString.wrap(StaticUtils.getBytes(shortName));
+ }
+
+ /**
+ * Remove a record from the entry database.
+ *
+ * @param txn The database transaction or null if none.
+ * @param index The index storing the trusted state info.
+ * @return true if the entry was removed, false if it was not.
+ * @throws StorageRuntimeException If an error occurs in the JE database.
+ */
+ public boolean removeIndexTrustState(WriteableStorage txn, DatabaseContainer index)
+ throws StorageRuntimeException
+ {
+ ByteString key = keyForIndex(index);
+ return delete(txn, key);
+ }
+
+ /**
+ * Fetch index state from the database.
+ * @param txn The database transaction or null if none.
+ * @param index The index storing the trusted state info.
+ * @return The trusted state of the index in the database.
+ * @throws StorageRuntimeException If an error occurs in the JE database.
+ */
+ public boolean getIndexTrustState(ReadableStorage txn, DatabaseContainer index)
+ throws StorageRuntimeException
+ {
+ ByteString key = keyForIndex(index);
+ ByteString value = read(txn, key, false);
+
+ if (value != null)
+ {
+ return value.equals(trueBytes);
+ }
+ return false;
+ }
+
+ /**
+ * Put index state to database.
+ * @param txn The database transaction or null if none.
+ * @param index The index storing the trusted state info.
+ * @param trusted The state value to put into the database.
+ * @throws StorageRuntimeException If an error occurs in the JE database.
+ */
+ public void putIndexTrustState(WriteableStorage txn, DatabaseContainer index, boolean trusted)
+ throws StorageRuntimeException
+ {
+ ByteString key = keyForIndex(index);
+
+ txn.put(treeName, key, trusted ? trueBytes : falseBytes);
+ }
+
+}
diff --git a/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/VLVIndex.java b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/VLVIndex.java
new file mode 100644
index 0000000..2eaf778
--- /dev/null
+++ b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/VLVIndex.java
@@ -0,0 +1,1428 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License"). You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at legal-notices/CDDLv1_0.txt
+ * or http://forgerock.org/license/CDDLv1.0.html.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at legal-notices/CDDLv1_0.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information:
+ * Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ *
+ *
+ * Copyright 2006-2008 Sun Microsystems, Inc.
+ * Portions Copyright 2011-2014 ForgeRock AS
+ */
+package org.opends.server.backends.pluggable;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.TreeSet;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.forgerock.i18n.LocalizableMessage;
+import org.forgerock.i18n.slf4j.LocalizedLogger;
+import org.forgerock.opendj.config.server.ConfigException;
+import org.forgerock.opendj.ldap.ByteSequence;
+import org.forgerock.opendj.ldap.ByteString;
+import org.forgerock.opendj.ldap.ByteStringBuilder;
+import org.forgerock.opendj.ldap.DecodeException;
+import org.forgerock.opendj.ldap.ResultCode;
+import org.forgerock.opendj.ldap.SearchScope;
+import org.forgerock.opendj.ldap.SearchScope.Enum;
+import org.forgerock.opendj.ldap.schema.MatchingRule;
+import org.opends.server.admin.server.ConfigurationChangeListener;
+import org.opends.server.admin.std.meta.LocalDBVLVIndexCfgDefn.Scope;
+import org.opends.server.admin.std.server.LocalDBVLVIndexCfg;
+import org.opends.server.backends.pluggable.BackendImpl.Cursor;
+import org.opends.server.backends.pluggable.BackendImpl.ReadableStorage;
+import org.opends.server.backends.pluggable.BackendImpl.Storage;
+import org.opends.server.backends.pluggable.BackendImpl.StorageRuntimeException;
+import org.opends.server.backends.pluggable.BackendImpl.WriteableStorage;
+import org.opends.server.controls.ServerSideSortRequestControl;
+import org.opends.server.controls.VLVRequestControl;
+import org.opends.server.controls.VLVResponseControl;
+import org.opends.server.core.DirectoryServer;
+import org.opends.server.core.SearchOperation;
+import org.opends.server.protocols.ldap.LDAPResultCode;
+import org.opends.server.types.Attribute;
+import org.opends.server.types.AttributeType;
+import org.opends.server.types.ConfigChangeResult;
+import org.opends.server.types.DN;
+import org.opends.server.types.DirectoryException;
+import org.opends.server.types.Entry;
+import org.opends.server.types.Modification;
+import org.opends.server.types.SearchFilter;
+import org.opends.server.types.SortKey;
+import org.opends.server.types.SortOrder;
+import org.opends.server.util.StaticUtils;
+
+import com.sleepycat.je.LockMode;
+
+import static org.opends.messages.JebMessages.*;
+import static org.opends.server.util.StaticUtils.*;
+
+/**
+ * This class represents a VLV index. Each database record is a sorted list
+ * of entry IDs followed by sets of attribute values used to sort the entries.
+ * The entire set of entry IDs are broken up into sorted subsets to decrease
+ * the number of database retrievals needed for a range lookup. The records are
+ * keyed by the last entry's first sort attribute value. The list of entries
+ * in a particular database record maintains the property where the first sort
+ * attribute value is bigger then the previous key but smaller or equal
+ * to its own key.
+ */
+public class VLVIndex extends DatabaseContainer
+ implements ConfigurationChangeListener<LocalDBVLVIndexCfg>
+{
+ private static final LocalizedLogger logger = LocalizedLogger.getLoggerForThisClass();
+
+ /** The comparator for vlvIndex keys. */
+ public VLVKeyComparator comparator;
+ /** The limit on the number of entry IDs that may be indexed by one key. */
+ private int sortedSetCapacity = 4000;
+ /** The SortOrder in use by this VLV index to sort the entries. */
+ public SortOrder sortOrder;
+
+ /** The cached count of entries in this index. */
+ private final AtomicInteger count;
+
+ private final State state;
+ /**
+ * A flag to indicate if this vlvIndex should be trusted to be consistent
+ * with the entries database.
+ */
+ private boolean trusted;
+ /** A flag to indicate if a rebuild process is running on this vlvIndex. */
+ private boolean rebuildRunning;
+
+ /** The VLV vlvIndex configuration. */
+ private LocalDBVLVIndexCfg config;
+
+ private DN baseDN;
+ private SearchFilter filter;
+ private SearchScope scope;
+
+
+ /**
+ * Create a new VLV vlvIndex object.
+ *
+ * @param config The VLV index config object to use for this VLV
+ * index.
+ * @param state The state database to persist vlvIndex state info.
+ * @param env The JE Storage
+ * @param entryContainer The database entryContainer holding this vlvIndex.
+ * @throws StorageRuntimeException
+ * If an error occurs in the JE database.
+ * @throws ConfigException if a error occurs while reading the VLV index
+ * configuration
+ */
+ public VLVIndex(LocalDBVLVIndexCfg config, State state, Storage env,
+ EntryContainer entryContainer)
+ throws StorageRuntimeException, ConfigException
+ {
+ super(entryContainer.getDatabasePrefix().child("vlv."+config.getName()),
+ env, entryContainer);
+
+ this.config = config;
+ this.baseDN = config.getBaseDN();
+ this.scope = valueOf(config.getScope());
+ this.sortedSetCapacity = config.getMaxBlockSize();
+
+ try
+ {
+ this.filter = SearchFilter.createFilterFromString(config.getFilter());
+ }
+ catch(Exception e)
+ {
+ LocalizableMessage msg = ERR_JEB_CONFIG_VLV_INDEX_BAD_FILTER.get(
+ config.getFilter(), treeName, stackTraceToSingleLineString(e));
+ throw new ConfigException(msg);
+ }
+
+ String[] sortAttrs = config.getSortOrder().split(" ");
+ SortKey[] sortKeys = new SortKey[sortAttrs.length];
+ MatchingRule[] orderingRules = new MatchingRule[sortAttrs.length];
+ boolean[] ascending = new boolean[sortAttrs.length];
+ for(int i = 0; i < sortAttrs.length; i++)
+ {
+ try
+ {
+ if(sortAttrs[i].startsWith("-"))
+ {
+ ascending[i] = false;
+ sortAttrs[i] = sortAttrs[i].substring(1);
+ }
+ else
+ {
+ ascending[i] = true;
+ if(sortAttrs[i].startsWith("+"))
+ {
+ sortAttrs[i] = sortAttrs[i].substring(1);
+ }
+ }
+ }
+ catch(Exception e)
+ {
+ throw new ConfigException(ERR_JEB_CONFIG_VLV_INDEX_UNDEFINED_ATTR.get(sortKeys[i], treeName));
+ }
+
+ AttributeType attrType =
+ DirectoryServer.getAttributeType(sortAttrs[i].toLowerCase());
+ if(attrType == null)
+ {
+ LocalizableMessage msg =
+ ERR_JEB_CONFIG_VLV_INDEX_UNDEFINED_ATTR.get(sortAttrs[i], treeName);
+ throw new ConfigException(msg);
+ }
+ sortKeys[i] = new SortKey(attrType, ascending[i]);
+ orderingRules[i] = attrType.getOrderingMatchingRule();
+ }
+
+ this.sortOrder = new SortOrder(sortKeys);
+ this.comparator = new VLVKeyComparator(orderingRules, ascending);
+
+ this.state = state;
+ this.trusted = state.getIndexTrustState(null, this);
+ if (!trusted && entryContainer.getHighestEntryID().longValue() == 0)
+ {
+ // If there are no entries in the entry container then there
+ // is no reason why this vlvIndex can't be upgraded to trusted.
+ setTrusted(null, true);
+ }
+
+ this.count = new AtomicInteger(0);
+ this.config.addChangeListener(this);
+ }
+
+ private SearchScope valueOf(Scope cfgScope)
+ {
+ final Enum toFind = SearchScope.Enum.valueOf(cfgScope.name());
+ for (SearchScope scope : SearchScope.values())
+ {
+ if (scope.asEnum() == toFind)
+ {
+ return scope;
+ }
+ }
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void open() throws StorageRuntimeException
+ {
+ super.open();
+
+ Cursor cursor = storage.openCursor(treeName);
+ try
+ {
+ while (cursor.next())
+ {
+ count.getAndAdd(SortValuesSet.getEncodedSize(cursor.getValue(), 0));
+ }
+ }
+ finally
+ {
+ cursor.close();
+ }
+ }
+
+ /**
+ * Close the VLV index.
+ *
+ * @throws StorageRuntimeException if a JE database error occurs while
+ * closing the index.
+ */
+ @Override
+ public void close() throws StorageRuntimeException
+ {
+ super.close();
+ this.config.removeChangeListener(this);
+ }
+
+ /**
+ * Update the vlvIndex for a new entry.
+ *
+ * @param txn A database transaction, or null if none is required.
+ * @param entryID The entry ID.
+ * @param entry The entry to be indexed.
+ * @return True if the entry ID for the entry are added. False if
+ * the entry ID already exists.
+ * @throws StorageRuntimeException If an error occurs in the JE database.
+ * @throws org.opends.server.types.DirectoryException If a Directory Server
+ * error occurs.
+ * @throws JebException If an error occurs in the JE backend.
+ */
+ public boolean addEntry(WriteableStorage txn, EntryID entryID, Entry entry)
+ throws StorageRuntimeException, DirectoryException, JebException
+ {
+ return shouldInclude(entry)
+ && insertValues(txn, entryID.longValue(), entry);
+ }
+
+ /**
+ * Update the vlvIndex for a new entry.
+ *
+ * @param buffer The index buffer to buffer the changes.
+ * @param entryID The entry ID.
+ * @param entry The entry to be indexed.
+ * @return True if the entry ID for the entry are added. False if
+ * the entry ID already exists.
+ * @throws DirectoryException If a Directory Server
+ * error occurs.
+ */
+ public boolean addEntry(IndexBuffer buffer, EntryID entryID, Entry entry)
+ throws DirectoryException
+ {
+ if (shouldInclude(entry))
+ {
+ final SortValues sortValues = new SortValues(entryID, entry, sortOrder);
+ buffer.getVLVIndex(this).addValues(sortValues);
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * Update the vlvIndex for a deleted entry.
+ *
+ * @param buffer The database transaction to be used for the deletions
+ * @param entryID The entry ID
+ * @param entry The contents of the deleted entry.
+ * @return True if the entry was successfully removed from this VLV index
+ * or False otherwise.
+ * @throws DirectoryException If a Directory Server error occurs.
+ */
+ public boolean removeEntry(IndexBuffer buffer, EntryID entryID, Entry entry)
+ throws DirectoryException
+ {
+ if (shouldInclude(entry))
+ {
+ final SortValues sortValues = new SortValues(entryID, entry, sortOrder);
+ buffer.getVLVIndex(this).deleteValues(sortValues);
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * Update the vlvIndex to reflect a sequence of modifications in a Modify
+ * operation.
+ *
+ * @param buffer The database transaction to be used for the deletions
+ * @param entryID The ID of the entry that was modified.
+ * @param oldEntry The entry before the modifications were applied.
+ * @param newEntry The entry after the modifications were applied.
+ * @param mods The sequence of modifications in the Modify operation.
+ * @return True if the modification was successfully processed or False
+ * otherwise.
+ * @throws StorageRuntimeException If an error occurs during an operation on a
+ * JE database.
+ * @throws DirectoryException If a Directory Server error occurs.
+ */
+ public boolean modifyEntry(IndexBuffer buffer,
+ EntryID entryID,
+ Entry oldEntry,
+ Entry newEntry,
+ List<Modification> mods)
+ throws StorageRuntimeException, DirectoryException
+ {
+ if (shouldInclude(oldEntry))
+ {
+ if (shouldInclude(newEntry))
+ {
+ // The entry should still be indexed. See if any sorted attributes are
+ // changed.
+ if (isSortAttributeModified(mods))
+ {
+ boolean success;
+ // Sorted attributes have changed. Reindex the entry;
+ success = removeEntry(buffer, entryID, oldEntry);
+ success &= addEntry(buffer, entryID, newEntry);
+ return success;
+ }
+ }
+ else
+ {
+ // The modifications caused the new entry to be unindexed. Remove from
+ // vlvIndex.
+ return removeEntry(buffer, entryID, oldEntry);
+ }
+ }
+ else
+ {
+ if (shouldInclude(newEntry))
+ {
+ // The modifications caused the new entry to be indexed. Add to vlvIndex
+ return addEntry(buffer, entryID, newEntry);
+ }
+ }
+
+ // The modifications does not affect this vlvIndex
+ return true;
+ }
+
+ private boolean isSortAttributeModified(List<Modification> mods)
+ {
+ for (SortKey sortKey : sortOrder.getSortKeys())
+ {
+ AttributeType attributeType = sortKey.getAttributeType();
+ Iterable<AttributeType> subTypes = DirectoryServer.getSchema().getSubTypes(attributeType);
+ for (Modification mod : mods)
+ {
+ AttributeType modAttrType = mod.getAttribute().getAttributeType();
+ if (modAttrType.equals(attributeType))
+ {
+ return true;
+ }
+ for (AttributeType subType : subTypes)
+ {
+ if (modAttrType.equals(subType))
+ {
+ return true;
+ }
+ }
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Get a sorted values set that should contain the entry with the given
+ * information.
+ *
+ * @param txn The transaction to use when retrieving the set or NULL if it is
+ * not required.
+ * @param entryID The entry ID to use.
+ * @param values The values to use.
+ * @param types The types of the values to use.
+ * @return The SortValuesSet that should contain the entry with the given
+ * information.
+ * @throws StorageRuntimeException If an error occurs during an operation on a
+ * JE database.
+ * @throws DirectoryException If a Directory Server error occurs.
+ */
+ public SortValuesSet getSortValuesSet(ReadableStorage txn, long entryID,
+ ByteString[] values, AttributeType[] types) throws StorageRuntimeException,
+ DirectoryException
+ {
+ ByteString key = encodeKey(entryID, values, types);
+ return getSortValuesSet(txn, key, false);
+ }
+
+ private SortValuesSet getSortValuesSet(ReadableStorage txn, ByteString key, boolean isRMW)
+ {
+ ByteString value = isRMW ? txn.getRMW(treeName, key) : txn.get(treeName, key);
+ if (value == null)
+ {
+ // There are no records in the database
+ if (logger.isTraceEnabled())
+ {
+ logger.trace("No sort values set exist in VLV vlvIndex %s. "
+ + "Creating unbound set.", config.getName());
+ }
+ // this could not be found, so clean the key for later reuse
+ return new SortValuesSet(this);
+ }
+
+ if (logger.isTraceEnabled())
+ {
+ logSearchKeyResult(key);
+ }
+ return new SortValuesSet(key, value, this);
+ }
+
+ private void logSearchKeyResult(ByteString key)
+ {
+ StringBuilder searchKeyHex = new StringBuilder();
+ StaticUtils.byteArrayToHexPlusAscii(searchKeyHex, key.toByteArray(), 4);
+ StringBuilder foundKeyHex = new StringBuilder();
+ StaticUtils.byteArrayToHexPlusAscii(foundKeyHex, key.toByteArray(), 4);
+ logger.trace("Retrieved a sort values set in VLV vlvIndex %s\n" +
+ "Search Key:%s\nFound Key:%s\n",
+ config.getName(), searchKeyHex, foundKeyHex);
+ }
+
+ /**
+ * Search for entries matching the entry ID and attribute values and
+ * return its entry ID.
+ *
+ * @param txn The JE transaction to use for database updates.
+ * @param entryID The entry ID to search for.
+ * @param values The values to search for.
+ * @param types The types of the values to search for.
+ * @return The index of the entry ID matching the values or -1 if its not
+ * found.
+ * @throws StorageRuntimeException If an error occurs during an operation on a
+ * JE database.
+ * @throws JebException If an error occurs during an operation on a
+ * JE database.
+ * @throws DirectoryException If a Directory Server error occurs.
+ */
+ public boolean containsValues(ReadableStorage txn, long entryID,
+ ByteString[] values, AttributeType[] types) throws JebException,
+ StorageRuntimeException, DirectoryException
+ {
+ SortValuesSet valuesSet = getSortValuesSet(txn, entryID, values, types);
+ int pos = valuesSet.binarySearch(entryID, values);
+ return pos >= 0;
+ }
+
+ private boolean insertValues(WriteableStorage txn, long entryID, Entry entry)
+ throws JebException, StorageRuntimeException, DirectoryException
+ {
+ ByteString[] values = getSortValues(entry);
+ AttributeType[] types = getSortTypes();
+ ByteString key = encodeKey(entryID, values, types);
+
+ SortValuesSet sortValuesSet = getSortValuesSet(txn, key, true);
+ boolean success = sortValuesSet.add(entryID, values, types);
+
+ int newSize = sortValuesSet.size();
+ if(newSize >= sortedSetCapacity)
+ {
+ SortValuesSet splitSortValuesSet = sortValuesSet.split(newSize / 2);
+ put(txn, splitSortValuesSet); // splitAfter
+ put(txn, sortValuesSet); // after
+
+ if(logger.isTraceEnabled())
+ {
+ logger.trace("SortValuesSet with key %s has reached" +
+ " the entry size of %d. Spliting into two sets with " +
+ " keys %s and %s.", splitSortValuesSet.getKeySortValues(),
+ newSize, sortValuesSet.getKeySortValues(),
+ splitSortValuesSet.getKeySortValues());
+ }
+ }
+ else
+ {
+ ByteString after = sortValuesSet.toByteString();
+ put(txn, key, after);
+ // TODO: What about phantoms?
+ }
+
+ if(success)
+ {
+ count.getAndIncrement();
+ }
+
+ return success;
+ }
+
+ private void put(WriteableStorage txn, SortValuesSet set) throws DirectoryException
+ {
+ put(txn, set.getKeyBytes(), set.toByteString());
+ }
+
+ /**
+ * Gets the types of the attribute values to sort.
+ *
+ * @return The types of the attribute values to sort on.
+ */
+ AttributeType[] getSortTypes()
+ {
+ SortKey[] sortKeys = sortOrder.getSortKeys();
+ AttributeType[] types = new AttributeType[sortKeys.length];
+ for (int i = 0; i < sortKeys.length; i++)
+ {
+ types[i] = sortKeys[i].getAttributeType();
+ }
+ return types;
+ }
+
+ private boolean getSearchKeyRange(ReadableStorage txn, ByteString key)
+ {
+ Cursor cursor = txn.openCursor(treeName);
+ try
+ {
+ return cursor.positionToKeyOrNext(key);
+ }
+ finally
+ {
+ cursor.close();
+ }
+ }
+
+ /**
+ * Update the vlvIndex with the specified values to add and delete.
+ *
+ * @param txn A database transaction, or null if none is required.
+ * @param addedValues The values to add to the VLV index.
+ * @param deletedValues The values to delete from the VLV index.
+ * @throws StorageRuntimeException If an error occurs in the JE database.
+ * @throws DirectoryException If a Directory Server
+ * error occurs.
+ */
+ public void updateIndex(WriteableStorage txn,
+ TreeSet<SortValues> addedValues,
+ TreeSet<SortValues> deletedValues)
+ throws DirectoryException, StorageRuntimeException
+ {
+ // Handle cases where nothing is changed early to avoid
+ // DB access.
+ if((addedValues == null || addedValues.isEmpty()) &&
+ (deletedValues == null || deletedValues.isEmpty()))
+ {
+ return;
+ }
+
+ Iterator<SortValues> aValues = null;
+ Iterator<SortValues> dValues = null;
+ SortValues av = null;
+ SortValues dv = null;
+
+ if(addedValues != null)
+ {
+ aValues = addedValues.iterator();
+ av = aValues.next();
+ }
+ if(deletedValues != null)
+ {
+ dValues = deletedValues.iterator();
+ dv = dValues.next();
+ }
+
+ while(true)
+ {
+ ByteString key;
+ if(av != null)
+ {
+ if(dv != null)
+ {
+ // Start from the smallest values from either set.
+ if(av.compareTo(dv) < 0)
+ {
+ key = encodeKey(av);
+ }
+ else
+ {
+ key = encodeKey(dv);
+ }
+ }
+ else
+ {
+ key = encodeKey(av);
+ }
+ }
+ else if(dv != null)
+ {
+ key = encodeKey(dv);
+ }
+ else
+ {
+ break;
+ }
+
+ final SortValuesSet sortValuesSet = getSortValuesSet(txn, key, true);
+ int oldSize = sortValuesSet.size();
+ if(key.length() == 0)
+ {
+ // This is the last unbounded set.
+ while(av != null)
+ {
+ sortValuesSet.add(av.getEntryID(), av.getValues(), av.getTypes());
+ av = moveToNextSortValues(aValues);
+ }
+
+ while(dv != null)
+ {
+ sortValuesSet.remove(dv.getEntryID(), dv.getValues());
+ dv = moveToNextSortValues(dValues);
+ }
+ }
+ else
+ {
+ SortValues maxValues = decodeKey(sortValuesSet.getKeyBytes());
+
+ while(av != null && av.compareTo(maxValues) <= 0)
+ {
+ sortValuesSet.add(av.getEntryID(), av.getValues(), av.getTypes());
+ av = moveToNextSortValues(aValues);
+ }
+
+ while(dv != null && dv.compareTo(maxValues) <= 0)
+ {
+ sortValuesSet.remove(dv.getEntryID(), dv.getValues());
+ dv = moveToNextSortValues(dValues);
+ }
+ }
+
+ int newSize = sortValuesSet.size();
+ if(newSize >= sortedSetCapacity)
+ {
+ SortValuesSet splitSortValuesSet = sortValuesSet.split(newSize / 2);
+ put(txn, splitSortValuesSet); // splitAfter
+ put(txn, sortValuesSet); // after
+
+ if(logger.isTraceEnabled())
+ {
+ logger.trace("SortValuesSet with key %s has reached" +
+ " the entry size of %d. Spliting into two sets with " +
+ " keys %s and %s.", splitSortValuesSet.getKeySortValues(),
+ newSize, sortValuesSet.getKeySortValues(),
+ splitSortValuesSet.getKeySortValues());
+ }
+ }
+ else if(newSize == 0)
+ {
+ delete(txn, key);
+ }
+ else
+ {
+ ByteString after = sortValuesSet.toByteString();
+ put(txn, key, after);
+ }
+
+ count.getAndAdd(newSize - oldSize);
+ }
+ }
+
+ private SortValues moveToNextSortValues(Iterator<SortValues> sortValues)
+ {
+ sortValues.remove();
+ if (sortValues.hasNext())
+ {
+ return sortValues.next();
+ }
+ return null;
+ }
+
+ private ByteString encodeKey(SortValues sv) throws DirectoryException
+ {
+ return encodeKey(sv.getEntryID(), sv.getValues(), sv.getTypes());
+ }
+
+ /**
+ * Evaluate a search with sort control using this VLV index.
+ *
+ * @param txn The transaction to used when reading the index or NULL if it is
+ * not required.
+ * @param searchOperation The search operation to evaluate.
+ * @param sortControl The sort request control to evaluate.
+ * @param vlvRequest The VLV request control to evaluate or NULL if VLV is not
+ * requested.
+ * @param debugBuilder If not null, a diagnostic string will be written
+ * which will help determine how this index contributed
+ * to this search.
+ * @return The sorted EntryIDSet containing the entry IDs that match the
+ * search criteria.
+ * @throws DirectoryException If a Directory Server error occurs.
+ * @throws StorageRuntimeException If an error occurs in the JE database.
+ */
+ public EntryIDSet evaluate(ReadableStorage txn,
+ SearchOperation searchOperation,
+ ServerSideSortRequestControl sortControl,
+ VLVRequestControl vlvRequest,
+ StringBuilder debugBuilder)
+ throws DirectoryException, StorageRuntimeException
+ {
+ if (!trusted || rebuildRunning
+ || !searchOperation.getBaseDN().equals(baseDN)
+ || !searchOperation.getScope().equals(scope)
+ || !searchOperation.getFilter().equals(filter)
+ || !sortControl.getSortOrder().equals(sortOrder))
+ {
+ return null;
+ }
+
+ if (debugBuilder != null)
+ {
+ debugBuilder.append("vlv=");
+ debugBuilder.append("[INDEX:");
+ debugBuilder.append(treeName.replace(entryContainer.getDatabasePrefix() + "_", ""));
+ debugBuilder.append("]");
+ }
+
+ long[] selectedIDs = new long[0];
+ if(vlvRequest != null)
+ {
+ int currentCount = count.get();
+ int beforeCount = vlvRequest.getBeforeCount();
+ int afterCount = vlvRequest.getAfterCount();
+
+ if (vlvRequest.getTargetType() == VLVRequestControl.TYPE_TARGET_BYOFFSET)
+ {
+ int targetOffset = vlvRequest.getOffset();
+ if (targetOffset < 0)
+ {
+ // The client specified a negative target offset. This should never
+ // be allowed.
+ searchOperation.addResponseControl(
+ new VLVResponseControl(targetOffset, currentCount,
+ LDAPResultCode.OFFSET_RANGE_ERROR));
+
+ LocalizableMessage message = ERR_ENTRYIDSORTER_NEGATIVE_START_POS.get();
+ throw new DirectoryException(ResultCode.VIRTUAL_LIST_VIEW_ERROR,
+ message);
+ }
+ else if (targetOffset == 0)
+ {
+ // This is an easy mistake to make, since VLV offsets start at 1
+ // instead of 0. We'll assume the client meant to use 1.
+ targetOffset = 1;
+ }
+ int listOffset = targetOffset - 1; // VLV offsets start at 1, not 0.
+ int startPos = listOffset - beforeCount;
+ if (startPos < 0)
+ {
+ // This can happen if beforeCount >= offset, and in this case we'll
+ // just adjust the start position to ignore the range of beforeCount
+ // that doesn't exist.
+ startPos = 0;
+ beforeCount = listOffset;
+ }
+ else if(startPos >= currentCount)
+ {
+ // The start position is beyond the end of the list. In this case,
+ // we'll assume that the start position was one greater than the
+ // size of the list and will only return the beforeCount entries.
+ // The start position is beyond the end of the list. In this case,
+ // we'll assume that the start position was one greater than the
+ // size of the list and will only return the beforeCount entries.
+ targetOffset = currentCount + 1;
+ listOffset = currentCount;
+ startPos = listOffset - beforeCount;
+ afterCount = 0;
+ }
+
+ int count = 1 + beforeCount + afterCount;
+ selectedIDs = new long[count];
+
+ Cursor cursor = txn.openCursor(treeName);
+ try
+ {
+ //Locate the set that contains the target entry.
+ int cursorCount = 0;
+ int selectedPos = 0;
+ while (cursor.next())
+ {
+ if(logger.isTraceEnabled())
+ {
+ logSearchKeyResult(cursor.getKey());
+ }
+ long[] IDs = SortValuesSet.getEncodedIDs(cursor.getValue(), 0);
+ for(int i = startPos + selectedPos - cursorCount;
+ i < IDs.length && selectedPos < count;
+ i++, selectedPos++)
+ {
+ selectedIDs[selectedPos] = IDs[i];
+ }
+ cursorCount += IDs.length;
+ }
+
+ if (selectedPos < count)
+ {
+ // We don't have enough entries in the set to meet the requested
+ // page size, so we'll need to shorten the array.
+ long[] newIDArray = new long[selectedPos];
+ System.arraycopy(selectedIDs, 0, newIDArray, 0, selectedPos);
+ selectedIDs = newIDArray;
+ }
+
+ searchOperation.addResponseControl(
+ new VLVResponseControl(targetOffset, currentCount,
+ LDAPResultCode.SUCCESS));
+
+ if(debugBuilder != null)
+ {
+ debugBuilder.append("[COUNT:");
+ debugBuilder.append(cursorCount);
+ debugBuilder.append("]");
+ }
+ }
+ finally
+ {
+ cursor.close();
+ }
+ }
+ else
+ {
+ int targetOffset = 0;
+ int includedBeforeCount = 0;
+ int includedAfterCount = 0;
+ LinkedList<EntryID> idList = new LinkedList<EntryID>();
+
+ Cursor cursor = openCursor(txn);
+ try
+ {
+ LockMode lockMode = LockMode.DEFAULT;
+ ByteSequence vBytes = vlvRequest.getGreaterThanOrEqualAssertion();
+ ByteStringBuilder keyBytes = new ByteStringBuilder(vBytes.length() + 4);
+ keyBytes.appendBERLength(vBytes.length());
+ vBytes.copyTo(keyBytes);
+
+ boolean success = cursor.positionToKeyOrNext(keyBytes);
+ if (success)
+ {
+ if(logger.isTraceEnabled())
+ {
+ logSearchKeyResult(cursor.getKey());
+ }
+ SortValuesSet sortValuesSet =
+ new SortValuesSet(cursor.getKey(), cursor.getValue(), this);
+
+ int adjustedTargetOffset = sortValuesSet.binarySearch(
+ -1, vlvRequest.getGreaterThanOrEqualAssertion());
+ if(adjustedTargetOffset < 0)
+ {
+ // For a negative return value r, the vlvIndex -(r+1) gives the
+ // array index of the ID that is greater then the assertion value.
+ adjustedTargetOffset = -(adjustedTargetOffset+1);
+ }
+
+ targetOffset = adjustedTargetOffset;
+
+ // Iterate through all the sort values sets before this one to find
+ // the target offset in the index.
+ int lastOffset = adjustedTargetOffset - 1;
+ long[] lastIDs = sortValuesSet.getEntryIDs();
+ while(true)
+ {
+ for(int i = lastOffset;
+ i >= 0 && includedBeforeCount < beforeCount; i--)
+ {
+ idList.addFirst(new EntryID(lastIDs[i]));
+ includedBeforeCount++;
+ }
+
+ success = cursor.previous();
+ if (success)
+ {
+ break;
+ }
+
+ if(includedBeforeCount < beforeCount)
+ {
+ lastIDs = SortValuesSet.getEncodedIDs(cursor.getValue(), 0);
+ lastOffset = lastIDs.length - 1;
+ targetOffset += lastIDs.length;
+ }
+ else
+ {
+ targetOffset += SortValuesSet.getEncodedSize(cursor.getValue(), 0);
+ }
+ }
+
+
+ // Set the cursor back to the position of the target entry set
+ cursor.positionToKey(sortValuesSet.getKeyBytes());
+
+ // Add the target and after count entries if the target was found.
+ lastOffset = adjustedTargetOffset;
+ lastIDs = sortValuesSet.getEntryIDs();
+ int afterIDCount = 0;
+ while(true)
+ {
+ for(int i = lastOffset;
+ i < lastIDs.length && includedAfterCount < afterCount + 1;
+ i++)
+ {
+ idList.addLast(new EntryID(lastIDs[i]));
+ includedAfterCount++;
+ }
+
+ if(includedAfterCount >= afterCount + 1)
+ {
+ break;
+ }
+
+ success = cursor.next();
+ if (success)
+ {
+ break;
+ }
+
+ lastIDs = SortValuesSet.getEncodedIDs(cursor.getValue(), 0);
+ lastOffset = 0;
+ afterIDCount += lastIDs.length;
+ }
+
+ selectedIDs = new long[idList.size()];
+ Iterator<EntryID> idIterator = idList.iterator();
+ for (int i=0; i < selectedIDs.length; i++)
+ {
+ selectedIDs[i] = idIterator.next().longValue();
+ }
+
+ searchOperation.addResponseControl(
+ new VLVResponseControl(targetOffset + 1, currentCount,
+ LDAPResultCode.SUCCESS));
+
+ if(debugBuilder != null)
+ {
+ debugBuilder.append("[COUNT:");
+ debugBuilder.append(targetOffset + afterIDCount + 1);
+ debugBuilder.append("]");
+ }
+ }
+ }
+ finally
+ {
+ cursor.close();
+ }
+ }
+ }
+ else
+ {
+ LinkedList<long[]> idSets = new LinkedList<long[]>();
+ int currentCount = 0;
+
+ Cursor cursor = openCursor(txn);
+ try
+ {
+ while (cursor.next())
+ {
+ if(logger.isTraceEnabled())
+ {
+ logSearchKeyResult(cursor.getKey());
+ }
+ long[] ids = SortValuesSet.getEncodedIDs(cursor.getValue(), 0);
+ idSets.add(ids);
+ currentCount += ids.length;
+ }
+ }
+ finally
+ {
+ cursor.close();
+ }
+
+ selectedIDs = new long[currentCount];
+ int pos = 0;
+ for(long[] id : idSets)
+ {
+ System.arraycopy(id, 0, selectedIDs, pos, id.length);
+ pos += id.length;
+ }
+
+ if(debugBuilder != null)
+ {
+ debugBuilder.append("[COUNT:");
+ debugBuilder.append(currentCount);
+ debugBuilder.append("]");
+ }
+ }
+ return new EntryIDSet(selectedIDs, 0, selectedIDs.length);
+ }
+
+ /**
+ * Set the vlvIndex trust state.
+ * @param txn A database transaction, or null if none is required.
+ * @param trusted True if this vlvIndex should be trusted or false
+ * otherwise.
+ * @throws StorageRuntimeException If an error occurs in the JE database.
+ */
+ public synchronized void setTrusted(WriteableStorage txn, boolean trusted)
+ throws StorageRuntimeException
+ {
+ this.trusted = trusted;
+ state.putIndexTrustState(txn, this, trusted);
+ }
+
+ /**
+ * Return true iff this index is trusted.
+ * @return the trusted state of this index
+ */
+ public boolean isTrusted()
+ {
+ return trusted;
+ }
+
+ /**
+ * Set the rebuild status of this vlvIndex.
+ * @param rebuildRunning True if a rebuild process on this vlvIndex
+ * is running or False otherwise.
+ */
+ public synchronized void setRebuildStatus(boolean rebuildRunning)
+ {
+ this.rebuildRunning = rebuildRunning;
+ }
+
+ /**
+ * Gets the values to sort on from the entry.
+ *
+ * @param entry The entry to get the values from.
+ * @return The attribute values to sort on.
+ */
+ ByteString[] getSortValues(Entry entry)
+ {
+ SortKey[] sortKeys = sortOrder.getSortKeys();
+ ByteString[] values = new ByteString[sortKeys.length];
+ for (int i=0; i < sortKeys.length; i++)
+ {
+ SortKey sortKey = sortKeys[i];
+ List<Attribute> attrList = entry.getAttribute(sortKey.getAttributeType());
+ if (attrList != null)
+ {
+ // There may be multiple versions of this attribute in the target entry
+ // (e.g., with different sets of options), and it may also be a
+ // multivalued attribute. In that case, we need to find the value that
+ // is the best match for the corresponding sort key (i.e., for sorting
+ // in ascending order, we want to find the lowest value; for sorting in
+ // descending order, we want to find the highest value). This is
+ // handled by the SortKey.compareValues method.
+ ByteString sortValue = null;
+ for (Attribute a : attrList)
+ {
+ for (ByteString v : a)
+ {
+ if (sortValue == null || sortKey.compareValues(v, sortValue) < 0)
+ {
+ sortValue = v;
+ }
+ }
+ }
+
+ values[i] = sortValue;
+ }
+ }
+ return values;
+ }
+
+ /**
+ * Encode a VLV database key with the given information.
+ *
+ * @param entryID The entry ID to encode.
+ * @param values The values to encode.
+ * @param types The types of the values to encode.
+ * @return The encoded bytes.
+ * @throws DirectoryException If a Directory Server error occurs.
+ */
+ ByteString encodeKey(long entryID, ByteString[] values, AttributeType[] types)
+ throws DirectoryException
+ {
+ try
+ {
+ final ByteStringBuilder builder = new ByteStringBuilder();
+
+ for (int i = 0; i < values.length; i++)
+ {
+ final ByteString v = values[i];
+ if (v == null)
+ {
+ builder.appendBERLength(0);
+ }
+ else
+ {
+ final MatchingRule eqRule = types[i].getEqualityMatchingRule();
+ final ByteString nv = eqRule.normalizeAttributeValue(v);
+ builder.appendBERLength(nv.length());
+ builder.append(nv);
+ }
+ }
+ builder.append(entryID);
+ builder.trimToSize();
+
+ return builder.toByteString();
+ }
+ catch (DecodeException e)
+ {
+ throw new DirectoryException(
+ ResultCode.INVALID_ATTRIBUTE_SYNTAX, e.getMessageObject(), e);
+ }
+ }
+
+ /**
+ * Decode a VLV database key.
+ *
+ * @param keyBytes The byte array to decode.
+ * @return The sort values represented by the key bytes.
+ * @throws DirectoryException If a Directory Server error occurs.
+ */
+ private SortValues decodeKey(ByteString keyBytes) throws DirectoryException
+ {
+ if(keyBytes == null || keyBytes.length() == 0)
+ {
+ return null;
+ }
+
+ ByteString[] attributeValues = new ByteString[sortOrder.getSortKeys().length];
+ int vBytesPos = 0;
+
+ for(int i = 0; i < attributeValues.length; i++)
+ {
+ int valueLength = keyBytes.byteAt(vBytesPos) & 0x7F;
+ if (valueLength != keyBytes.byteAt(vBytesPos++))
+ {
+ int valueLengthBytes = valueLength;
+ valueLength = 0;
+ for (int j=0; j < valueLengthBytes; j++, vBytesPos++)
+ {
+ valueLength = (valueLength << 8) | (keyBytes.byteAt(vBytesPos) & 0xFF);
+ }
+ }
+
+ if(valueLength == 0)
+ {
+ attributeValues[i] = null;
+ }
+ else
+ {
+ byte[] valueBytes = new byte[valueLength];
+ System.arraycopy(keyBytes, vBytesPos, valueBytes, 0, valueLength);
+ attributeValues[i] = ByteString.wrap(valueBytes);
+ }
+
+ vBytesPos += valueLength;
+ }
+
+ final long id = JebFormat.toLong(keyBytes.toByteArray(), vBytesPos, keyBytes.length());
+ return new SortValues(new EntryID(id), attributeValues, sortOrder);
+ }
+
+ /**
+ * Get the sorted set capacity configured for this VLV index.
+ *
+ * @return The sorted set capacity.
+ */
+ public int getSortedSetCapacity()
+ {
+ return sortedSetCapacity;
+ }
+
+ /**
+ * Indicates if the given entry should belong in this VLV index.
+ *
+ * @param entry The entry to check.
+ * @return True if the given entry should belong in this VLV index or False
+ * otherwise.
+ * @throws DirectoryException If a Directory Server error occurs.
+ */
+ public boolean shouldInclude(Entry entry) throws DirectoryException
+ {
+ DN entryDN = entry.getName();
+ return entryDN.matchesBaseAndScope(baseDN, scope)
+ && filter.matchesEntry(entry);
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public synchronized boolean isConfigurationChangeAcceptable(
+ LocalDBVLVIndexCfg cfg,
+ List<LocalizableMessage> unacceptableReasons)
+ {
+ try
+ {
+ this.filter = SearchFilter.createFilterFromString(cfg.getFilter());
+ }
+ catch(Exception e)
+ {
+ LocalizableMessage msg = ERR_JEB_CONFIG_VLV_INDEX_BAD_FILTER.get(
+ cfg.getFilter(), treeName,
+ stackTraceToSingleLineString(e));
+ unacceptableReasons.add(msg);
+ return false;
+ }
+
+ String[] sortAttrs = cfg.getSortOrder().split(" ");
+ SortKey[] sortKeys = new SortKey[sortAttrs.length];
+ MatchingRule[] orderingRules = new MatchingRule[sortAttrs.length];
+ boolean[] ascending = new boolean[sortAttrs.length];
+ for(int i = 0; i < sortAttrs.length; i++)
+ {
+ try
+ {
+ if(sortAttrs[i].startsWith("-"))
+ {
+ ascending[i] = false;
+ sortAttrs[i] = sortAttrs[i].substring(1);
+ }
+ else
+ {
+ ascending[i] = true;
+ if(sortAttrs[i].startsWith("+"))
+ {
+ sortAttrs[i] = sortAttrs[i].substring(1);
+ }
+ }
+ }
+ catch(Exception e)
+ {
+ unacceptableReasons.add(ERR_JEB_CONFIG_VLV_INDEX_UNDEFINED_ATTR.get(sortKeys[i], treeName));
+ return false;
+ }
+
+ AttributeType attrType = DirectoryServer.getAttributeType(sortAttrs[i].toLowerCase());
+ if(attrType == null)
+ {
+ LocalizableMessage msg = ERR_JEB_CONFIG_VLV_INDEX_UNDEFINED_ATTR.get(sortAttrs[i], treeName);
+ unacceptableReasons.add(msg);
+ return false;
+ }
+ sortKeys[i] = new SortKey(attrType, ascending[i]);
+ orderingRules[i] = attrType.getOrderingMatchingRule();
+ }
+
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public synchronized ConfigChangeResult applyConfigurationChange(
+ LocalDBVLVIndexCfg cfg)
+ {
+ ResultCode resultCode = ResultCode.SUCCESS;
+ boolean adminActionRequired = false;
+ ArrayList<LocalizableMessage> messages = new ArrayList<LocalizableMessage>();
+
+ // Update base DN only if changed..
+ if(!config.getBaseDN().equals(cfg.getBaseDN()))
+ {
+ this.baseDN = cfg.getBaseDN();
+ adminActionRequired = true;
+ }
+
+ // Update scope only if changed.
+ if(!config.getScope().equals(cfg.getScope()))
+ {
+ this.scope = SearchScope.valueOf(cfg.getScope().name());
+ adminActionRequired = true;
+ }
+
+ // Update sort set capacity only if changed.
+ if (config.getMaxBlockSize() != cfg.getMaxBlockSize())
+ {
+ this.sortedSetCapacity = cfg.getMaxBlockSize();
+
+ // Require admin action only if the new capacity is larger. Otherwise,
+ // we will lazyly update the sorted sets.
+ if (config.getMaxBlockSize() < cfg.getMaxBlockSize())
+ {
+ adminActionRequired = true;
+ }
+ }
+
+ // Update the filter only if changed.
+ if(!config.getFilter().equals(cfg.getFilter()))
+ {
+ try
+ {
+ this.filter = SearchFilter.createFilterFromString(cfg.getFilter());
+ adminActionRequired = true;
+ }
+ catch(Exception e)
+ {
+ LocalizableMessage msg = ERR_JEB_CONFIG_VLV_INDEX_BAD_FILTER.get(
+ config.getFilter(), treeName,
+ stackTraceToSingleLineString(e));
+ messages.add(msg);
+ if(resultCode == ResultCode.SUCCESS)
+ {
+ resultCode = ResultCode.INVALID_ATTRIBUTE_SYNTAX;
+ }
+ }
+ }
+
+ // Update the sort order only if changed.
+ if (!config.getSortOrder().equals(cfg.getSortOrder()))
+ {
+ String[] sortAttrs = cfg.getSortOrder().split(" ");
+ SortKey[] sortKeys = new SortKey[sortAttrs.length];
+ MatchingRule[] orderingRules = new MatchingRule[sortAttrs.length];
+ boolean[] ascending = new boolean[sortAttrs.length];
+ for(int i = 0; i < sortAttrs.length; i++)
+ {
+ try
+ {
+ if(sortAttrs[i].startsWith("-"))
+ {
+ ascending[i] = false;
+ sortAttrs[i] = sortAttrs[i].substring(1);
+ }
+ else
+ {
+ ascending[i] = true;
+ if(sortAttrs[i].startsWith("+"))
+ {
+ sortAttrs[i] = sortAttrs[i].substring(1);
+ }
+ }
+ }
+ catch(Exception e)
+ {
+ messages.add(ERR_JEB_CONFIG_VLV_INDEX_UNDEFINED_ATTR.get(sortKeys[i], treeName));
+ if(resultCode == ResultCode.SUCCESS)
+ {
+ resultCode = ResultCode.INVALID_ATTRIBUTE_SYNTAX;
+ }
+ }
+
+ AttributeType attrType =
+ DirectoryServer.getAttributeType(sortAttrs[i].toLowerCase());
+ if(attrType == null)
+ {
+ messages.add(ERR_JEB_CONFIG_VLV_INDEX_UNDEFINED_ATTR.get(sortKeys[i], treeName));
+ if(resultCode == ResultCode.SUCCESS)
+ {
+ resultCode = ResultCode.INVALID_ATTRIBUTE_SYNTAX;
+ }
+ }
+ else
+ {
+ sortKeys[i] = new SortKey(attrType, ascending[i]);
+ orderingRules[i] = attrType.getOrderingMatchingRule();
+ }
+ }
+
+ this.sortOrder = new SortOrder(sortKeys);
+ this.comparator = new VLVKeyComparator(orderingRules, ascending);
+
+ // We have to close the database and open it using the new comparator.
+ entryContainer.exclusiveLock.lock();
+ try
+ {
+ close();
+ open();
+ }
+ catch(StorageRuntimeException de)
+ {
+ messages.add(LocalizableMessage.raw(StaticUtils.stackTraceToSingleLineString(de)));
+ if(resultCode == ResultCode.SUCCESS)
+ {
+ resultCode = DirectoryServer.getServerErrorResultCode();
+ }
+ }
+ finally
+ {
+ entryContainer.exclusiveLock.unlock();
+ }
+
+ adminActionRequired = true;
+ }
+
+
+ if(adminActionRequired)
+ {
+ trusted = false;
+ messages.add(NOTE_JEB_INDEX_ADD_REQUIRES_REBUILD.get(treeName));
+ try
+ {
+ state.putIndexTrustState(null, this, false);
+ }
+ catch(StorageRuntimeException de)
+ {
+ messages.add(LocalizableMessage.raw(StaticUtils.stackTraceToSingleLineString(de)));
+ if(resultCode == ResultCode.SUCCESS)
+ {
+ resultCode = DirectoryServer.getServerErrorResultCode();
+ }
+ }
+ }
+
+ this.config = cfg;
+ return new ConfigChangeResult(resultCode, adminActionRequired, messages);
+ }
+}
diff --git a/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/VLVKeyComparator.java b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/VLVKeyComparator.java
new file mode 100644
index 0000000..0c3f86b
--- /dev/null
+++ b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/VLVKeyComparator.java
@@ -0,0 +1,351 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License"). You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at legal-notices/CDDLv1_0.txt
+ * or http://forgerock.org/license/CDDLv1.0.html.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at legal-notices/CDDLv1_0.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information:
+ * Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ *
+ *
+ * Copyright 2006-2008 Sun Microsystems, Inc.
+ * Portions Copyright 2014 ForgeRock AS
+ */
+package org.opends.server.backends.pluggable;
+
+import java.util.Comparator;
+
+import org.forgerock.opendj.ldap.ByteSequence;
+import org.forgerock.opendj.ldap.ByteString;
+import org.forgerock.opendj.ldap.DecodeException;
+import org.forgerock.opendj.ldap.ResultCode;
+import org.forgerock.opendj.ldap.schema.MatchingRule;
+import org.opends.server.backends.pluggable.BackendImpl.StorageRuntimeException;
+import org.opends.server.core.DirectoryServer;
+import org.opends.server.types.DirectoryException;
+
+/**
+ * This class is used to compare the keys used in a VLV index. Each key is
+ * made up the sort values and the entry ID of the largest entry in the sorted
+ * set stored in the data for the key.
+ */
+public class VLVKeyComparator implements DatabaseComparator
+{
+ /**
+ * The serial version identifier required to satisfy the compiler because this
+ * class implements the <CODE>java.io.Serializable</CODE> interface. This
+ * value was generated using the <CODE>serialver</CODE> command-line utility
+ * included with the Java SDK.
+ */
+ static final long serialVersionUID = 1585167927344130604L;
+
+ /** Matching rules are not serializable. */
+ private transient MatchingRule[] orderingRules;
+
+ /**
+ * Only oids of matching rules are recorded for serialization. Oids allow to
+ * retrieve matching rules after deserialization, through
+ * {@code initialize(ClassLoader)} method.
+ */
+ private String[] orderingRuleOids;
+
+ private boolean[] ascending;
+
+ /**
+ * Construct a new VLV Key Comparator object.
+ *
+ * @param orderingRules The array of ordering rules to use when comparing
+ * the decoded values in the key.
+ * @param ascending The array of booleans indicating the ordering for
+ * each value.
+ */
+ public VLVKeyComparator(MatchingRule[] orderingRules, boolean[] ascending)
+ {
+ this.orderingRules = orderingRules;
+ this.orderingRuleOids = new String[orderingRules.length];
+ for (int i = 0; i < orderingRules.length; i++)
+ {
+ orderingRuleOids[i] = orderingRules[i].getOID();
+ }
+ this.ascending = ascending;
+ }
+
+ /**
+ * Compares the contents of the provided byte arrays to determine their
+ * relative order. A key in the VLV index contains the sorted attribute values
+ * in order followed by the 8 byte entry ID. A attribute value of length 0
+ * means that value is null and the attribute type was not part of the entry.
+ * A null value is always considered greater then a non null value. If all
+ * attribute values are the same, the entry ID will be used to determine the
+ * ordering.
+ *
+ * When comparing partial keys (ie. keys with only the first attribute value
+ * encoded for evaluating VLV assertion value offsets or keys with no entry
+ * IDs), only information available in both byte keys will be used to
+ * determine the ordering. If all available information is the same, 0 will
+ * be returned.
+ *
+ * @param b1 The first byte array to use in the comparison.
+ * @param b2 The second byte array to use in the comparison.
+ *
+ * @return A negative integer if <CODE>b1</CODE> should come before
+ * <CODE>b2</CODE> in ascending order, a positive integer if
+ * <CODE>b1</CODE> should come after <CODE>b2</CODE> in ascending
+ * order, or zero if there is no difference between the values with
+ * regard to ordering.
+ */
+ @Override
+ public int compare(byte[] b1, byte[] b2)
+ {
+ // A 0 length byte array is a special key used for the unbound max
+ // sort values set. It always comes after a non length byte array.
+ if(b1.length == 0)
+ {
+ if(b2.length == 0)
+ {
+ return 0;
+ }
+ else
+ {
+ return 1;
+ }
+ }
+ else if(b2.length == 0)
+ {
+ return -1;
+ }
+
+ int b1Pos = 0;
+ int b2Pos = 0;
+ for (int j=0;
+ j < orderingRules.length && b1Pos < b1.length && b2Pos < b2.length;
+ j++)
+ {
+ int b1Length = b1[b1Pos] & 0x7F;
+ if (b1[b1Pos++] != b1Length)
+ {
+ int b1NumLengthBytes = b1Length;
+ b1Length = 0;
+ for (int k=0; k < b1NumLengthBytes; k++, b1Pos++)
+ {
+ b1Length = (b1Length << 8) |
+ (b1[b1Pos] & 0xFF);
+ }
+ }
+
+ int b2Length = b2[b2Pos] & 0x7F;
+ if (b2[b2Pos++] != b2Length)
+ {
+ int b2NumLengthBytes = b2Length;
+ b2Length = 0;
+ for (int k=0; k < b2NumLengthBytes; k++, b2Pos++)
+ {
+ b2Length = (b2Length << 8) |
+ (b2[b2Pos] & 0xFF);
+ }
+ }
+
+ byte[] b1Bytes;
+ byte[] b2Bytes;
+ if(b1Length > 0)
+ {
+ b1Bytes = new byte[b1Length];
+ System.arraycopy(b1, b1Pos, b1Bytes, 0, b1Length);
+ b1Pos += b1Length;
+ }
+ else
+ {
+ b1Bytes = null;
+ }
+
+ if(b2Length > 0)
+ {
+ b2Bytes = new byte[b2Length];
+ System.arraycopy(b2, b2Pos, b2Bytes, 0, b2Length);
+ b2Pos += b2Length;
+ }
+ else
+ {
+ b2Bytes = null;
+ }
+
+ // A null value will always come after a non-null value.
+ if (b1Bytes == null)
+ {
+ if (b2Bytes == null)
+ {
+ continue;
+ }
+ else
+ {
+ return 1;
+ }
+ }
+ else if (b2Bytes == null)
+ {
+ return -1;
+ }
+
+ final Comparator<ByteSequence> comp = orderingRules[j].comparator();
+ final ByteString val1 = ByteString.valueOf(b1Bytes);
+ final ByteString val2 = ByteString.valueOf(b2Bytes);
+ final int result = ascending[j] ? comp.compare(val1, val2) : comp.compare(val2, val1);
+
+ if(result != 0)
+ {
+ return result;
+ }
+ }
+
+ // If we've gotten here, then we can't tell a difference between the sets
+ // of available values, so sort based on entry ID if its in the key.
+
+ if(b1Pos + 8 <= b1.length && b2Pos + 8 <= b2.length)
+ {
+ long b1ID = JebFormat.toLong(b1, b1Pos, b1Pos + 8);
+ long b2ID = JebFormat.toLong(b2, b2Pos, b2Pos + 8);
+ return compare(b1ID, b2ID);
+ }
+
+ // If we've gotten here, then we can't tell the difference between the sets
+ // of available values and entry IDs are not all available, so just return 0
+ return 0;
+ }
+
+ /**
+ * Compares the contents in the provided values set with the given values to
+ * determine their relative order. A null value is always considered greater
+ * then a non null value. If all attribute values are the same, the entry ID
+ * will be used to determine the ordering.
+ *
+ * If the given attribute values array does not contain all the values in the
+ * sort order, any missing values will be considered as a unknown or
+ * wildcard value instead of a non existent value. When comparing partial
+ * information, only values available in both the values set and the
+ * given values will be used to determine the ordering. If all available
+ * information is the same, 0 will be returned.
+ *
+ * @param set The sort values set to containing the values.
+ * @param index The index of the values in the set.
+ * @param entryID The entry ID to use in the comparison.
+ * @param values The values to use in the comparison.
+ * @return A negative integer if the values in the set should come before
+ * the given values in ascending order, a positive integer if
+ * the values in the set should come after the given values in
+ * ascending order, or zero if there is no difference between the
+ * values with regard to ordering.
+ * @throws StorageRuntimeException If an error occurs during an operation on a
+ * JE database.
+ * @throws DirectoryException If an error occurs while trying to
+ * normalize the value (e.g., if it is
+ * not acceptable for use with the
+ * associated equality matching rule).
+ */
+ public int compare(SortValuesSet set, int index, long entryID,
+ ByteSequence... values) throws StorageRuntimeException, DirectoryException
+ {
+ for (int j=0; j < orderingRules.length; j++)
+ {
+ if(j >= values.length)
+ {
+ break;
+ }
+
+ ByteString b1Bytes = set.getValue((index * orderingRules.length) + j);
+ ByteString b2Bytes = null;
+
+ if(values[j] != null)
+ {
+ try
+ {
+ b2Bytes = orderingRules[j].normalizeAttributeValue(values[j]);
+ }
+ catch (DecodeException e)
+ {
+ throw new DirectoryException(
+ ResultCode.INVALID_ATTRIBUTE_SYNTAX, e.getMessageObject(), e);
+ }
+ }
+
+ // A null value will always come after a non-null value.
+ if (b1Bytes == null)
+ {
+ if (b2Bytes == null)
+ {
+ continue;
+ }
+ else
+ {
+ return 1;
+ }
+ }
+ else if (b2Bytes == null)
+ {
+ return -1;
+ }
+
+ final Comparator<ByteSequence> comp = orderingRules[j].comparator();
+ final int result = ascending[j] ? comp.compare(b1Bytes, b2Bytes) : comp.compare(b2Bytes, b1Bytes);
+
+ if(result != 0)
+ {
+ return result;
+ }
+ }
+
+ if(entryID != -1)
+ {
+ // If we've gotten here, then we can't tell a difference between the sets
+ // of values, so sort based on entry ID.
+ return compare(set.getEntryIDs()[index], entryID);
+ }
+
+ // If we've gotten here, then we can't tell the difference between the sets
+ // of available values and the entry ID is not available. Just return 0.
+ return 0;
+ }
+
+ private int compare(long l1, long l2)
+ {
+ final long difference = l1 - l2;
+ if (difference < 0)
+ {
+ return -1;
+ }
+ else if (difference > 0)
+ {
+ return 1;
+ }
+ else
+ {
+ return 0;
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void initialize(ClassLoader loader)
+ {
+ if (orderingRules == null)
+ {
+ orderingRules = new MatchingRule[orderingRuleOids.length];
+ for (int i = 0; i < orderingRuleOids.length; i++)
+ {
+ orderingRules[i] = DirectoryServer.getSchema().getMatchingRule(orderingRuleOids[i]);
+ }
+ }
+ }
+}
diff --git a/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/VerifyConfig.java b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/VerifyConfig.java
new file mode 100644
index 0000000..712e582
--- /dev/null
+++ b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/VerifyConfig.java
@@ -0,0 +1,116 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License"). You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at legal-notices/CDDLv1_0.txt
+ * or http://forgerock.org/license/CDDLv1.0.html.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at legal-notices/CDDLv1_0.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information:
+ * Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ *
+ *
+ * Copyright 2006-2008 Sun Microsystems, Inc.
+ * Portions Copyright 2014 ForgeRock AS
+ */
+package org.opends.server.backends.pluggable;
+
+import org.opends.server.types.DN;
+
+import java.util.ArrayList;
+
+/**
+ * This class represents the configuration of a JE backend verification process.
+ */
+public class VerifyConfig
+{
+ /**
+ * The base DN to be verified.
+ */
+ private DN baseDN;
+
+ /**
+ * The names of indexes to be verified for completeness.
+ */
+ private ArrayList<String> completeList;
+
+ /**
+ * The names of indexes to be verified for cleanliness.
+ */
+ private ArrayList<String> cleanList;
+
+ /**
+ * Create a new verify configuration.
+ */
+ public VerifyConfig()
+ {
+ baseDN = null;
+ completeList = new ArrayList<String>();
+ cleanList = new ArrayList<String>();
+ }
+
+ /**
+ * Get the base DN to be verified.
+ * @return The base DN to be verified.
+ */
+ public DN getBaseDN()
+ {
+ return baseDN;
+ }
+
+ /**
+ * Set the base DN to be verified.
+ * @param baseDN The base DN to be verified.
+ */
+ public void setBaseDN(DN baseDN)
+ {
+ this.baseDN = baseDN;
+ }
+
+ /**
+ * Get the names of indexes to be verified for completeness.
+ * @return The names of indexes to be verified for completeness.
+ */
+ public ArrayList<String> getCompleteList()
+ {
+ return completeList;
+ }
+
+ /**
+ * Add the name of an index to those indexes to be verified for completeness.
+ * @param index The name of an index to be verified for completeness.
+ */
+ public void addCompleteIndex(String index)
+ {
+ completeList.add(index);
+ }
+
+ /**
+ * Get the names of indexes to be verified for cleanliness.
+ * @return The names of indexes to be verified for cleanliness.
+ */
+ public ArrayList<String> getCleanList()
+ {
+ return cleanList;
+ }
+
+ /**
+ * Add the name of an index to those indexes to be verified for cleanliness.
+ * @param index The name of an index to be verified for cleanliness.
+ */
+ public void addCleanIndex(String index)
+ {
+ cleanList.add(index);
+ }
+}
diff --git a/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/VerifyJob.java b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/VerifyJob.java
new file mode 100644
index 0000000..a836d8d
--- /dev/null
+++ b/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/VerifyJob.java
@@ -0,0 +1,1808 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License"). You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at legal-notices/CDDLv1_0.txt
+ * or http://forgerock.org/license/CDDLv1.0.html.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at legal-notices/CDDLv1_0.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information:
+ * Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ *
+ *
+ * Copyright 2006-2010 Sun Microsystems, Inc.
+ * Portions Copyright 2011-2014 ForgeRock AS
+ */
+package org.opends.server.backends.pluggable;
+import java.util.AbstractSet;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.IdentityHashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.Timer;
+import java.util.TimerTask;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.forgerock.i18n.LocalizableMessage;
+import org.forgerock.i18n.slf4j.LocalizedLogger;
+import org.forgerock.opendj.ldap.ByteSequence;
+import org.forgerock.opendj.ldap.ByteString;
+import org.forgerock.opendj.ldap.ConditionResult;
+import org.forgerock.opendj.ldap.DecodeException;
+import org.forgerock.opendj.ldap.ResultCode;
+import org.forgerock.opendj.ldap.schema.MatchingRule;
+import org.forgerock.opendj.ldap.spi.IndexingOptions;
+import org.opends.server.backends.pluggable.BackendImpl.Cursor;
+import org.opends.server.backends.pluggable.BackendImpl.ReadOperation;
+import org.opends.server.backends.pluggable.BackendImpl.ReadableStorage;
+import org.opends.server.backends.pluggable.BackendImpl.Storage;
+import org.opends.server.backends.pluggable.BackendImpl.StorageRuntimeException;
+import org.opends.server.core.DirectoryServer;
+import org.opends.server.types.Attribute;
+import org.opends.server.types.AttributeType;
+import org.opends.server.types.Attributes;
+import org.opends.server.types.DN;
+import org.opends.server.types.DirectoryException;
+import org.opends.server.types.Entry;
+import org.opends.server.util.ServerConstants;
+import org.opends.server.util.StaticUtils;
+
+import com.sleepycat.je.EnvironmentStats;
+import com.sleepycat.je.StatsConfig;
+
+import static org.opends.messages.JebMessages.*;
+import static org.opends.server.backends.pluggable.JebFormat.*;
+
+/**
+ * This class is used to run an index verification process on the backend.
+ */
+public class VerifyJob
+{
+ private static final LocalizedLogger logger = LocalizedLogger.getLoggerForThisClass();
+
+ /** The verify configuration. */
+ private final VerifyConfig verifyConfig;
+ /** The root container used for the verify job. */
+ private RootContainer rootContainer;
+
+ /** The number of milliseconds between job progress reports. */
+ private final long progressInterval = 10000;
+ /** The number of index keys processed. */
+ private long keyCount;
+ /** The number of errors found. */
+ private long errorCount;
+ /** The number of records that have exceeded the entry limit. */
+ private long entryLimitExceededCount;
+ /** The number of records that reference more than one entry. */
+ private long multiReferenceCount;
+ /** The total number of entry references. */
+ private long entryReferencesCount;
+ /** The maximum number of references per record. */
+ private long maxEntryPerValue;
+
+ /**
+ * This map is used to gather some statistics about values that have
+ * exceeded the entry limit.
+ */
+ private IdentityHashMap<Index, HashMap<ByteString, Long>> entryLimitMap =
+ new IdentityHashMap<Index, HashMap<ByteString, Long>>();
+
+ /** Indicates whether the DN database is to be verified. */
+ private boolean verifyDN2ID;
+ /** Indicates whether the children database is to be verified. */
+ private boolean verifyID2Children;
+ /** Indicates whether the subtree database is to be verified. */
+ private boolean verifyID2Subtree;
+
+ /** The entry database. */
+ private ID2Entry id2entry;
+ /** The DN database. */
+ private DN2ID dn2id;
+ /** The children database. */
+ private Index id2c;
+ /** The subtree database. */
+ private Index id2s;
+
+ /**
+ * A list of the attribute indexes to be verified.
+ */
+ private final ArrayList<AttributeIndex> attrIndexList = new ArrayList<AttributeIndex>();
+
+ /**
+ * A list of the VLV indexes to be verified.
+ */
+ private final ArrayList<VLVIndex> vlvIndexList = new ArrayList<VLVIndex>();
+
+ /**
+ * Construct a VerifyJob.
+ *
+ * @param verifyConfig The verify configuration.
+ */
+ public VerifyJob(VerifyConfig verifyConfig)
+ {
+ this.verifyConfig = verifyConfig;
+ }
+
+ /**
+ * Verify the backend.
+ *
+ * @param rootContainer The root container that holds the entries to verify.
+ * @param statEntry Optional statistics entry.
+ * @return The error count.
+ * @throws StorageRuntimeException If an error occurs in the JE database.
+ * @throws JebException If an error occurs in the JE backend.
+ * @throws DirectoryException If an error occurs while verifying the backend.
+ */
+ public long verifyBackend(final RootContainer rootContainer, final Entry statEntry) throws StorageRuntimeException,
+ JebException, DirectoryException
+ {
+ Storage s;
+ try
+ {
+ return s.read(new ReadOperation<Long>()
+ {
+ @Override
+ public Long run(ReadableStorage txn) throws Exception
+ {
+ return verifyBackend0(txn, rootContainer, statEntry);
+ }
+ });
+ }
+ catch (Exception e)
+ {
+ throw new StorageRuntimeException(e);
+ }
+ }
+
+ private long verifyBackend0(ReadableStorage txn, RootContainer rootContainer, Entry statEntry)
+ throws StorageRuntimeException, JebException, DirectoryException
+ {
+ this.rootContainer = rootContainer;
+ EntryContainer entryContainer =
+ rootContainer.getEntryContainer(verifyConfig.getBaseDN());
+
+ entryContainer.sharedLock.lock();
+ try
+ {
+ ArrayList<String> completeList = verifyConfig.getCompleteList();
+ ArrayList<String> cleanList = verifyConfig.getCleanList();
+
+ boolean cleanMode = false;
+ if (completeList.isEmpty() && cleanList.isEmpty())
+ {
+ verifyDN2ID = true;
+ if (rootContainer.getConfiguration().isSubordinateIndexesEnabled())
+ {
+ verifyID2Children = true;
+ verifyID2Subtree = true;
+ }
+ attrIndexList.addAll(entryContainer.getAttributeIndexes());
+ }
+ else
+ {
+ ArrayList<String> list;
+ if (!completeList.isEmpty())
+ {
+ list = completeList;
+ }
+ else
+ {
+ list = cleanList;
+ cleanMode = true;
+ }
+
+ for (String index : list)
+ {
+ String lowerName = index.toLowerCase();
+ if ("dn2id".equals(lowerName))
+ {
+ verifyDN2ID = true;
+ }
+ else if ("id2children".equals(lowerName))
+ {
+ if (rootContainer.getConfiguration().isSubordinateIndexesEnabled())
+ {
+ verifyID2Children = true;
+ }
+ else
+ {
+ LocalizableMessage msg = NOTE_JEB_SUBORDINATE_INDEXES_DISABLED
+ .get(rootContainer.getConfiguration().getBackendId());
+ throw new JebException(msg);
+ }
+ }
+ else if ("id2subtree".equals(lowerName))
+ {
+ if (rootContainer.getConfiguration().isSubordinateIndexesEnabled())
+ {
+ verifyID2Subtree = true;
+ }
+ else
+ {
+ LocalizableMessage msg = NOTE_JEB_SUBORDINATE_INDEXES_DISABLED
+ .get(rootContainer.getConfiguration().getBackendId());
+ throw new JebException(msg);
+ }
+ }
+ else if(lowerName.startsWith("vlv."))
+ {
+ if(lowerName.length() < 5)
+ {
+ LocalizableMessage msg = ERR_JEB_VLV_INDEX_NOT_CONFIGURED.get(lowerName);
+ throw new JebException(msg);
+ }
+
+ VLVIndex vlvIndex =
+ entryContainer.getVLVIndex(lowerName.substring(4));
+ if(vlvIndex == null)
+ {
+ LocalizableMessage msg =
+ ERR_JEB_VLV_INDEX_NOT_CONFIGURED.get(lowerName.substring(4));
+ throw new JebException(msg);
+ }
+
+ vlvIndexList.add(vlvIndex);
+ }
+ else
+ {
+ AttributeType attrType =
+ DirectoryServer.getAttributeType(lowerName);
+ if (attrType == null)
+ {
+ LocalizableMessage msg = ERR_JEB_ATTRIBUTE_INDEX_NOT_CONFIGURED.get(index);
+ throw new JebException(msg);
+ }
+ AttributeIndex attrIndex =
+ entryContainer.getAttributeIndex(attrType);
+ if (attrIndex == null)
+ {
+ LocalizableMessage msg = ERR_JEB_ATTRIBUTE_INDEX_NOT_CONFIGURED.get(index);
+ throw new JebException(msg);
+ }
+ attrIndexList.add(attrIndex);
+ }
+ }
+ }
+
+ entryLimitMap =
+ new IdentityHashMap<Index,HashMap<ByteString,Long>>(
+ attrIndexList.size());
+
+ // We will be updating these files independently of the indexes
+ // so we need direct access to them rather than going through
+ // the entry entryContainer methods.
+ id2entry = entryContainer.getID2Entry();
+ dn2id = entryContainer.getDN2ID();
+ id2c = entryContainer.getID2Children();
+ id2s = entryContainer.getID2Subtree();
+
+ // Make a note of the time we started.
+ long startTime = System.currentTimeMillis();
+
+ // Start a timer for the progress report.
+ Timer timer = new Timer();
+ TimerTask progressTask = new ProgressTask();
+ if (cleanMode)
+ {
+ // Create a new progressTask based on the index count.
+ progressTask = new ProgressTask(true);
+ }
+ timer.scheduleAtFixedRate(progressTask, progressInterval,
+ progressInterval);
+
+ // Iterate through the index keys.
+ try
+ {
+ if (cleanMode)
+ {
+ iterateIndex(txn);
+ }
+ else
+ {
+ iterateID2Entry(txn);
+
+ // Make sure the vlv indexes are in correct order.
+ for(VLVIndex vlvIndex : vlvIndexList)
+ {
+ iterateVLVIndex(txn, vlvIndex, false);
+ }
+ }
+ }
+ finally
+ {
+ timer.cancel();
+ }
+
+ long finishTime = System.currentTimeMillis();
+ long totalTime = finishTime - startTime;
+
+ float rate = 0;
+ if (totalTime > 0)
+ {
+ rate = 1000f*keyCount / totalTime;
+ }
+
+ addStatEntry(statEntry, "verify-error-count", String.valueOf(errorCount));
+ addStatEntry(statEntry, "verify-key-count", String.valueOf(keyCount));
+ if (cleanMode)
+ {
+ logger.info(NOTE_JEB_VERIFY_CLEAN_FINAL_STATUS, keyCount, errorCount, totalTime/1000, rate);
+
+ if (multiReferenceCount > 0)
+ {
+ float averageEntryReferences = 0;
+ if (keyCount > 0)
+ {
+ averageEntryReferences = entryReferencesCount/keyCount;
+ }
+
+ logger.debug(INFO_JEB_VERIFY_MULTIPLE_REFERENCE_COUNT, multiReferenceCount);
+ addStatEntry(statEntry, "verify-multiple-reference-count",
+ String.valueOf(multiReferenceCount));
+
+ logger.debug(INFO_JEB_VERIFY_ENTRY_LIMIT_EXCEEDED_COUNT, entryLimitExceededCount);
+ addStatEntry(statEntry, "verify-entry-limit-exceeded-count",
+ String.valueOf(entryLimitExceededCount));
+
+ logger.debug(INFO_JEB_VERIFY_AVERAGE_REFERENCE_COUNT, averageEntryReferences);
+ addStatEntry(statEntry, "verify-average-reference-count",
+ String.valueOf(averageEntryReferences));
+
+ logger.debug(INFO_JEB_VERIFY_MAX_REFERENCE_COUNT, maxEntryPerValue);
+ addStatEntry(statEntry, "verify-max-reference-count",
+ String.valueOf(maxEntryPerValue));
+ }
+ }
+ else
+ {
+ logger.info(NOTE_JEB_VERIFY_FINAL_STATUS, keyCount, errorCount, totalTime/1000, rate);
+ //TODO add entry-limit-stats to the statEntry
+ if (entryLimitMap.size() > 0)
+ {
+ logger.debug(INFO_JEB_VERIFY_ENTRY_LIMIT_STATS_HEADER);
+
+ for (Map.Entry<Index,HashMap<ByteString,Long>> mapEntry :
+ entryLimitMap.entrySet())
+ {
+ Index index = mapEntry.getKey();
+ Long[] values = mapEntry.getValue().values().toArray(new Long[0]);
+
+ // Calculate the median value for entry limit exceeded.
+ Arrays.sort(values);
+ long medianValue;
+ int x = values.length / 2;
+ if (values.length % 2 == 0)
+ {
+ medianValue = (values[x] + values[x-1]) / 2;
+ }
+ else
+ {
+ medianValue = values[x];
+ }
+
+ logger.debug(INFO_JEB_VERIFY_ENTRY_LIMIT_STATS_ROW, index, values.length, values[0],
+ values[values.length-1], medianValue);
+ }
+ }
+ }
+ }
+ finally
+ {
+ entryContainer.sharedLock.unlock();
+ }
+ return errorCount;
+ }
+
+ /**
+ * Iterate through the entries in id2entry to perform a check for
+ * index completeness. We check that the ID for the entry is indeed
+ * present in the indexes for the appropriate values.
+ *
+ * @throws StorageRuntimeException If an error occurs in the JE database.
+ */
+ private void iterateID2Entry(ReadableStorage txn) throws StorageRuntimeException
+ {
+ Cursor cursor = id2entry.openCursor(txn);
+ try
+ {
+ long storedEntryCount = id2entry.getRecordCount();
+ while (cursor.next())
+ {
+ ByteString key = cursor.getKey();
+ ByteString value = cursor.getValue();
+
+ EntryID entryID;
+ try
+ {
+ entryID = new EntryID(key);
+ }
+ catch (Exception e)
+ {
+ errorCount++;
+ if (logger.isTraceEnabled())
+ {
+ logger.traceException(e);
+
+ logger.trace("Malformed id2entry ID %s.%n", StaticUtils.bytesToHex(key));
+ }
+ continue;
+ }
+
+ keyCount++;
+
+ Entry entry;
+ try
+ {
+ entry = ID2Entry.entryFromDatabase(value, rootContainer.getCompressedSchema());
+ }
+ catch (Exception e)
+ {
+ errorCount++;
+ if (logger.isTraceEnabled())
+ {
+ logger.traceException(e);
+
+ logger.trace("Malformed id2entry record for ID %d:%n%s%n", entryID, StaticUtils.bytesToHex(value));
+ }
+ continue;
+ }
+
+ verifyEntry(txn, entryID, entry);
+ }
+ if (keyCount != storedEntryCount)
+ {
+ errorCount++;
+ if (logger.isTraceEnabled())
+ {
+ logger.trace("The stored entry count in id2entry (%d) does " +
+ "not agree with the actual number of entry " +
+ "records found (%d).%n", storedEntryCount, keyCount);
+ }
+ }
+ }
+ finally
+ {
+ cursor.close();
+ }
+ }
+
+ /**
+ * Iterate through the entries in an index to perform a check for
+ * index cleanliness. For each ID in the index we check that the
+ * entry it refers to does indeed contain the expected value.
+ *
+ * @throws JebException If an error occurs in the JE backend.
+ * @throws StorageRuntimeException If an error occurs in the JE database.
+ * @throws DirectoryException If an error occurs reading values in the index.
+ */
+ private void iterateIndex(ReadableStorage txn)
+ throws JebException, StorageRuntimeException, DirectoryException
+ {
+ if (verifyDN2ID)
+ {
+ iterateDN2ID(txn);
+ }
+ else if (verifyID2Children)
+ {
+ iterateID2Children(txn);
+ }
+ else if (verifyID2Subtree)
+ {
+ iterateID2Subtree(txn);
+ }
+ else if (attrIndexList.size() > 0)
+ {
+ AttributeIndex attrIndex = attrIndexList.get(0);
+ final IndexingOptions options = attrIndex.getIndexingOptions();
+ iterateAttrIndex(txn, attrIndex.getEqualityIndex(), options);
+ iterateAttrIndex(txn, attrIndex.getPresenceIndex(), options);
+ iterateAttrIndex(txn, attrIndex.getSubstringIndex(), options);
+ iterateAttrIndex(txn, attrIndex.getOrderingIndex(), options);
+ iterateAttrIndex(txn, attrIndex.getApproximateIndex(), options);
+ // TODO: Need to iterate through ExtendedMatchingRules indexes.
+ }
+ else if (vlvIndexList.size() > 0)
+ {
+ iterateVLVIndex(txn, vlvIndexList.get(0), true);
+ }
+ }
+
+ /**
+ * Iterate through the entries in DN2ID to perform a check for
+ * index cleanliness.
+ *
+ * @throws StorageRuntimeException If an error occurs in the JE database.
+ */
+ private void iterateDN2ID(ReadableStorage txn) throws StorageRuntimeException
+ {
+ Cursor cursor = dn2id.openCursor(txn);
+ try
+ {
+ while (cursor.next())
+ {
+ keyCount++;
+
+ ByteString key = cursor.getKey();
+ ByteString value = cursor.getValue();
+
+ EntryID entryID;
+ try
+ {
+ entryID = new EntryID(value);
+ }
+ catch (Exception e)
+ {
+ errorCount++;
+ if (logger.isTraceEnabled())
+ {
+ logger.traceException(e);
+
+ logger.trace("File dn2id has malformed ID for DN <%s>:%n%s%n", key, StaticUtils.bytesToHex(value));
+ }
+ continue;
+ }
+
+ Entry entry;
+ try
+ {
+ entry = id2entry.get(txn, entryID, false);
+ }
+ catch (Exception e)
+ {
+ errorCount++;
+ logger.traceException(e);
+ continue;
+ }
+
+ if (entry == null)
+ {
+ errorCount++;
+ if (logger.isTraceEnabled())
+ {
+ logger.trace("File dn2id has DN <%s> referencing unknown ID %d%n", key, entryID);
+ }
+ }
+ else if (!key.equals(dnToDNKey(entry.getName(), verifyConfig.getBaseDN().size())))
+ {
+ errorCount++;
+ if (logger.isTraceEnabled())
+ {
+ logger.trace("File dn2id has DN <%s> referencing entry with wrong DN <%s>%n", key, entry.getName());
+ }
+ }
+ }
+ }
+ finally
+ {
+ cursor.close();
+ }
+ }
+
+ /**
+ * Iterate through the entries in ID2Children to perform a check for
+ * index cleanliness.
+ *
+ * @throws JebException If an error occurs in the JE backend.
+ * @throws StorageRuntimeException If an error occurs in the JE database.
+ */
+ private void iterateID2Children(ReadableStorage txn) throws JebException, StorageRuntimeException
+ {
+ Cursor cursor = id2c.openCursor(txn);
+ try
+ {
+ while (cursor.next())
+ {
+ keyCount++;
+
+ ByteString key = cursor.getKey();
+ ByteString value = cursor.getValue();
+
+ EntryID entryID;
+ try
+ {
+ entryID = new EntryID(key);
+ }
+ catch (Exception e)
+ {
+ errorCount++;
+ if (logger.isTraceEnabled())
+ {
+ logger.traceException(e);
+
+ logger.trace("File id2children has malformed ID %s%n", StaticUtils.bytesToHex(key));
+ }
+ continue;
+ }
+
+ EntryIDSet entryIDList;
+
+ try
+ {
+ JebFormat.entryIDListFromDatabase(value);
+ entryIDList = new EntryIDSet(key, value);
+ }
+ catch (Exception e)
+ {
+ errorCount++;
+ if (logger.isTraceEnabled())
+ {
+ logger.traceException(e);
+
+ logger.trace("File id2children has malformed ID list for ID %s:%n%s%n",
+ entryID, StaticUtils.bytesToHex(value));
+ }
+ continue;
+ }
+
+ updateIndexStats(entryIDList);
+
+ if (entryIDList.isDefined())
+ {
+ Entry entry;
+ try
+ {
+ entry = id2entry.get(txn, entryID, false);
+ }
+ catch (Exception e)
+ {
+ logger.traceException(e);
+ errorCount++;
+ continue;
+ }
+
+ if (entry == null)
+ {
+ errorCount++;
+ if (logger.isTraceEnabled())
+ {
+ logger.trace("File id2children has unknown ID %d%n", entryID);
+ }
+ continue;
+ }
+
+ for (EntryID id : entryIDList)
+ {
+ Entry childEntry;
+ try
+ {
+ childEntry = id2entry.get(txn, id, false);
+ }
+ catch (Exception e)
+ {
+ logger.traceException(e);
+ errorCount++;
+ continue;
+ }
+
+ if (childEntry == null)
+ {
+ errorCount++;
+ if (logger.isTraceEnabled())
+ {
+ logger.trace("File id2children has ID %d referencing " +
+ "unknown ID %d%n", entryID, id);
+ }
+ continue;
+ }
+
+ if (!childEntry.getName().isDescendantOf(entry.getName()) ||
+ childEntry.getName().size() !=
+ entry.getName().size() + 1)
+ {
+ errorCount++;
+ if (logger.isTraceEnabled())
+ {
+ logger.trace("File id2children has ID %d with DN <%s> " +
+ "referencing ID %d with non-child DN <%s>%n",
+ entryID, entry.getName(), id, childEntry.getName());
+ }
+ }
+ }
+ }
+ }
+ }
+ finally
+ {
+ cursor.close();
+ }
+ }
+
+ /**
+ * Iterate through the entries in ID2Subtree to perform a check for
+ * index cleanliness.
+ *
+ * @throws JebException If an error occurs in the JE backend.
+ * @throws StorageRuntimeException If an error occurs in the JE database.
+ */
+ private void iterateID2Subtree(ReadableStorage txn) throws JebException, StorageRuntimeException
+ {
+ Cursor cursor = id2s.openCursor(txn);
+ try
+ {
+ while (cursor.next())
+ {
+ keyCount++;
+
+ ByteString key = cursor.getKey();
+ ByteString value = cursor.getValue();
+
+ EntryID entryID;
+ try
+ {
+ entryID = new EntryID(key);
+ }
+ catch (Exception e)
+ {
+ errorCount++;
+ if (logger.isTraceEnabled())
+ {
+ logger.traceException(e);
+
+ logger.trace("File id2subtree has malformed ID %s%n", StaticUtils.bytesToHex(key));
+ }
+ continue;
+ }
+
+ EntryIDSet entryIDList;
+ try
+ {
+ JebFormat.entryIDListFromDatabase(value);
+ entryIDList = new EntryIDSet(key, value);
+ }
+ catch (Exception e)
+ {
+ errorCount++;
+ if (logger.isTraceEnabled())
+ {
+ logger.traceException(e);
+
+ logger.trace("File id2subtree has malformed ID list " +
+ "for ID %s:%n%s%n", entryID,
+ StaticUtils
+ .bytesToHex(value));
+ }
+ continue;
+ }
+
+ updateIndexStats(entryIDList);
+
+ if (entryIDList.isDefined())
+ {
+ Entry entry;
+ try
+ {
+ entry = id2entry.get(txn, entryID, false);
+ }
+ catch (Exception e)
+ {
+ logger.traceException(e);
+ errorCount++;
+ continue;
+ }
+
+ if (entry == null)
+ {
+ errorCount++;
+ if (logger.isTraceEnabled())
+ {
+ logger.trace("File id2subtree has unknown ID %d%n", entryID);
+ }
+ continue;
+ }
+
+ for (EntryID id : entryIDList)
+ {
+ Entry subordEntry;
+ try
+ {
+ subordEntry = id2entry.get(txn, id, false);
+ }
+ catch (Exception e)
+ {
+ logger.traceException(e);
+ errorCount++;
+ continue;
+ }
+
+ if (subordEntry == null)
+ {
+ errorCount++;
+ if (logger.isTraceEnabled())
+ {
+ logger.trace("File id2subtree has ID %d referencing " +
+ "unknown ID %d%n", entryID, id);
+ }
+ continue;
+ }
+
+ if (!subordEntry.getName().isDescendantOf(entry.getName()))
+ {
+ errorCount++;
+ if (logger.isTraceEnabled())
+ {
+ logger.trace("File id2subtree has ID %d with DN <%s> " +
+ "referencing ID %d with non-subordinate DN <%s>%n",
+ entryID, entry.getName(), id, subordEntry
+ .getName());
+ }
+ }
+ }
+ }
+ }
+ }
+ finally
+ {
+ cursor.close();
+ }
+ }
+
+ /**
+ * Increment the counter for a key that has exceeded the
+ * entry limit. The counter gives the number of entries that have
+ * referenced the key.
+ *
+ * @param index The index containing the key.
+ * @param key A key that has exceeded the entry limit.
+ */
+ private void incrEntryLimitStats(Index index, ByteString key)
+ {
+ HashMap<ByteString,Long> hashMap = entryLimitMap.get(index);
+ if (hashMap == null)
+ {
+ hashMap = new HashMap<ByteString, Long>();
+ entryLimitMap.put(index, hashMap);
+ }
+ Long counter = hashMap.get(key);
+ if (counter != null)
+ {
+ counter++;
+ }
+ else
+ {
+ counter = 1L;
+ }
+ hashMap.put(key, counter);
+ }
+
+ /**
+ * Update the statistical information for an index record.
+ *
+ * @param entryIDSet The set of entry IDs for the index record.
+ */
+ private void updateIndexStats(EntryIDSet entryIDSet)
+ {
+ if (!entryIDSet.isDefined())
+ {
+ entryLimitExceededCount++;
+ multiReferenceCount++;
+ }
+ else
+ {
+ if (entryIDSet.size() > 1)
+ {
+ multiReferenceCount++;
+ }
+ entryReferencesCount += entryIDSet.size();
+ maxEntryPerValue = Math.max(maxEntryPerValue, entryIDSet.size());
+ }
+ }
+
+ /**
+ * Iterate through the entries in a VLV index to perform a check for index
+ * cleanliness.
+ *
+ * @param vlvIndex The VLV index to perform the check against.
+ * @param verifyID True to verify the IDs against id2entry.
+ * @throws JebException If an error occurs in the JE backend.
+ * @throws StorageRuntimeException If an error occurs in the JE database.
+ * @throws DirectoryException If an error occurs reading values in the index.
+ */
+ private void iterateVLVIndex(ReadableStorage txn, VLVIndex vlvIndex, boolean verifyID)
+ throws JebException, StorageRuntimeException, DirectoryException
+ {
+ if(vlvIndex == null)
+ {
+ return;
+ }
+
+ Cursor cursor = vlvIndex.openCursor(txn);
+ try
+ {
+ SortValues lastValues = null;
+ while (cursor.next())
+ {
+ ByteString key = cursor.getKey();
+ ByteString value = cursor.getValue();
+
+ SortValuesSet sortValuesSet = new SortValuesSet(key, value, vlvIndex);
+ for(int i = 0; i < sortValuesSet.getEntryIDs().length; i++)
+ {
+ keyCount++;
+ SortValues values = sortValuesSet.getSortValues(i);
+ if(lastValues != null && lastValues.compareTo(values) >= 1)
+ {
+ // Make sure the values is larger then the previous one.
+ if(logger.isTraceEnabled())
+ {
+ logger.trace("Values %s and %s are incorrectly ordered",
+ lastValues, values, keyDump(vlvIndex,
+ sortValuesSet.getKeySortValues()));
+ }
+ errorCount++;
+ }
+ if (i == sortValuesSet.getEntryIDs().length - 1 && key.length() != 0)
+ {
+ // If this is the last one in a bounded set, make sure it is the
+ // same as the database key.
+ ByteString encodedKey = vlvIndex.encodeKey(
+ values.getEntryID(), values.getValues(), values.getTypes());
+ if (!key.equals(encodedKey))
+ {
+ if(logger.isTraceEnabled())
+ {
+ logger.trace("Incorrect key for SortValuesSet in VLV " +
+ "index %s. Last values bytes %s, Key bytes %s",
+ vlvIndex.getName(), encodedKey, key);
+ }
+ errorCount++;
+ }
+ }
+ lastValues = values;
+
+ if(verifyID)
+ {
+ Entry entry;
+ EntryID id = new EntryID(values.getEntryID());
+ try
+ {
+ entry = id2entry.get(txn, id, false);
+ }
+ catch (Exception e)
+ {
+ logger.traceException(e);
+ errorCount++;
+ continue;
+ }
+
+ if (entry == null)
+ {
+ errorCount++;
+ if (logger.isTraceEnabled())
+ {
+ logger.trace("Reference to unknown ID %d%n%s",
+ id, keyDump(vlvIndex, sortValuesSet.getKeySortValues()));
+ }
+ continue;
+ }
+
+ SortValues entryValues = new SortValues(id, entry, vlvIndex.sortOrder);
+ if(entryValues.compareTo(values) != 0)
+ {
+ errorCount++;
+ if(logger.isTraceEnabled())
+ {
+ logger.trace("Reference to entry ID %d " +
+ "which does not match the values%n%s",
+ id, keyDump(vlvIndex, sortValuesSet.getKeySortValues()));
+ }
+ }
+ }
+ }
+ }
+ }
+ finally
+ {
+ cursor.close();
+ }
+ }
+
+ /**
+ * Iterate through the entries in an attribute index to perform a check for
+ * index cleanliness.
+ * @param index The index database to be checked.
+ * @throws JebException If an error occurs in the JE backend.
+ * @throws StorageRuntimeException If an error occurs in the JE database.
+ */
+ private void iterateAttrIndex(ReadableStorage txn, Index index, IndexingOptions options)
+ throws JebException, StorageRuntimeException
+ {
+ if (index == null)
+ {
+ return;
+ }
+
+ Cursor cursor = index.openCursor(txn);
+ try
+ {
+ while (cursor.next())
+ {
+ keyCount++;
+
+ final ByteString key = cursor.getKey();
+ ByteString value = cursor.getValue();
+
+ EntryIDSet entryIDList;
+ try
+ {
+ JebFormat.entryIDListFromDatabase(value);
+ entryIDList = new EntryIDSet(key, value);
+ }
+ catch (Exception e)
+ {
+ errorCount++;
+ if (logger.isTraceEnabled())
+ {
+ logger.traceException(e);
+
+ logger.trace("Malformed ID list: %s%n%s",
+ StaticUtils.bytesToHex(value), keyDump(index, key));
+ }
+ continue;
+ }
+
+ updateIndexStats(entryIDList);
+
+ if (entryIDList.isDefined())
+ {
+ EntryID prevID = null;
+
+ for (EntryID id : entryIDList)
+ {
+ if (prevID != null && id.equals(prevID) && logger.isTraceEnabled())
+ {
+ logger.trace("Duplicate reference to ID %d%n%s", id, keyDump(index, key));
+ }
+ prevID = id;
+
+ Entry entry;
+ try
+ {
+ entry = id2entry.get(txn, id, false);
+ }
+ catch (Exception e)
+ {
+ logger.traceException(e);
+ errorCount++;
+ continue;
+ }
+
+ if (entry == null)
+ {
+ errorCount++;
+ if (logger.isTraceEnabled())
+ {
+ logger.trace("Reference to unknown ID %d%n%s", id, keyDump(index, key));
+ }
+ continue;
+ }
+
+ // As an optimization avoid passing in a real set and wasting time
+ // hashing and comparing a potentially large set of values, as well
+ // as using up memory. Instead just intercept the add() method and
+ // detect when an equivalent value has been added.
+
+ // We need to use an AtomicBoolean here since anonymous classes
+ // require referenced external variables to be final.
+ final AtomicBoolean foundMatchingKey = new AtomicBoolean(false);
+
+ Set<ByteString> dummySet = new AbstractSet<ByteString>()
+ {
+ @Override
+ public Iterator<ByteString> iterator()
+ {
+ // The set is always empty.
+ return Collections.<ByteString> emptySet().iterator();
+ }
+
+ @Override
+ public int size()
+ {
+ // The set is always empty.
+ return 0;
+ }
+
+ @Override
+ public boolean add(ByteString e)
+ {
+ if (key.equals(e))
+ {
+ // We could terminate processing at this point by throwing an
+ // UnsupportedOperationException, but this optimization is
+ // already ugly enough.
+ foundMatchingKey.set(true);
+ }
+ return true;
+ }
+
+ };
+
+ index.indexer.indexEntry(entry, dummySet, options);
+
+ if (!foundMatchingKey.get())
+ {
+ errorCount++;
+ if (logger.isTraceEnabled())
+ {
+ logger.trace("Reference to entry "
+ + "<%s> which does not match the value%n%s",
+ entry.getName(), keyDump(index, key));
+ }
+ }
+ }
+ }
+ }
+ }
+ finally
+ {
+ cursor.close();
+ }
+ }
+
+ /**
+ * Check that an index is complete for a given entry.
+ *
+ * @param entryID The entry ID.
+ * @param entry The entry to be checked.
+ */
+ private void verifyEntry(ReadableStorage txn, EntryID entryID, Entry entry)
+ {
+ if (verifyDN2ID)
+ {
+ verifyDN2ID(txn, entryID, entry);
+ }
+ if (verifyID2Children)
+ {
+ verifyID2Children(txn, entryID, entry);
+ }
+ if (verifyID2Subtree)
+ {
+ verifyID2Subtree(txn, entryID, entry);
+ }
+ verifyIndex(txn, entryID, entry);
+ }
+
+ /**
+ * Check that the DN2ID index is complete for a given entry.
+ *
+ * @param entryID The entry ID.
+ * @param entry The entry to be checked.
+ */
+ private void verifyDN2ID(ReadableStorage txn, EntryID entryID, Entry entry)
+ {
+ DN dn = entry.getName();
+
+ // Check the ID is in dn2id with the correct DN.
+ try
+ {
+ EntryID id = dn2id.get(txn, dn, false);
+ if (id == null)
+ {
+ if (logger.isTraceEnabled())
+ {
+ logger.trace("File dn2id is missing key %s.%n", dn);
+ }
+ errorCount++;
+ }
+ else if (!id.equals(entryID))
+ {
+ if (logger.isTraceEnabled())
+ {
+ logger.trace("File dn2id has ID %d instead of %d for key %s.%n", id, entryID, dn);
+ }
+ errorCount++;
+ }
+ }
+ catch (Exception e)
+ {
+ if (logger.isTraceEnabled())
+ {
+ logger.traceException(e);
+ logger.trace("File dn2id has error reading key %s: %s.%n", dn, e.getMessage());
+ }
+ errorCount++;
+ }
+
+ // Check the parent DN is in dn2id.
+ DN parentDN = getParent(dn);
+ if (parentDN != null)
+ {
+ try
+ {
+ EntryID id = dn2id.get(txn, parentDN, false);
+ if (id == null)
+ {
+ if (logger.isTraceEnabled())
+ {
+ logger.trace("File dn2id is missing key %s.%n", parentDN);
+ }
+ errorCount++;
+ }
+ }
+ catch (Exception e)
+ {
+ if (logger.isTraceEnabled())
+ {
+ logger.traceException(e);
+ logger.trace("File dn2id has error reading key %s: %s.%n", parentDN, e.getMessage());
+ }
+ errorCount++;
+ }
+ }
+ }
+
+ /**
+ * Check that the ID2Children index is complete for a given entry.
+ *
+ * @param entryID The entry ID.
+ * @param entry The entry to be checked.
+ */
+ private void verifyID2Children(ReadableStorage txn, EntryID entryID, Entry entry)
+ {
+ DN dn = entry.getName();
+
+ DN parentDN = getParent(dn);
+ if (parentDN != null)
+ {
+ EntryID parentID = null;
+ try
+ {
+ parentID = dn2id.get(txn, parentDN, false);
+ if (parentID == null)
+ {
+ if (logger.isTraceEnabled())
+ {
+ logger.trace("File dn2id is missing key %s.%n", parentDN);
+ }
+ errorCount++;
+ }
+ }
+ catch (Exception e)
+ {
+ if (logger.isTraceEnabled())
+ {
+ logger.traceException(e);
+ logger.trace("File dn2id has error reading key %s: %s.", parentDN, e.getMessage());
+ }
+ errorCount++;
+ }
+ if (parentID != null)
+ {
+ try
+ {
+ ConditionResult cr = id2c.containsID(null, parentID.toByteString(), entryID);
+ if (cr == ConditionResult.FALSE)
+ {
+ if (logger.isTraceEnabled())
+ {
+ logger.trace("File id2children is missing ID %d for key %d.%n", entryID, parentID);
+ }
+ errorCount++;
+ }
+ else if (cr == ConditionResult.UNDEFINED)
+ {
+ incrEntryLimitStats(id2c, parentID.toByteString());
+ }
+ }
+ catch (StorageRuntimeException e)
+ {
+ if (logger.isTraceEnabled())
+ {
+ logger.traceException(e);
+
+ logger.trace("File id2children has error reading key %d: %s.", parentID, e.getMessage());
+ }
+ errorCount++;
+ }
+ }
+ }
+ }
+
+ /**
+ * Check that the ID2Subtree index is complete for a given entry.
+ *
+ * @param entryID The entry ID.
+ * @param entry The entry to be checked.
+ */
+ private void verifyID2Subtree(ReadableStorage txn, EntryID entryID, Entry entry)
+ {
+ for (DN dn = getParent(entry.getName()); dn != null; dn = getParent(dn))
+ {
+ EntryID id = null;
+ try
+ {
+ id = dn2id.get(txn, dn, false);
+ if (id == null)
+ {
+ if (logger.isTraceEnabled())
+ {
+ logger.trace("File dn2id is missing key %s.%n", dn);
+ }
+ errorCount++;
+ }
+ }
+ catch (Exception e)
+ {
+ if (logger.isTraceEnabled())
+ {
+ logger.traceException(e);
+ logger.trace("File dn2id has error reading key %s: %s.%n", dn, e.getMessage());
+ }
+ errorCount++;
+ }
+ if (id != null)
+ {
+ try
+ {
+ ConditionResult cr;
+ cr = id2s.containsID(null, id.toByteString(), entryID);
+ if (cr == ConditionResult.FALSE)
+ {
+ if (logger.isTraceEnabled())
+ {
+ logger.trace("File id2subtree is missing ID %d for key %d.%n", entryID, id);
+ }
+ errorCount++;
+ }
+ else if (cr == ConditionResult.UNDEFINED)
+ {
+ incrEntryLimitStats(id2s, id.toByteString());
+ }
+ }
+ catch (StorageRuntimeException e)
+ {
+ if (logger.isTraceEnabled())
+ {
+ logger.traceException(e);
+
+ logger.trace("File id2subtree has error reading key %d: %s.%n", id, e.getMessage());
+ }
+ errorCount++;
+ }
+ }
+ }
+ }
+
+ /**
+ * Construct a printable string from a raw key value.
+ *
+ * @param index
+ * The index database containing the key value.
+ * @param key
+ * The bytes of the key.
+ * @return A string that may be logged or printed.
+ */
+ private String keyDump(Index index, ByteSequence key)
+ {
+ StringBuilder buffer = new StringBuilder(128);
+ buffer.append("File: ");
+ buffer.append(index);
+ buffer.append(ServerConstants.EOL);
+ buffer.append("Key:");
+ buffer.append(ServerConstants.EOL);
+ StaticUtils.byteArrayToHexPlusAscii(buffer, key.toByteArray(), 6);
+ return buffer.toString();
+ }
+
+ /**
+ * Construct a printable string from a raw key value.
+ *
+ * @param vlvIndex The vlvIndex database containing the key value.
+ * @param keySortValues THe sort values that is being used as the key.
+ * @return A string that may be logged or printed.
+ */
+ private String keyDump(VLVIndex vlvIndex, SortValues keySortValues)
+ {
+ StringBuilder buffer = new StringBuilder(128);
+ buffer.append("File: ");
+ buffer.append(vlvIndex);
+ buffer.append(ServerConstants.EOL);
+ buffer.append("Key (last sort values):");
+ if(keySortValues != null)
+ {
+ buffer.append(keySortValues);
+ }
+ else
+ {
+ buffer.append("UNBOUNDED (0x00)");
+ }
+ return buffer.toString();
+ }
+
+ /**
+ * Check that an attribute index is complete for a given entry.
+ *
+ * @param entryID The entry ID.
+ * @param entry The entry to be checked.
+ */
+ private void verifyIndex(ReadableStorage txn, EntryID entryID, Entry entry)
+ {
+ for (AttributeIndex attrIndex : attrIndexList)
+ {
+ try
+ {
+ List<Attribute> attrList =
+ entry.getAttribute(attrIndex.getAttributeType());
+ if (attrList != null)
+ {
+ verifyAttribute(attrIndex, entryID, attrList);
+ }
+ }
+ catch (DirectoryException e)
+ {
+ if (logger.isTraceEnabled())
+ {
+ logger.traceException(e);
+
+ logger.trace("Error normalizing values of attribute %s in " +
+ "entry <%s>: %s.%n",
+ attrIndex.getAttributeType(), entry.getName(), e.getMessageObject());
+ }
+ }
+ }
+
+ for (VLVIndex vlvIndex : vlvIndexList)
+ {
+ try
+ {
+ if (vlvIndex.shouldInclude(entry)
+ && !vlvIndex.containsValues(null, entryID.longValue(),
+ vlvIndex.getSortValues(entry), vlvIndex.getSortTypes()))
+ {
+ if(logger.isTraceEnabled())
+ {
+ logger.trace("Missing entry %s in VLV index %s", entry.getName(), vlvIndex.getName());
+ }
+ errorCount++;
+ }
+ }
+ catch (DirectoryException e)
+ {
+ if (logger.isTraceEnabled())
+ {
+ logger.traceException(e);
+ logger.trace("Error checking entry %s against filter or base DN for VLV index %s: %s",
+ entry.getName(), vlvIndex.getName(), e.getMessageObject());
+ }
+ errorCount++;
+ }
+ catch (StorageRuntimeException e)
+ {
+ if (logger.isTraceEnabled())
+ {
+ logger.traceException(e);
+ logger.trace("Error reading VLV index %s for entry %s: %s",
+ vlvIndex.getName(), entry.getName(), StaticUtils.getBacktrace(e));
+ }
+ errorCount++;
+ }
+ catch (JebException e)
+ {
+ if (logger.isTraceEnabled())
+ {
+ logger.traceException(e);
+ logger.trace("Error reading VLV index %s for entry %s: %s",
+ vlvIndex.getName(), entry.getName(), StaticUtils.getBacktrace(e));
+ }
+ errorCount++;
+ }
+ }
+ }
+
+ /**
+ * Check that an attribute index is complete for a given attribute.
+ *
+ * @param attrIndex The attribute index to be checked.
+ * @param entryID The entry ID.
+ * @param attrList The attribute to be checked.
+ * @throws DirectoryException If a Directory Server error occurs.
+ */
+ private void verifyAttribute(AttributeIndex attrIndex, EntryID entryID,
+ List<Attribute> attrList)
+ throws DirectoryException
+ {
+ if (attrList == null || attrList.isEmpty())
+ {
+ return;
+ }
+
+ ReadableStorage txn = null; // FIXME JNR
+ Index equalityIndex = attrIndex.getEqualityIndex();
+ Index presenceIndex = attrIndex.getPresenceIndex();
+ Index substringIndex = attrIndex.getSubstringIndex();
+ Index orderingIndex = attrIndex.getOrderingIndex();
+ Index approximateIndex = attrIndex.getApproximateIndex();
+ // TODO: Add support for Extended Matching Rules indexes.
+
+ if (presenceIndex != null)
+ {
+ verifyAttributeInIndex(presenceIndex, txn, PresenceIndexer.presenceKey, entryID);
+ }
+
+ for (Attribute attr : attrList)
+ {
+ final AttributeType attrType = attr.getAttributeType();
+ MatchingRule equalityRule = attrType.getEqualityMatchingRule();
+ for (ByteString value : attr)
+ {
+ ByteString normalizedBytes = normalize(equalityRule, value);
+
+ if (equalityIndex != null)
+ {
+ verifyAttributeInIndex(equalityIndex, txn, normalizedBytes, entryID);
+ }
+
+ if (substringIndex != null)
+ {
+ for (ByteString key : attrIndex.substringKeys(normalizedBytes))
+ {
+ verifyAttributeInIndex(substringIndex, txn, key, entryID);
+ }
+ }
+
+ if (orderingIndex != null)
+ {
+ ByteString key = normalize(attrType.getOrderingMatchingRule(), value);
+ verifyAttributeInIndex(orderingIndex, txn, key, entryID);
+ }
+
+ if (approximateIndex != null)
+ {
+ ByteString key = normalize(attrType.getApproximateMatchingRule(), value);
+ verifyAttributeInIndex(approximateIndex, txn, key, entryID);
+ }
+ }
+ }
+ }
+
+ private void verifyAttributeInIndex(Index index, ReadableStorage txn,
+ ByteString key, EntryID entryID)
+ {
+ try
+ {
+ ConditionResult cr = index.containsID(txn, key, entryID);
+ if (cr == ConditionResult.FALSE)
+ {
+ if (logger.isTraceEnabled())
+ {
+ logger.trace("Missing ID %d%n%s", entryID, keyDump(index, key));
+ }
+ errorCount++;
+ }
+ else if (cr == ConditionResult.UNDEFINED)
+ {
+ incrEntryLimitStats(index, key);
+ }
+ }
+ catch (StorageRuntimeException e)
+ {
+ if (logger.isTraceEnabled())
+ {
+ logger.traceException(e);
+
+ logger.trace("Error reading database: %s%n%s", e.getMessage(), keyDump(index, key));
+ }
+ errorCount++;
+ }
+ }
+
+ private ByteString normalize(MatchingRule matchingRule, ByteString value) throws DirectoryException
+ {
+ try
+ {
+ return matchingRule.normalizeAttributeValue(value);
+ }
+ catch (DecodeException e)
+ {
+ throw new DirectoryException(ResultCode.INVALID_ATTRIBUTE_SYNTAX,
+ e.getMessageObject(), e);
+ }
+ }
+
+ /**
+ * Get the parent DN of a given DN.
+ *
+ * @param dn The DN.
+ * @return The parent DN or null if the given DN is a base DN.
+ */
+ private DN getParent(DN dn)
+ {
+ if (dn.equals(verifyConfig.getBaseDN()))
+ {
+ return null;
+ }
+ return dn.getParentDNInSuffix();
+ }
+
+ /**
+ * This class reports progress of the verify job at fixed intervals.
+ */
+ private class ProgressTask extends TimerTask
+ {
+ /**
+ * The total number of records to process.
+ */
+ private long totalCount;
+
+ /**
+ * The number of records that had been processed at the time of the
+ * previous progress report.
+ */
+ private long previousCount;
+
+ /**
+ * The time in milliseconds of the previous progress report.
+ */
+ private long previousTime;
+
+ /**
+ * The environment statistics at the time of the previous report.
+ */
+ private EnvironmentStats prevEnvStats;
+
+ /**
+ * The number of bytes in a megabyte.
+ * Note that 1024*1024 bytes may eventually become known as a mebibyte(MiB).
+ */
+ private static final int bytesPerMegabyte = 1024*1024;
+
+ /**
+ * Create a new verify progress task.
+ * @throws StorageRuntimeException An error occurred while accessing the JE
+ * database.
+ */
+ public ProgressTask() throws StorageRuntimeException
+ {
+ previousTime = System.currentTimeMillis();
+ prevEnvStats =
+ rootContainer.getEnvironmentStats(new StatsConfig());
+ totalCount = rootContainer.getEntryContainer(
+ verifyConfig.getBaseDN()).getEntryCount();
+ }
+
+ /**
+ * Create a new verify progress task.
+ * @param indexIterator boolean, indicates if the task is iterating
+ * through indexes or the entries.
+ * @throws StorageRuntimeException An error occurred while accessing the JE
+ * database.
+ */
+ private ProgressTask(boolean indexIterator) throws StorageRuntimeException
+ {
+ previousTime = System.currentTimeMillis();
+ prevEnvStats = rootContainer.getEnvironmentStats(new StatsConfig());
+
+ if (indexIterator)
+ {
+ if (verifyDN2ID)
+ {
+ totalCount = dn2id.getRecordCount();
+ }
+ else if (verifyID2Children)
+ {
+ totalCount = id2c.getRecordCount();
+ }
+ else if (verifyID2Subtree)
+ {
+ totalCount = id2s.getRecordCount();
+ }
+ else if(attrIndexList.size() > 0)
+ {
+ AttributeIndex attrIndex = attrIndexList.get(0);
+ totalCount = 0;
+ if (attrIndex.getEqualityIndex() != null)
+ {
+ totalCount += attrIndex.getEqualityIndex().getRecordCount();
+ }
+ if (attrIndex.getPresenceIndex() != null)
+ {
+ totalCount += attrIndex.getPresenceIndex().getRecordCount();
+ }
+ if (attrIndex.getSubstringIndex() != null)
+ {
+ totalCount += attrIndex.getSubstringIndex().getRecordCount();
+ }
+ if (attrIndex.getOrderingIndex() != null)
+ {
+ totalCount += attrIndex.getOrderingIndex().getRecordCount();
+ }
+ if (attrIndex.getApproximateIndex() != null)
+ {
+ totalCount += attrIndex.getApproximateIndex().getRecordCount();
+ }
+ // TODO: Add support for Extended Matching Rules indexes.
+ }
+ else if (vlvIndexList.size() > 0)
+ {
+ totalCount = vlvIndexList.get(0).getRecordCount();
+ }
+ }
+ else
+ {
+ totalCount = rootContainer.getEntryContainer(
+ verifyConfig.getBaseDN()).getEntryCount();
+ }
+ }
+
+ /**
+ * The action to be performed by this timer task.
+ */
+ @Override
+ public void run()
+ {
+ long latestCount = keyCount;
+ long deltaCount = latestCount - previousCount;
+ long latestTime = System.currentTimeMillis();
+ long deltaTime = latestTime - previousTime;
+
+ if (deltaTime == 0)
+ {
+ return;
+ }
+
+ float rate = 1000f*deltaCount / deltaTime;
+
+ logger.info(NOTE_JEB_VERIFY_PROGRESS_REPORT, latestCount, totalCount, errorCount, rate);
+
+ try
+ {
+ Runtime runtime = Runtime.getRuntime();
+ long freeMemory = runtime.freeMemory() / bytesPerMegabyte;
+
+ EnvironmentStats envStats =
+ rootContainer.getEnvironmentStats(new StatsConfig());
+ long nCacheMiss =
+ envStats.getNCacheMiss() - prevEnvStats.getNCacheMiss();
+
+ float cacheMissRate = 0;
+ if (deltaCount > 0)
+ {
+ cacheMissRate = nCacheMiss/(float)deltaCount;
+ }
+
+ logger.debug(INFO_JEB_VERIFY_CACHE_AND_MEMORY_REPORT, freeMemory, cacheMissRate);
+
+ prevEnvStats = envStats;
+ }
+ catch (StorageRuntimeException e)
+ {
+ logger.traceException(e);
+ }
+
+
+ previousCount = latestCount;
+ previousTime = latestTime;
+ }
+ }
+
+ /**
+ * Adds an attribute of type t and value v to the statEntry, only if the
+ * statEntry is not null.
+ * @param statEntry passed in from backentryImpl.verifyBackend.
+ * @param t String to be used as the attribute type.
+ * @param v String to be used as the attribute value.
+ */
+ private void addStatEntry(Entry statEntry, String t, String v)
+ {
+ if (statEntry != null)
+ {
+ Attribute a = Attributes.create(t, v);
+ statEntry.addAttribute(a, null);
+ }
+ }
+}
diff --git a/opendj3-server-dev/src/server/org/opends/server/util/StaticUtils.java b/opendj3-server-dev/src/server/org/opends/server/util/StaticUtils.java
index 04f2533..411ecd4 100644
--- a/opendj3-server-dev/src/server/org/opends/server/util/StaticUtils.java
+++ b/opendj3-server-dev/src/server/org/opends/server/util/StaticUtils.java
@@ -773,7 +773,6 @@
{
return (char) b;
}
-
return ' ';
}
@@ -813,7 +812,6 @@
* array using hexadecimal characters and a space between each byte.
*
* @param b The byte array containing the data.
- *
* @return A string representation of the contents of the provided byte
* array using hexadecimal characters.
*/
@@ -837,6 +835,34 @@
return buffer.toString();
}
+ /**
+ * Retrieves a string representation of the contents of the provided byte
+ * sequence using hexadecimal characters and a space between each byte.
+ *
+ * @param b The byte sequence containing the data.
+ * @return A string representation of the contents of the provided byte
+ * sequence using hexadecimal characters.
+ */
+ public static String bytesToHex(ByteSequence b)
+ {
+ if ((b == null) || (b.length() == 0))
+ {
+ return "";
+ }
+
+ int arrayLength = b.length();
+ StringBuilder buffer = new StringBuilder((arrayLength - 1) * 3 + 2);
+ buffer.append(byteToHex(b.byteAt(0)));
+
+ for (int i=1; i < arrayLength; i++)
+ {
+ buffer.append(" ");
+ buffer.append(byteToHex(b.byteAt(i)));
+ }
+
+ return buffer.toString();
+ }
+
/**
@@ -4199,6 +4225,7 @@
}
catch (NamingException ignored)
{
+ // ignore
}
}
}
@@ -4219,6 +4246,7 @@
}
catch (InterruptedException wokenUp)
{
+ // ignore
}
}
@@ -4348,7 +4376,6 @@
{
return new Iterable<T>()
{
-
@Override
public Iterator<T> iterator()
{
--
Gitblit v1.10.0