opendj-sdk/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/AttributeIndex.java
@@ -118,7 +118,7 @@ /** The mapping from names to indexes. */ private final Map<String, Index> nameToIndexes = new HashMap<String, Index>(); private final IndexQueryFactory<IndexQuery> indexQueryFactory; private final IndexingOptions indexingOptions; /** * The mapping from extensible index types (e.g. "substring" or "shared") to list of indexes. @@ -144,8 +144,7 @@ buildIndexes(txn, IndexType.APPROXIMATE); buildExtensibleIndexes(txn); final JEIndexConfig config = new JEIndexConfig(indexConfig.getSubstringLength()); indexQueryFactory = new IndexQueryFactoryImpl(nameToIndexes, config); indexingOptions = new JEIndexConfig(indexConfig.getSubstringLength()); extensibleIndexesMapping = computeExtensibleIndexesMapping(); } @@ -294,7 +293,7 @@ */ public IndexingOptions getIndexingOptions() { return indexQueryFactory.getIndexingOptions(); return indexingOptions; } /** @@ -318,10 +317,9 @@ public void addEntry(IndexBuffer buffer, EntryID entryID, Entry entry) throws StorageRuntimeException, DirectoryException { final IndexingOptions options = indexQueryFactory.getIndexingOptions(); for (Index index : nameToIndexes.values()) { index.addEntry(buffer, entryID, entry, options); index.addEntry(buffer, entryID, entry, indexingOptions); } } @@ -337,10 +335,9 @@ public void removeEntry(IndexBuffer buffer, EntryID entryID, Entry entry) throws StorageRuntimeException, DirectoryException { final IndexingOptions options = indexQueryFactory.getIndexingOptions(); for (Index index : nameToIndexes.values()) { index.removeEntry(buffer, entryID, entry, options); index.removeEntry(buffer, entryID, entry, indexingOptions); } } @@ -363,10 +360,9 @@ List<Modification> mods) throws StorageRuntimeException { final IndexingOptions options = indexQueryFactory.getIndexingOptions(); for (Index index : nameToIndexes.values()) { index.modifyEntry(buffer, entryID, oldEntry, newEntry, mods, options); index.modifyEntry(buffer, entryID, oldEntry, newEntry, mods, indexingOptions); } } @@ -451,6 +447,8 @@ * Retrieve the entry IDs that might match two filters that restrict a value * to both a lower bound and an upper bound. * * @param indexQueryFactory * The index query factory to use for the evaluation * @param filter1 * The first filter, that is either a less-or-equal filter or a * greater-or-equal filter. @@ -466,8 +464,8 @@ * filter usage statistics. * @return The candidate entry IDs that might contain match both filters. */ public EntryIDSet evaluateBoundedRange(SearchFilter filter1, SearchFilter filter2, StringBuilder debugBuffer, DatabaseEnvironmentMonitor monitor) public EntryIDSet evaluateBoundedRange(IndexQueryFactory<IndexQuery> indexQueryFactory, SearchFilter filter1, SearchFilter filter2, StringBuilder debugBuffer, DatabaseEnvironmentMonitor monitor) { // TODO : this implementation is not optimal // as it implies two separate evaluations instead of a single one, @@ -475,22 +473,24 @@ // in IndexFilter#evaluateLogicalAndFilter method. // One solution could be to implement a boundedRangeAssertion that combine // the two operations in one. EntryIDSet results = evaluate(filter1, debugBuffer, monitor); EntryIDSet results2 = evaluate(filter2, debugBuffer, monitor); EntryIDSet results = evaluate(indexQueryFactory, filter1, debugBuffer, monitor); EntryIDSet results2 = evaluate(indexQueryFactory, filter2, debugBuffer, monitor); results.retainAll(results2); return results; } private EntryIDSet evaluate(SearchFilter filter, StringBuilder debugBuffer, DatabaseEnvironmentMonitor monitor) private EntryIDSet evaluate(IndexQueryFactory<IndexQuery> indexQueryFactory, SearchFilter filter, StringBuilder debugBuffer, DatabaseEnvironmentMonitor monitor) { boolean isLessOrEqual = filter.getFilterType() == FilterType.LESS_OR_EQUAL; IndexFilterType indexFilterType = isLessOrEqual ? IndexFilterType.LESS_OR_EQUAL : IndexFilterType.GREATER_OR_EQUAL; return evaluateFilter(indexFilterType, filter, debugBuffer, monitor); return evaluateFilter(indexQueryFactory, indexFilterType, filter, debugBuffer, monitor); } /** * Retrieve the entry IDs that might match a filter. * * @param indexQueryFactory the index query factory to use for the evaluation * @param indexFilterType the index type filter * @param filter The filter. * @param debugBuffer If not null, a diagnostic string will be written @@ -501,12 +501,12 @@ * @return The candidate entry IDs that might contain a value * that matches the filter type. */ public EntryIDSet evaluateFilter(IndexFilterType indexFilterType, SearchFilter filter, StringBuilder debugBuffer, DatabaseEnvironmentMonitor monitor) public EntryIDSet evaluateFilter(IndexQueryFactory<IndexQuery> indexQueryFactory, IndexFilterType indexFilterType, SearchFilter filter, StringBuilder debugBuffer, DatabaseEnvironmentMonitor monitor) { try { final IndexQuery indexQuery = getIndexQuery(indexFilterType, filter); final IndexQuery indexQuery = getIndexQuery(indexQueryFactory, indexFilterType, filter); return evaluateIndexQuery(indexQuery, indexFilterType.toString(), filter, debugBuffer, monitor); } catch (DecodeException e) @@ -516,7 +516,8 @@ } } private IndexQuery getIndexQuery(IndexFilterType indexFilterType, SearchFilter filter) throws DecodeException private IndexQuery getIndexQuery(IndexQueryFactory<IndexQuery> indexQueryFactory, IndexFilterType indexFilterType, SearchFilter filter) throws DecodeException { MatchingRule rule; Assertion assertion; @@ -907,7 +908,7 @@ * @return The equality index. */ public Index getEqualityIndex() { return nameToIndexes.get(IndexType.EQUALITY.toString()); return getIndexById(IndexType.EQUALITY.toString()); } /** @@ -916,7 +917,7 @@ * @return The approximate index. */ public Index getApproximateIndex() { return nameToIndexes.get(IndexType.APPROXIMATE.toString()); return getIndexById(IndexType.APPROXIMATE.toString()); } /** @@ -925,7 +926,7 @@ * @return The ordering index. */ public Index getOrderingIndex() { return nameToIndexes.get(IndexType.ORDERING.toString()); return getIndexById(IndexType.ORDERING.toString()); } /** @@ -934,7 +935,7 @@ * @return The substring index. */ public Index getSubstringIndex() { return nameToIndexes.get(IndexType.SUBSTRING.toString()); return getIndexById(IndexType.SUBSTRING.toString()); } /** @@ -943,7 +944,23 @@ * @return The presence index. */ public Index getPresenceIndex() { return nameToIndexes.get(IndexType.PRESENCE.toString()); return getIndexById(IndexType.PRESENCE.toString()); } /** * Return the index identified by the provided identifier. * <p> * Common index identifiers are "presence", "equality", "substring", * "ordering" and "approximate". * * @param indexId * the identifier of the requested index * @return The index identified by the provided identifier, or null if no such * index exists */ public Index getIndexById(String indexId) { return nameToIndexes.get(indexId); } /** @@ -1003,6 +1020,7 @@ /** * Retrieve the entry IDs that might match an extensible filter. * * @param indexQueryFactory the index query factory to use for the evaluation * @param filter The extensible filter. * @param debugBuffer If not null, a diagnostic string will be written * which will help determine how the indexes contributed @@ -1012,9 +1030,8 @@ * @return The candidate entry IDs that might contain the filter * assertion value. */ public EntryIDSet evaluateExtensibleFilter(SearchFilter filter, StringBuilder debugBuffer, DatabaseEnvironmentMonitor monitor) public EntryIDSet evaluateExtensibleFilter(IndexQueryFactory<IndexQuery> indexQueryFactory, SearchFilter filter, StringBuilder debugBuffer, DatabaseEnvironmentMonitor monitor) { //Get the Matching Rule OID of the filter. String matchRuleOID = filter.getMatchingRuleID(); @@ -1029,7 +1046,7 @@ || matchRuleOID.equalsIgnoreCase(eqRule.getNameOrOID())) { //No matching rule is defined; use the default equality matching rule. return evaluateFilter(IndexFilterType.EQUALITY, filter, debugBuffer, monitor); return evaluateFilter(indexQueryFactory, IndexFilterType.EQUALITY, filter, debugBuffer, monitor); } MatchingRule rule = DirectoryServer.getMatchingRule(matchRuleOID); opendj-sdk/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/EntryContainer.java
@@ -929,8 +929,8 @@ } // Create an index filter to get the search result candidate entries IndexFilter indexFilter = new IndexFilter(EntryContainer.this, searchOperation, debugBuffer, rootContainer.getMonitorProvider()); IndexFilter indexFilter = new IndexFilter( EntryContainer.this, txn, searchOperation, debugBuffer, rootContainer.getMonitorProvider()); // Evaluate the filter against the attribute indexes. entryIDList = indexFilter.evaluate(); opendj-sdk/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/ExportJob.java
@@ -35,7 +35,8 @@ import org.forgerock.i18n.slf4j.LocalizedLogger; import org.forgerock.opendj.ldap.ByteString; import org.opends.server.backends.pluggable.spi.Cursor; import org.opends.server.backends.pluggable.spi.Storage; import org.opends.server.backends.pluggable.spi.ReadOperation; import org.opends.server.backends.pluggable.spi.ReadableStorage; import org.opends.server.backends.pluggable.spi.StorageRuntimeException; import org.opends.server.types.DN; import org.opends.server.types.Entry; @@ -95,14 +96,12 @@ throws IOException, LDIFException, StorageRuntimeException { List<DN> includeBranches = exportConfig.getIncludeBranches(); DN baseDN; ArrayList<EntryContainer> exportContainers = new ArrayList<EntryContainer>(); final ArrayList<EntryContainer> exportContainers = new ArrayList<EntryContainer>(); for (EntryContainer entryContainer : rootContainer.getEntryContainers()) { // Skip containers that are not covered by the include branches. baseDN = entryContainer.getBaseDN(); DN baseDN = entryContainer.getBaseDN(); if (includeBranches == null || includeBranches.isEmpty()) { @@ -128,29 +127,40 @@ // Start a timer for the progress report. Timer timer = new Timer(); TimerTask progressTask = new ProgressTask(); timer.scheduleAtFixedRate(progressTask, progressInterval, progressInterval); timer.scheduleAtFixedRate(progressTask, progressInterval, progressInterval); // Iterate through the containers. try { for (EntryContainer exportContainer : exportContainers) rootContainer.getStorage().read(new ReadOperation<Void>() { if (exportConfig.isCancelled()) @Override public Void run(ReadableStorage txn) throws Exception { break; } for (EntryContainer exportContainer : exportContainers) { if (exportConfig.isCancelled()) { break; } exportContainer.sharedLock.lock(); try { exportContainer(exportContainer); exportContainer.sharedLock.lock(); try { exportContainer(txn, exportContainer); } finally { exportContainer.sharedLock.unlock(); } } return null; } finally { exportContainer.sharedLock.unlock(); } } }); } catch (Exception e) { throw new StorageRuntimeException(e); } finally { @@ -181,12 +191,10 @@ * @throws LDIFException If an error occurs while trying to determine * whether to write an entry. */ private void exportContainer(EntryContainer entryContainer) private void exportContainer(ReadableStorage txn, EntryContainer entryContainer) throws StorageRuntimeException, IOException, LDIFException { Storage storage = entryContainer.getStorage(); Cursor cursor = storage.openCursor(entryContainer.getID2Entry().getName()); Cursor cursor = txn.openCursor(entryContainer.getID2Entry().getName()); try { while (cursor.next()) opendj-sdk/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/Index.java
@@ -492,8 +492,8 @@ * specified. * @return The set of entry IDs. */ public EntryIDSet readRange(ByteSequence lower, ByteSequence upper, boolean lowerIncluded, boolean upperIncluded) public EntryIDSet readRange(ReadableStorage txn, ByteSequence lower, ByteSequence upper, boolean lowerIncluded, boolean upperIncluded) { // If this index is not trusted, then just return an undefined id set. if(rebuildRunning || !trusted) @@ -508,7 +508,7 @@ ArrayList<EntryIDSet> lists = new ArrayList<EntryIDSet>(); Cursor cursor = storage.openCursor(treeName); Cursor cursor = txn.openCursor(treeName); try { ByteSequence key = ByteString.empty(); opendj-sdk/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/IndexFilter.java
@@ -32,6 +32,7 @@ import java.util.Map; import org.opends.server.backends.pluggable.AttributeIndex.IndexFilterType; import org.opends.server.backends.pluggable.spi.ReadableStorage; import org.opends.server.core.SearchOperation; import org.opends.server.types.AttributeType; import org.opends.server.types.FilterType; @@ -51,10 +52,9 @@ */ public static final int FILTER_CANDIDATE_THRESHOLD = 10; /** * The entry entryContainer holding the attribute indexes. */ /** The entry container holding the attribute indexes. */ private final EntryContainer entryContainer; private final ReadableStorage txn; /** * The search operation provides the search base, scope and filter. @@ -67,7 +67,6 @@ * how the indexed contributed to the search operation. */ private final StringBuilder buffer; private final DatabaseEnvironmentMonitor monitor; /** @@ -76,17 +75,15 @@ * @param entryContainer The entry entryContainer. * @param searchOp The search operation to be evaluated. * @param monitor The monitor to gather filter usage stats. * * @param debugBuilder If not null, a diagnostic string will be written * which will help determine how the indexes contributed * to this search. */ public IndexFilter(EntryContainer entryContainer, SearchOperation searchOp, StringBuilder debugBuilder, DatabaseEnvironmentMonitor monitor) public IndexFilter(EntryContainer entryContainer, ReadableStorage txn, SearchOperation searchOp, StringBuilder debugBuilder, DatabaseEnvironmentMonitor monitor) { this.entryContainer = entryContainer; this.txn = txn; this.searchOp = searchOp; this.buffer = debugBuilder; this.monitor = monitor; @@ -265,7 +262,8 @@ continue; } EntryIDSet set = attributeIndex.evaluateBoundedRange(filter1, filter2, buffer, monitor); final IndexQueryFactoryImpl indexQueryFactory = new IndexQueryFactoryImpl(txn, attributeIndex); EntryIDSet set = attributeIndex.evaluateBoundedRange(indexQueryFactory, filter1, filter2, buffer, monitor); if(monitor.isFilterUseEnabled() && set.isDefined()) { monitor.updateStats(SearchFilter.createANDFilter(rangeList), set.size()); @@ -356,7 +354,8 @@ AttributeIndex attributeIndex = entryContainer.getAttributeIndex(filter.getAttributeType()); if (attributeIndex != null) { return attributeIndex.evaluateFilter(indexFilterType, filter, buffer, monitor); final IndexQueryFactoryImpl indexQueryFactory = new IndexQueryFactoryImpl(txn, attributeIndex); return attributeIndex.evaluateFilter(indexQueryFactory, indexFilterType, filter, buffer, monitor); } if (monitor.isFilterUseEnabled()) @@ -386,7 +385,8 @@ AttributeIndex attributeIndex = entryContainer.getAttributeIndex(extensibleFilter.getAttributeType()); if (attributeIndex != null) { return attributeIndex.evaluateExtensibleFilter(extensibleFilter, buffer, monitor); final IndexQueryFactoryImpl indexQueryFactory = new IndexQueryFactoryImpl(txn, attributeIndex); return attributeIndex.evaluateExtensibleFilter(indexQueryFactory, extensibleFilter, buffer, monitor); } return IndexQuery.createNullIndexQuery().evaluate(null); } opendj-sdk/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/IndexQueryFactoryImpl.java
@@ -27,12 +27,12 @@ package org.opends.server.backends.pluggable; import java.util.Collection; import java.util.Map; import org.forgerock.i18n.LocalizableMessageBuilder; import org.forgerock.opendj.ldap.ByteSequence; import org.forgerock.opendj.ldap.spi.IndexQueryFactory; import org.forgerock.opendj.ldap.spi.IndexingOptions; import org.opends.server.backends.pluggable.spi.ReadableStorage; import static org.opends.messages.JebMessages.*; @@ -40,47 +40,41 @@ * This class is an implementation of IndexQueryFactory which creates * IndexQuery objects as part of the query of the JEB index. */ public final class IndexQueryFactoryImpl implements IndexQueryFactory<IndexQuery> public final class IndexQueryFactoryImpl implements IndexQueryFactory<IndexQuery> { private static final String PRESENCE_INDEX_KEY = "presence"; /** * The Map containing the string type identifier and the corresponding index. */ private final Map<String, Index> indexMap; private final IndexingOptions indexingOptions; private final ReadableStorage txn; /** The Map containing the string type identifier and the corresponding index. */ private final AttributeIndex attributeIndex; /** * Creates a new IndexQueryFactoryImpl object. * * @param indexMap * A map containing the index id and the corresponding index. * @param indexingOptions * The options to use for indexing * @param txn * The readable storage * @param attributeIndex * The targeted attribute index */ public IndexQueryFactoryImpl(Map<String, Index> indexMap, IndexingOptions indexingOptions) public IndexQueryFactoryImpl(ReadableStorage txn, AttributeIndex attributeIndex) { this.indexMap = indexMap; this.indexingOptions = indexingOptions; this.txn = txn; this.attributeIndex = attributeIndex; } /** {@inheritDoc} */ @Override public IndexQuery createExactMatchQuery(final String indexID, final ByteSequence key) { return new IndexQuery() { @Override public EntryIDSet evaluate(LocalizableMessageBuilder debugMessage) { // Read the database and get Record for the key. // Select the right index to be used. Index index = indexMap.get(indexID); final Index index = attributeIndex.getIndexById(indexID); if (index == null) { if(debugMessage != null) @@ -89,7 +83,8 @@ } return createMatchAllQuery().evaluate(debugMessage); } EntryIDSet entrySet = index.readKey(key, null); final EntryIDSet entrySet = index.readKey(key, null); if(debugMessage != null && !entrySet.isDefined()) { updateStatsUndefinedResults(debugMessage, index); @@ -99,8 +94,6 @@ }; } /** {@inheritDoc} */ @Override public IndexQuery createRangeMatchQuery(final String indexID, @@ -109,12 +102,11 @@ { return new IndexQuery() { @Override public EntryIDSet evaluate(LocalizableMessageBuilder debugMessage) { // Find the right index. Index index = indexMap.get(indexID); final Index index = attributeIndex.getIndexById(indexID); if (index == null) { if(debugMessage != null) @@ -123,7 +115,8 @@ } return createMatchAllQuery().evaluate(debugMessage); } EntryIDSet entrySet = index.readRange(lowerBound, upperBound, final EntryIDSet entrySet = index.readRange(txn, lowerBound, upperBound, includeLowerBound, includeUpperBound); if(debugMessage != null && !entrySet.isDefined()) { @@ -134,8 +127,6 @@ }; } /** {@inheritDoc} */ @Override public IndexQuery createIntersectionQuery(Collection<IndexQuery> subqueries) @@ -143,8 +134,6 @@ return IndexQuery.createIntersectionIndexQuery(subqueries); } /** {@inheritDoc} */ @Override public IndexQuery createUnionQuery(Collection<IndexQuery> subqueries) @@ -152,8 +141,6 @@ return IndexQuery.createUnionIndexQuery(subqueries); } /** * {@inheritDoc} * <p> @@ -168,8 +155,8 @@ @Override public EntryIDSet evaluate(LocalizableMessageBuilder debugMessage) { final String indexID = PRESENCE_INDEX_KEY; final Index index = indexMap.get(indexID); final String indexID = PRESENCE_INDEX_KEY; final Index index = attributeIndex.getIndexById(indexID); if (index == null) { if(debugMessage != null) @@ -179,7 +166,7 @@ return new EntryIDSet(); } EntryIDSet entrySet = index.readKey(PresenceIndexer.presenceKey, null); final EntryIDSet entrySet = index.readKey(PresenceIndexer.presenceKey, null); if (debugMessage != null && !entrySet.isDefined()) { updateStatsUndefinedResults(debugMessage, index); @@ -209,6 +196,6 @@ @Override public IndexingOptions getIndexingOptions() { return indexingOptions; return attributeIndex.getIndexingOptions(); } } opendj-sdk/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/NullIndex.java
@@ -109,8 +109,8 @@ /** {@inheritDoc} */ @Override public EntryIDSet readRange(ByteSequence lower, ByteSequence upper, boolean lowerIncluded, boolean upperIncluded) public EntryIDSet readRange(ReadableStorage txn, ByteSequence lower, ByteSequence upper, boolean lowerIncluded, boolean upperIncluded) { return new EntryIDSet(); } opendj-sdk/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/spi/Storage.java
@@ -44,8 +44,6 @@ void write(WriteOperation updateTransaction) throws Exception; Cursor openCursor(TreeName name); @Override void close(); }