opendj3-server-dev/src/server/org/opends/server/api/ExtensibleIndexer.java
@@ -26,16 +26,7 @@ */ package org.opends.server.api; import java.util.Collection; import java.util.Set; import org.forgerock.opendj.ldap.ByteSequence; import org.forgerock.opendj.ldap.ByteString; import org.forgerock.opendj.ldap.DecodeException; import org.forgerock.opendj.ldap.schema.Schema; import org.forgerock.opendj.ldap.spi.Indexer; import org.forgerock.opendj.ldap.spi.IndexingOptions; import org.opends.server.types.AttributeValue; /** * This class is registered with a Backend and it provides call- backs @@ -63,25 +54,4 @@ */ public abstract String getExtensibleIndexID(); /** * Generates the set of index keys for an attribute. * * @param value * The attribute value for which keys are required. * @param keys * The set into which the generated keys will be inserted. */ public abstract void getKeys(AttributeValue value, Set<byte[]> keys); /** {@inheritDoc} */ @Override public void createKeys(Schema schema, ByteSequence value, IndexingOptions options, Collection<ByteString> keys) throws DecodeException { throw new RuntimeException("Not implemented yet"); } } opendj3-server-dev/src/server/org/opends/server/backends/jeb/ApproximateIndexer.java
@@ -26,21 +26,22 @@ */ package org.opends.server.backends.jeb; import java.util.Set; import java.util.Collection; import org.forgerock.i18n.slf4j.LocalizedLogger; import org.forgerock.opendj.ldap.ByteSequence; import org.forgerock.opendj.ldap.ByteString; import org.forgerock.opendj.ldap.DecodeException; import org.forgerock.opendj.ldap.schema.Schema; import org.forgerock.opendj.ldap.spi.IndexingOptions; import org.opends.server.api.ApproximateMatchingRule; import org.opends.server.api.ExtensibleIndexer; import org.opends.server.types.AttributeType; import org.opends.server.types.AttributeValue; /** * An implementation of an Indexer for attribute approximate matching. */ public class ApproximateIndexer extends ExtensibleIndexer { private static final LocalizedLogger logger = LocalizedLogger.getLoggerForThisClass(); /** * The attribute type approximate matching rule. @@ -75,16 +76,11 @@ /** {@inheritDoc} */ @Override public void getKeys(AttributeValue value, Set<byte[]> keys) public void createKeys(Schema schema, ByteSequence value, IndexingOptions options, Collection<ByteString> keys) throws DecodeException { try { keys.add(approximateRule.normalizeAttributeValue(value.getValue()).toByteArray()); } catch (DecodeException e) { logger.traceException(e); } keys.add(approximateRule.normalizeAttributeValue(value)); } } opendj3-server-dev/src/server/org/opends/server/backends/jeb/AttributeIndex.java
@@ -155,7 +155,7 @@ LocalDBIndexCfgDefn.IndexType.EQUALITY)) { this.equalityIndex = buildExtIndex( name, attrType, attrType.getEqualityMatchingRule(), new EqualityIndexer()); name, attrType, attrType.getEqualityMatchingRule(), new EqualityIndexer(attrType)); } if (indexConfig.getIndexType().contains( @@ -1591,7 +1591,7 @@ { if (equalityIndex == null) { EqualityIndexer indexer = new EqualityIndexer(); EqualityIndexer indexer = new EqualityIndexer(attrType); Indexer extIndexer = new JEExtensibleIndexer(attrType, attrType.getEqualityMatchingRule(), indexer); equalityIndex = openNewIndex(name, extIndexer, indexer, adminActionRequired, messages); } opendj3-server-dev/src/server/org/opends/server/backends/jeb/EqualityIndexer.java
@@ -26,19 +26,38 @@ */ package org.opends.server.backends.jeb; import java.util.Set; import java.util.Collection; import org.forgerock.i18n.slf4j.LocalizedLogger; import org.forgerock.opendj.ldap.ByteSequence; import org.forgerock.opendj.ldap.ByteString; import org.forgerock.opendj.ldap.DecodeException; import org.forgerock.opendj.ldap.schema.Schema; import org.forgerock.opendj.ldap.spi.IndexingOptions; import org.opends.server.api.EqualityMatchingRule; import org.opends.server.api.ExtensibleIndexer; import org.opends.server.types.AttributeValue; import org.opends.server.types.DirectoryException; import org.opends.server.types.AttributeType; /** * An implementation of an Indexer for attribute equality. */ public class EqualityIndexer extends ExtensibleIndexer { private static final LocalizedLogger logger = LocalizedLogger.getLoggerForThisClass(); /** * The attribute type equality matching rule which is also the * comparator for the index keys generated by this class. */ private EqualityMatchingRule equalityRule; /** * Create a new attribute equality indexer for the given index configuration. * @param attributeType The attribute type for which an indexer is * required. */ public EqualityIndexer(AttributeType attributeType) { this.equalityRule = attributeType.getEqualityMatchingRule(); } /** {@inheritDoc} */ @Override @@ -57,16 +76,11 @@ /** {@inheritDoc} */ @Override public void getKeys(AttributeValue value, Set<byte[]> keys) public void createKeys(Schema schema, ByteSequence value, IndexingOptions options, Collection<ByteString> keys) throws DecodeException { try { keys.add(value.getNormalizedValue().toByteArray()); } catch (DirectoryException e) { logger.traceException(e); } keys.add(equalityRule.normalizeAttributeValue(value)); } } opendj3-server-dev/src/server/org/opends/server/backends/jeb/JEExtensibleIndexer.java
@@ -31,7 +31,11 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeSet; import org.forgerock.i18n.slf4j.LocalizedLogger; import org.forgerock.opendj.ldap.ByteString; import org.forgerock.opendj.ldap.DecodeException; import org.opends.server.api.ExtensibleIndexer; import org.opends.server.api.MatchingRule; import org.opends.server.types.AttributeType; @@ -45,6 +49,8 @@ */ public final class JEExtensibleIndexer extends Indexer { private static final LocalizedLogger logger = LocalizedLogger.getLoggerForThisClass(); /** * The comparator for index keys generated by this class. */ @@ -173,16 +179,28 @@ { if (attrList == null) return; Set<ByteString> keysBS = new TreeSet<ByteString>(); for (Attribute attr : attrList) { if (!attr.isVirtual()) { for (AttributeValue value : attr) { extensibleIndexer.getKeys(value, keys); try { extensibleIndexer.createKeys(null, value.getValue(), null, keysBS); } catch (DecodeException e) { logger.traceException(e); } } } } for (ByteString key : keysBS) { keys.add(key.toByteArray()); } } opendj3-server-dev/src/server/org/opends/server/backends/jeb/OrderingIndexer.java
@@ -26,21 +26,22 @@ */ package org.opends.server.backends.jeb; import java.util.Set; import java.util.Collection; import org.forgerock.i18n.slf4j.LocalizedLogger; import org.forgerock.opendj.ldap.ByteSequence; import org.forgerock.opendj.ldap.ByteString; import org.forgerock.opendj.ldap.DecodeException; import org.forgerock.opendj.ldap.schema.Schema; import org.forgerock.opendj.ldap.spi.IndexingOptions; import org.opends.server.api.ExtensibleIndexer; import org.opends.server.api.OrderingMatchingRule; import org.opends.server.types.AttributeType; import org.opends.server.types.AttributeValue; /** * An implementation of an Indexer for attribute ordering. */ public class OrderingIndexer extends ExtensibleIndexer { private static final LocalizedLogger logger = LocalizedLogger.getLoggerForThisClass(); /** * The attribute type ordering matching rule which is also the @@ -76,16 +77,11 @@ /** {@inheritDoc} */ @Override public void getKeys(AttributeValue value, Set<byte[]> keys) public void createKeys(Schema schema, ByteSequence value, IndexingOptions options, Collection<ByteString> keys) throws DecodeException { try { keys.add(orderingRule.normalizeAttributeValue(value.getValue()).toByteArray()); } catch (DecodeException e) { logger.traceException(e); } keys.add(orderingRule.normalizeAttributeValue(value)); } } opendj3-server-dev/src/server/org/opends/server/backends/jeb/SubstringIndexer.java
@@ -26,22 +26,22 @@ */ package org.opends.server.backends.jeb; import java.util.Set; import java.util.Collection; import org.forgerock.i18n.slf4j.LocalizedLogger; import org.forgerock.opendj.ldap.ByteSequence; import org.forgerock.opendj.ldap.ByteString; import org.forgerock.opendj.ldap.DecodeException; import org.forgerock.opendj.ldap.schema.Schema; import org.forgerock.opendj.ldap.spi.IndexingOptions; import org.opends.server.api.ExtensibleIndexer; import org.opends.server.api.SubstringMatchingRule; import org.opends.server.types.AttributeType; import org.opends.server.types.AttributeValue; /** * An implementation of an Indexer for attribute substrings. */ public class SubstringIndexer extends ExtensibleIndexer { private static final LocalizedLogger logger = LocalizedLogger.getLoggerForThisClass(); private SubstringMatchingRule substringRule; private IndexingOptions indexingOptions; @@ -76,54 +76,27 @@ return "substring"; } /** * Decompose an attribute value into a set of substring index keys. * The ID of the entry containing this value should be inserted * into the list of each of these keys. * * @param attrValue A byte array containing the normalized attribute value * @param keys A set into which the keys will be inserted. */ /** {@inheritDoc} */ @Override public void getKeys(AttributeValue attrValue, Set<byte[]> keys) { // TODO merge with ExtensibleIndexer.getKeys(attrValue, keys); try { byte[] value = substringRule.normalizeAttributeValue(attrValue.getValue()).toByteArray(); public void createKeys(Schema schema, ByteSequence value, IndexingOptions options, Collection<ByteString> keys) throws DecodeException { // FIXME Code similar to // AbstractSubstringMatchingRuleImpl.SubstringIndexer.createKeys() ByteString normValue = substringRule.normalizeAttributeValue(value); final int substringKeySize = indexingOptions.substringKeySize(); // Example: The value is ABCDE and the substring length is 3. // We produce the keys ABC BCD CDE DE E // To find values containing a short substring such as DE, // iterate through keys with prefix DE. To find values // containing a longer substring such as BCDE, read keys // BCD and CDE. final int substringKeySize = indexingOptions.substringKeySize(); for (int i = 0, remain = value.length; remain > 0; i++, remain--) { int len = Math.min(substringKeySize, remain); keys.add(makeSubstringKey(value, i, len)); } } catch (DecodeException e) // Example: The value is ABCDE and the substring length is 3. // We produce the keys ABC BCD CDE DE E // To find values containing a short substring such as DE, // iterate through keys with prefix DE. To find values // containing a longer substring such as BCDE, read keys // BCD and CDE. for (int i = 0, remain = normValue.length(); remain > 0; i++, remain--) { logger.traceException(e); int len = Math.min(substringKeySize, remain); keys.add(normValue.subSequence(i, i + len)); } } /** * Makes a byte array representing a substring index key for * one substring of a value. * * @param bytes The byte array containing the value * @param pos The starting position of the substring * @param len The length of the substring * @return A byte array containing a substring key */ private byte[] makeSubstringKey(byte[] bytes, int pos, int len) { byte[] keyBytes = new byte[len]; System.arraycopy(bytes, pos, keyBytes, 0, len); return keyBytes; } } opendj3-server-dev/src/server/org/opends/server/schema/CollationMatchingRuleFactory.java
@@ -33,21 +33,16 @@ import org.forgerock.i18n.LocalizableMessage; import org.forgerock.i18n.slf4j.LocalizedLogger; import org.forgerock.opendj.ldap.ByteSequence; import org.forgerock.opendj.ldap.ByteString; import org.forgerock.opendj.ldap.ConditionResult; import org.forgerock.opendj.ldap.DecodeException; import org.forgerock.opendj.ldap.ResultCode; import org.forgerock.opendj.ldap.*; import org.forgerock.opendj.ldap.schema.Schema; import org.forgerock.opendj.ldap.spi.IndexQueryFactory; import org.forgerock.opendj.ldap.spi.IndexingOptions; import org.opends.server.admin.server.ConfigurationChangeListener; import org.opends.server.admin.std.meta.CollationMatchingRuleCfgDefn.MatchingRuleType; import org.opends.server.admin.std.server.CollationMatchingRuleCfg; import org.opends.server.api.*; import org.opends.server.backends.jeb.AttributeIndex; import org.opends.server.config.ConfigException; import org.opends.server.core.DirectoryServer; import org.opends.server.types.AttributeValue; import org.opends.server.types.ConfigChangeResult; import org.opends.server.types.DirectoryException; import org.opends.server.types.InitializationException; @@ -1569,16 +1564,15 @@ * The starting position of the substring. * @param len * The length of the substring. * @return A byte array containing a substring key. * @return A byte string containing a substring key. */ private byte[] makeSubstringKey(String value, int pos, int len) private ByteString makeSubstringKey(String value, int pos, int len) { String sub = value.substring(pos, pos + len); CollationKey col = collator.getCollationKey(sub); byte[] origKey = col.toByteArray(); byte[] newKey = new byte[origKey.length - 4]; System.arraycopy(origKey, 0, newKey, 0, newKey.length); return newKey; byte[] key = col.toByteArray(); // truncate the key return ByteString.wrap(key).subSequence(0, key.length - 4); } @@ -1593,34 +1587,36 @@ */ private <T> T matchInitialSubstring(String value, IndexQueryFactory<T> factory) { // Use the shared equality indexer. return createRangeMatchQuery(value, factory, this.indexer); } private <T> T createRangeMatchQuery(String value, IndexQueryFactory<T> factory, ExtensibleIndexer indexer) { // FIXME Code similar to // AbstractSubstringMatchingRuleImpl.DefaultSubstringAssertion.rangeMatch() byte[] lower = makeSubstringKey(value, 0, value.length()); byte[] upper = new byte[lower.length]; System.arraycopy(lower, 0, upper, 0, lower.length); for (int i = upper.length - 1; i >= 0; i--) ByteString lower = makeSubstringKey(value, 0, value.length()); ByteStringBuilder upper = new ByteStringBuilder(lower); for (int i = upper.length() - 1; i >= 0; i--) { if (upper[i] == 0xFF) if (upper.byteAt(i) == 0xFF) { // We have to carry the overflow to the more significant byte. upper[i] = 0; upper.setByte(i, (byte) 0); } else { // No overflow, we can stop. upper[i] = (byte) (upper[i] + 1); upper.setByte(i, (byte) (upper.byteAt(i) + 1)); break; } } // Use the shared equality indexer. return factory.createRangeMatchQuery(indexer .getExtensibleIndexID(), ByteString.wrap(lower), ByteString .wrap(upper), true, false); // Read the range: lower <= keys < upper. return factory.createRangeMatchQuery( indexer.getExtensibleIndexID(), lower, upper, true, false); } /** * Retrieves the Index Records that might contain a given substring. * @@ -1636,58 +1632,29 @@ IndexQueryFactory<T> factory) { // FIXME Code similar to // AbstractSubstringMatchingRuleImpl.DefaultSubstringAssertion.substringMatch() T intersectionQuery; int substrLength = subIndexer.getSubstringLength(); if (value.length() < substrLength) { byte[] lower = makeSubstringKey(value, 0, value.length()); byte[] upper = makeSubstringKey(value, 0, value.length()); for (int i = upper.length - 1; i >= 0; i--) { if (upper[i] == 0xFF) { // We have to carry the overflow to the more significant // byte. upper[i] = 0; } else { // No overflow, we can stop. upper[i] = (byte) (upper[i] + 1); break; } } // Read the range: lower <= keys < upper. intersectionQuery = factory.createRangeMatchQuery(subIndexer .getExtensibleIndexID(), ByteString.wrap(lower), ByteString.wrap(upper), true, false); return createRangeMatchQuery(value, factory, subIndexer); } else List<T> queryList = new ArrayList<T>(); Set<ByteString> set = new TreeSet<ByteString>(); for (int first = 0, last = substrLength; last <= value.length(); first++, last++) { List<T> queryList = new ArrayList<T>(); Set<byte[]> set = new TreeSet<byte[]>(new AttributeIndex.KeyComparator()); for (int first = 0, last = substrLength; last <= value.length(); first++, last++) { set.add(makeSubstringKey(value, first, substrLength)); } for (byte[] keyBytes : set) { queryList.add(factory.createExactMatchQuery( subIndexer.getExtensibleIndexID(), ByteString.wrap(keyBytes))); } intersectionQuery = factory.createIntersectionQuery(queryList); set.add(makeSubstringKey(value, first, substrLength)); } return intersectionQuery; for (ByteString keyBytes : set) { queryList.add(factory.createExactMatchQuery( subIndexer.getExtensibleIndexID(), keyBytes)); } return factory.createIntersectionQuery(queryList); } /** * {@inheritDoc} */ @@ -2091,17 +2058,11 @@ * {@inheritDoc} */ @Override public final void getKeys(AttributeValue value, Set<byte[]> keys) public final void createKeys(Schema schema, ByteSequence value, IndexingOptions options, Collection<ByteString> keys) throws DecodeException { try { ByteString key = matchingRule.normalizeAttributeValue(value.getValue()); keys.add(key.toByteArray()); } catch (DecodeException e) { logger.traceException(e); } keys.add(matchingRule.normalizeAttributeValue(value)); } /** {@inheritDoc} */ @@ -2148,15 +2109,15 @@ * {@inheritDoc} */ @Override public void getKeys(AttributeValue attValue, Set<byte[]> keys) { // TODO merge with ExtensibleIndexer.getKeys(attrValue, keys); // TODO and with AbstractSubstringMatchingRuleImpl.SubstringIndexer.createKeys(); String value = attValue.toString(); public void createKeys(Schema schema, ByteSequence value, IndexingOptions options, Collection<ByteString> keys) { // TODO merge with AbstractSubstringMatchingRuleImpl.SubstringIndexer.createKeys(); String normValue = value.toString(); int keyLength = substringLen; for (int i = 0, remain = value.length(); remain > 0; i++, remain--) for (int i = 0, remain = normValue.length(); remain > 0; i++, remain--) { int len = Math.min(keyLength, remain); keys.add(matchingRule.makeSubstringKey(value, i, len)); keys.add(matchingRule.makeSubstringKey(normValue, i, len)); } } opendj3-server-dev/src/server/org/opends/server/schema/TimeBasedMatchingRuleFactory.java
@@ -38,13 +38,13 @@ import org.forgerock.opendj.ldap.ByteStringBuilder; import org.forgerock.opendj.ldap.ConditionResult; import org.forgerock.opendj.ldap.DecodeException; import org.forgerock.opendj.ldap.schema.Schema; import org.forgerock.opendj.ldap.spi.IndexQueryFactory; import org.forgerock.opendj.ldap.spi.IndexingOptions; import org.opends.server.admin.std.server.MatchingRuleCfg; import org.opends.server.api.*; import org.opends.server.config.ConfigException; import org.opends.server.core.DirectoryServer; import org.opends.server.types.AttributeValue; import org.opends.server.types.DirectoryException; import org.opends.server.types.InitializationException; import org.opends.server.util.StaticUtils; @@ -89,20 +89,10 @@ //Constants for generating keys. private static final char SECOND = 's'; private static final char MINUTE = 'm'; private static final char HOUR = 'h'; private static final char MONTH = 'M'; private static final char DATE = 'D'; private static final char YEAR = 'Y'; @@ -619,17 +609,11 @@ * {@inheritDoc} */ @Override public final void getKeys(AttributeValue value, Set<byte[]> keys) public final void createKeys(Schema schema, ByteSequence value2, IndexingOptions options, Collection<ByteString> keys) throws DecodeException { try { ByteString key = matchingRule.normalizeAttributeValue(value.getValue()); keys.add(key.toByteArray()); } catch (DecodeException de) { //don't do anything. } keys.add(matchingRule.normalizeAttributeValue(value2)); } /** {@inheritDoc} */ @@ -1108,7 +1092,7 @@ * @param set * A set into which the keys will be inserted. */ private void timeKeys(ByteString attributeValue, Set<byte[]> keys) private void timeKeys(ByteSequence attributeValue, Collection<ByteString> keys) { long timeInMS = 0L; try @@ -1124,46 +1108,23 @@ //Build the information from the attribute value. GregorianCalendar cal = new GregorianCalendar(TIME_ZONE_UTC_OBJ); cal.setTimeInMillis(timeInMS); int second = cal.get(Calendar.SECOND); int minute = cal.get(Calendar.MINUTE); int hour = cal.get(Calendar.HOUR_OF_DAY); int date = cal.get(Calendar.DATE); int month = cal.get(Calendar.MONTH); int year = cal.get(Calendar.YEAR); if (second >=0) { keys.add(getKey(second,SECOND).toByteArray()); } if(minute >=0) { keys.add(getKey(minute,MINUTE).toByteArray()); } if(hour >=0) { keys.add(getKey(hour,HOUR).toByteArray()); } //Insert date. if(date > 0) { keys.add(getKey(date,DATE).toByteArray()); } //Insert month. if(month >=0) { keys.add(getKey(month,MONTH).toByteArray()); } if(year > 0) { keys.add(getKey(year,YEAR).toByteArray()); } addKeyIfNotZero(keys, cal, Calendar.SECOND, SECOND); addKeyIfNotZero(keys, cal, Calendar.MINUTE, MINUTE); addKeyIfNotZero(keys, cal, Calendar.HOUR_OF_DAY, HOUR); addKeyIfNotZero(keys, cal, Calendar.DATE, DATE); addKeyIfNotZero(keys, cal, Calendar.MONTH, MONTH); addKeyIfNotZero(keys, cal, Calendar.YEAR, YEAR); } private void addKeyIfNotZero(Collection<ByteString> keys, GregorianCalendar cal, int calField, char type) { int value = cal.get(calField); if (value >= 0) { keys.add(getKey(value, type)); } } private ByteString getKey(int value, char type) { @@ -1205,9 +1166,10 @@ * {@inheritDoc} */ @Override public void getKeys(AttributeValue value, Set<byte[]> keys) public void createKeys(Schema schema, ByteSequence value, IndexingOptions options, Collection<ByteString> keys) { matchingRule.timeKeys(value.getValue(), keys); matchingRule.timeKeys(value, keys); } /** {@inheritDoc} */ opendj3-server-dev/tests/unit-tests-testng/src/server/org/opends/server/backends/jeb/TestBackendImpl.java
@@ -870,7 +870,7 @@ private Indexer newEqualityIndexer(AttributeIndex index) { AttributeType attrType = index.getAttributeType(); ExtensibleIndexer extIndexer = new EqualityIndexer(); ExtensibleIndexer extIndexer = new EqualityIndexer(index.getAttributeType()); return new JEExtensibleIndexer(attrType, attrType.getSubstringMatchingRule(), extIndexer); }