opendj3-server-dev/src/server/org/opends/server/backends/jeb/AttributeIndex.java
@@ -419,46 +419,32 @@ Entry entry) throws DatabaseException, DirectoryException { boolean success = true; boolean success = false; if (equalityIndex != null) if (equalityIndex != null && !equalityIndex.addEntry(buffer, entryID, entry)) { if(!equalityIndex.addEntry(buffer, entryID, entry)) { success = false; } success = false; } if (presenceIndex != null) if (presenceIndex != null && !presenceIndex.addEntry(buffer, entryID, entry)) { if(!presenceIndex.addEntry(buffer, entryID, entry)) { success = false; } success = false; } if (substringIndex != null) if (substringIndex != null && !substringIndex.addEntry(buffer, entryID, entry)) { if(!substringIndex.addEntry(buffer, entryID, entry)) { success = false; } success = false; } if (orderingIndex != null) if (orderingIndex != null && !orderingIndex.addEntry(buffer, entryID, entry)) { if(!orderingIndex.addEntry(buffer, entryID, entry)) { success = false; } success = false; } if (approximateIndex != null) if (approximateIndex != null && !approximateIndex.addEntry(buffer, entryID, entry)) { if(!approximateIndex.addEntry(buffer, entryID, entry)) { success = false; } success = false; } if(extensibleIndexes!=null) @@ -492,44 +478,26 @@ { boolean success = true; if (equalityIndex != null) if (equalityIndex != null && !equalityIndex.addEntry(txn, entryID, entry)) { if(!equalityIndex.addEntry(txn, entryID, entry)) { success = false; } success = false; } if (presenceIndex != null) if (presenceIndex != null && !presenceIndex.addEntry(txn, entryID, entry)) { if(!presenceIndex.addEntry(txn, entryID, entry)) { success = false; } success = false; } if (substringIndex != null) if (substringIndex != null && !substringIndex.addEntry(txn, entryID, entry)) { if(!substringIndex.addEntry(txn, entryID, entry)) { success = false; } success = false; } if (orderingIndex != null) if (orderingIndex != null && !orderingIndex.addEntry(txn, entryID, entry)) { if(!orderingIndex.addEntry(txn, entryID, entry)) { success = false; } success = false; } if (approximateIndex != null) if (approximateIndex != null && !approximateIndex.addEntry(txn, entryID, entry)) { if(!approximateIndex.addEntry(txn, entryID, entry)) { success = false; } success = false; } if(extensibleIndexes!=null) @@ -1259,7 +1227,7 @@ if (filter.getSubInitialElement() != null) { // Use the equality index for initial substrings if possible. if ((equalityIndex != null) && (matchRule != null)) if (equalityIndex != null && matchRule != null) { ByteString normValue = matchRule.normalizeSubstring(filter.getSubInitialElement()); @@ -1460,7 +1428,7 @@ * The default lexicographic byte array comparator. * Is there one available in the Java platform? */ static public class KeyComparator implements Comparator<byte[]> public static class KeyComparator implements Comparator<byte[]> { /** * Compares its two arguments for order. Returns a negative integer, @@ -1721,9 +1689,7 @@ return getName(); } /** * {@inheritDoc} */ /** {@inheritDoc} */ @Override public synchronized boolean isConfigurationChangeAcceptable( LocalDBIndexCfg cfg, @@ -1731,46 +1697,39 @@ { AttributeType attrType = cfg.getAttribute(); if (cfg.getIndexType().contains(LocalDBIndexCfgDefn.IndexType.EQUALITY)) if (cfg.getIndexType().contains(LocalDBIndexCfgDefn.IndexType.EQUALITY) && equalityIndex == null && attrType.getEqualityMatchingRule() == null) { if (equalityIndex == null && attrType.getEqualityMatchingRule() == null) { unacceptableReasons.add(ERR_CONFIG_INDEX_TYPE_NEEDS_MATCHING_RULE.get(attrType, "equality")); return false; } unacceptableReasons.add(ERR_CONFIG_INDEX_TYPE_NEEDS_MATCHING_RULE.get(attrType, "equality")); return false; } if (cfg.getIndexType().contains(LocalDBIndexCfgDefn.IndexType.SUBSTRING)) if (cfg.getIndexType().contains(LocalDBIndexCfgDefn.IndexType.SUBSTRING) && substringIndex == null && attrType.getSubstringMatchingRule() == null) { if (substringIndex == null && attrType.getSubstringMatchingRule() == null) { unacceptableReasons.add(ERR_CONFIG_INDEX_TYPE_NEEDS_MATCHING_RULE.get(attrType, "substring")); return false; } unacceptableReasons.add(ERR_CONFIG_INDEX_TYPE_NEEDS_MATCHING_RULE.get(attrType, "substring")); return false; } if (cfg.getIndexType().contains(LocalDBIndexCfgDefn.IndexType.ORDERING)) if (cfg.getIndexType().contains(LocalDBIndexCfgDefn.IndexType.ORDERING) && orderingIndex == null && attrType.getOrderingMatchingRule() == null) { if (orderingIndex == null && attrType.getOrderingMatchingRule() == null) { unacceptableReasons.add(ERR_CONFIG_INDEX_TYPE_NEEDS_MATCHING_RULE.get(attrType, "ordering")); return false; } unacceptableReasons.add(ERR_CONFIG_INDEX_TYPE_NEEDS_MATCHING_RULE.get(attrType, "ordering")); return false; } if (cfg.getIndexType().contains(LocalDBIndexCfgDefn.IndexType.APPROXIMATE)) if (cfg.getIndexType().contains(LocalDBIndexCfgDefn.IndexType.APPROXIMATE) && approximateIndex == null && attrType.getApproximateMatchingRule() == null) { if (approximateIndex == null && attrType.getApproximateMatchingRule() == null) { unacceptableReasons.add(ERR_CONFIG_INDEX_TYPE_NEEDS_MATCHING_RULE.get(attrType, "approximate")); return false; } unacceptableReasons.add(ERR_CONFIG_INDEX_TYPE_NEEDS_MATCHING_RULE.get(attrType, "approximate")); return false; } if (cfg.getIndexType().contains(LocalDBIndexCfgDefn.IndexType.EXTENSIBLE)) { Set<String> newRules = cfg.getIndexExtensibleMatchingRule(); Set<String> newRules = cfg.getIndexExtensibleMatchingRule(); if (newRules == null || newRules.isEmpty()) { unacceptableReasons.add(ERR_CONFIG_INDEX_TYPE_NEEDS_MATCHING_RULE.get(attrType, "extensible")); @@ -1781,14 +1740,11 @@ return true; } /** * {@inheritDoc} */ /** {@inheritDoc} */ @Override public synchronized ConfigChangeResult applyConfigurationChange( LocalDBIndexCfg cfg) { ConfigChangeResult ccr; boolean adminActionRequired = false; ArrayList<LocalizableMessage> messages = new ArrayList<LocalizableMessage>(); try @@ -1850,9 +1806,8 @@ { messages.add(LocalizableMessage.raw( StaticUtils.stackTraceToSingleLineString(de))); ccr = new ConfigChangeResult( return new ConfigChangeResult( DirectoryServer.getServerErrorResultCode(), false, messages); return ccr; } finally { @@ -1911,9 +1866,8 @@ { messages.add(LocalizableMessage.raw( StaticUtils.stackTraceToSingleLineString(de))); ccr = new ConfigChangeResult( return new ConfigChangeResult( DirectoryServer.getServerErrorResultCode(), false, messages); return ccr; } finally { @@ -1980,9 +1934,8 @@ { messages.add(LocalizableMessage.raw( StaticUtils.stackTraceToSingleLineString(de))); ccr = new ConfigChangeResult( return new ConfigChangeResult( DirectoryServer.getServerErrorResultCode(), false, messages); return ccr; } finally { @@ -2041,9 +1994,8 @@ { messages.add(LocalizableMessage.raw( StaticUtils.stackTraceToSingleLineString(de))); ccr = new ConfigChangeResult( return new ConfigChangeResult( DirectoryServer.getServerErrorResultCode(), false, messages); return ccr; } finally { @@ -2103,9 +2055,8 @@ { messages.add( LocalizableMessage.raw(StaticUtils.stackTraceToSingleLineString(de))); ccr = new ConfigChangeResult( return new ConfigChangeResult( DirectoryServer.getServerErrorResultCode(), false, messages); return ccr; } finally { @@ -2259,9 +2210,8 @@ { messages.add( LocalizableMessage.raw(StaticUtils.stackTraceToSingleLineString(de))); ccr = new ConfigChangeResult( DirectoryServer.getServerErrorResultCode(), false, messages); return ccr; return new ConfigChangeResult( DirectoryServer.getServerErrorResultCode(), false, messages); } finally { @@ -2287,9 +2237,8 @@ { messages.add( LocalizableMessage.raw(StaticUtils.stackTraceToSingleLineString(de))); ccr = new ConfigChangeResult( DirectoryServer.getServerErrorResultCode(), false, messages); return ccr; return new ConfigChangeResult( DirectoryServer.getServerErrorResultCode(), false, messages); } finally { @@ -2306,10 +2255,8 @@ catch(Exception e) { messages.add(LocalizableMessage.raw(StaticUtils.stackTraceToSingleLineString(e))); ccr = new ConfigChangeResult(DirectoryServer.getServerErrorResultCode(), adminActionRequired, messages); return ccr; return new ConfigChangeResult( DirectoryServer.getServerErrorResultCode(), adminActionRequired, messages); } } @@ -2444,18 +2391,15 @@ } /** * Get the JE database name prefix for indexes in this attribute * index. * Get the JE database name prefix for indexes in this attribute index. * * @return JE database name for this database container. */ public String getName() { StringBuilder builder = new StringBuilder(); builder.append(entryContainer.getDatabasePrefix()); builder.append("_"); builder.append(indexConfig.getAttribute().getNameOrOID()); return builder.toString(); return entryContainer.getDatabasePrefix() + "_" + indexConfig.getAttribute().getNameOrOID(); } /** @@ -2680,13 +2624,12 @@ new JEIndexConfig(indexConfig.getSubstringLength()); for(ExtensibleIndexer indexer : rule.getIndexers(config)) { String indexerID = indexer.getExtensibleIndexID(); String indexName = indexer.getPreferredIndexName(); String indexID = " " + extensibleFilter.getAttributeType().getNameOrOID() + "." + indexName + "." +indexerID; debugBuffer.append(indexID); debugBuffer.append(" ") .append(extensibleFilter.getAttributeType().getNameOrOID()) .append(".") .append(indexer.getPreferredIndexName()) .append(".") .append(indexer.getExtensibleIndexID()); } debugBuffer.append("]"); } @@ -2727,7 +2670,7 @@ * This class manages all the configured extensible matching rules and * their corresponding indexes. */ private class ExtensibleMatchingRuleIndex private static class ExtensibleMatchingRuleIndex { /** * The mapping of index ID and Index database. @@ -2960,7 +2903,7 @@ */ private class JEIndexConfig extends IndexConfig { //The length of the substring index. /** The length of the substring index. */ private int substringLength; @@ -2978,10 +2921,10 @@ * Returns the length of the substring. * @return the length of the substring. */ @Override public int getSubstringLength() { return substringLength; } @Override public int getSubstringLength() { return substringLength; } } } opendj3-server-dev/src/server/org/opends/server/backends/jeb/VerifyJob.java
@@ -62,7 +62,7 @@ /** * The root container used for the verify job. */ RootContainer rootContainer; private RootContainer rootContainer; /** * The number of milliseconds between job progress reports. @@ -82,28 +82,28 @@ /** * The number of records that have exceeded the entry limit. */ long entryLimitExceededCount = 0; private long entryLimitExceededCount = 0; /** * The number of records that reference more than one entry. */ long multiReferenceCount = 0; private long multiReferenceCount = 0; /** * The total number of entry references. */ long entryReferencesCount = 0; private long entryReferencesCount = 0; /** * The maximum number of references per record. */ long maxEntryPerValue = 0; private long maxEntryPerValue = 0; /** * This map is used to gather some statistics about values that have * exceeded the entry limit. */ IdentityHashMap<Index,HashMap<ByteString,Long>> entryLimitMap = private IdentityHashMap<Index, HashMap<ByteString, Long>> entryLimitMap = new IdentityHashMap<Index, HashMap<ByteString, Long>>(); /** @@ -124,37 +124,38 @@ /** * The entry database. */ ID2Entry id2entry = null; private ID2Entry id2entry; /** * The DN database. */ DN2ID dn2id = null; private DN2ID dn2id; /** * The children database. */ Index id2c = null; private Index id2c; /** * The subtree database. */ Index id2s = null; private Index id2s; /** * A list of the attribute indexes to be verified. */ ArrayList<AttributeIndex> attrIndexList = new ArrayList<AttributeIndex>(); private ArrayList<AttributeIndex> attrIndexList = new ArrayList<AttributeIndex>(); /** * A list of the VLV indexes to be verified. */ ArrayList<VLVIndex> vlvIndexList = new ArrayList<VLVIndex>(); private ArrayList<VLVIndex> vlvIndexList = new ArrayList<VLVIndex>(); /** * The types of indexes that are verifiable. */ enum IndexType /** * The types of indexes that are verifiable. */ private enum IndexType { PRES, EQ, SUBSTRING, ORDERING, APPROXIMATE } @@ -1475,21 +1476,9 @@ */ private String keyDump(Index index, byte[] keyBytes) { /* String str; try { str = new String(keyBytes, "UTF-8"); } catch (UnsupportedEncodingException e) { str = StaticUtils.bytesToHex(keyBytes); } return str; */ StringBuilder buffer = new StringBuilder(128); buffer.append("File: "); buffer.append(index.toString()); buffer.append(index); buffer.append(ServerConstants.EOL); buffer.append("Key:"); buffer.append(ServerConstants.EOL); @@ -1506,21 +1495,9 @@ */ private String keyDump(VLVIndex vlvIndex, SortValues keySortValues) { /* String str; try { str = new String(keyBytes, "UTF-8"); } catch (UnsupportedEncodingException e) { str = StaticUtils.bytesToHex(keyBytes); } return str; */ StringBuilder buffer = new StringBuilder(128); buffer.append("File: "); buffer.append(vlvIndex.toString()); buffer.append(vlvIndex); buffer.append(ServerConstants.EOL); buffer.append("Key (last sort values):"); if(keySortValues == null) @@ -1529,7 +1506,7 @@ } else { buffer.append(keySortValues.toString()); buffer.append(keySortValues); } return buffer.toString(); } @@ -1630,6 +1607,11 @@ List<Attribute> attrList) throws DirectoryException { if (attrList == null || attrList.isEmpty()) { return; } Transaction txn = null; Index equalityIndex = attrIndex.getEqualityIndex(); Index presenceIndex = attrIndex.getPresenceIndex(); @@ -1637,219 +1619,102 @@ Index orderingIndex = attrIndex.getOrderingIndex(); Index approximateIndex = attrIndex.getApproximateIndex(); // TODO: Add support for Extended Matching Rules indexes. DatabaseEntry presenceKey = AttributeIndex.presenceKey; // Presence index. if ((attrList != null) && !attrList.isEmpty() && presenceIndex != null) if (presenceIndex != null) { try DatabaseEntry presenceKey = AttributeIndex.presenceKey; verifyAttributeInIndex(presenceIndex, txn, presenceKey, entryID); } for (Attribute attr : attrList) { for (AttributeValue value : attr) { ConditionResult cr; cr = presenceIndex.containsID(txn, presenceKey, entryID); if (cr == ConditionResult.FALSE) byte[] normalizedBytes = value.getNormalizedValue().toByteArray(); // Equality index. if (equalityIndex != null) { if (logger.isTraceEnabled()) { logger.trace("Missing ID %d%n%s", entryID.longValue(), keyDump(presenceIndex, presenceKey.getData())); } errorCount++; DatabaseEntry key = new DatabaseEntry(normalizedBytes); verifyAttributeInIndex(equalityIndex, txn, key, entryID); } else if (cr == ConditionResult.UNDEFINED) // Substring index. if (substringIndex != null) { incrEntryLimitStats(presenceIndex, presenceKey.getData()); Set<ByteString> keyBytesSet = attrIndex.substringKeys(normalizedBytes); DatabaseEntry key = new DatabaseEntry(); for (ByteString keyBytes : keyBytesSet) { key.setData(keyBytes.toByteArray()); verifyAttributeInIndex(substringIndex, txn, key, entryID); } } // Ordering index. if (orderingIndex != null) { // Use the ordering matching rule to normalize the value. OrderingMatchingRule orderingRule = attr.getAttributeType().getOrderingMatchingRule(); normalizedBytes = normalizeAttributeValue(orderingRule, value); DatabaseEntry key = new DatabaseEntry(normalizedBytes); verifyAttributeInIndex(orderingIndex, txn, key, entryID); } // Approximate index. if (approximateIndex != null) { // Use the approximate matching rule to normalize the value. ApproximateMatchingRule approximateRule = attr.getAttributeType().getApproximateMatchingRule(); normalizedBytes = normalizeAttributeValue(approximateRule, value); DatabaseEntry key = new DatabaseEntry(normalizedBytes); verifyAttributeInIndex(approximateIndex, txn, key, entryID); } } catch (DatabaseException e) } } private void verifyAttributeInIndex(Index index, Transaction txn, DatabaseEntry key, EntryID entryID) { try { ConditionResult cr = index.containsID(txn, key, entryID); if (cr == ConditionResult.FALSE) { if (logger.isTraceEnabled()) { logger.traceException(e); logger.trace("Error reading database: %s%n%s", e.getMessage(), keyDump(presenceIndex, presenceKey.getData())); logger.trace("Missing ID %d%n%s", entryID.longValue(), keyDump(index, key.getData())); } errorCount++; } } if (attrList != null) { for (Attribute attr : attrList) else if (cr == ConditionResult.UNDEFINED) { for (AttributeValue value : attr) { byte[] normalizedBytes = value.getNormalizedValue().toByteArray(); // Equality index. if (equalityIndex != null) { DatabaseEntry key = new DatabaseEntry(normalizedBytes); try { ConditionResult cr; cr = equalityIndex.containsID(txn, key, entryID); if (cr == ConditionResult.FALSE) { if (logger.isTraceEnabled()) { logger.trace("Missing ID %d%n%s", entryID.longValue(), keyDump(equalityIndex, normalizedBytes)); } errorCount++; } else if (cr == ConditionResult.UNDEFINED) { incrEntryLimitStats(equalityIndex, normalizedBytes); } } catch (DatabaseException e) { if (logger.isTraceEnabled()) { logger.traceException(e); logger.trace("Error reading database: %s%n%s", e.getMessage(), keyDump(equalityIndex, normalizedBytes)); } errorCount++; } } // Substring index. if (substringIndex != null) { Set<ByteString> keyBytesSet = attrIndex.substringKeys(normalizedBytes); DatabaseEntry key = new DatabaseEntry(); for (ByteString keyBytes : keyBytesSet) { key.setData(keyBytes.toByteArray()); try { ConditionResult cr; cr = substringIndex.containsID(txn, key, entryID); if (cr == ConditionResult.FALSE) { if (logger.isTraceEnabled()) { logger.trace("Missing ID %d%n%s", entryID.longValue(), keyDump(substringIndex, key.getData())); } errorCount++; } else if (cr == ConditionResult.UNDEFINED) { incrEntryLimitStats(substringIndex, key.getData()); } } catch (DatabaseException e) { if (logger.isTraceEnabled()) { logger.traceException(e); logger.trace("Error reading database: %s%n%s", e.getMessage(), keyDump(substringIndex, key.getData())); } errorCount++; } } } // Ordering index. if (orderingIndex != null) { // Use the ordering matching rule to normalize the value. OrderingMatchingRule orderingRule = attr.getAttributeType().getOrderingMatchingRule(); normalizedBytes = normalizeAttributeValue(orderingRule, value); DatabaseEntry key = new DatabaseEntry(normalizedBytes); try { ConditionResult cr; cr = orderingIndex.containsID(txn, key, entryID); if (cr == ConditionResult.FALSE) { if (logger.isTraceEnabled()) { logger.trace("Missing ID %d%n%s", entryID.longValue(), keyDump(orderingIndex, normalizedBytes)); } errorCount++; } else if (cr == ConditionResult.UNDEFINED) { incrEntryLimitStats(orderingIndex, normalizedBytes); } } catch (DatabaseException e) { if (logger.isTraceEnabled()) { logger.traceException(e); logger.trace("Error reading database: %s%n%s", e.getMessage(), keyDump(orderingIndex, normalizedBytes)); } errorCount++; } } // Approximate index. if (approximateIndex != null) { // Use the approximate matching rule to normalize the value. ApproximateMatchingRule approximateRule = attr.getAttributeType().getApproximateMatchingRule(); normalizedBytes = normalizeAttributeValue(approximateRule, value); DatabaseEntry key = new DatabaseEntry(normalizedBytes); try { ConditionResult cr; cr = approximateIndex.containsID(txn, key, entryID); if (cr == ConditionResult.FALSE) { if (logger.isTraceEnabled()) { logger.trace("Missing ID %d%n%s", entryID.longValue(), keyDump(orderingIndex, normalizedBytes)); } errorCount++; } else if (cr == ConditionResult.UNDEFINED) { incrEntryLimitStats(orderingIndex, normalizedBytes); } } catch (DatabaseException e) { if (logger.isTraceEnabled()) { logger.traceException(e); logger.trace("Error reading database: %s%n%s", e.getMessage(), keyDump(approximateIndex, normalizedBytes)); } errorCount++; } } } incrEntryLimitStats(index, key.getData()); } } catch (DatabaseException e) { if (logger.isTraceEnabled()) { logger.traceException(e); logger.trace("Error reading database: %s%n%s", e.getMessage(), keyDump(index, key.getData())); } errorCount++; } } private byte[] normalizeAttributeValue(MatchingRule matchingRule,