From 325b2ee4a27d0c24aa0a539f7bd0a8cf24905ff7 Mon Sep 17 00:00:00 2001
From: boli <boli@localhost>
Date: Tue, 10 Apr 2007 20:41:27 +0000
Subject: [PATCH] Added the following capabilities to OpenDS: - Index rebuilding capabilities. All indexes including system and attribute indexes can be rebuilt. Each index will be rebuilt by a seperate thread to increase performance. A max number of rebuild threads could be set to limit the resources used by large rebuild jobs. Partial rebuilds of attribute indexes could also be done by specifying the attribute index type after the attribute type (ie. sn.approximate). - Index rebuilding standalone tool. Rebuilding of attribute indexes could be done with the backend online. However, rebuilds including system indexes must be done with the backend offline. - Index rebuilding task. Rebuilding of attribute indexes are done with the backend online. Rebuilds that include system indexes will be performed after bring the backend offline. The user must have index-rebuild privilages to rebuild indexes. - Approxitae indexing capability. The value of the attribute will be normalized using the approximate maching rule of that attribute type. This is used as the key for the index. Approximate indexes are fully supported by the index verify, rebuild, and import jobs. - Fixed bug in build.xml where weave is enabled even if a test.* property is set. - Consolidated some common tool messages. - Consolidated some JE backend methods common to all tools. - Added unit tests for rebuild job and approximate indexes.
---
opends/src/server/org/opends/server/backends/jeb/VerifyJob.java | 150 +++++++++++++++++++++++++++++++++++++++++++++++--
1 files changed, 142 insertions(+), 8 deletions(-)
diff --git a/opends/src/server/org/opends/server/backends/jeb/VerifyJob.java b/opends/src/server/org/opends/server/backends/jeb/VerifyJob.java
index b6edbeb..b1fb472 100644
--- a/opends/src/server/org/opends/server/backends/jeb/VerifyJob.java
+++ b/opends/src/server/org/opends/server/backends/jeb/VerifyJob.java
@@ -39,6 +39,7 @@
import com.sleepycat.je.Transaction;
import org.opends.server.api.OrderingMatchingRule;
+import org.opends.server.api.ApproximateMatchingRule;
import org.opends.server.core.DirectoryServer;
import org.opends.server.protocols.asn1.ASN1OctetString;
import org.opends.server.types.Attribute;
@@ -180,7 +181,7 @@
*/
enum IndexType
{
- PRES, EQ, SUBSTRING, ORDERING
+ PRES, EQ, SUBSTRING, ORDERING, APPROXIMATE
}
/**
@@ -529,6 +530,8 @@
attrIndex.substringIndex, IndexType.SUBSTRING);
iterateAttrIndex(attrIndex.getAttributeType(),
attrIndex.orderingIndex, IndexType.ORDERING);
+ iterateAttrIndex(attrIndex.getAttributeType(),
+ attrIndex.approximateIndex, IndexType.APPROXIMATE);
}
}
@@ -993,6 +996,12 @@
DatabaseEntry key = new DatabaseEntry();
DatabaseEntry data = new DatabaseEntry();
+ OrderingMatchingRule orderingMatchingRule =
+ attrType.getOrderingMatchingRule();
+ ApproximateMatchingRule approximateMatchingRule =
+ attrType.getApproximateMatchingRule();
+ ASN1OctetString previousValue = null;
+
OperationStatus status;
for (status = cursor.getFirst(key, data, LockMode.DEFAULT);
status == OperationStatus.SUCCESS;
@@ -1026,6 +1035,7 @@
{
byte[] value = key.getData();
SearchFilter sf;
+ AttributeValue assertionValue;
switch (indexType)
{
@@ -1037,13 +1047,55 @@
sf = SearchFilter.createSubstringFilter(attrType,null,
subAnyElements,null);
break;
- /* TODO
- * This ORDERING case needs further study
- * about what type of SearchFilter should be created.
- * case ORDERING:
- * */
+ case ORDERING:
+ // Ordering index checking is two fold:
+ // 1. Make sure the entry has an attribute value that is the same
+ // as the key. This is done by falling through to the next
+ // case and create an equality filter.
+ // 2. Make sure the key value is greater then the previous key
+ // value.
+ assertionValue =
+ new AttributeValue(attrType, new ASN1OctetString(value));
+
+ sf = SearchFilter.createEqualityFilter(attrType,assertionValue);
+
+ if(orderingMatchingRule != null && previousValue != null)
+ {
+ ASN1OctetString thisValue = new ASN1OctetString(value);
+ int order = orderingMatchingRule.compareValues(thisValue,
+ previousValue);
+ if(order > 0)
+ {
+ errorCount++;
+ if(debugEnabled())
+ {
+ debugError("Reversed ordering of index keys " +
+ "(keys dumped in the order found in database)%n" +
+ "Key 1:%n%s%nKey 2:%n%s",
+ keyDump(index, thisValue.value()),
+ keyDump(index,previousValue.value()));
+ }
+ continue;
+ }
+ else if(order == 0)
+ {
+ errorCount++;
+ if(debugEnabled())
+ {
+ debugError("Duplicate index keys%nKey 1:%n%s%nKey2:%n%s",
+ keyDump(index, thisValue.value()),
+ keyDump(index,previousValue.value()));
+ }
+ continue;
+ }
+ else
+ {
+ previousValue = thisValue;
+ }
+ }
+ break;
case EQ:
- AttributeValue assertionValue =
+ assertionValue =
new AttributeValue(attrType, new ASN1OctetString(value));
sf = SearchFilter.createEqualityFilter(attrType,assertionValue);
@@ -1053,6 +1105,12 @@
sf = SearchFilter.createPresenceFilter(attrType);
break;
+ case APPROXIMATE:
+ // This must be handled differently since we can't use a search
+ // filter to see if the key matches.
+ sf = null;
+ break;
+
default:
errorCount++;
if (debugEnabled())
@@ -1103,7 +1161,39 @@
try
{
- if (!sf.matchesEntry(entry))
+ boolean match = false;
+ if(indexType != IndexType.APPROXIMATE)
+ {
+ match = sf.matchesEntry(entry);
+ }
+ else
+ {
+ ByteString normalizedValue = new ASN1OctetString(value);
+ List<Attribute> attrs = entry.getAttribute(attrType);
+ if ((attrs != null) && (!attrs.isEmpty()))
+ {
+ for (Attribute a : attrs)
+ {
+ for (AttributeValue v : a.getValues())
+ {
+ ByteString nv =
+ approximateMatchingRule.normalizeValue(v.getValue());
+ match = approximateMatchingRule.
+ approximatelyMatch(nv, normalizedValue);
+ if(match)
+ {
+ break;
+ }
+ }
+ if(match)
+ {
+ break;
+ }
+ }
+ }
+ }
+
+ if (!match)
{
errorCount++;
if (debugEnabled())
@@ -1464,6 +1554,7 @@
Index presenceIndex = attrIndex.presenceIndex;
Index substringIndex = attrIndex.substringIndex;
Index orderingIndex = attrIndex.orderingIndex;
+ Index approximateIndex = attrIndex.approximateIndex;
IndexConfig indexConfig = attrIndex.indexConfig;
DatabaseEntry presenceKey = AttributeIndex.presenceKey;
@@ -1635,6 +1726,49 @@
errorCount++;
}
}
+ // Approximate index.
+ if (indexConfig.isApproximateIndex())
+ {
+ // Use the approximate matching rule to normalize the value.
+ ApproximateMatchingRule approximateRule =
+ attr.getAttributeType().getApproximateMatchingRule();
+
+ normalizedBytes =
+ approximateRule.normalizeValue(value.getValue()).value();
+
+ DatabaseEntry key = new DatabaseEntry(normalizedBytes);
+ try
+ {
+ ConditionResult cr;
+ cr = approximateIndex.containsID(txn, key, entryID);
+ if (cr == ConditionResult.FALSE)
+ {
+ if (debugEnabled())
+ {
+ debugError("Missing ID %d%n%s",
+ entryID.longValue(),
+ keyDump(orderingIndex, normalizedBytes));
+ }
+ errorCount++;
+ }
+ else if (cr == ConditionResult.UNDEFINED)
+ {
+ incrEntryLimitStats(orderingIndex, normalizedBytes);
+ }
+ }
+ catch (DatabaseException e)
+ {
+ if (debugEnabled())
+ {
+ debugCaught(DebugLogLevel.ERROR, e);
+
+ debugError("Error reading database: %s%n%s",
+ e.getMessage(),
+ keyDump(approximateIndex, normalizedBytes));
+ }
+ errorCount++;
+ }
+ }
}
}
}
--
Gitblit v1.10.0