From 9f0904fda87bfcf921deeccdbaeafe834fbad696 Mon Sep 17 00:00:00 2001
From: Yannick Lecaillez <yannick.lecaillez@forgerock.com>
Date: Fri, 24 Apr 2015 14:30:47 +0000
Subject: [PATCH] OPENDJ-1725: Persistit: very long recovery and many discarded txns after addrate test
---
opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/VerifyJob.java | 562 +++++++++----------------------------------------------
1 files changed, 94 insertions(+), 468 deletions(-)
diff --git a/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/VerifyJob.java b/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/VerifyJob.java
index c7f1559..1da0f1c 100644
--- a/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/VerifyJob.java
+++ b/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/VerifyJob.java
@@ -38,8 +38,10 @@
import java.util.HashSet;
import java.util.IdentityHashMap;
import java.util.Iterator;
+import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import java.util.Queue;
import java.util.Set;
import java.util.Timer;
import java.util.TimerTask;
@@ -73,7 +75,7 @@
/** The verify configuration. */
private final VerifyConfig verifyConfig;
/** The root container used for the verify job. */
- private RootContainer rootContainer;
+ private final RootContainer rootContainer;
/** The number of milliseconds between job progress reports. */
private final long progressInterval = 10000;
@@ -99,19 +101,15 @@
/** Indicates whether the DN database is to be verified. */
private boolean verifyDN2ID;
- /** Indicates whether the children database is to be verified. */
- private boolean verifyID2Children;
- /** Indicates whether the subtree database is to be verified. */
- private boolean verifyID2Subtree;
+ /** Indicates whether the children count database is to be verified. */
+ private boolean verifyID2ChildrenCount;
/** The entry database. */
private ID2Entry id2entry;
/** The DN database. */
private DN2ID dn2id;
/** The children database. */
- private Index id2c;
- /** The subtree database. */
- private Index id2s;
+ private ID2Count id2childrenCount;
/** A list of the attribute indexes to be verified. */
private final ArrayList<AttributeIndex> attrIndexList = new ArrayList<AttributeIndex>();
@@ -123,8 +121,9 @@
*
* @param verifyConfig The verify configuration.
*/
- VerifyJob(VerifyConfig verifyConfig)
+ VerifyJob(RootContainer rootContainer, VerifyConfig verifyConfig)
{
+ this.rootContainer = rootContainer;
this.verifyConfig = verifyConfig;
}
@@ -136,7 +135,7 @@
* @throws StorageRuntimeException If an error occurs in the database.
* @throws DirectoryException If an error occurs while verifying the backend.
*/
- long verifyBackend(final RootContainer rootContainer) throws StorageRuntimeException,
+ long verifyBackend() throws StorageRuntimeException,
DirectoryException
{
try
@@ -146,7 +145,7 @@
@Override
public Long run(ReadableTransaction txn) throws Exception
{
- return verifyBackend0(txn, rootContainer);
+ return verifyBackend0(txn);
}
});
}
@@ -160,10 +159,8 @@
}
}
- private long verifyBackend0(ReadableTransaction txn, RootContainer rootContainer)
- throws StorageRuntimeException, DirectoryException
+ private long verifyBackend0(ReadableTransaction txn) throws StorageRuntimeException, DirectoryException
{
- this.rootContainer = rootContainer;
EntryContainer entryContainer =
rootContainer.getEntryContainer(verifyConfig.getBaseDN());
@@ -177,11 +174,7 @@
if (completeList.isEmpty() && cleanList.isEmpty())
{
verifyDN2ID = true;
- if (rootContainer.getConfiguration().isSubordinateIndexesEnabled())
- {
- verifyID2Children = true;
- verifyID2Subtree = true;
- }
+ verifyID2ChildrenCount = true;
attrIndexList.addAll(entryContainer.getAttributeIndexes());
}
else
@@ -204,31 +197,9 @@
{
verifyDN2ID = true;
}
- else if ("id2children".equals(lowerName))
+ if ("id2childrencount".equals(lowerName))
{
- if (rootContainer.getConfiguration().isSubordinateIndexesEnabled())
- {
- verifyID2Children = true;
- }
- else
- {
- LocalizableMessage msg = NOTE_JEB_SUBORDINATE_INDEXES_DISABLED
- .get(rootContainer.getConfiguration().getBackendId());
- throw new StorageRuntimeException(msg.toString());
- }
- }
- else if ("id2subtree".equals(lowerName))
- {
- if (rootContainer.getConfiguration().isSubordinateIndexesEnabled())
- {
- verifyID2Subtree = true;
- }
- else
- {
- LocalizableMessage msg = NOTE_JEB_SUBORDINATE_INDEXES_DISABLED
- .get(rootContainer.getConfiguration().getBackendId());
- throw new StorageRuntimeException(msg.toString());
- }
+ verifyID2ChildrenCount = true;
}
else if(lowerName.startsWith("vlv."))
{
@@ -275,8 +246,7 @@
// the entry entryContainer methods.
id2entry = entryContainer.getID2Entry();
dn2id = entryContainer.getDN2ID();
- id2c = entryContainer.getID2Children();
- id2s = entryContainer.getID2Subtree();
+ id2childrenCount = entryContainer.getID2ChildrenCount();
// Make a note of the time we started.
long startTime = System.currentTimeMillis();
@@ -387,8 +357,7 @@
*/
private void iterateID2Entry(ReadableTransaction txn) throws StorageRuntimeException
{
- Cursor<ByteString, ByteString> cursor = txn.openCursor(id2entry.getName());
- try
+ try(final Cursor<ByteString, ByteString> cursor = txn.openCursor(id2entry.getName()))
{
long storedEntryCount = id2entry.getRecordCount(txn);
while (cursor.next())
@@ -445,10 +414,6 @@
}
}
}
- finally
- {
- cursor.close();
- }
}
/**
@@ -465,13 +430,9 @@
{
iterateDN2ID(txn);
}
- else if (verifyID2Children)
+ else if (verifyID2ChildrenCount)
{
- iterateID2Children(txn);
- }
- else if (verifyID2Subtree)
- {
- iterateID2Subtree(txn);
+ iterateID2ChildrenCount(txn);
}
else if (attrIndexList.size() > 0)
{
@@ -496,34 +457,31 @@
*/
private void iterateDN2ID(ReadableTransaction txn) throws StorageRuntimeException
{
- Cursor<ByteString, ByteString> cursor = txn.openCursor(dn2id.getName());
- try
+ final Queue<ChildrenCount> childrenCounters = new LinkedList<>();
+ ChildrenCount currentNode = null;
+
+ try(final Cursor<ByteString, ByteString> cursor = txn.openCursor(dn2id.getName()))
{
while (cursor.next())
{
keyCount++;
- ByteString key = cursor.getKey();
- ByteString value = cursor.getValue();
-
- EntryID entryID;
+ final ByteString key = cursor.getKey();
+ final EntryID entryID;
try
{
- entryID = new EntryID(value);
+ entryID = new EntryID(cursor.getValue());
}
catch (Exception e)
{
errorCount++;
- if (logger.isTraceEnabled())
- {
- logger.traceException(e);
-
- logger.trace("File dn2id has malformed ID for DN <%s>:%n%s%n", key, StaticUtils.bytesToHex(value));
- }
+ logger.trace("File dn2id has malformed ID for DN <%s>", key, e);
continue;
}
- Entry entry;
+ currentNode = verifyID2ChildrenCount(txn, childrenCounters, key, entryID);
+
+ final Entry entry;
try
{
entry = id2entry.get(txn, entryID);
@@ -538,261 +496,68 @@
if (entry == null)
{
errorCount++;
- if (logger.isTraceEnabled())
- {
- logger.trace("File dn2id has DN <%s> referencing unknown ID %d%n", key, entryID);
- }
+ logger.trace("File dn2id has DN <%s> referencing unknown ID %d%n", key, entryID);
}
else if (!key.equals(dnToDNKey(entry.getName(), verifyConfig.getBaseDN().size())))
{
errorCount++;
- if (logger.isTraceEnabled())
- {
- logger.trace("File dn2id has DN <%s> referencing entry with wrong DN <%s>%n", key, entry.getName());
- }
+ logger.trace("File dn2id has DN <%s> referencing entry with wrong DN <%s>%n", key, entry.getName());
}
}
- }
- finally
- {
- cursor.close();
+
+ while ((currentNode = childrenCounters.poll()) != null)
+ {
+ verifyID2ChildrenCount(txn, currentNode);
+ }
}
}
- /**
- * Iterate through the entries in ID2Children to perform a check for
- * index cleanliness.
- *
- * @throws StorageRuntimeException If an error occurs in the database.
- */
- private void iterateID2Children(ReadableTransaction txn) throws StorageRuntimeException
+ private ChildrenCount verifyID2ChildrenCount(ReadableTransaction txn, final Queue<ChildrenCount> childrenCounters,
+ final ByteString key, final EntryID entryID)
{
- Cursor<ByteString, EntryIDSet> cursor = id2c.openCursor(txn);
- try
+ while (childrenCounters.peek() != null && !DN2ID.isChild(childrenCounters.peek().baseDN, key))
{
- while (cursor.next())
- {
- keyCount++;
-
- ByteString key = cursor.getKey();
-
- EntryID entryID;
- try
- {
- entryID = new EntryID(key);
- }
- catch (Exception e)
- {
- errorCount++;
- if (logger.isTraceEnabled())
- {
- logger.traceException(e);
-
- logger.trace("File id2children has malformed ID %s%n", StaticUtils.bytesToHex(key));
- }
- continue;
- }
-
- EntryIDSet entryIDSet;
-
- try
- {
- entryIDSet = cursor.getValue();
- }
- catch (Exception e)
- {
- errorCount++;
- logger.traceException(e);
- logger.trace("File id2children has malformed ID list for ID %s", entryID);
- continue;
- }
-
- updateIndexStats(entryIDSet);
-
- if (entryIDSet.isDefined())
- {
- Entry entry;
- try
- {
- entry = id2entry.get(txn, entryID);
- }
- catch (Exception e)
- {
- logger.traceException(e);
- errorCount++;
- continue;
- }
-
- if (entry == null)
- {
- errorCount++;
- if (logger.isTraceEnabled())
- {
- logger.trace("File id2children has unknown ID %d%n", entryID);
- }
- continue;
- }
-
- for (EntryID id : entryIDSet)
- {
- Entry childEntry;
- try
- {
- childEntry = id2entry.get(txn, id);
- }
- catch (Exception e)
- {
- logger.traceException(e);
- errorCount++;
- continue;
- }
-
- if (childEntry == null)
- {
- errorCount++;
- if (logger.isTraceEnabled())
- {
- logger.trace("File id2children has ID %d referencing unknown ID %d%n", entryID, id);
- }
- continue;
- }
-
- if (!childEntry.getName().isDescendantOf(entry.getName()) ||
- childEntry.getName().size() !=
- entry.getName().size() + 1)
- {
- errorCount++;
- if (logger.isTraceEnabled())
- {
- logger.trace("File id2children has ID %d with DN <%s> " +
- "referencing ID %d with non-child DN <%s>%n",
- entryID, entry.getName(), id, childEntry.getName());
- }
- }
- }
- }
- }
+ // This subtree is fully processed, pop the counter of the parent DN from the stack and verify it's value
+ verifyID2ChildrenCount(txn, childrenCounters.remove());
}
- finally
+ if (childrenCounters.peek() != null)
{
- cursor.close();
+ childrenCounters.peek().numberOfChildren++;
+ }
+ final ChildrenCount node = new ChildrenCount(key, entryID);
+ childrenCounters.add(node);
+ return node;
+ }
+
+ private void verifyID2ChildrenCount(ReadableTransaction txn, ChildrenCount parent) {
+ final long expected = parent.numberOfChildren;
+ final long currentValue = id2childrenCount.getCount(txn, parent.entryID);
+ if (expected != currentValue)
+ {
+ errorCount++;
+ logger.trace("File id2childrenCount has wrong number of children for DN <%s> (got %d, expecting %d)",
+ parent.baseDN, currentValue, expected);
}
}
- /**
- * Iterate through the entries in ID2Subtree to perform a check for
- * index cleanliness.
- *
- * @throws StorageRuntimeException If an error occurs in the database.
- */
- private void iterateID2Subtree(ReadableTransaction txn) throws StorageRuntimeException
+ private void iterateID2ChildrenCount(ReadableTransaction txn) throws StorageRuntimeException
{
- Cursor<ByteString, EntryIDSet> cursor = id2s.openCursor(txn);
- try
- {
- while (cursor.next())
- {
- keyCount++;
-
- ByteString key = cursor.getKey();
- EntryID entryID;
- try
- {
- entryID = new EntryID(key);
- }
- catch (Exception e)
- {
- errorCount++;
- if (logger.isTraceEnabled())
- {
- logger.traceException(e);
-
- logger.trace("File id2subtree has malformed ID %s%n", StaticUtils.bytesToHex(key));
- }
- continue;
- }
-
- EntryIDSet entryIDSet;
- try
- {
- entryIDSet = cursor.getValue();
- }
- catch (Exception e)
- {
- errorCount++;
- logger.traceException(e);
- logger.trace("File id2subtree has malformed ID list for ID %s", entryID);
- continue;
- }
-
- updateIndexStats(entryIDSet);
-
- if (entryIDSet.isDefined())
- {
- Entry entry;
- try
- {
- entry = id2entry.get(txn, entryID);
- }
- catch (Exception e)
- {
- logger.traceException(e);
- errorCount++;
- continue;
- }
-
- if (entry == null)
- {
- errorCount++;
- if (logger.isTraceEnabled())
- {
- logger.trace("File id2subtree has unknown ID %d%n", entryID);
- }
- continue;
- }
-
- for (EntryID id : entryIDSet)
- {
- Entry subordEntry;
- try
- {
- subordEntry = id2entry.get(txn, id);
- }
- catch (Exception e)
- {
- logger.traceException(e);
- errorCount++;
- continue;
- }
-
- if (subordEntry == null)
- {
- errorCount++;
- if (logger.isTraceEnabled())
- {
- logger.trace("File id2subtree has ID %d referencing " +
- "unknown ID %d%n", entryID, id);
- }
- continue;
- }
-
- if (!subordEntry.getName().isDescendantOf(entry.getName()))
- {
- errorCount++;
- if (logger.isTraceEnabled())
- {
- logger.trace("File id2subtree has ID %d with DN <%s> " +
- "referencing ID %d with non-subordinate DN <%s>%n",
- entryID, entry.getName(), id, subordEntry.getName());
- }
- }
- }
- }
- }
+ Cursor<EntryID, Long> cursor = id2childrenCount.openCursor(txn);
+ if (!cursor.next()) {
+ return;
}
- finally
- {
- cursor.close();
+
+ EntryID currentEntryID = new EntryID(-1);
+ while(cursor.next()) {
+ if (cursor.getKey().equals(currentEntryID)) {
+ /** Sharded cursor may return the same EntryID multiple times */
+ continue;
+ }
+ currentEntryID = cursor.getKey();
+ if (!id2entry.containsEntryID(txn, currentEntryID)) {
+ logger.trace("File id2ChildrenCount reference non-existing EntryID <%d>%n", currentEntryID);
+ errorCount++;
+ }
}
}
@@ -864,8 +629,7 @@
return;
}
- Cursor<ByteString, ByteString> cursor = txn.openCursor(vlvIndex.getName());
- try
+ try(final Cursor<ByteString, ByteString> cursor = txn.openCursor(vlvIndex.getName()))
{
while (cursor.next())
{
@@ -906,10 +670,6 @@
}
}
- finally
- {
- cursor.close();
- }
}
/**
@@ -926,8 +686,7 @@
return;
}
- Cursor<ByteString,EntryIDSet> cursor = index.openCursor(txn);
- try
+ try(final Cursor<ByteString,EntryIDSet> cursor = index.openCursor(txn))
{
while (cursor.next())
{
@@ -1042,10 +801,6 @@
}
}
}
- finally
- {
- cursor.close();
- }
}
/**
@@ -1060,14 +815,6 @@
{
verifyDN2ID(txn, entryID, entry);
}
- if (verifyID2Children)
- {
- verifyID2Children(txn, entryID, entry);
- }
- if (verifyID2Subtree)
- {
- verifyID2Subtree(txn, entryID, entry);
- }
verifyIndex(txn, entryID, entry);
}
@@ -1141,137 +888,6 @@
}
/**
- * Check that the ID2Children index is complete for a given entry.
- *
- * @param entryID The entry ID.
- * @param entry The entry to be checked.
- */
- private void verifyID2Children(ReadableTransaction txn, EntryID entryID, Entry entry)
- {
- DN dn = entry.getName();
-
- DN parentDN = getParent(dn);
- if (parentDN != null)
- {
- EntryID parentID = null;
- try
- {
- parentID = dn2id.get(txn, parentDN);
- if (parentID == null)
- {
- if (logger.isTraceEnabled())
- {
- logger.trace("File dn2id is missing key %s.%n", parentDN);
- }
- errorCount++;
- }
- }
- catch (Exception e)
- {
- if (logger.isTraceEnabled())
- {
- logger.traceException(e);
- logger.trace("File dn2id has error reading key %s: %s.", parentDN, e.getMessage());
- }
- errorCount++;
- }
- if (parentID != null)
- {
- try
- {
- ConditionResult cr = indexContainsID(id2c, txn, parentID.toByteString(), entryID);
- if (cr == ConditionResult.FALSE)
- {
- if (logger.isTraceEnabled())
- {
- logger.trace("File id2children is missing ID %d for key %d.%n", entryID, parentID);
- }
- errorCount++;
- }
- else if (cr == ConditionResult.UNDEFINED)
- {
- incrEntryLimitStats(id2c, parentID.toByteString());
- }
- }
- catch (StorageRuntimeException e)
- {
- if (logger.isTraceEnabled())
- {
- logger.traceException(e);
-
- logger.trace("File id2children has error reading key %d: %s.", parentID, e.getMessage());
- }
- errorCount++;
- }
- }
- }
- }
-
- /**
- * Check that the ID2Subtree index is complete for a given entry.
- *
- * @param entryID The entry ID.
- * @param entry The entry to be checked.
- */
- private void verifyID2Subtree(ReadableTransaction txn, EntryID entryID, Entry entry)
- {
- for (DN dn = getParent(entry.getName()); dn != null; dn = getParent(dn))
- {
- EntryID id = null;
- try
- {
- id = dn2id.get(txn, dn);
- if (id == null)
- {
- if (logger.isTraceEnabled())
- {
- logger.trace("File dn2id is missing key %s.%n", dn);
- }
- errorCount++;
- }
- }
- catch (Exception e)
- {
- if (logger.isTraceEnabled())
- {
- logger.traceException(e);
- logger.trace("File dn2id has error reading key %s: %s.%n", dn, e.getMessage());
- }
- errorCount++;
- }
- if (id != null)
- {
- try
- {
- ConditionResult cr = indexContainsID(id2s, txn, id.toByteString(), entryID);
- if (cr == ConditionResult.FALSE)
- {
- if (logger.isTraceEnabled())
- {
- logger.trace("File id2subtree is missing ID %d for key %d.%n", entryID, id);
- }
- errorCount++;
- }
- else if (cr == ConditionResult.UNDEFINED)
- {
- incrEntryLimitStats(id2s, id.toByteString());
- }
- }
- catch (StorageRuntimeException e)
- {
- if (logger.isTraceEnabled())
- {
- logger.traceException(e);
-
- logger.trace("File id2subtree has error reading key %d: %s.%n", id, e.getMessage());
- }
- errorCount++;
- }
- }
- }
- }
-
- /**
* Construct a printable string from a raw key value.
*
* @param indexName
@@ -1280,7 +896,7 @@
* The bytes of the key.
* @return A string that may be logged or printed.
*/
- private String keyDump(String indexName, ByteSequence key)
+ private static String keyDump(String indexName, ByteSequence key)
{
StringBuilder buffer = new StringBuilder(128);
buffer.append("Index: ");
@@ -1391,7 +1007,7 @@
}
}
- private ConditionResult indexContainsID(Index index, ReadableTransaction txn, ByteString key, EntryID entryID)
+ private static ConditionResult indexContainsID(Index index, ReadableTransaction txn, ByteString key, EntryID entryID)
{
EntryIDSet entryIDSet = index.get(txn, key);
if (entryIDSet.isDefined())
@@ -1416,6 +1032,20 @@
return dn.getParentDNInSuffix();
}
+ /**
+ * This class maintain the number of children for a given dn
+ */
+ private static final class ChildrenCount {
+ private final ByteString baseDN;
+ private final EntryID entryID;
+ private long numberOfChildren;
+
+ private ChildrenCount(ByteString dn, EntryID id) {
+ this.baseDN = dn;
+ this.entryID = id;
+ }
+ }
+
/** This class reports progress of the verify job at fixed intervals. */
private final class ProgressTask extends TimerTask
{
@@ -1454,13 +1084,9 @@
{
totalCount = dn2id.getRecordCount(txn);
}
- else if (verifyID2Children)
+ else if (verifyID2ChildrenCount)
{
- totalCount = id2c.getRecordCount(txn);
- }
- else if (verifyID2Subtree)
- {
- totalCount = id2s.getRecordCount(txn);
+ totalCount = id2childrenCount.getRecordCount(txn);
}
else if (!attrIndexList.isEmpty())
{
@@ -1478,7 +1104,7 @@
}
else
{
- totalCount = rootContainer.getEntryContainer(verifyConfig.getBaseDN()).getEntryCount(txn);
+ totalCount = rootContainer.getEntryContainer(verifyConfig.getBaseDN()).getNumberOfEntriesInBaseDN();
}
}
--
Gitblit v1.10.0