From b12119c55b89ece2495e84fba229d96439e8219b Mon Sep 17 00:00:00 2001
From: ludovicp <ludovicp@localhost>
Date: Fri, 30 Jul 2010 14:52:40 +0000
Subject: [PATCH] Implements index analysis features. There are 2 components: Index Filter Analyzer : Part of DatabaseEnvironmentProvider, it gathers search filter statistics and displays the index filter, number of hits, max matching entries, and a message. Compound filters are broken down to their basic elements. f an index was not utilized while evaluating the search filter, max matching entries will be -1 and a diagnostic message will be included. The monitor entry also includes when the analyzer was enabled along with the number of total index and unindexed searches processed.
---
opends/src/server/org/opends/server/monitors/DatabaseEnvironmentMonitor.java | 250 ++++++++++++++++++++++++++++++++++++++++++++++++-
1 files changed, 243 insertions(+), 7 deletions(-)
diff --git a/opends/src/server/org/opends/server/monitors/DatabaseEnvironmentMonitor.java b/opends/src/server/org/opends/server/monitors/DatabaseEnvironmentMonitor.java
index ad259c1..8e2550f 100644
--- a/opends/src/server/org/opends/server/monitors/DatabaseEnvironmentMonitor.java
+++ b/opends/src/server/org/opends/server/monitors/DatabaseEnvironmentMonitor.java
@@ -31,27 +31,28 @@
import static org.opends.server.loggers.debug.DebugLogger.*;
import java.lang.reflect.Method;
-import java.util.ArrayList;
+import java.util.*;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.opends.messages.Message;
import org.opends.server.admin.std.server.MonitorProviderCfg;
import org.opends.server.api.AttributeSyntax;
import org.opends.server.api.MonitorProvider;
+import org.opends.server.backends.jeb.DatabaseContainer;
+import org.opends.server.backends.jeb.EntryContainer;
+import org.opends.server.backends.jeb.Index;
import org.opends.server.backends.jeb.RootContainer;
import org.opends.server.config.ConfigException;
import org.opends.server.core.DirectoryServer;
import org.opends.server.loggers.debug.DebugTracer;
-import org.opends.server.types.Attribute;
-import org.opends.server.types.AttributeType;
-import org.opends.server.types.Attributes;
-import org.opends.server.types.DebugLogLevel;
-import org.opends.server.types.InitializationException;
+import org.opends.server.types.*;
import com.sleepycat.je.DatabaseException;
import com.sleepycat.je.EnvironmentStats;
import com.sleepycat.je.JEVersion;
import com.sleepycat.je.StatsConfig;
import com.sleepycat.je.TransactionStats;
-
+import org.opends.server.util.TimeThread;
/**
@@ -67,7 +68,38 @@
*/
private static final DebugTracer TRACER = getTracer();
+ /**
+ * Represents the statistical information kept for each search filter.
+ */
+ private static class FilterStats implements Comparable<FilterStats>
+ {
+ private volatile Message failureReason = Message.EMPTY;
+ private long maxMatchingEntries = -1;
+ private final AtomicInteger hits = new AtomicInteger();
+ public int compareTo(FilterStats that) {
+ return this.hits.get() - that.hits.get();
+ }
+
+ private void update(int hitCount, Message failureReason)
+ {
+ this.hits.getAndAdd(hitCount);
+ this.failureReason = failureReason;
+ }
+
+ private void update(int hitCount, long matchingEntries)
+ {
+ this.hits.getAndAdd(hitCount);
+ this.failureReason = Message.EMPTY;
+ synchronized(this)
+ {
+ if(matchingEntries > maxMatchingEntries)
+ {
+ maxMatchingEntries = matchingEntries;
+ }
+ }
+ }
+ }
/**
* The name of this monitor instance.
@@ -79,6 +111,14 @@
*/
private RootContainer rootContainer;
+ private int maxEntries = 1024;
+ private boolean filterUseEnabled = false;
+ private String startTimeStamp;
+ private final HashMap<SearchFilter, FilterStats> filterToStats =
+ new HashMap<SearchFilter, FilterStats>();
+ private final AtomicInteger indexedSearchCount = new AtomicInteger();
+ private final AtomicInteger unindexedSearchCount = new AtomicInteger();
+
/**
* Creates a new database environment monitor.
* @param name The monitor instance name.
@@ -208,6 +248,202 @@
addAttributesForStatsObject(monitorAttrs, environmentStats, "Environment");
addAttributesForStatsObject(monitorAttrs, transactionStats, "Transaction");
+ AttributeBuilder needReindex = new AttributeBuilder("need-reindex");
+ for(EntryContainer ec : rootContainer.getEntryContainers())
+ {
+ List<DatabaseContainer> databases = new ArrayList<DatabaseContainer>();
+ ec.listDatabases(databases);
+ for(DatabaseContainer dc : databases)
+ {
+ if(dc instanceof Index && !((Index)dc).isTrusted())
+ {
+ needReindex.add(dc.getName());
+ }
+ }
+ }
+ if(needReindex.size() > 0)
+ {
+ monitorAttrs.add(needReindex.toAttribute());
+ }
+
+ if(filterUseEnabled)
+ {
+ monitorAttrs.add(Attributes.create("filter-use-startTime",
+ startTimeStamp));
+ AttributeBuilder builder = new AttributeBuilder("filter-use");
+
+ StringBuilder stringBuilder = new StringBuilder();
+ synchronized(filterToStats)
+ {
+ for(Map.Entry<SearchFilter, FilterStats> entry :
+ filterToStats.entrySet())
+ {
+ entry.getKey().toString(stringBuilder);
+ stringBuilder.append(" hits:");
+ stringBuilder.append(entry.getValue().hits.get());
+ stringBuilder.append(" maxmatches:");
+ stringBuilder.append(entry.getValue().maxMatchingEntries);
+ stringBuilder.append(" message:");
+ stringBuilder.append(entry.getValue().failureReason);
+ builder.add(stringBuilder.toString());
+ stringBuilder.setLength(0);
+ }
+ }
+ monitorAttrs.add(builder.toAttribute());
+ monitorAttrs.add(Attributes.create("filter-use-indexed",
+ String.valueOf(indexedSearchCount.get())));
+ monitorAttrs.add(Attributes.create("filter-use-unindexed",
+ String.valueOf(unindexedSearchCount.get())));
+ }
+
return monitorAttrs;
}
+
+
+ /**
+ * Updates the index filter statistics with this latest search filter
+ * and the reason why an index was not used.
+ *
+ * @param searchFilter The search filter that was evaluated.
+ * @param failureMessage The reason why an index was not used.
+ */
+ public void updateStats(SearchFilter searchFilter, Message failureMessage)
+ {
+ if(!filterUseEnabled)
+ {
+ return;
+ }
+
+ FilterStats stats;
+ synchronized(filterToStats)
+ {
+ stats = filterToStats.get(searchFilter);
+
+
+ if(stats != null)
+ {
+ stats.update(1, failureMessage);
+ }
+ else
+ {
+ stats = new FilterStats();
+ stats.update(1, failureMessage);
+ removeLowestHit();
+ filterToStats.put(searchFilter, stats);
+ }
+ }
+ }
+
+ /**
+ * Updates the index filter statistics with this latest search filter
+ * and the number of entries matched by the index lookup.
+ *
+ * @param searchFilter The search filter that was evaluated.
+ * @param matchingEntries The number of entries matched by the successful
+ * index lookup.
+ */
+ public void updateStats(SearchFilter searchFilter, long matchingEntries)
+ {
+ if(!filterUseEnabled)
+ {
+ return;
+ }
+
+ FilterStats stats;
+ synchronized(filterToStats)
+ {
+ stats = filterToStats.get(searchFilter);
+
+
+ if(stats != null)
+ {
+ stats.update(1, matchingEntries);
+ }
+ else
+ {
+ stats = new FilterStats();
+ stats.update(1, matchingEntries);
+ removeLowestHit();
+ filterToStats.put(searchFilter, stats);
+ }
+ }
+ }
+
+ /**
+ * Enable or disable index filter statistics gathering.
+ *
+ * @param enabled <code>true></code> to enable index filter statics gathering.
+ */
+ public void enableFilterUseStats(boolean enabled)
+ {
+ if(enabled && !filterUseEnabled)
+ {
+ startTimeStamp = TimeThread.getGMTTime();
+ indexedSearchCount.set(0);
+ unindexedSearchCount.set(0);
+ }
+ else if(!enabled)
+ {
+ filterToStats.clear();
+ }
+ filterUseEnabled = enabled;
+ }
+
+ /**
+ * Indicates if index filter statistics gathering is enabled.
+ *
+ * @return <code>true</code> If index filter statistics gathering is enabled.
+ */
+ public boolean isFilterUseEnabled()
+ {
+ return filterUseEnabled;
+ }
+
+ /**
+ * Sets the maximum number of search filters statistics entries to keep
+ * before ones with the least hits will be removed.
+ *
+ * @param maxEntries The maximum number of search filters statistics
+ * entries to keep
+ */
+ public void setMaxEntries(int maxEntries) {
+ this.maxEntries = maxEntries;
+ }
+
+ /**
+ * Updates the statistics counter to include an indexed search.
+ */
+ public void updateIndexedSearchCount()
+ {
+ indexedSearchCount.getAndIncrement();
+ }
+
+ /**
+ * Updates the statistics counter to include an unindexed search.
+ */
+ public void updateUnindexedSearchCount()
+ {
+ unindexedSearchCount.getAndIncrement();
+ }
+
+ private void removeLowestHit()
+ {
+ while(!filterToStats.isEmpty() && filterToStats.size() > maxEntries)
+ {
+ Iterator<Map.Entry<SearchFilter, FilterStats>> i =
+ filterToStats.entrySet().iterator();
+ Map.Entry<SearchFilter, FilterStats> lowest = i.next();
+ Map.Entry<SearchFilter, FilterStats> entry;
+ while(lowest.getValue().hits.get() > 1 && i.hasNext())
+ {
+ entry = i.next();
+ if(entry.getValue().hits.get() < lowest.getValue().hits.get())
+ {
+ lowest = entry;
+ }
+ }
+
+ filterToStats.remove(lowest.getKey());
+ }
+ }
}
--
Gitblit v1.10.0