Implements index analysis features. There are 2 components: Index Filter An...

ludovicp

30.52.2010 b12119c55b89ece2495e84fba229d96439e8219b

 opends/resource/schema/02-config.ldif

@@ -2480,6 +2480,11 @@
  SYNTAX 1.3.6.1.4.1.1466.115.121.1.15
  SINGLE-VALUE
  X-ORIGIN 'OpenDS Directory Server' )
attributeTypes: ( 1.3.6.1.4.1.26027.1.1.612
  NAME 'ds-cfg-index-filter-analyzer-enabled'
  SYNTAX 1.3.6.1.4.1.1466.115.121.1.7
  SINGLE-VALUE
  X-ORIGIN 'OpenDS Directory Server' )
objectClasses: ( 1.3.6.1.4.1.26027.1.2.1
  NAME 'ds-cfg-access-control-handler'
  SUP top
@@ -2551,7 +2556,9 @@
        ds-cfg-compact-encoding $
        ds-cfg-je-property $
        ds-cfg-disk-full-threshold $
        ds-cfg-disk-low-threshold )
        ds-cfg-disk-low-threshold $
        ds-cfg-index-filter-analyzer-enabled $
        ds-cfg-max-entries )
  X-ORIGIN 'OpenDS Directory Server' )
objectClasses: ( 1.3.6.1.4.1.26027.1.2.7
  NAME 'ds-cfg-local-db-index'

 opends/src/admin/defn/org/opends/server/admin/std/LocalDBBackendConfiguration.xml

@@ -879,4 +879,58 @@
      </ldap:attribute>
    </adm:profile>
  </adm:property>
  <adm:property name="index-filter-analyzer-enabled" advanced="true">
    <adm:synopsis>
      Indicates whether to gather statistical information about the search
        filters processed by the Directory Server while evaluating the usage of
        indexes.
    </adm:synopsis>
    <adm:description>
      Analyzing indexes requires gathering search filter usage patterns from
        user requests, especially for values as specified in the filters and
        subsequently looking the status of those values into the index files.
        When a search requests is processed, internal or user generated, a
        first phase uses indexes to find potential entries to be returned.
        Depending on the search filter, if the index of one of the specified
        attributes matches too many entries (exceeds the index entry limit),
        the search becomes non-indexed. In any case, all entries thus
        gathered (or the entire DIT) are matched against the filter for
        actually returning the search result.
    </adm:description>
    <adm:default-behavior>
      <adm:defined>
        <adm:value>false</adm:value>
      </adm:defined>
    </adm:default-behavior>
    <adm:syntax>
      <adm:boolean />
    </adm:syntax>
    <adm:profile name="ldap">
      <ldap:attribute>
        <ldap:name>ds-cfg-index-filter-analyzer-enabled</ldap:name>
      </ldap:attribute>
    </adm:profile>
  </adm:property>
  <adm:property name="max-entries" advanced="true">
    <adm:synopsis>
      The maximum number of search filter statistics to keep.
    </adm:synopsis>
    <adm:description>
      When the maximum number of search filter is reached, the least used one
      will be deleted.
    </adm:description>
    <adm:default-behavior>
      <adm:defined>
        <adm:value>25</adm:value>
      </adm:defined>
    </adm:default-behavior>
    <adm:syntax>
      <adm:integer lower-limit="1" />
    </adm:syntax>
    <adm:profile name="ldap">
      <ldap:attribute>
        <ldap:name>ds-cfg-max-entries</ldap:name>
      </ldap:attribute>
    </adm:profile>
  </adm:property>
</adm:managed-object>

 opends/src/admin/messages/LocalDBBackendCfgDefn.properties

@@ -63,9 +63,13 @@
property.index-entry-limit.synopsis=Specifies the maximum number of entries that is allowed to match a given index key before that particular index key is no longer maintained.
property.index-entry-limit.description=This property is analogous to the ALL IDs threshold in the Sun Java System Directory Server. Note that this is the default limit for the backend, and it may be overridden on a per-attribute basis.A value of 0 means there is no limit.
property.index-entry-limit.requires-admin-action.synopsis=If any index keys have already reached this limit, indexes need to be rebuilt before they are allowed to use the new limit.
property.index-filter-analyzer-enabled.synopsis=Indicates whether to gather statistical information about the search filters processed by the Directory Server while evaluating the usage of indexes.
property.index-filter-analyzer-enabled.description=Analyzing indexes requires gathering search filter usage patterns from user requests, especially for values as specified in the filters and subsequently looking the status of those values into the index files. When a search requests is processed, internal or user generated, a first phase uses indexes to find potential entries to be returned. Depending on the search filter, if the index of one of the specified attributes matches too many entries (exceeds the index entry limit), the search becomes non-indexed. In any case, all entries thus gathered (or the entire DIT) are matched against the filter for actually returning the search result.
property.java-class.synopsis=Specifies the fully-qualified name of the Java class that provides the backend implementation.
property.je-property.synopsis=Specifies the database and environment properties for the Berkeley DB Java Edition database serving the data for this backend.
property.je-property.description=Any Berkeley DB Java Edition property can be specified using the following form: property-name=property-value. Refer to OpenDS documentation for further information on related properties, their implications, and range values. The definitive identification of all the property parameters is available in the example.properties file of Berkeley DB Java Edition distribution.
property.max-entries.synopsis=The maximum number of search filter statistics to keep.
property.max-entries.description=When the maximum number of search filter is reached, the least used one will be deleted.
property.preload-time-limit.synopsis=Specifies the length of time that the backend is allowed to spend "pre-loading" data when it is initialized.
property.preload-time-limit.description=The pre-load process is used to pre-populate the database cache, so that it can be more quickly available when the server is processing requests. A duration of zero means there is no pre-load.
property.writability-mode.synopsis=Specifies the behavior that the backend should use when processing write operations.

 opends/src/messages/messages/jeb.properties

@@ -437,3 +437,12 @@
SEVERE_ERR_REBUILD_INDEX_LACK_DISK_224=The disk containing directory \
  %s is full (%d bytes free). Rebuild index can not continue until the free \
  space rises above the threshold (%d bytes)
INFO_JEB_INDEX_FILTER_INDEX_TYPE_DISABLED_225=%s index type is disabled for \
  the %s attribute
INFO_JEB_INDEX_FILTER_INDEX_NOT_TRUSTED_226=%s index is invalid and needs to \
  be rebuilt
INFO_JEB_INDEX_FILTER_INDEX_REBUILD_IN_PROGRESS_227=%s index is being rebuilt
INFO_JEB_INDEX_FILTER_INDEX_LIMIT_EXCEEDED_228=The filter value exceeded the \
  index entry limit for the %s index
INFO_JEB_INDEX_FILTER_MATCHING_RULE_NOT_INDEXED_229=Matching rule %s is \
  disabled for the extensible index of the %s attribute

 opends/src/messages/messages/tools.properties

@@ -1985,7 +1985,7 @@
INFO_LABEL_DBTEST_JE_RECORD_COUNT_1356=Record Count
INFO_LABEL_DBTEST_INDEX_NAME_1357=Index Name
INFO_LABEL_DBTEST_INDEX_TYPE_1358=Index Type
INFO_LABEL_DBTEST_INDEX_STATUS_1359=Index Status
INFO_LABEL_DBTEST_INDEX_STATUS_1359=Index Valid
INFO_LABEL_DBTEST_KEY_1360=Key
INFO_LABEL_DBTEST_DATA_1361=Data
SEVERE_WARN_DBTEST_CANNOT_UNLOCK_BACKEND_1362=An error occurred while \

 opends/src/server/org/opends/server/backends/jeb/AttributeIndex.java

@@ -36,6 +36,8 @@
import org.opends.server.api.ApproximateMatchingRule;

import static org.opends.server.loggers.debug.DebugLogger.*;

import org.opends.server.monitors.DatabaseEnvironmentMonitor;
import org.opends.server.loggers.debug.DebugTracer;
import org.opends.server.types.*;
import org.opends.server.admin.std.server.LocalDBIndexCfg;
@@ -945,14 +947,23 @@
   * @param debugBuffer If not null, a diagnostic string will be written
   *                     which will help determine how the indexes contributed
   *                     to this search.
   * @param monitor The database environment monitor provider that will keep
   *                index filter usage statistics.
   * @return The candidate entry IDs that might contain the filter
   *         assertion value.
   */
  public EntryIDSet evaluateEqualityFilter(SearchFilter equalityFilter,
                                           StringBuilder debugBuffer)
                                           StringBuilder debugBuffer,
                                           DatabaseEnvironmentMonitor monitor)
  {
    if (equalityIndex == null)
    {
      if(monitor.isFilterUseEnabled())
      {
        monitor.updateStats(equalityFilter,
            INFO_JEB_INDEX_FILTER_INDEX_TYPE_DISABLED.get("equality",
                indexConfig.getAttribute().getNameOrOID()));
      }
      return new EntryIDSet();
    }

@@ -972,7 +983,33 @@
      }

      // Read the key.
      return equalityIndex.readKey(key, null, LockMode.DEFAULT);
      EntryIDSet idSet = equalityIndex.readKey(key, null, LockMode.DEFAULT);
      if(monitor.isFilterUseEnabled())
      {
        if(idSet.isDefined())
        {
          monitor.updateStats(equalityFilter, idSet.size());
        }
        else if(!equalityIndex.isTrusted())
        {
          monitor.updateStats(equalityFilter,
              INFO_JEB_INDEX_FILTER_INDEX_NOT_TRUSTED.get(
                  equalityIndex.getName()));
        }
        else if(equalityIndex.isRebuildRunning())
        {
          monitor.updateStats(equalityFilter,
              INFO_JEB_INDEX_FILTER_INDEX_REBUILD_IN_PROGRESS.get(
                  equalityIndex.getName()));
        }
        else
        {
          monitor.updateStats(equalityFilter,
              INFO_JEB_INDEX_FILTER_INDEX_LIMIT_EXCEEDED.get(
                  equalityIndex.getName()));
        }
      }
      return idSet;
    }
    catch (DirectoryException e)
    {
@@ -991,14 +1028,23 @@
   * @param debugBuffer If not null, a diagnostic string will be written
   *                     which will help determine how the indexes contributed
   *                     to this search.
   * @param monitor The database environment monitor provider that will keep
   *                index filter usage statistics.
   * @return The candidate entry IDs that might contain one or more
   *         values of the attribute type in the filter.
   */
  public EntryIDSet evaluatePresenceFilter(SearchFilter filter,
                                           StringBuilder debugBuffer)
                                           StringBuilder debugBuffer,
                                           DatabaseEnvironmentMonitor monitor)
  {
    if (presenceIndex == null)
    {
      if(monitor.isFilterUseEnabled())
      {
        monitor.updateStats(filter,
            INFO_JEB_INDEX_FILTER_INDEX_TYPE_DISABLED.get("presence",
                indexConfig.getAttribute().getNameOrOID()));
      }
      return new EntryIDSet();
    }

@@ -1011,7 +1057,34 @@
    }

    // Read the presence key
    return presenceIndex.readKey(presenceKey, null, LockMode.DEFAULT);
    EntryIDSet idSet =
        presenceIndex.readKey(presenceKey, null, LockMode.DEFAULT);
    if(monitor.isFilterUseEnabled())
    {
      if(idSet.isDefined())
      {
        monitor.updateStats(filter, idSet.size());
      }
      else if(!presenceIndex.isTrusted())
      {
        monitor.updateStats(filter,
            INFO_JEB_INDEX_FILTER_INDEX_NOT_TRUSTED.get(
                  presenceIndex.getName()));
      }
      else if(presenceIndex.isRebuildRunning())
      {
        monitor.updateStats(filter,
            INFO_JEB_INDEX_FILTER_INDEX_REBUILD_IN_PROGRESS.get(
                  presenceIndex.getName()));
      }
      else
      {
        monitor.updateStats(filter,
            INFO_JEB_INDEX_FILTER_INDEX_LIMIT_EXCEEDED.get(
                  presenceIndex.getName()));
      }
    }
    return idSet;
  }

  /**
@@ -1021,14 +1094,23 @@
   * @param debugBuffer If not null, a diagnostic string will be written
   *                     which will help determine how the indexes contributed
   *                     to this search.
   * @param monitor The database environment monitor provider that will keep
   *                index filter usage statistics.
   * @return The candidate entry IDs that might contain a value
   *         greater than or equal to the filter assertion value.
   */
  public EntryIDSet evaluateGreaterOrEqualFilter(SearchFilter filter,
                                                 StringBuilder debugBuffer)
  public EntryIDSet evaluateGreaterOrEqualFilter(
      SearchFilter filter, StringBuilder debugBuffer,
      DatabaseEnvironmentMonitor monitor)
  {
    if (orderingIndex == null)
    {
      if(monitor.isFilterUseEnabled())
      {
        monitor.updateStats(filter,
            INFO_JEB_INDEX_FILTER_INDEX_TYPE_DISABLED.get("ordering",
                indexConfig.getAttribute().getNameOrOID()));
      }
      return new EntryIDSet();
    }

@@ -1054,7 +1136,33 @@
      }

      // Read the range: lower <= keys < upper.
      return orderingIndex.readRange(lower, upper, true, false);
      EntryIDSet idSet = orderingIndex.readRange(lower, upper, true, false);
      if(monitor.isFilterUseEnabled())
      {
        if(idSet.isDefined())
        {
          monitor.updateStats(filter, idSet.size());
        }
        else if(!orderingIndex.isTrusted())
        {
          monitor.updateStats(filter,
              INFO_JEB_INDEX_FILTER_INDEX_NOT_TRUSTED.get(
                  orderingIndex.getName()));
        }
        else if(orderingIndex.isRebuildRunning())
        {
          monitor.updateStats(filter,
              INFO_JEB_INDEX_FILTER_INDEX_REBUILD_IN_PROGRESS.get(
                  orderingIndex.getName()));
        }
        else
        {
          monitor.updateStats(filter,
              INFO_JEB_INDEX_FILTER_INDEX_LIMIT_EXCEEDED.get(
                  orderingIndex.getName()));
        }
      }
      return idSet;
    }
    catch (DirectoryException e)
    {
@@ -1073,14 +1181,23 @@
   * @param debugBuffer If not null, a diagnostic string will be written
   *                     which will help determine how the indexes contributed
   *                     to this search.
   * @param monitor The database environment monitor provider that will keep
   *                index filter usage statistics.
   * @return The candidate entry IDs that might contain a value
   *         less than or equal to the filter assertion value.
   */
  public EntryIDSet evaluateLessOrEqualFilter(SearchFilter filter,
                                              StringBuilder debugBuffer)
                                              StringBuilder debugBuffer,
                                             DatabaseEnvironmentMonitor monitor)
  {
    if (orderingIndex == null)
    {
      if(monitor.isFilterUseEnabled())
      {
        monitor.updateStats(filter,
            INFO_JEB_INDEX_FILTER_INDEX_TYPE_DISABLED.get("ordering",
                indexConfig.getAttribute().getNameOrOID()));
      }
      return new EntryIDSet();
    }

@@ -1106,7 +1223,33 @@
      }

      // Read the range: lower < keys <= upper.
      return orderingIndex.readRange(lower, upper, false, true);
      EntryIDSet idSet = orderingIndex.readRange(lower, upper, false, true);
      if(monitor.isFilterUseEnabled())
      {
        if(idSet.isDefined())
        {
          monitor.updateStats(filter, idSet.size());
        }
        else if(!orderingIndex.isTrusted())
        {
          monitor.updateStats(filter,
              INFO_JEB_INDEX_FILTER_INDEX_NOT_TRUSTED.get(
                  orderingIndex.getName()));
        }
        else if(orderingIndex.isRebuildRunning())
        {
          monitor.updateStats(filter,
              INFO_JEB_INDEX_FILTER_INDEX_REBUILD_IN_PROGRESS.get(
                  orderingIndex.getName()));
        }
        else
        {
          monitor.updateStats(filter,
              INFO_JEB_INDEX_FILTER_INDEX_LIMIT_EXCEEDED.get(
                  orderingIndex.getName()));
        }
      }
      return idSet;
    }
    catch (DirectoryException e)
    {
@@ -1125,11 +1268,14 @@
   * @param debugBuffer If not null, a diagnostic string will be written
   *                     which will help determine how the indexes contributed
   *                     to this search.
   * @param monitor The database environment monitor provider that will keep
   *                index filter usage statistics.
   * @return The candidate entry IDs that might contain a value
   *         that matches the filter substrings.
   */
  public EntryIDSet evaluateSubstringFilter(SearchFilter filter,
                                            StringBuilder debugBuffer)
                                            StringBuilder debugBuffer,
                                            DatabaseEnvironmentMonitor monitor)
  {
    SubstringMatchingRule matchRule =
         filter.getAttributeType().getSubstringMatchingRule();
@@ -1164,6 +1310,10 @@
              debugBuffer.append("equality]");
            }

            if(monitor.isFilterUseEnabled())
            {
              monitor.updateStats(filter, results.size());
            }
            return results;
          }
        }
@@ -1175,6 +1325,49 @@

      if (substringIndex == null)
      {
        if(monitor.isFilterUseEnabled())
        {
          if(!results.isDefined())
          {
            if(filter.getSubInitialElement() != null)
            {
              if(equalityIndex == null)
              {
                monitor.updateStats(filter,
                    INFO_JEB_INDEX_FILTER_INDEX_TYPE_DISABLED.get("equality",
                        indexConfig.getAttribute().getNameOrOID()));
              }
              else if(!equalityIndex.isTrusted())
              {
                monitor.updateStats(filter,
                    INFO_JEB_INDEX_FILTER_INDEX_NOT_TRUSTED.get(
                        equalityIndex.getName()));
              }
              else if(equalityIndex.isRebuildRunning())
              {
                monitor.updateStats(filter,
                    INFO_JEB_INDEX_FILTER_INDEX_REBUILD_IN_PROGRESS.get(
                        equalityIndex.getName()));
              }
              else
              {
                monitor.updateStats(filter,
                    INFO_JEB_INDEX_FILTER_INDEX_LIMIT_EXCEEDED.get(
                        equalityIndex.getName()));
              }
            }
            else
            {
              monitor.updateStats(filter,
                  INFO_JEB_INDEX_FILTER_INDEX_TYPE_DISABLED.get("substring",
                      indexConfig.getAttribute().getNameOrOID()));
            }
          }
          else
          {
            monitor.updateStats(filter, results.size());
          }
        }
        return results;
      }

@@ -1217,6 +1410,32 @@
        debugBuffer.append("substring]");
      }

      if(monitor.isFilterUseEnabled())
      {
        if(results.isDefined())
        {
          monitor.updateStats(filter, results.size());
        }
        else if(!substringIndex.isTrusted())
        {
          monitor.updateStats(filter,
              INFO_JEB_INDEX_FILTER_INDEX_NOT_TRUSTED.get(
                  substringIndex.getName()));
        }
        else if(substringIndex.isRebuildRunning())
        {
          monitor.updateStats(filter,
              INFO_JEB_INDEX_FILTER_INDEX_REBUILD_IN_PROGRESS.get(
                  substringIndex.getName()));
        }
        else
        {
          monitor.updateStats(filter,
              INFO_JEB_INDEX_FILTER_INDEX_LIMIT_EXCEEDED.get(
                  substringIndex.getName()));
        }
      }

      return results;
    }
    catch (DirectoryException e)
@@ -1326,14 +1545,23 @@
   * @param debugBuffer If not null, a diagnostic string will be written
   *                     which will help determine how the indexes contributed
   *                     to this search.
   * @param monitor The database environment monitor provider that will keep
   *                index filter usage statistics.
   * @return The candidate entry IDs that might contain the filter
   *         assertion value.
   */
  public EntryIDSet evaluateApproximateFilter(SearchFilter approximateFilter,
                                              StringBuilder debugBuffer)
                                              StringBuilder debugBuffer,
                                             DatabaseEnvironmentMonitor monitor)
  {
    if (approximateIndex == null)
    {
      if(monitor.isFilterUseEnabled())
      {
        monitor.updateStats(approximateFilter,
            INFO_JEB_INDEX_FILTER_INDEX_TYPE_DISABLED.get("approximate",
                indexConfig.getAttribute().getNameOrOID()));
      }
      return new EntryIDSet();
    }

@@ -1356,7 +1584,33 @@
      }

      // Read the key.
      return approximateIndex.readKey(key, null, LockMode.DEFAULT);
      EntryIDSet idSet = approximateIndex.readKey(key, null, LockMode.DEFAULT);
      if(monitor.isFilterUseEnabled())
      {
        if(idSet.isDefined())
        {
          monitor.updateStats(approximateFilter, idSet.size());
        }
        else if(!approximateIndex.isTrusted())
        {
          monitor.updateStats(approximateFilter,
              INFO_JEB_INDEX_FILTER_INDEX_NOT_TRUSTED.get(
                  approximateIndex.getName()));
        }
        else if(approximateIndex.isRebuildRunning())
        {
          monitor.updateStats(approximateFilter,
              INFO_JEB_INDEX_FILTER_INDEX_REBUILD_IN_PROGRESS.get(
                  approximateIndex.getName()));
        }
        else
        {
          monitor.updateStats(approximateFilter,
              INFO_JEB_INDEX_FILTER_INDEX_LIMIT_EXCEEDED.get(
                  approximateIndex.getName()));
        }
      }
      return idSet;
    }
    catch (DirectoryException e)
    {
@@ -2366,11 +2620,14 @@
   * @param debugBuffer If not null, a diagnostic string will be written
   *                     which will help determine how the indexes contributed
   *                     to this search.
   * @param monitor The database environment monitor provider that will keep
   *                index filter usage statistics.
   * @return The candidate entry IDs that might contain the filter
   *         assertion value.
   */
  public EntryIDSet evaluateExtensibleFilter(SearchFilter extensibleFilter,
                                              StringBuilder debugBuffer)
                                             StringBuilder debugBuffer,
                                             DatabaseEnvironmentMonitor monitor)
  {
    //Get the Matching Rule OID of the filter.
    String nOID  = extensibleFilter.getMatchingRuleID();
@@ -2388,7 +2645,13 @@
      if(equalityIndex == null)
      {
        // There is no index on this matching rule.
        return IndexQuery.createNullIndexQuery().evaluate();
        if(monitor.isFilterUseEnabled())
        {
          monitor.updateStats(extensibleFilter,
              INFO_JEB_INDEX_FILTER_INDEX_TYPE_DISABLED.get("equality",
                  indexConfig.getAttribute().getNameOrOID()));
        }
        return IndexQuery.createNullIndexQuery().evaluate(null);
      }
      try
      {
@@ -2407,7 +2670,33 @@
        }

        // Read the key.
        return equalityIndex.readKey(key, null, LockMode.DEFAULT);
        EntryIDSet idSet = equalityIndex.readKey(key, null, LockMode.DEFAULT);
        if(monitor.isFilterUseEnabled())
        {
          if(idSet.isDefined())
          {
            monitor.updateStats(extensibleFilter, idSet.size());
          }
          else if(!equalityIndex.isTrusted())
          {
            monitor.updateStats(extensibleFilter,
                INFO_JEB_INDEX_FILTER_INDEX_NOT_TRUSTED.get(
                  equalityIndex.getName()));
          }
          else if(equalityIndex.isRebuildRunning())
          {
            monitor.updateStats(extensibleFilter,
                INFO_JEB_INDEX_FILTER_INDEX_REBUILD_IN_PROGRESS.get(
                  equalityIndex.getName()));
          }
          else
          {
            monitor.updateStats(extensibleFilter,
                INFO_JEB_INDEX_FILTER_INDEX_LIMIT_EXCEEDED.get(
                  equalityIndex.getName()));
          }
        }
        return idSet;
      }
      catch (DirectoryException e)
      {
@@ -2415,7 +2704,7 @@
        {
          TRACER.debugCaught(DebugLogLevel.ERROR, e);
        }
        return IndexQuery.createNullIndexQuery().evaluate();
        return IndexQuery.createNullIndexQuery().evaluate(null);
      }
    }
    ExtensibleMatchingRule rule =
@@ -2425,7 +2714,13 @@
            || (factory = extensibleIndexes.getQueryFactory(rule))==null)
    {
      // There is no index on this matching rule.
      return IndexQuery.createNullIndexQuery().evaluate();
      if(monitor.isFilterUseEnabled())
      {
        monitor.updateStats(extensibleFilter,
            INFO_JEB_INDEX_FILTER_MATCHING_RULE_NOT_INDEXED.get(nOID,
                indexConfig.getAttribute().getNameOrOID()));
      }
      return IndexQuery.createNullIndexQuery().evaluate(null);
    }

    try
@@ -2451,7 +2746,28 @@
      ByteString assertionValue =
              extensibleFilter.getAssertionValue().getValue();
      IndexQuery expression = rule.createIndexQuery(assertionValue, factory);
      return expression.evaluate();
      List<Message> debugMessages =
          monitor.isFilterUseEnabled() ? new ArrayList<Message>() : null;
      EntryIDSet idSet = expression.evaluate(debugMessages);
      if(monitor.isFilterUseEnabled())
      {
        if(idSet.isDefined())
        {
          monitor.updateStats(extensibleFilter, idSet.size());
        }
        else
        {
          if(debugMessages != null && !debugMessages.isEmpty())
          {
            monitor.updateStats(extensibleFilter, debugMessages.get(0));
          }
          else
          {
            monitor.updateStats(extensibleFilter, Message.EMPTY);
          }
        }
      }
      return idSet;
    }
    catch (DirectoryException e)
    {
@@ -2459,7 +2775,7 @@
      {
        TRACER.debugCaught(DebugLogLevel.ERROR, e);
      }
      return IndexQuery.createNullIndexQuery().evaluate();
      return IndexQuery.createNullIndexQuery().evaluate(null);
    }
  }


 opends/src/server/org/opends/server/backends/jeb/EntryContainer.java

@@ -1016,7 +1016,8 @@
    {
      // Create an index filter to get the search result candidate entries.
      IndexFilter indexFilter =
        new IndexFilter(this, searchOperation, debugBuffer);
        new IndexFilter(this, searchOperation, debugBuffer,
            rootContainer.getMonitorProvider());

      // Evaluate the filter against the attribute indexes.
      entryIDList = indexFilter.evaluate();
@@ -1129,11 +1130,19 @@

    if (entryIDList.isDefined())
    {
      if(rootContainer.getMonitorProvider().isFilterUseEnabled())
      {
        rootContainer.getMonitorProvider().updateIndexedSearchCount();
      }
      searchIndexed(entryIDList, candidatesAreInScope, searchOperation,
          pageRequest);
    }
    else
    {
      if(rootContainer.getMonitorProvider().isFilterUseEnabled())
      {
        rootContainer.getMonitorProvider().updateUnindexedSearchCount();
      }
      // See if we could use a virtual attribute rule to process the search.
      for (VirtualAttributeRule rule : DirectoryServer.getVirtualAttributes())
      {

 opends/src/server/org/opends/server/backends/jeb/Index.java

@@ -22,7 +22,7 @@
 * CDDL HEADER END
 *
 *
 *      Copyright 2006-2009 Sun Microsystems, Inc.
 *      Copyright 2006-2010 Sun Microsystems, Inc.
 */
package org.opends.server.backends.jeb;

@@ -1441,6 +1441,15 @@
  }

  /**
   * Return <code>true</code> iff this index is being rebuilt.
   * @return The rebuild state of this index
   */
  public synchronized boolean isRebuildRunning()
  {
    return rebuildRunning;
  }

  /**
   * Set the rebuild status of this index.
   * @param rebuildRunning True if a rebuild process on this index
   *                       is running or False otherwise.

 opends/src/server/org/opends/server/backends/jeb/IndexFilter.java

@@ -22,15 +22,18 @@
 * CDDL HEADER END
 *
 *
 *      Copyright 2006-2009 Sun Microsystems, Inc.
 *      Copyright 2006-2010 Sun Microsystems, Inc.
 */
package org.opends.server.backends.jeb;

import org.opends.server.core.SearchOperation;
import org.opends.server.monitors.DatabaseEnvironmentMonitor;
import org.opends.server.types.AttributeType;
import org.opends.server.types.FilterType;
import org.opends.server.types.SearchFilter;

import static org.opends.messages.JebMessages.*;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
@@ -50,25 +53,28 @@
  /**
   * The entry entryContainer holding the attribute indexes.
   */
  private EntryContainer entryContainer;
  private final EntryContainer entryContainer;

  /**
   * The search operation provides the search base, scope and filter.
   * It can also be checked periodically for cancellation.
   */
  private SearchOperation searchOp;
  private final SearchOperation searchOp;

  /**
   * A string builder to hold a diagnostic string which helps determine
   * how the indexed contributed to the search operation.
   */
  private StringBuilder buffer;
  private final StringBuilder buffer;

  private final DatabaseEnvironmentMonitor monitor;

  /**
   * Construct an index filter for a search operation.
   *
   * @param entryContainer The entry entryContainer.
   * @param searchOp       The search operation to be evaluated.
   * @param monitor        The monitor to gather filter usage stats.
   *
   * @param debugBuilder If not null, a diagnostic string will be written
   *                     which will help determine how the indexes contributed
@@ -76,11 +82,13 @@
   */
  public IndexFilter(EntryContainer entryContainer,
                     SearchOperation searchOp,
                     StringBuilder debugBuilder)
                     StringBuilder debugBuilder,
                     DatabaseEnvironmentMonitor monitor)
  {
    this.entryContainer = entryContainer;
    this.searchOp = searchOp;
    this.buffer = debugBuilder;
    this.monitor = monitor;
  }

  /**
@@ -296,6 +304,12 @@
             entryContainer.getAttributeIndex(rangeEntry.getKey());
        if (attributeIndex == null)
        {
          if(monitor.isFilterUseEnabled())
          {
            monitor.updateStats(SearchFilter.createANDFilter(rangeList),
                INFO_JEB_INDEX_FILTER_INDEX_TYPE_DISABLED.get("ordering",
                    rangeEntry.getKey().getNameOrOID()));
          }
          continue;
        }

@@ -314,6 +328,33 @@
            set.toString(buffer);
          }

          if(monitor.isFilterUseEnabled())
          {
            if(set.isDefined())
            {
              monitor.updateStats(SearchFilter.createANDFilter(rangeList),
                  set.size());
            }
            else if(!attributeIndex.orderingIndex.isTrusted())
            {
              monitor.updateStats(SearchFilter.createANDFilter(rangeList),
                  INFO_JEB_INDEX_FILTER_INDEX_NOT_TRUSTED.get(
                      attributeIndex.orderingIndex.getName()));
            }
            else if(attributeIndex.orderingIndex.isRebuildRunning())
            {
              monitor.updateStats(SearchFilter.createANDFilter(rangeList),
                  INFO_JEB_INDEX_FILTER_INDEX_REBUILD_IN_PROGRESS.get(
                      attributeIndex.orderingIndex.getName()));
            }
            else
            {
              monitor.updateStats(SearchFilter.createANDFilter(rangeList),
                  INFO_JEB_INDEX_FILTER_INDEX_LIMIT_EXCEEDED.get(
                      attributeIndex.orderingIndex.getName()));
            }
          }

          if (retainAll(results, set))
          {
            return results;
@@ -335,6 +376,34 @@
            set.toString(buffer);
          }

          if(monitor.isFilterUseEnabled())
          {
            if(set.isDefined())
            {
              monitor.updateStats(SearchFilter.createANDFilter(rangeList),
                  set.size());
            }
            else if(!attributeIndex.orderingIndex.isTrusted())
            {
              monitor.updateStats(SearchFilter.createANDFilter(rangeList),
                  INFO_JEB_INDEX_FILTER_INDEX_NOT_TRUSTED.get(
                      attributeIndex.orderingIndex.getName()));
            }
            else if(attributeIndex.orderingIndex.isRebuildRunning())
            {
              monitor.updateStats(SearchFilter.createANDFilter(rangeList),
                  INFO_JEB_INDEX_FILTER_INDEX_REBUILD_IN_PROGRESS.get(
                      attributeIndex.orderingIndex.getName()));
            }
            else
            {
              monitor.updateStats(SearchFilter.createANDFilter(rangeList),
                  INFO_JEB_INDEX_FILTER_INDEX_LIMIT_EXCEEDED.get(
                      attributeIndex.orderingIndex.getName()));
            }
          }


          if (retainAll(results, set))
          {
            return results;
@@ -378,12 +447,8 @@
    // We may have reached the point of diminishing returns where
    // it is quicker to stop now and process the current small number of
    // candidates.
    if (a.isDefined() && a.size() <= FILTER_CANDIDATE_THRESHOLD)
    {
      return true;
    }
    return a.isDefined() && a.size() <= FILTER_CANDIDATE_THRESHOLD;

    return false;
  }

  /**
@@ -423,12 +488,18 @@
         entryContainer.getAttributeIndex(equalityFilter.getAttributeType());
    if (attributeIndex == null)
    {
      if(monitor.isFilterUseEnabled())
      {
        monitor.updateStats(equalityFilter,
            INFO_JEB_INDEX_FILTER_INDEX_TYPE_DISABLED.get("equality",
                equalityFilter.getAttributeType().getNameOrOID()));
      }
      candidates = new EntryIDSet();
    }
    else
    {
      candidates =
          attributeIndex.evaluateEqualityFilter(equalityFilter, buffer);
      candidates = attributeIndex.evaluateEqualityFilter(equalityFilter,
          buffer, monitor);
    }
    return candidates;
  }
@@ -446,11 +517,18 @@
         entryContainer.getAttributeIndex(filter.getAttributeType());
    if (attributeIndex == null)
    {
      if(monitor.isFilterUseEnabled())
      {
        monitor.updateStats(filter,
            INFO_JEB_INDEX_FILTER_INDEX_TYPE_DISABLED.get("presence",
                filter.getAttributeType().getNameOrOID()));
      }
      candidates = new EntryIDSet();
    }
    else
    {
      candidates = attributeIndex.evaluatePresenceFilter(filter, buffer);
      candidates = attributeIndex.evaluatePresenceFilter(filter, buffer,
          monitor);
    }
    return candidates;
  }
@@ -468,11 +546,18 @@
         entryContainer.getAttributeIndex(filter.getAttributeType());
    if (attributeIndex == null)
    {
      if(monitor.isFilterUseEnabled())
      {
        monitor.updateStats(filter,
            INFO_JEB_INDEX_FILTER_INDEX_TYPE_DISABLED.get("ordering",
                filter.getAttributeType().getNameOrOID()));
      }
      candidates = new EntryIDSet();
    }
    else
    {
      candidates = attributeIndex.evaluateGreaterOrEqualFilter(filter, buffer);
      candidates = attributeIndex.evaluateGreaterOrEqualFilter(filter,
          buffer, monitor);
    }
    return candidates;
  }
@@ -490,11 +575,18 @@
         entryContainer.getAttributeIndex(filter.getAttributeType());
    if (attributeIndex == null)
    {
      if(monitor.isFilterUseEnabled())
      {
        monitor.updateStats(filter,
            INFO_JEB_INDEX_FILTER_INDEX_TYPE_DISABLED.get("ordering",
                filter.getAttributeType().getNameOrOID()));
      }
      candidates = new EntryIDSet();
    }
    else
    {
      candidates = attributeIndex.evaluateLessOrEqualFilter(filter, buffer);
      candidates = attributeIndex.evaluateLessOrEqualFilter(filter, buffer,
          monitor);
    }
    return candidates;
  }
@@ -512,11 +604,19 @@
         entryContainer.getAttributeIndex(filter.getAttributeType());
    if (attributeIndex == null)
    {
      if(monitor.isFilterUseEnabled())
      {
        monitor.updateStats(filter,
            INFO_JEB_INDEX_FILTER_INDEX_TYPE_DISABLED.get(
                "substring or equality",
                filter.getAttributeType().getNameOrOID()));
      }
      candidates = new EntryIDSet();
    }
    else
    {
      candidates = attributeIndex.evaluateSubstringFilter(filter, buffer);
      candidates = attributeIndex.evaluateSubstringFilter(filter,
          buffer, monitor);
    }
    return candidates;
  }
@@ -534,12 +634,18 @@
         entryContainer.getAttributeIndex(approximateFilter.getAttributeType());
    if (attributeIndex == null)
    {
      if(monitor.isFilterUseEnabled())
      {
        monitor.updateStats(approximateFilter,
            INFO_JEB_INDEX_FILTER_INDEX_TYPE_DISABLED.get("approximate",
                approximateFilter.getAttributeType().getNameOrOID()));
      }
      candidates = new EntryIDSet();
    }
    else
    {
      candidates =
          attributeIndex.evaluateApproximateFilter(approximateFilter, buffer);
      candidates = attributeIndex.evaluateApproximateFilter(approximateFilter,
          buffer, monitor);
    }
    return candidates;
  }
@@ -557,12 +663,13 @@
         entryContainer.getAttributeIndex(extensibleFilter.getAttributeType());
    if (attributeIndex == null)
    {
      candidates = IndexQuery.createNullIndexQuery().evaluate();
      candidates = IndexQuery.createNullIndexQuery().evaluate(null);
    }
    else
    {
      candidates =
          attributeIndex.evaluateExtensibleFilter(extensibleFilter, buffer);
          attributeIndex.evaluateExtensibleFilter(extensibleFilter, buffer,
              monitor);
    }
    return candidates;
  }

 opends/src/server/org/opends/server/backends/jeb/IndexQuery.java

@@ -22,14 +22,18 @@
 * CDDL HEADER END
 *
 *
 *      Copyright 2009 Sun Microsystems, Inc.
 *      Copyright 2009-2010 Sun Microsystems, Inc.
 */

package org.opends.server.backends.jeb;



import org.opends.messages.Message;

import java.util.Collection;
import java.util.List;

import static org.opends.server.backends.jeb.IndexFilter.*;


@@ -47,9 +51,12 @@
  /**
   * Evaluates the index query and returns the EntryIDSet.
   *
   * @param debugMessages If not null, diagnostic messages will be written
   *                      which will help to determine why the returned
   *                      EntryIDSet is not defined.
   * @return The EntryIDSet as a result of evaulation of this query.
   */
  public abstract EntryIDSet evaluate();
  public abstract EntryIDSet evaluate(List<Message> debugMessages);



@@ -107,9 +114,10 @@
  {
    /**
     * {@inheritDoc}
     * @param debugMessages
     */
    @Override
    public EntryIDSet evaluate()
    public EntryIDSet evaluate(List<Message> debugMessages)
    {
      return new EntryIDSet();
    }
@@ -143,20 +151,21 @@

    /**
     * {@inheritDoc}
     * @param debugMessages
     */
    @Override
    public EntryIDSet evaluate()
    public EntryIDSet evaluate(List<Message> debugMessages)
    {
      EntryIDSet entryIDs = null;
      for (IndexQuery query : subIndexQueries)
      {
        if (entryIDs == null)
        {
          entryIDs = query.evaluate();
          entryIDs = query.evaluate(debugMessages);
        }
        else
        {
          entryIDs.retainAll(query.evaluate());
          entryIDs.retainAll(query.evaluate(debugMessages));
        }
        if (entryIDs.isDefined()
            && entryIDs.size() <= FILTER_CANDIDATE_THRESHOLD)
@@ -195,20 +204,21 @@

    /**
     * {@inheritDoc}
     * @param debugMessages
     */
    @Override
    public EntryIDSet evaluate()
    public EntryIDSet evaluate(List<Message> debugMessages)
    {
      EntryIDSet entryIDs = null;
      for (IndexQuery query : subIndexQueries)
      {
        if (entryIDs == null)
        {
          entryIDs = query.evaluate();
          entryIDs = query.evaluate(debugMessages);
        }
        else
        {
          entryIDs.addAll(query.evaluate());
          entryIDs.addAll(query.evaluate(debugMessages));
        }
        if (entryIDs.isDefined()
            && entryIDs.size() <= FILTER_CANDIDATE_THRESHOLD)

 opends/src/server/org/opends/server/backends/jeb/IndexQueryFactoryImpl.java

@@ -22,7 +22,7 @@
 * CDDL HEADER END
 *
 *
 *      Copyright 2009 Sun Microsystems, Inc.
 *      Copyright 2009-2010 Sun Microsystems, Inc.
 */

package org.opends.server.backends.jeb;
@@ -32,10 +32,19 @@
import com.sleepycat.je.DatabaseEntry;
import com.sleepycat.je.LockMode;
import java.util.Collection;
import java.util.List;
import java.util.Map;

import org.opends.messages.Message;
import org.opends.server.api.IndexQueryFactory;
import org.opends.server.types.ByteSequence;

import static org.opends.messages.JebMessages.
    INFO_JEB_INDEX_FILTER_INDEX_LIMIT_EXCEEDED;
import static org.opends.messages.JebMessages.
    INFO_JEB_INDEX_FILTER_INDEX_NOT_TRUSTED;
import static org.opends.messages.JebMessages.
    INFO_JEB_INDEX_FILTER_INDEX_REBUILD_IN_PROGRESS;


/**
@@ -76,7 +85,7 @@
      {

        @Override
        public EntryIDSet evaluate()
        public EntryIDSet evaluate(List<Message> debugMessages)
        {
          // Read the database and get Record for the key.
          DatabaseEntry key = new DatabaseEntry(value.toByteArray());
@@ -85,6 +94,27 @@
          Index index = indexMap.get(indexID);
          EntryIDSet entrySet =
              index.readKey(key, null, LockMode.DEFAULT);
          if(debugMessages != null && !entrySet.isDefined())
          {
            if(!index.isTrusted())
            {
              debugMessages.add(
                  INFO_JEB_INDEX_FILTER_INDEX_NOT_TRUSTED.get(
                      index.getName()));
            }
            else if(index.isRebuildRunning())
            {
              debugMessages.add(
                  INFO_JEB_INDEX_FILTER_INDEX_REBUILD_IN_PROGRESS.get(
                      index.getName()));
            }
            else
            {
              debugMessages.add(
                  INFO_JEB_INDEX_FILTER_INDEX_LIMIT_EXCEEDED.get(
                      index.getName()));
            }
          }
          return entrySet;
        }
      };
@@ -103,13 +133,34 @@
      {

        @Override
        public EntryIDSet evaluate()
        public EntryIDSet evaluate(List<Message> debugMessages)
        {
          // Find the right index.
          Index index = indexMap.get(indexID);
          EntryIDSet entrySet =
              index.readRange(lowerBound.toByteArray(), upperBound
                  .toByteArray(), includeLowerBound, includeUpperBound);
          if(debugMessages != null && !entrySet.isDefined())
          {
            if(!index.isTrusted())
            {
              debugMessages.add(
                  INFO_JEB_INDEX_FILTER_INDEX_NOT_TRUSTED.get(
                      index.getName()));
            }
            else if(index.isRebuildRunning())
            {
              debugMessages.add(
                  INFO_JEB_INDEX_FILTER_INDEX_REBUILD_IN_PROGRESS.get(
                      index.getName()));
            }
            else
            {
              debugMessages.add(
                  INFO_JEB_INDEX_FILTER_INDEX_LIMIT_EXCEEDED.get(
                      index.getName()));
            }
          }
          return entrySet;
        }
      };
@@ -150,10 +201,10 @@
      {

        @Override
        public EntryIDSet evaluate()
        public EntryIDSet evaluate(List<Message> debugMessages)
        {
          return new EntryIDSet();
        }
      };
  }
}
}

 opends/src/server/org/opends/server/backends/jeb/RootContainer.java

@@ -124,6 +124,10 @@
    this.config = config;
    this.compressedSchema = null;

    getMonitorProvider().enableFilterUseStats(
        config.isIndexFilterAnalyzerEnabled());
    getMonitorProvider().setMaxEntries(config.getMaxEntries());

    config.addLocalDBChangeListener(this);
    importForceCheckPoint.setForce(true);
  }
@@ -1005,6 +1009,10 @@
        }
      }

      getMonitorProvider().enableFilterUseStats(
          cfg.isIndexFilterAnalyzerEnabled());
      getMonitorProvider().setMaxEntries(cfg.getMaxEntries());

      this.config = cfg;
    }
    catch (Exception e)

 opends/src/server/org/opends/server/monitors/DatabaseEnvironmentMonitor.java

@@ -31,27 +31,28 @@
import static org.opends.server.loggers.debug.DebugLogger.*;

import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;

import org.opends.messages.Message;
import org.opends.server.admin.std.server.MonitorProviderCfg;
import org.opends.server.api.AttributeSyntax;
import org.opends.server.api.MonitorProvider;
import org.opends.server.backends.jeb.DatabaseContainer;
import org.opends.server.backends.jeb.EntryContainer;
import org.opends.server.backends.jeb.Index;
import org.opends.server.backends.jeb.RootContainer;
import org.opends.server.config.ConfigException;
import org.opends.server.core.DirectoryServer;
import org.opends.server.loggers.debug.DebugTracer;
import org.opends.server.types.Attribute;
import org.opends.server.types.AttributeType;
import org.opends.server.types.Attributes;
import org.opends.server.types.DebugLogLevel;
import org.opends.server.types.InitializationException;
import org.opends.server.types.*;

import com.sleepycat.je.DatabaseException;
import com.sleepycat.je.EnvironmentStats;
import com.sleepycat.je.JEVersion;
import com.sleepycat.je.StatsConfig;
import com.sleepycat.je.TransactionStats;

import org.opends.server.util.TimeThread;


/**
@@ -67,7 +68,38 @@
   */
  private static final DebugTracer TRACER = getTracer();

  /**
   * Represents the statistical information kept for each search filter.
   */
  private static class FilterStats implements Comparable<FilterStats>
  {
    private volatile Message failureReason = Message.EMPTY;
    private long maxMatchingEntries = -1;
    private final AtomicInteger hits = new AtomicInteger();

    public int compareTo(FilterStats that) {
      return this.hits.get() - that.hits.get();
    }

    private void update(int hitCount, Message failureReason)
    {
      this.hits.getAndAdd(hitCount);
      this.failureReason = failureReason;
    }

    private void update(int hitCount, long matchingEntries)
    {
      this.hits.getAndAdd(hitCount);
      this.failureReason = Message.EMPTY;
      synchronized(this)
      {
        if(matchingEntries > maxMatchingEntries)
        {
          maxMatchingEntries = matchingEntries;
        }
      }
    }
  }

  /**
   * The name of this monitor instance.
@@ -79,6 +111,14 @@
   */
  private RootContainer rootContainer;

  private int maxEntries = 1024;
  private boolean filterUseEnabled = false;
  private String startTimeStamp;
  private final HashMap<SearchFilter, FilterStats> filterToStats =
      new HashMap<SearchFilter, FilterStats>();
  private final AtomicInteger indexedSearchCount = new AtomicInteger();
  private final AtomicInteger unindexedSearchCount = new AtomicInteger();

  /**
   * Creates a new database environment monitor.
   * @param name The monitor instance name.
@@ -208,6 +248,202 @@
    addAttributesForStatsObject(monitorAttrs, environmentStats, "Environment");
    addAttributesForStatsObject(monitorAttrs, transactionStats, "Transaction");

    AttributeBuilder needReindex = new AttributeBuilder("need-reindex");
    for(EntryContainer ec : rootContainer.getEntryContainers())
    {
      List<DatabaseContainer> databases = new ArrayList<DatabaseContainer>();
      ec.listDatabases(databases);
      for(DatabaseContainer dc : databases)
      {
        if(dc instanceof Index && !((Index)dc).isTrusted())
        {
          needReindex.add(dc.getName());
        }
      }
    }
    if(needReindex.size() > 0)
    {
      monitorAttrs.add(needReindex.toAttribute());
    }

    if(filterUseEnabled)
    {
      monitorAttrs.add(Attributes.create("filter-use-startTime",
          startTimeStamp));
      AttributeBuilder builder = new AttributeBuilder("filter-use");

      StringBuilder stringBuilder = new StringBuilder();
      synchronized(filterToStats)
      {
        for(Map.Entry<SearchFilter, FilterStats> entry :
            filterToStats.entrySet())
        {
          entry.getKey().toString(stringBuilder);
          stringBuilder.append(" hits:");
          stringBuilder.append(entry.getValue().hits.get());
          stringBuilder.append(" maxmatches:");
          stringBuilder.append(entry.getValue().maxMatchingEntries);
          stringBuilder.append(" message:");
          stringBuilder.append(entry.getValue().failureReason);
          builder.add(stringBuilder.toString());
          stringBuilder.setLength(0);
        }
      }
      monitorAttrs.add(builder.toAttribute());
      monitorAttrs.add(Attributes.create("filter-use-indexed",
          String.valueOf(indexedSearchCount.get())));
      monitorAttrs.add(Attributes.create("filter-use-unindexed",
          String.valueOf(unindexedSearchCount.get())));
    }

    return monitorAttrs;
  }


  /**
   * Updates the index filter statistics with this latest search filter
   * and the reason why an index was not used.
   *
   * @param searchFilter The search filter that was evaluated.
   * @param failureMessage The reason why an index was not used.
   */
  public void updateStats(SearchFilter searchFilter, Message failureMessage)
  {
    if(!filterUseEnabled)
    {
      return;
    }

    FilterStats stats;
    synchronized(filterToStats)
    {
      stats = filterToStats.get(searchFilter);


      if(stats != null)
      {
        stats.update(1, failureMessage);
      }
      else
      {
        stats = new FilterStats();
        stats.update(1, failureMessage);
        removeLowestHit();
        filterToStats.put(searchFilter, stats);
      }
    }
  }

  /**
   * Updates the index filter statistics with this latest search filter
   * and the number of entries matched by the index lookup.
   *
   * @param searchFilter The search filter that was evaluated.
   * @param matchingEntries The number of entries matched by the successful
   *                        index lookup.
   */
  public void updateStats(SearchFilter searchFilter, long matchingEntries)
  {
    if(!filterUseEnabled)
    {
      return;
    }

    FilterStats stats;
    synchronized(filterToStats)
    {
      stats = filterToStats.get(searchFilter);


      if(stats != null)
      {
        stats.update(1, matchingEntries);
      }
      else
      {
        stats = new FilterStats();
        stats.update(1, matchingEntries);
        removeLowestHit();
        filterToStats.put(searchFilter, stats);
      }
    }
  }

  /**
   * Enable or disable index filter statistics gathering.
   *
   * @param enabled <code>true></code> to enable index filter statics gathering.
   */
  public void enableFilterUseStats(boolean enabled)
  {
    if(enabled && !filterUseEnabled)
    {
      startTimeStamp = TimeThread.getGMTTime();
      indexedSearchCount.set(0);
      unindexedSearchCount.set(0);
    }
    else if(!enabled)
    {
      filterToStats.clear();
    }
    filterUseEnabled = enabled;
  }

  /**
   * Indicates if index filter statistics gathering is enabled.
   *
   * @return <code>true</code> If index filter statistics gathering is enabled.
   */
  public boolean isFilterUseEnabled()
  {
    return filterUseEnabled;
  }

  /**
   * Sets the maximum number of search filters statistics entries to keep
   * before ones with the least hits will be removed.
   *
   * @param maxEntries The maximum number of search filters statistics
   * entries to keep
   */
  public void setMaxEntries(int maxEntries) {
    this.maxEntries = maxEntries;
  }

  /**
   * Updates the statistics counter to include an indexed search.
   */
  public void updateIndexedSearchCount()
  {
    indexedSearchCount.getAndIncrement();
  }

  /**
   * Updates the statistics counter to include an unindexed search.
   */
  public void updateUnindexedSearchCount()
  {
    unindexedSearchCount.getAndIncrement();
  }

  private void removeLowestHit()
  {
    while(!filterToStats.isEmpty() && filterToStats.size() > maxEntries)
    {
      Iterator<Map.Entry<SearchFilter, FilterStats>> i =
          filterToStats.entrySet().iterator();
      Map.Entry<SearchFilter, FilterStats> lowest = i.next();
      Map.Entry<SearchFilter, FilterStats> entry;
      while(lowest.getValue().hits.get() > 1 && i.hasNext())
      {
        entry = i.next();
        if(entry.getValue().hits.get() < lowest.getValue().hits.get())
        {
          lowest = entry;
        }
      }

      filterToStats.remove(lowest.getKey());
    }
  }
}

 opends/src/server/org/opends/server/tools/DBTest.java

@@ -962,6 +962,13 @@
      builder.appendHeading(INFO_LABEL_DBTEST_INDEX_TYPE.get());
      builder.appendHeading(INFO_LABEL_DBTEST_JE_DATABASE_NAME.get());
      builder.appendHeading(INFO_LABEL_DBTEST_INDEX_STATUS.get());
      builder.appendHeading(INFO_LABEL_DBTEST_JE_RECORD_COUNT.get());
      builder.appendHeading(
          INFO_LABEL_DBTEST_INDEX_UNDEFINED_RECORD_COUNT.get());
      builder.appendHeading(Message.raw("95%"));
      builder.appendHeading(Message.raw("90%"));
      builder.appendHeading(Message.raw("85%"));


      EntryContainer ec = rc.getEntryContainer(base);

@@ -974,6 +981,8 @@

      ArrayList<DatabaseContainer> databaseContainers =
          new ArrayList<DatabaseContainer>();
      Map<Index, StringBuilder> undefinedKeys =
          new HashMap<Index, StringBuilder>();
      ec.listDatabases(databaseContainers);
      for(DatabaseContainer dc : databaseContainers)
      {
@@ -994,6 +1003,86 @@
            builder.appendCell(ec.getState().getIndexTrustState(null,
                                                               ((VLVIndex)dc)));
          }
          builder.appendCell(dc.getRecordCount());

          if(dc instanceof Index)
          {
            Index index = (Index)dc;
            long undefined = 0, ninetyFive = 0, ninety = 0, eighty = 0;
            DatabaseEntry key = new DatabaseEntry();
            DatabaseEntry data = new DatabaseEntry();
            LockMode lockMode = LockMode.DEFAULT;
            OperationStatus status;

            Cursor cursor = dc.openCursor(null, CursorConfig.DEFAULT);
            status = cursor.getFirst(key, data, lockMode);
            while(status == OperationStatus.SUCCESS)
            {
              byte[] bytes = data.getData();
              if (bytes.length == 0 || ((bytes[0] & 0x80) == 0x80))
              {
                // Entry limit has exceeded and there is no encoded
                //  undefined set size.
                undefined ++;
                StringBuilder keyList = undefinedKeys.get(index);
                if(keyList == null)
                {
                  keyList = new StringBuilder();
                  undefinedKeys.put(index, keyList);
                }
                else
                {
                  keyList.append(" ");
                }
                if(index == ec.getID2Children() || index == ec.getID2Subtree())
                {
                  keyList.append("[").append(
                    JebFormat.entryIDFromDatabase(key.getData())).append("]");
                }
                else
                {
                  keyList.append("[").append(
                    new String(key.getData())).append("]");
                }
              }
              else
              {
                // Seems like entry limit has not been exceeded and the bytes
                // is a list of entry IDs.
                double percentFull =
                    (bytes.length / 8) / index.getIndexEntryLimit();
                if(percentFull >= .8)
                {
                  if(percentFull < .9)
                  {
                    eighty++;
                  }
                  else if(percentFull < .95)
                  {
                    ninety++;
                  }
                  else
                  {
                    ninetyFive++;
                  }
                }
              }
              status = cursor.getNext(key, data, lockMode);
            }
            builder.appendCell(undefined);
            builder.appendCell(ninetyFive);
            builder.appendCell(ninety);
            builder.appendCell(eighty);
            cursor.close();
          }
          else
          {
            builder.appendCell("-");
            builder.appendCell("-");
            builder.appendCell("-");
            builder.appendCell("-");
          }

          count++;
        }
      }
@@ -1001,6 +1090,12 @@
      TextTablePrinter printer = new TextTablePrinter(out);
      builder.print(printer);
      out.format("%nTotal: %d%n", count);
      for(Map.Entry<Index, StringBuilder> e : undefinedKeys.entrySet())
      {
        out.format("%nIndex: %s%n",
            e.getKey().getName().replace(ec.getDatabasePrefix()+"_", ""));
        out.format("Undefined keys: %s%n", e.getValue().toString());
      }
      return 0;
    }
    catch(DatabaseException de)

 opends/tests/unit-tests-testng/src/server/org/opends/server/admin/ValidateConfigDefinitionsTest.java

@@ -124,6 +124,7 @@
  // Exceptions to properties ending in -enabled being exactly 'enabled'.
  private static final List<String> ENABLED_PROPERTY_EXCEPTIONS =
          Arrays.asList(new String[]{
                  "index-filter-analyzer-enabled"
                  // e.g. "prop-name-ending-with-enabled"
          });