Fix for issue 1794: - Attribute index no longer need the backend index entr...

boli

24.41.2007 21223912856469a2eac9be72e23bcac4cc07c71a

Fix for issue 1794:
- Attribute index no longer need the backend index entry limit parameter on construction. This is now handled by the admin framework.
- Updated test cases to reflect the change

Fix for issue 1846
- The debug string returned when using the debugsearchindex attribute now includes the indexes used during evaluation.

 opends/src/admin/defn/org/opends/server/admin/std/JEIndexConfiguration.xml

@@ -92,12 +92,10 @@
      </adm:other>
    </adm:requires-admin-action>
    <adm:default-behavior>
      <adm:alias>
        <adm:synopsis>
          The index entry limit specified in the JE backend-wide
          configuration will be used.
        </adm:synopsis>
      </adm:alias>
      <adm:inherited>
        <adm:relative property-name="backend-index-entry-limit"
                      offset="1" managed-object-name="je-backend" />
      </adm:inherited>
    </adm:default-behavior>
    <adm:syntax>
      <adm:integer lower-limit="0" upper-limit="2147483647">

 opends/src/server/org/opends/server/backends/jeb/AttributeIndex.java

@@ -120,39 +120,27 @@

  private int cursorEntryLimit = 100000;

  private int backendIndexEntryLimit = 4000;

  /**
   * Create a new attribute index object.
   * @param entryContainer The entryContainer of this attribute index.
   * @param state The state database to persist index state info.
   * @param env The JE environment handle.
   * @param indexConfig The attribute index configuration.
   * @param backendIndexEntryLimit The backend index entry limit to use
   *        if none is specified for this attribute index.
   * @throws DatabaseException if a JE database error occurs.
   * @throws ConfigException if a configuration related error occurs.
   */
  public AttributeIndex(JEIndexCfg indexConfig, State state,
                        int backendIndexEntryLimit,
                        Environment env,
  public AttributeIndex(JEIndexCfg indexConfig, State state, Environment env,
                        EntryContainer entryContainer)
      throws DatabaseException, ConfigException
  {
    this.entryContainer = entryContainer;
    this.env = env;
    this.indexConfig = indexConfig;
    this.backendIndexEntryLimit = backendIndexEntryLimit;
    this.state = state;

    AttributeType attrType = indexConfig.getIndexAttribute();
    String name = attrType.getNameOrOID();
    int indexEntryLimit = backendIndexEntryLimit;

    if(indexConfig.getIndexEntryLimit() != null)
    {
      indexEntryLimit = indexConfig.getIndexEntryLimit();
    }
    int indexEntryLimit = indexConfig.getIndexEntryLimit();

    if (indexConfig.getIndexType().contains(JEIndexCfgDefn.IndexType.EQUALITY))
    {
@@ -655,10 +643,14 @@
   * Retrieve the entry IDs that might match an equality filter.
   *
   * @param equalityFilter The equality filter.
   * @param debugBuffer If not null, a diagnostic string will be written
   *                     which will help determine how the indexes contributed
   *                     to this search.
   * @return The candidate entry IDs that might contain the filter
   *         assertion value.
   */
  public EntryIDSet evaluateEqualityFilter(SearchFilter equalityFilter)
  public EntryIDSet evaluateEqualityFilter(SearchFilter equalityFilter,
                                           StringBuilder debugBuffer)
  {
    if (equalityIndex == null)
    {
@@ -672,6 +664,14 @@
           equalityFilter.getAssertionValue().getNormalizedValue().value();
      DatabaseEntry key = new DatabaseEntry(keyBytes);

      if(debugBuffer != null)
      {
        debugBuffer.append("[INDEX:");
        debugBuffer.append(indexConfig.getIndexAttribute().getNameOrOID());
        debugBuffer.append(".");
        debugBuffer.append("equality]");
      }

      // Read the key.
      return equalityIndex.readKey(key, null, LockMode.DEFAULT);
    }
@@ -689,16 +689,28 @@
   * Retrieve the entry IDs that might match a presence filter.
   *
   * @param filter The presence filter.
   * @param debugBuffer If not null, a diagnostic string will be written
   *                     which will help determine how the indexes contributed
   *                     to this search.
   * @return The candidate entry IDs that might contain one or more
   *         values of the attribute type in the filter.
   */
  public EntryIDSet evaluatePresenceFilter(SearchFilter filter)
  public EntryIDSet evaluatePresenceFilter(SearchFilter filter,
                                           StringBuilder debugBuffer)
  {
    if (presenceIndex == null)
    {
      return new EntryIDSet();
    }

    if(debugBuffer != null)
    {
      debugBuffer.append("[INDEX:");
      debugBuffer.append(indexConfig.getIndexAttribute().getNameOrOID());
      debugBuffer.append(".");
      debugBuffer.append("presence]");
    }

    // Read the presence key
    return presenceIndex.readKey(presenceKey, null, LockMode.DEFAULT);
  }
@@ -707,10 +719,14 @@
   * Retrieve the entry IDs that might match a greater-or-equal filter.
   *
   * @param filter The greater-or-equal filter.
   * @param debugBuffer If not null, a diagnostic string will be written
   *                     which will help determine how the indexes contributed
   *                     to this search.
   * @return The candidate entry IDs that might contain a value
   *         greater than or equal to the filter assertion value.
   */
  public EntryIDSet evaluateGreaterOrEqualFilter(SearchFilter filter)
  public EntryIDSet evaluateGreaterOrEqualFilter(SearchFilter filter,
                                                 StringBuilder debugBuffer)
  {
    if (orderingIndex == null)
    {
@@ -730,6 +746,14 @@
      // bound.
      byte[] upper = new byte[0];

      if(debugBuffer != null)
      {
        debugBuffer.append("[INDEX:");
        debugBuffer.append(indexConfig.getIndexAttribute().getNameOrOID());
        debugBuffer.append(".");
        debugBuffer.append("ordering]");
      }

      // Read the range: lower <= keys < upper.
      return orderingIndex.readRange(lower, upper, true, false);
    }
@@ -747,10 +771,14 @@
   * Retrieve the entry IDs that might match a less-or-equal filter.
   *
   * @param filter The less-or-equal filter.
   * @param debugBuffer If not null, a diagnostic string will be written
   *                     which will help determine how the indexes contributed
   *                     to this search.
   * @return The candidate entry IDs that might contain a value
   *         less than or equal to the filter assertion value.
   */
  public EntryIDSet evaluateLessOrEqualFilter(SearchFilter filter)
  public EntryIDSet evaluateLessOrEqualFilter(SearchFilter filter,
                                              StringBuilder debugBuffer)
  {
    if (orderingIndex == null)
    {
@@ -770,6 +798,14 @@
      byte[] upper = orderingRule.normalizeValue(
           filter.getAssertionValue().getValue()).value();

      if(debugBuffer != null)
      {
        debugBuffer.append("[INDEX:");
        debugBuffer.append(indexConfig.getIndexAttribute().getNameOrOID());
        debugBuffer.append(".");
        debugBuffer.append("ordering]");
      }

      // Read the range: lower < keys <= upper.
      return orderingIndex.readRange(lower, upper, false, true);
    }
@@ -787,10 +823,14 @@
   * Retrieve the entry IDs that might match a substring filter.
   *
   * @param filter The substring filter.
   * @param debugBuffer If not null, a diagnostic string will be written
   *                     which will help determine how the indexes contributed
   *                     to this search.
   * @return The candidate entry IDs that might contain a value
   *         that matches the filter substrings.
   */
  public EntryIDSet evaluateSubstringFilter(SearchFilter filter)
  public EntryIDSet evaluateSubstringFilter(SearchFilter filter,
                                            StringBuilder debugBuffer)
  {
    SubstringMatchingRule matchRule =
         filter.getAttributeType().getSubstringMatchingRule();
@@ -815,6 +855,16 @@
          if (results.isDefined() &&
               results.size() <= IndexFilter.FILTER_CANDIDATE_THRESHOLD)
          {

            if(debugBuffer != null)
            {
              debugBuffer.append("[INDEX:");
              debugBuffer.append(indexConfig.getIndexAttribute().
                  getNameOrOID());
              debugBuffer.append(".");
              debugBuffer.append("equality]");
            }

            return results;
          }
        }
@@ -860,6 +910,14 @@
        }
      }

      if(debugBuffer != null)
      {
        debugBuffer.append("[INDEX:");
        debugBuffer.append(indexConfig.getIndexAttribute().getNameOrOID());
        debugBuffer.append(".");
        debugBuffer.append("substring]");
      }

      return results;
    }
    catch (DirectoryException e)
@@ -964,10 +1022,14 @@
   * Retrieve the entry IDs that might match an approximate filter.
   *
   * @param approximateFilter The approximate filter.
   * @param debugBuffer If not null, a diagnostic string will be written
   *                     which will help determine how the indexes contributed
   *                     to this search.
   * @return The candidate entry IDs that might contain the filter
   *         assertion value.
   */
  public EntryIDSet evaluateApproximateFilter(SearchFilter approximateFilter)
  public EntryIDSet evaluateApproximateFilter(SearchFilter approximateFilter,
                                              StringBuilder debugBuffer)
  {
    if (approximateIndex == null)
    {
@@ -984,6 +1046,14 @@
               approximateFilter.getAssertionValue().getValue()).value();
      DatabaseEntry key = new DatabaseEntry(keyBytes);

      if(debugBuffer != null)
      {
        debugBuffer.append("[INDEX:");
        debugBuffer.append(indexConfig.getIndexAttribute().getNameOrOID());
        debugBuffer.append(".");
        debugBuffer.append("approximate]");
      }

      // Read the key.
      return approximateIndex.readKey(key, null, LockMode.DEFAULT);
    }
@@ -1078,57 +1148,6 @@
  }

  /**
   * Set the index entry limit used by the backend using this attribute index.
   * This index will use the backend entry limit only if there is not one
   * specified for this index.
   *
   * @param backendIndexEntryLimit The backend index entry limit.
   * @return True if a rebuild is required or false otherwise.
   */
  public synchronized boolean setBackendIndexEntryLimit(
      int backendIndexEntryLimit)
  {
    // Only update if there is no limit specified for this index.
    boolean rebuildRequired = false;
    if(indexConfig.getIndexEntryLimit() == null)
    {
      if(equalityIndex != null)
      {
        rebuildRequired |=
            equalityIndex.setIndexEntryLimit(backendIndexEntryLimit);
      }

      if(presenceIndex != null)
      {
        rebuildRequired |=
            presenceIndex.setIndexEntryLimit(backendIndexEntryLimit);
      }

      if(substringIndex != null)
      {
        rebuildRequired |=
            substringIndex.setIndexEntryLimit(backendIndexEntryLimit);
      }

      if(orderingIndex != null)
      {
        rebuildRequired |=
            orderingIndex.setIndexEntryLimit(backendIndexEntryLimit);
      }

      if(approximateIndex != null)
      {
        rebuildRequired |=
            approximateIndex.setIndexEntryLimit(backendIndexEntryLimit);
      }
    }

    this.backendIndexEntryLimit = backendIndexEntryLimit;

    return rebuildRequired;
  }

  /**
   * {@inheritDoc}
   */
  public synchronized boolean isConfigurationChangeAcceptable(
@@ -1198,12 +1217,7 @@
    {
      AttributeType attrType = cfg.getIndexAttribute();
      String name = attrType.getNameOrOID();
      int indexEntryLimit = backendIndexEntryLimit;

      if(cfg.getIndexEntryLimit() != null)
      {
        indexEntryLimit = cfg.getIndexEntryLimit();
      }
      int indexEntryLimit = cfg.getIndexEntryLimit();

      if (cfg.getIndexType().contains(JEIndexCfgDefn.IndexType.EQUALITY))
      {

 opends/src/server/org/opends/server/backends/jeb/EntryContainer.java

@@ -273,9 +273,7 @@
        //TODO: When issue 1793 is fixed, use inherited default values in
        //admin framework instead for the entry limit.
        AttributeIndex index =
            new AttributeIndex(indexCfg, state,
                               indexEntryLimit,
                               env, this);
            new AttributeIndex(indexCfg, state, env, this);
        index.open();
        attrIndexMap.put(indexCfg.getIndexAttribute(), index);
      }
@@ -3782,19 +3780,6 @@

    if(config.getBackendIndexEntryLimit() != cfg.getBackendIndexEntryLimit())
    {
      for(AttributeIndex index : attrIndexMap.values())
      {
        if(index.setBackendIndexEntryLimit(cfg.getBackendIndexEntryLimit()))
        {
          adminActionRequired = true;
          int msgID = MSGID_JEB_CONFIG_INDEX_ENTRY_LIMIT_REQUIRES_REBUILD;
          String message = getMessage(msgID,
                                      index.getAttributeType().getNameOrOID());
          messages.add(message);
        }
        index.setBackendIndexEntryLimit(cfg.getBackendIndexEntryLimit());
      }

      if(id2children.setIndexEntryLimit(cfg.getBackendIndexEntryLimit()))
      {
        adminActionRequired = true;
@@ -3847,9 +3832,7 @@
    try
    {
      AttributeIndex index =
          new AttributeIndex(cfg, state,
                             indexEntryLimit,
                             env, this);
          new AttributeIndex(cfg, state, env, this);
      index.open();
      attrIndexMap.put(cfg.getIndexAttribute(), index);
    }

 opends/src/server/org/opends/server/backends/jeb/IndexFilter.java

@@ -420,7 +420,8 @@
    }
    else
    {
      candidates = attributeIndex.evaluateEqualityFilter(equalityFilter);
      candidates =
          attributeIndex.evaluateEqualityFilter(equalityFilter, buffer);
    }
    return candidates;
  }
@@ -442,7 +443,7 @@
    }
    else
    {
      candidates = attributeIndex.evaluatePresenceFilter(filter);
      candidates = attributeIndex.evaluatePresenceFilter(filter, buffer);
    }
    return candidates;
  }
@@ -464,7 +465,7 @@
    }
    else
    {
      candidates = attributeIndex.evaluateGreaterOrEqualFilter(filter);
      candidates = attributeIndex.evaluateGreaterOrEqualFilter(filter, buffer);
    }
    return candidates;
  }
@@ -486,7 +487,7 @@
    }
    else
    {
      candidates = attributeIndex.evaluateLessOrEqualFilter(filter);
      candidates = attributeIndex.evaluateLessOrEqualFilter(filter, buffer);
    }
    return candidates;
  }
@@ -508,7 +509,7 @@
    }
    else
    {
      candidates = attributeIndex.evaluateSubstringFilter(filter);
      candidates = attributeIndex.evaluateSubstringFilter(filter, buffer);
    }
    return candidates;
  }
@@ -530,7 +531,8 @@
    }
    else
    {
      candidates = attributeIndex.evaluateApproximateFilter(approximateFilter);
      candidates =
          attributeIndex.evaluateApproximateFilter(approximateFilter, buffer);
    }
    return candidates;
  }

 opends/tests/unit-tests-testng/src/server/org/opends/server/backends/jeb/TestBackendImpl.java

@@ -30,9 +30,7 @@

import org.opends.server.TestCaseUtils;
import org.opends.server.admin.std.server.JEBackendCfg;
import org.opends.server.admin.std.server.JEIndexCfg;
import org.opends.server.admin.std.meta.JEBackendCfgDefn;
import org.opends.server.admin.std.meta.JEIndexCfgDefn;
import org.opends.server.admin.server.AdminTestCaseUtils;
import org.opends.server.core.ModifyDNOperationBasis;
import org.opends.server.core.DirectoryServer;
@@ -631,7 +629,7 @@
    //Only one index should be used because it is below the FILTER_CANDIDATEassertEquals(ec.getDN2URI().)_THRESHOLD.
    debugString =
        result.get(0).getAttribute("debugsearchindex").get(0).getValues().toString();
    assertTrue(debugString.split("cn").length <= 2);
    assertTrue(debugString.split("cn").length <= 3);
    finalStartPos = debugString.indexOf("final=") + 13;
    finalEndPos = debugString.indexOf("]", finalStartPos);
    finalCount = Integer.valueOf(debugString.substring(finalStartPos,
@@ -1238,26 +1236,20 @@
      "testSearchNotIndexed",
      "testModifyDNNewSuperior", "testMatchedDN"})
  public void testApplyIndexConfig() throws Exception {
    Entry configEntry = TestCaseUtils.makeEntry(
    int resultCode = TestCaseUtils.applyModifications(
        "dn: ds-cfg-index-attribute=givenName,cn=Index," +
            "ds-cfg-backend-id=indexRoot,cn=Backends,cn=config",
        "objectClass: top",
        "objectClass: ds-cfg-je-index",
        "ds-cfg-index-attribute: givenName",
        "changetype: modify",
        "replace: ds-cfg-index-type",
        "ds-cfg-index-type: approximate");

    JEIndexCfg cfg = AdminTestCaseUtils.getConfiguration(
        JEIndexCfgDefn.getInstance(), configEntry);
    assertEquals(resultCode, 0);

    RootContainer rootContainer = backend.getRootContainer();
    EntryContainer ec = rootContainer.getEntryContainer(DN.decode("dc=test,dc=com"));

    AttributeIndex index =
        ec.getAttributeIndex(DirectoryServer.getAttributeType("givenname"));
    ConfigChangeResult ccr = index.applyConfigurationChange(cfg);
    assertTrue(ccr.getResultCode().equals(ResultCode.SUCCESS));
    assertTrue(ccr.adminActionRequired());
    assertFalse(ccr.getMessages().isEmpty());
    assertNull(index.equalityIndex);
    assertNull(index.presenceIndex);
    assertNull(index.substringIndex);
@@ -1325,24 +1317,18 @@
        result.get(0).getAttribute("debugsearchindex").get(0).getValues().toString();
    assertTrue(debugString.contains("NOT-INDEXED"));

    configEntry = TestCaseUtils.makeEntry(
    resultCode = TestCaseUtils.applyModifications(
        "dn: ds-cfg-index-attribute=givenName,cn=Index," +
            "ds-cfg-backend-id=indexRoot,cn=Backends,cn=config",
        "objectClass: top",
        "objectClass: ds-cfg-je-index",
        "ds-cfg-index-attribute: givenName",
        "changetype: modify",
        "replace: ds-cfg-index-type",
        "ds-cfg-index-type: equality",
        "ds-cfg-index-type: presence",
        "ds-cfg-index-type: ordering",
        "ds-cfg-index-type: substring");

    cfg = AdminTestCaseUtils.getConfiguration(
        JEIndexCfgDefn.getInstance(), configEntry);
    assertEquals(resultCode, 0);

    ccr = index.applyConfigurationChange(cfg);
    assertTrue(ccr.getResultCode().equals(ResultCode.SUCCESS));
    assertTrue(ccr.adminActionRequired());
    assertFalse(ccr.getMessages().isEmpty());
    assertNotNull(index.equalityIndex);
    assertNotNull(index.presenceIndex);
    assertNotNull(index.substringIndex);
@@ -1385,8 +1371,13 @@
    assertFalse(apfound);

    // Delete the entries attribute index.
    ccr = ec.applyConfigurationDelete(cfg);
    assertTrue(ccr.getResultCode().equals(ResultCode.SUCCESS));
    resultCode = TestCaseUtils.applyModifications(
        "dn: ds-cfg-index-attribute=givenName,cn=Index," +
            "ds-cfg-backend-id=indexRoot,cn=Backends,cn=config",
        "changetype: delete");

    assertEquals(resultCode, 0);

    assertNull(ec.getAttributeIndex(
        DirectoryServer.getAttributeType("givenname")));
    databases = new ArrayList<DatabaseContainer>();
@@ -1397,10 +1388,20 @@
    }

    // Add it back
    ccr = ec.applyConfigurationAdd(cfg);
    assertTrue(ccr.getResultCode().equals(ResultCode.SUCCESS));
    assertTrue(ccr.adminActionRequired());
    assertFalse(ccr.getMessages().isEmpty());
    resultCode = TestCaseUtils.applyModifications(
        "dn: ds-cfg-index-attribute=givenName,cn=Index," +
            "ds-cfg-backend-id=indexRoot,cn=Backends,cn=config",
        "changetype: add",
        "objectClass: top",
        "objectClass: ds-cfg-je-index",
        "ds-cfg-index-attribute: givenName",
        "ds-cfg-index-type: equality",
        "ds-cfg-index-type: presence",
        "ds-cfg-index-type: ordering",
        "ds-cfg-index-type: substring");

    assertEquals(resultCode, 0);

    assertNotNull(ec.getAttributeIndex(
        DirectoryServer.getAttributeType("givenname")));
    databases = new ArrayList<DatabaseContainer>();
@@ -1441,53 +1442,24 @@

    // Make sure changing the index entry limit on an index where the limit
    // is already exceeded causes warnings.
    configEntry = TestCaseUtils.makeEntry(
    resultCode = TestCaseUtils.applyModifications(
        "dn: ds-cfg-index-attribute=mail,cn=Index," +
            "ds-cfg-backend-id=indexRoot,cn=Backends,cn=config",
        "objectClass: top",
        "objectClass: ds-cfg-je-index",
        "ds-cfg-index-attribute: mail",
        "ds-cfg-index-type: presence",
        "ds-cfg-index-type: equality",
        "ds-cfg-index-type: ordering",
        "ds-cfg-index-type: substring",
        "ds-cfg-index-type: approximate",
        "changetype: modify",
        "replace: ds-cfg-index-entry-limit",
        "ds-cfg-index-entry-limit: 30");

    cfg = AdminTestCaseUtils.getConfiguration(
        JEIndexCfgDefn.getInstance(), configEntry);
    assertEquals(resultCode, 0);

    // Make sure removing a index entry limit for an index makes it use the
    // backend wide setting.
    index =
        ec.getAttributeIndex(DirectoryServer.getAttributeType("mail"));
    ccr = index.applyConfigurationChange(cfg);
    assertTrue(ccr.getResultCode().equals(ResultCode.SUCCESS));
    assertTrue(ccr.adminActionRequired());
    assertFalse(ccr.getMessages().isEmpty());

    configEntry = TestCaseUtils.makeEntry(
    resultCode = TestCaseUtils.applyModifications(
        "dn: ds-cfg-index-attribute=mail,cn=Index," +
            "ds-cfg-backend-id=indexRoot,cn=Backends,cn=config",
        "objectClass: top",
        "objectClass: ds-cfg-je-index",
        "ds-cfg-index-attribute: mail",
        "ds-cfg-index-type: presence",
        "ds-cfg-index-type: equality",
        "ds-cfg-index-type: ordering",
        "ds-cfg-index-type: substring",
        "ds-cfg-index-type: approximate");
        "changetype: modify",
        "delete: ds-cfg-index-entry-limit");

    cfg = AdminTestCaseUtils.getConfiguration(
        JEIndexCfgDefn.getInstance(), configEntry);

   index =
        ec.getAttributeIndex(DirectoryServer.getAttributeType("mail"));
    ccr = index.applyConfigurationChange(cfg);
    assertTrue(ccr.getResultCode().equals(ResultCode.SUCCESS));
    assertFalse(ccr.adminActionRequired());
    assertTrue(ccr.getMessages().isEmpty());

    assertEquals(resultCode, 0);
  }