Bug# 1462:server deadlock on subtree delete Fix: Introduce a batch paramete...

sin

10.50.2007 9d7725370d82557e35ed802a6a7bc206c44b993b

Bug# 1462:server deadlock on subtree delete
Fix: Introduce a batch parameter to create the additional transactions for a new batch.

 opends/resource/config/config.ldif

@@ -115,6 +115,7 @@
ds-cfg-backend-mode: 700
ds-cfg-backend-index-entry-limit: 4000
ds-cfg-backend-subtree-delete-size-limit: 100000
ds-cfg-backend-subtree-delete-batch-size: 5000
ds-cfg-backend-preload-time-limit: 0 seconds
ds-cfg-backend-import-temp-directory: importTmp
ds-cfg-backend-import-buffer-size: 256 megabytes

 opends/resource/schema/02-config.ldif

@@ -1279,6 +1279,10 @@
attributeTypes: ( 1.3.6.1.4.1.26027.1.1.380 NAME 'ds-sync-conflict'
  SYNTAX 1.3.6.1.4.1.1466.115.121.1.15
  USAGE directoryOperation X-ORIGIN 'OpenDS Directory Server' )
attributeTypes: ( 1.3.6.1.4.1.26027.1.1.381
  NAME 'ds-cfg-backend-subtree-delete-batch-size'
  SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE
  X-ORIGIN 'OpenDS Directory Server' )
attributeTypes: ( 1.3.6.1.4.1.26027.1.1.382
  NAME 'ds-cfg-index-substring-length'
  SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE
@@ -1504,7 +1508,7 @@
  ds-cfg-backend-import-queue-size $ ds-cfg-backend-import-thread-count $
  ds-cfg-backend-entries-compressed $ ds-cfg-backend-deadlock-retry-limit $
  ds-cfg-backend-import-pass-size $ ds-cfg-backend-mode $
  ds-cfg-database-cache-percent $
  ds-cfg-database-cache-percent $ ds-cfg-backend-subtree-delete-batch-size $
  ds-cfg-database-cache-size $ ds-cfg-database-txn-no-sync $
  ds-cfg-database-txn-write-no-sync $ ds-cfg-database-run-cleaner $
  ds-cfg-database-cleaner-min-utilization $ ds-cfg-database-evictor-lru-only $

 opends/src/admin/defn/org/opends/server/admin/std/JEBackendConfiguration.xml

@@ -469,6 +469,33 @@
      </ldap:attribute>
    </adm:profile>
  </adm:property>
  <adm:property name="backend-subtree-delete-batch-size"
    mandatory="false"
    multi-valued="false">
    <adm:synopsis>
      Specifies the maximum number of entries that may be deleted from the
      backend when using the subtree delete control within a single transaction.
    </adm:synopsis>
    <adm:description>
      If a subtree delete operation targets a subtree with more than this
      number of entries, then additional transactions are used to remove the
      remaining entries in that subtree.
    </adm:description>
    <adm:default-behavior>
      <adm:defined>
        <adm:value>5000</adm:value>
      </adm:defined>
    </adm:default-behavior>
    <adm:syntax>
      <adm:integer  lower-limit="0" />
    </adm:syntax>
    <adm:profile name="ldap">
      <ldap:attribute>
        <ldap:oid>1.3.6.1.4.1.26027.1.1.381</ldap:oid>
        <ldap:name>ds-cfg-backend-subtree-delete-batch-size</ldap:name>
      </ldap:attribute>
    </adm:profile>
  </adm:property>
  <adm:property name="database-cache-percent"
    mandatory="false"
    multi-valued="false">

 opends/src/server/org/opends/server/backends/jeb/EntryContainer.java

@@ -182,6 +182,8 @@

  private int subtreeDeleteSizeLimit;

  private int subtreeDeleteBatchSize;

  private int indexEntryLimit;

  /**
@@ -215,6 +217,7 @@
    this.rootContainer = rootContainer;
    this.deadlockRetryLimit = config.getBackendDeadlockRetryLimit();
    this.subtreeDeleteSizeLimit = config.getBackendSubtreeDeleteSizeLimit();
    this.subtreeDeleteBatchSize = config.getBackendSubtreeDeleteBatchSize();
    this.indexEntryLimit = config.getBackendIndexEntryLimit();

    // Instantiate the attribute indexes.
@@ -1678,24 +1681,33 @@
  {
    DeleteEntryTransaction operation =
        new DeleteEntryTransaction(entryDN, deleteOperation);

    invokeTransactedOperation(operation);

    if (operation.adminSizeLimitExceeded())
    boolean isComplete = false;
    while(!isComplete)
    {
      String message = getMessage(MSGID_JEB_SUBTREE_DELETE_SIZE_LIMIT_EXCEEDED,
      invokeTransactedOperation(operation);

      if (operation.adminSizeLimitExceeded())
      {
        String message = getMessage(
                MSGID_JEB_SUBTREE_DELETE_SIZE_LIMIT_EXCEEDED,
                                  operation.getDeletedEntryCount());
      throw new DirectoryException(
        throw new DirectoryException(
          ResultCode.ADMIN_LIMIT_EXCEEDED,
          message,
          MSGID_JEB_SUBTREE_DELETE_SIZE_LIMIT_EXCEEDED);
    }

    String message = getMessage(MSGID_JEB_DELETED_ENTRY_COUNT,
      }
      if(operation.batchSizeExceeded())
      {
        operation.resetBatchSize();
        continue;
      }
      isComplete = true;
      String message = getMessage(MSGID_JEB_DELETED_ENTRY_COUNT,
                                operation.getDeletedEntryCount());
    StringBuilder errorMessage = new StringBuilder();
    errorMessage.append(message);
    deleteOperation.setErrorMessage(errorMessage);
      StringBuilder errorMessage = new StringBuilder();
      errorMessage.append(message);
      deleteOperation.setErrorMessage(errorMessage);
    }
  }

  /**
@@ -1930,7 +1942,7 @@
    private DeleteOperation deleteOperation;

    /**
     * A list of the DNs of all entries deleted by this operation.
     * A list of the DNs of all entries deleted by this operation in a batch.
     * The subtree delete control can cause multiple entries to be deleted.
     */
    private ArrayList<DN> deletedDNList;
@@ -1941,6 +1953,18 @@
     */
    private boolean adminSizeLimitExceeded = false;


    /**
     * Indicates whether the subtree delete batch size has been exceeded.
     */
    private boolean batchSizeExceeded = false;


    /**
     * Indicates the count of deleted DNs in the Delete Operation.
     */
    private int countDeletedDN;

    /**
     * Create a new Delete Entry Transaction.
     * @param entryDN The entry or subtree to be deleted.
@@ -1963,12 +1987,31 @@
    }

    /**
     * Determine whether the subtree delete batch size has been exceeded.
     * @return true if the batch size has been exceeded.
     */
    public boolean batchSizeExceeded()
    {
      return batchSizeExceeded;
    }

    /**
     * Resets the batchSizeExceeded parameter to reuse the object
     * for multiple batches.
     */
    public void resetBatchSize()
    {
      batchSizeExceeded=false;
      deletedDNList.clear();
    }

    /**
     * Get the number of entries deleted during the operation.
     * @return The number of entries deleted.
     */
    public int getDeletedEntryCount()
    {
      return deletedDNList.size();
      return countDeletedDN;
    }

    /**
@@ -1999,6 +2042,7 @@

      // Determine whether this is a subtree delete.
      int adminSizeLimit = subtreeDeleteSizeLimit;
      int deleteBatchSize = subtreeDeleteBatchSize;
      boolean isSubtreeDelete = false;
      List<Control> controls = deleteOperation.getRequestControls();
      if (controls != null)
@@ -2082,12 +2126,19 @@
          }

          // Enforce any subtree delete size limit.
          if (adminSizeLimit > 0 && deletedDNList.size() >= adminSizeLimit)
          if (adminSizeLimit > 0 && countDeletedDN >= adminSizeLimit)
          {
            adminSizeLimitExceeded = true;
            break;
          }

          // Enforce any subtree delete batch size.
          if (deleteBatchSize > 0 && deletedDNList.size() >= deleteBatchSize)
          {
            batchSizeExceeded = true;
            break;
          }

          /*
           * Delete this entry which by now must be a leaf because
           * we have been deleting from the bottom of the tree upwards.
@@ -2098,7 +2149,7 @@
                     txn, subordinateDN, entryID);

          deletedDNList.add(subordinateDN);

          countDeletedDN++;
          status = cursor.getPrev(key, data, LockMode.DEFAULT);
        }
      }
@@ -2109,13 +2160,18 @@

      // Finally delete the target entry as it was not included
      // in the dn2id iteration.
      if (!adminSizeLimitExceeded)
      if (!adminSizeLimitExceeded && !batchSizeExceeded)
      {
        // Enforce any subtree delete size limit.
        if (adminSizeLimit > 0 && deletedDNList.size() >= adminSizeLimit)
        if (adminSizeLimit > 0 && countDeletedDN >= adminSizeLimit)
        {
          adminSizeLimitExceeded = true;
        }
        else if (deleteBatchSize > 0 &&
                                      deletedDNList.size() >= deleteBatchSize)
        {
          batchSizeExceeded = true;
        }
        else
        {
          boolean manageDsaIT;
@@ -2134,6 +2190,7 @@
          deleteTarget(manageDsaIT, id2cBuffered, id2sBuffered, txn, entryDN);

          deletedDNList.add(entryDN);
          countDeletedDN++;
        }
      }

@@ -3762,6 +3819,7 @@
    this.config = cfg;
    this.deadlockRetryLimit = config.getBackendDeadlockRetryLimit();
    this.subtreeDeleteSizeLimit = config.getBackendSubtreeDeleteSizeLimit();
    this.subtreeDeleteBatchSize = config.getBackendSubtreeDeleteBatchSize();
    this.indexEntryLimit = config.getBackendIndexEntryLimit();
    return new ConfigChangeResult(ResultCode.SUCCESS,
                                  adminActionRequired, messages);