Fix for OPENDJ-55: Failing modify operations causing memory leak. The memor...

Ludovic Poitou

25.47.2011 ad1e1e776a8f5386df8237af2bc413446c4d8762

Fix for OPENDJ-55: Failing modify operations causing memory leak.
The memory leak is happening when PreOperation plugins are aborting an update (like UniqueAttributePlugin).
Several issues with the whole plugin manager, preOperation and postOperation plugins.
1/ The postOperation method of a plugin was always called even when the PreOp was skipped. This is due to an error in the ModifyOperationWrapper equals method (A.equals(A) was always false). As a result, the concurrentHashMap of skipped plugins was never cleaned and leaked memory big time.
2/ The postOperation method for most of the registered plugin would not check if the operation was successful or not before processing. This could possibly create issues with Access Controls, Groups or Subentries. 
3/ The UniqueAttributePlugin would not clean the concurrentHashMap of attribute values being checked for uniqueness, on errors. On success, it's done in the PostOperation method. On errors, the PostOperation method is not called, so the PreOp needs to cleans it before returning. For this we're now keeping a list of values added to the ConcurrrentHashMap, and remove them before returning an error.

 opends/src/server/org/opends/server/authorization/dseecompat/AciListenerManager.java

@@ -23,6 +23,7 @@
 *
 *
 *      Copyright 2008-2010 Sun Microsystems, Inc.
 *      Portions Copyright 2011 ForgeRock AS
 */

package org.opends.server.authorization.dseecompat;
@@ -166,7 +167,12 @@
    public PostOperation doPostOperation(
        PostOperationAddOperation addOperation)
    {
      // Only do something if the operation is successful, meaning there
      // has been a change.
      if (addOperation.getResultCode() == ResultCode.SUCCESS)
      {
      doPostAdd(addOperation.getEntryToAdd());
      }

      // If we've gotten here, then everything is acceptable.
      return PluginResult.PostOperation.continueOperationProcessing();
@@ -180,7 +186,12 @@
    public PostOperation doPostOperation(
        PostOperationDeleteOperation deleteOperation)
    {
      // Only do something if the operation is successful, meaning there
      // has been a change.
      if (deleteOperation.getResultCode() == ResultCode.SUCCESS)
      {
      doPostDelete(deleteOperation.getEntryToDelete());
      }

      // If we've gotten here, then everything is acceptable.
      return PluginResult.PostOperation.continueOperationProcessing();
@@ -194,8 +205,13 @@
    public PostOperation doPostOperation(
        PostOperationModifyDNOperation modifyDNOperation)
    {
      // Only do something if the operation is successful, meaning there
      // has been a change.
      if (modifyDNOperation.getResultCode() == ResultCode.SUCCESS)
      {
      doPostModifyDN(modifyDNOperation.getOriginalEntry().getDN(),
          modifyDNOperation.getUpdatedEntry().getDN());
      }

      // If we've gotten here, then everything is acceptable.
      return PluginResult.PostOperation.continueOperationProcessing();
@@ -209,8 +225,13 @@
    public PostOperation doPostOperation(
        PostOperationModifyOperation modifyOperation)
    {
      // Only do something if the operation is successful, meaning there
      // has been a change.
      if (modifyOperation.getResultCode() == ResultCode.SUCCESS)
      {
      doPostModify(modifyOperation.getModifications(), modifyOperation
          .getCurrentEntry(), modifyOperation.getModifiedEntry());
      }

      // If we've gotten here, then everything is acceptable.
      return PluginResult.PostOperation.continueOperationProcessing();

 opends/src/server/org/opends/server/core/GroupManager.java

@@ -23,6 +23,7 @@
 *
 *
 *      Copyright 2007-2010 Sun Microsystems, Inc.
 *      Portions Copyright 2011 ForgeRock AS
 */
package org.opends.server.core;

@@ -970,7 +971,12 @@
  public PostOperation doPostOperation(
          PostOperationAddOperation addOperation)
  {
    // Only do something if the operation is successful, meaning there
    // has been a change.
    if (addOperation.getResultCode() == ResultCode.SUCCESS)
    {
    doPostAdd(addOperation, addOperation.getEntryToAdd());
    }

    // If we've gotten here, then everything is acceptable.
    return PluginResult.PostOperation.continueOperationProcessing();
@@ -983,7 +989,12 @@
  public PostOperation doPostOperation(
          PostOperationDeleteOperation deleteOperation)
  {
    // Only do something if the operation is successful, meaning there
    // has been a change.
    if (deleteOperation.getResultCode() == ResultCode.SUCCESS)
    {
    doPostDelete(deleteOperation, deleteOperation.getEntryToDelete());
    }

    // If we've gotten here, then everything is acceptable.
    return PluginResult.PostOperation.continueOperationProcessing();
@@ -996,9 +1007,14 @@
  public PostOperation doPostOperation(
          PostOperationModifyOperation modifyOperation)
  {
    // Only do something if the operation is successful, meaning there
    // has been a change.
    if (modifyOperation.getResultCode() == ResultCode.SUCCESS)
    {
    doPostModify(modifyOperation,
            modifyOperation.getCurrentEntry(),
            modifyOperation.getModifiedEntry());
    }

    // If we've gotten here, then everything is acceptable.
    return PluginResult.PostOperation.continueOperationProcessing();
@@ -1011,9 +1027,14 @@
  public PostOperation doPostOperation(
          PostOperationModifyDNOperation modifyDNOperation)
  {
    // Only do something if the operation is successful, meaning there
    // has been a change.
    if (modifyDNOperation.getResultCode() == ResultCode.SUCCESS)
    {
    doPostModifyDN(modifyDNOperation,
            modifyDNOperation.getOriginalEntry(),
            modifyDNOperation.getUpdatedEntry());
    }

    // If we've gotten here, then everything is acceptable.
    return PluginResult.PostOperation.continueOperationProcessing();

 opends/src/server/org/opends/server/core/ModifyOperationWrapper.java

@@ -23,6 +23,7 @@
 *
 *
 *      Copyright 2008 Sun Microsystems, Inc.
 *      Portions Copyright 2011 ForgeRock AS
 */
package org.opends.server.core;

@@ -74,14 +75,6 @@
  /**
   * {@inheritDoc}
   */
  public boolean equals(Object obj)
  {
    return modify.equals(obj);
  }

  /**
   * {@inheritDoc}
   */
  public DN getEntryDN()
  {
    return modify.getEntryDN();
@@ -114,14 +107,6 @@
  /**
   * {@inheritDoc}
   */
  public int hashCode()
  {
    return modify.hashCode();
  }

  /**
   * {@inheritDoc}
   */
  public void setRawEntryDN(ByteString rawEntryDN)
  {
    modify.setRawEntryDN(rawEntryDN);

 opends/src/server/org/opends/server/core/OperationWrapper.java

@@ -23,6 +23,7 @@
 *
 *
 *      Copyright 2006-2008 Sun Microsystems, Inc.
 *      Portions Copyright 2011 ForgeRock AS
 */
package org.opends.server.core;

@@ -479,6 +480,42 @@
    operation.setSynchronizationOperation(isSynchronizationOperation);
  }



  /**
   * {@inheritDoc}
   */
  public final int hashCode()
  {
    return getClientConnection().hashCode() * (int) getOperationID();
  }



  /**
   * {@inheritDoc}
   */
  public final boolean equals(Object obj)
  {
    if (this == obj)
    {
      return true;
    }

    if (obj instanceof Operation)
    {
      Operation other = (Operation) obj;
      if (other.getClientConnection().equals(getClientConnection()))
      {
        return other.getOperationID() == getOperationID();
      }
    }

    return false;
  }



  /**
   * {@inheritDoc}
   */

 opends/src/server/org/opends/server/core/PluginConfigManager.java

@@ -23,6 +23,7 @@
 *
 *
 *      Copyright 2006-2010 Sun Microsystems, Inc.
 *      Portions Copyright 2011 ForgeRock AS
 */
package org.opends.server.core;

@@ -3442,6 +3443,9 @@
    PluginResult.PostOperation result = null;
    PluginResult.PostOperation finalResult = null;

    ArrayList<DirectoryServerPlugin> skippedPlugins =
        skippedPreOperationPlugins.remove(addOperation);

    for (DirectoryServerPlugin p : postOperationAddPlugins)
    {
      if (addOperation.isInternalOperation() &&
@@ -3450,8 +3454,6 @@
        continue;
      }

      ArrayList<DirectoryServerPlugin> skippedPlugins =
          skippedPreOperationPlugins.remove(addOperation);
      if(skippedPlugins != null && skippedPlugins.contains(p))
      {
        continue;
@@ -3527,6 +3529,9 @@
    PluginResult.PostOperation result = null;
    PluginResult.PostOperation finalResult = null;

    ArrayList<DirectoryServerPlugin> skippedPlugins =
        skippedPreOperationPlugins.remove(bindOperation);

    for (DirectoryServerPlugin p : postOperationBindPlugins)
    {
      if (bindOperation.isInternalOperation() &&
@@ -3535,8 +3540,6 @@
        continue;
      }

      ArrayList<DirectoryServerPlugin> skippedPlugins =
          skippedPreOperationPlugins.remove(bindOperation);
      if(skippedPlugins != null && skippedPlugins.contains(p))
      {
        continue;
@@ -3613,6 +3616,9 @@
    PluginResult.PostOperation result = null;
    PluginResult.PostOperation finalResult = null;

    ArrayList<DirectoryServerPlugin> skippedPlugins =
        skippedPreOperationPlugins.remove(compareOperation);

    for (DirectoryServerPlugin p : postOperationComparePlugins)
    {
      if (compareOperation.isInternalOperation() &&
@@ -3621,8 +3627,6 @@
        continue;
      }

      ArrayList<DirectoryServerPlugin> skippedPlugins =
          skippedPreOperationPlugins.remove(compareOperation);
      if(skippedPlugins != null && skippedPlugins.contains(p))
      {
        continue;
@@ -3700,6 +3704,9 @@
    PluginResult.PostOperation result = null;
    PluginResult.PostOperation finalResult = null;

    ArrayList<DirectoryServerPlugin> skippedPlugins =
        skippedPreOperationPlugins.remove(deleteOperation);

    for (DirectoryServerPlugin p : postOperationDeletePlugins)
    {
      if (deleteOperation.isInternalOperation() &&
@@ -3708,8 +3715,6 @@
        continue;
      }

      ArrayList<DirectoryServerPlugin> skippedPlugins =
          skippedPreOperationPlugins.remove(deleteOperation);
      if(skippedPlugins != null && skippedPlugins.contains(p))
      {
        continue;
@@ -3787,6 +3792,9 @@
    PluginResult.PostOperation result = null;
    PluginResult.PostOperation finalResult = null;

    ArrayList<DirectoryServerPlugin> skippedPlugins =
        skippedPreOperationPlugins.remove(extendedOperation);

    for (DirectoryServerPlugin p : postOperationExtendedPlugins)
    {
      if (extendedOperation.isInternalOperation() &&
@@ -3795,8 +3803,6 @@
        continue;
      }

      ArrayList<DirectoryServerPlugin> skippedPlugins =
          skippedPreOperationPlugins.remove(extendedOperation);
      if(skippedPlugins != null && skippedPlugins.contains(p))
      {
        continue;
@@ -3874,6 +3880,9 @@
    PluginResult.PostOperation result = null;
    PluginResult.PostOperation finalResult = null;

    ArrayList<DirectoryServerPlugin> skippedPlugins =
        skippedPreOperationPlugins.remove(modifyOperation);

    for (DirectoryServerPlugin p : postOperationModifyPlugins)
    {
      if (modifyOperation.isInternalOperation() &&
@@ -3882,8 +3891,6 @@
        continue;
      }

      ArrayList<DirectoryServerPlugin> skippedPlugins =
          skippedPreOperationPlugins.remove(modifyOperation);
      if(skippedPlugins != null && skippedPlugins.contains(p))
      {
        continue;
@@ -3960,6 +3967,9 @@
    PluginResult.PostOperation result = null;
    PluginResult.PostOperation finalResult = null;

    ArrayList<DirectoryServerPlugin> skippedPlugins =
        skippedPreOperationPlugins.remove(modifyDNOperation);

    for (DirectoryServerPlugin p : postOperationModifyDNPlugins)
    {
      if (modifyDNOperation.isInternalOperation() &&
@@ -3968,8 +3978,6 @@
        continue;
      }

      ArrayList<DirectoryServerPlugin> skippedPlugins =
          skippedPreOperationPlugins.remove(modifyDNOperation);
      if(skippedPlugins != null && skippedPlugins.contains(p))
      {
        continue;
@@ -4047,6 +4055,9 @@
    PluginResult.PostOperation result = null;
    PluginResult.PostOperation finalResult = null;

    ArrayList<DirectoryServerPlugin> skippedPlugins =
        skippedPreOperationPlugins.remove(searchOperation);

    for (DirectoryServerPlugin p : postOperationSearchPlugins)
    {
      if (searchOperation.isInternalOperation() &&
@@ -4055,8 +4066,6 @@
        continue;
      }

      ArrayList<DirectoryServerPlugin> skippedPlugins =
          skippedPreOperationPlugins.remove(searchOperation);
      if(skippedPlugins != null && skippedPlugins.contains(p))
      {
        continue;
@@ -4134,6 +4143,9 @@
    PluginResult.PostOperation result = null;
    PluginResult.PostOperation finalResult = null;

    ArrayList<DirectoryServerPlugin> skippedPlugins =
        skippedPreOperationPlugins.remove(unbindOperation);

    for (DirectoryServerPlugin p : postOperationUnbindPlugins)
    {
      if (unbindOperation.isInternalOperation() &&
@@ -4142,8 +4154,6 @@
        continue;
      }

      ArrayList<DirectoryServerPlugin> skippedPlugins =
          skippedPreOperationPlugins.remove(unbindOperation);
      if(skippedPlugins != null && skippedPlugins.contains(p))
      {
        continue;

 opends/src/server/org/opends/server/core/SubentryManager.java

@@ -1166,7 +1166,12 @@
  public PostOperation doPostOperation(
          PostOperationAddOperation addOperation)
  {
    // Only do something if the operation is successful, meaning there
    // has been a change.
    if (addOperation.getResultCode() == ResultCode.SUCCESS)
    {
    doPostAdd(addOperation.getEntryToAdd());
    }

    // If we've gotten here, then everything is acceptable.
    return PluginResult.PostOperation.continueOperationProcessing();
@@ -1179,7 +1184,12 @@
  public PostOperation doPostOperation(
          PostOperationDeleteOperation deleteOperation)
  {
    // Only do something if the operation is successful, meaning there
    // has been a change.
    if (deleteOperation.getResultCode() == ResultCode.SUCCESS)
    {
    doPostDelete(deleteOperation.getEntryToDelete());
    }

    // If we've gotten here, then everything is acceptable.
    return PluginResult.PostOperation.continueOperationProcessing();
@@ -1192,8 +1202,13 @@
  public PostOperation doPostOperation(
          PostOperationModifyOperation modifyOperation)
  {
    // Only do something if the operation is successful, meaning there
    // has been a change.
    if (modifyOperation.getResultCode() == ResultCode.SUCCESS)
    {
    doPostModify(modifyOperation.getCurrentEntry(),
            modifyOperation.getModifiedEntry());
    }

    // If we've gotten here, then everything is acceptable.
    return PluginResult.PostOperation.continueOperationProcessing();
@@ -1206,8 +1221,13 @@
  public PostOperation doPostOperation(
          PostOperationModifyDNOperation modifyDNOperation)
  {
    // Only do something if the operation is successful, meaning there
    // has been a change.
    if (modifyDNOperation.getResultCode() == ResultCode.SUCCESS)
    {
    doPostModifyDN(modifyDNOperation.getOriginalEntry(),
            modifyDNOperation.getUpdatedEntry());
    }

    // If we've gotten here, then everything is acceptable.
    return PluginResult.PostOperation.continueOperationProcessing();

 opends/src/server/org/opends/server/plugins/UniqueAttributePlugin.java

@@ -23,11 +23,13 @@
 *
 *
 *      Copyright 2008-2009 Sun Microsystems, Inc.
 *      Portions Copyright 2011 ForgeRock AS
 */
package org.opends.server.plugins;



import java.util.LinkedList;
import java.util.ArrayList;
import java.util.concurrent.ConcurrentHashMap;
import java.util.LinkedHashMap;
@@ -220,6 +222,8 @@
      return PluginResult.PreOperation.continueOperationProcessing();
    }

    LinkedList<AttributeValue> recordedValues =
        new LinkedList<AttributeValue>();
    for (AttributeType t : config.getType())
    {
      List<Attribute> attrList = entry.getAttribute(t);
@@ -238,11 +242,19 @@
              if((conflictDN=
                      uniqueAttrValue2Dn.putIfAbsent(v, entry.getDN()))==null)
              {
                recordedValues.add(v);
                conflictDN = getConflictingEntryDN(baseDNs, entry.getDN(),
                                                   config, v);
              }
              if (conflictDN != null)
              {
                // Before returning, we need to remove all values added
                // in the uniqueAttrValue2Dn map, because PostOperation
                // plugin does not get called.
                for (AttributeValue v2 : recordedValues)
                {
                   uniqueAttrValue2Dn.remove(v2);
                }
                Message msg = ERR_PLUGIN_UNIQUEATTR_ATTR_NOT_UNIQUE.get(
                    t.getNameOrOID(), v.getValue().toString(),
                    conflictDN.toString());
@@ -261,6 +273,12 @@
                               de.getResultCode().toString(),
                               de.getMessageObject());

             // Try some cleanup before returning, to avoid memory leaks
             for (AttributeValue v2 : recordedValues)
             {
               uniqueAttrValue2Dn.remove(v2);
             }

              return PluginResult.PreOperation.stopProcessing(
                    DirectoryServer.getServerErrorResultCode(), m);
            }
@@ -291,6 +309,8 @@
      return PluginResult.PreOperation.continueOperationProcessing();
    }

    LinkedList<AttributeValue> recordedValues =
        new LinkedList<AttributeValue>();
    for (Modification m : modifyOperation.getModifications())
    {
      Attribute a = m.getAttribute();
@@ -316,11 +336,19 @@
              if((conflictDN=
                      uniqueAttrValue2Dn.putIfAbsent(v, entryDN))==null)
              {
                recordedValues.add(v);
               conflictDN = getConflictingEntryDN(baseDNs, entryDN, config,
                                                   v);
              }
              if (conflictDN != null)
              {
                // Before returning, we need to remove all values added
                // in the uniqueAttrValue2Dn map, because PostOperation
                // plugin does not get called.
                for (AttributeValue v2 : recordedValues)
                {
                   uniqueAttrValue2Dn.remove(v2);
                }
                Message msg = ERR_PLUGIN_UNIQUEATTR_ATTR_NOT_UNIQUE.get(
                    t.getNameOrOID(), v.getValue().toString(),
                    conflictDN.toString());
@@ -371,11 +399,19 @@
                  if((conflictDN=
                      uniqueAttrValue2Dn.putIfAbsent(v, entryDN))==null)
                  {
                    recordedValues.add(v);
                    conflictDN = getConflictingEntryDN(baseDNs, entryDN,
                                                        config, v);
                  }
                  if (conflictDN != null)
                  {
                    // Before returning, we need to remove all values added
                    // in the uniqueAttrValue2Dn map, because PostOperation
                    // plugin does not get called.
                    for (AttributeValue v2 : recordedValues)
                    {
                      uniqueAttrValue2Dn.remove(v2);
                    }
                    Message msg = ERR_PLUGIN_UNIQUEATTR_ATTR_NOT_UNIQUE.get(
                        t.getNameOrOID(), v.getValue().toString(),
                        conflictDN.toString());
@@ -394,6 +430,12 @@
                                         de.getResultCode().toString(),
                                         de.getMessageObject());

                  // Try some cleanup before returning, to avoid memory leaks
                  for (AttributeValue v2 : recordedValues)
                  {
                    uniqueAttrValue2Dn.remove(v2);
                  }

                  return PluginResult.PreOperation.stopProcessing(
                      DirectoryServer.getServerErrorResultCode(), message);
                }
@@ -431,6 +473,8 @@
      return PluginResult.PreOperation.continueOperationProcessing();
    }

    LinkedList<AttributeValue> recordedValues =
        new LinkedList<AttributeValue>();
    RDN newRDN = modifyDNOperation.getNewRDN();
    for (int i=0; i < newRDN.getNumValues(); i++)
    {
@@ -451,11 +495,19 @@
        if((conflictDN=uniqueAttrValue2Dn.putIfAbsent(
                              v, modifyDNOperation.getEntryDN()))==null)
        {
          recordedValues.add(v);
          conflictDN = getConflictingEntryDN(baseDNs,
            modifyDNOperation.getEntryDN(), config, v);
        }
        if (conflictDN != null)
        {
          // Before returning, we need to remove all values added
          // in the uniqueAttrValue2Dn map, because PostOperation
          // plugin does not get called.
          for (AttributeValue v2 : recordedValues)
          {
            uniqueAttrValue2Dn.remove(v2);
          }
          Message msg = ERR_PLUGIN_UNIQUEATTR_ATTR_NOT_UNIQUE.get(
              t.getNameOrOID(), v.getValue().toString(),
              conflictDN.toString());
@@ -474,6 +526,12 @@
                         de.getResultCode().toString(),
                         de.getMessageObject());

        // Try some cleanup before returning, to avoid memory leaks
        for (AttributeValue v2 : recordedValues)
        {
          uniqueAttrValue2Dn.remove(v2);
        }

        return PluginResult.PreOperation.stopProcessing(
            DirectoryServer.getServerErrorResultCode(), m);
      }

 opends/src/server/org/opends/server/types/AbstractOperation.java

@@ -23,6 +23,7 @@
 *
 *
 *      Copyright 2006-2010 Sun Microsystems, Inc.
 *      Portions Copyright 2011 ForgeRock AS
 */
package org.opends.server.types;
import org.opends.messages.Message;
@@ -1202,6 +1203,40 @@


  /**
   * {@inheritDoc}
   */
  public final int hashCode()
  {
    return clientConnection.hashCode() * (int) operationID;
  }



  /**
   * {@inheritDoc}
   */
  public final boolean equals(Object obj)
  {
    if (this == obj)
    {
      return true;
    }

    if (obj instanceof Operation)
    {
      Operation other = (Operation) obj;
      if (other.getClientConnection().equals(clientConnection))
      {
        return other.getOperationID() == operationID;
      }
    }

    return false;
  }



  /**
   * Invokes the post response callbacks that were registered with
   * this operation.
   */