Issue OPENDJ-262: Implement pass through authentication (PTA)

Matthew Swift

06.21.2011 7eec159c2d6083b89135ee061fd76126ae83831b

Issue OPENDJ-262: Implement pass through authentication (PTA)

Implement connection pooling, load-balancing, and fail-over support. Enable first set of unit tests.

 opends/src/server/org/opends/server/extensions/LDAPPassThroughAuthenticationPolicyFactory.java

@@ -34,14 +34,17 @@
import java.io.Closeable;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.Semaphore;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;

import org.opends.messages.Message;
import org.opends.server.admin.server.ConfigurationChangeListener;
import org.opends.server.admin.std.server.
    LDAPPassThroughAuthenticationPolicyCfg;
import org.opends.server.admin.std.server.*;
import org.opends.server.api.AuthenticationPolicy;
import org.opends.server.api.AuthenticationPolicyFactory;
import org.opends.server.api.AuthenticationPolicyState;
@@ -58,6 +61,11 @@
    AuthenticationPolicyFactory<LDAPPassThroughAuthenticationPolicyCfg>
{

  // TODO: retry operations transparently until all connections exhausted.
  // TODO: handle password policy response controls? AD?
  // TODO: periodically ping offline servers in order to detect when they come
  // back.

  /**
   * An LDAP connection which will be used in order to search for or
   * authenticate users.
@@ -75,9 +83,9 @@

    /**
     * Returns the name of the user whose entry matches the provided search
     * criteria.
     * <p>
     * TODO: define result codes used when no entries found or too many entries.
     * criteria. This will return CLIENT_SIDE_NO_RESULTS_RETURNED/NO_SUCH_OBJECT
     * if no search results were returned, or CLIENT_SIDE_MORE_RESULTS_TO_RETURN
     * if too many results were returned.
     *
     * @param baseDN
     *          The search base DN.
@@ -165,6 +173,483 @@
  private final class PolicyImpl extends AuthenticationPolicy implements
      ConfigurationChangeListener<LDAPPassThroughAuthenticationPolicyCfg>
  {

    /**
     * A factory which returns pre-authenticated connections for searches.
     */
    private final class AuthenticatedConnectionFactory implements
        ConnectionFactory
    {

      private final ConnectionFactory factory;



      private AuthenticatedConnectionFactory(final ConnectionFactory factory)
      {
        this.factory = factory;
      }



      /**
       * {@inheritDoc}
       */
      @Override
      public Connection getConnection() throws DirectoryException
      {
        final DN username = configuration.getMappedSearchBindDN();
        final String password = configuration.getMappedSearchBindPassword();

        final Connection connection = factory.getConnection();
        if (username != null && !username.isNullDN())
        {
          try
          {
            connection.simpleBind(ByteString.valueOf(username.toString()),
                ByteString.valueOf(password));
          }
          catch (final DirectoryException e)
          {
            connection.close();
            throw e;
          }
        }
        return connection;
      }



      /**
       * {@inheritDoc}
       */
      @Override
      public String toString()
      {
        final StringBuilder builder = new StringBuilder();
        builder.append("AuthenticationConnectionFactory(");
        builder.append(factory);
        builder.append(')');
        return builder.toString();
      }

    }



    /**
     * PTA connection pool.
     */
    private final class ConnectionPool implements ConnectionFactory, Closeable
    {

      /**
       * Pooled connection's intercept close and release connection back to the
       * pool.
       */
      private final class PooledConnection implements Connection
      {
        private final Connection connection;
        private boolean connectionIsClosed = false;



        private PooledConnection(final Connection connection)
        {
          this.connection = connection;
        }



        /**
         * {@inheritDoc}
         */
        @Override
        public void close()
        {
          if (!connectionIsClosed)
          {
            connectionIsClosed = true;

            // Guarded by PolicyImpl
            if (poolIsClosed)
            {
              connection.close();
            }
            else
            {
              pooledConnections.offer(this);
            }
            availableConnections.release();
          }
        }



        /**
         * {@inheritDoc}
         */
        @Override
        public ByteString search(final DN baseDN, final SearchScope scope,
            final SearchFilter filter) throws DirectoryException
        {
          try
          {
            return connection.search(baseDN, scope, filter);
          }
          catch (final DirectoryException e)
          {
            // Don't put the connection back in the pool if it has failed.
            closeConnectionOnFatalError(e);
            throw e;
          }
        }



        /**
         * {@inheritDoc}
         */
        @Override
        public void simpleBind(final ByteString username,
            final ByteString password) throws DirectoryException
        {
          try
          {
            connection.simpleBind(username, password);
          }
          catch (final DirectoryException e)
          {
            // Don't put the connection back in the pool if it has failed.
            closeConnectionOnFatalError(e);
            throw e;
          }
        }



        private void closeConnectionOnFatalError(final DirectoryException e)
        {
          if (isFatalResultCode(e.getResultCode()))
          {
            connectionIsClosed = true;
            connection.close();
            availableConnections.release();
          }
        }

      }



      // Guarded by PolicyImpl.lock.
      private boolean poolIsClosed = false;

      private final ConnectionFactory factory;
      private final int poolSize =
        Runtime.getRuntime().availableProcessors() * 2;
      private final Semaphore availableConnections = new Semaphore(poolSize);
      private final LinkedBlockingQueue<PooledConnection> pooledConnections =
        new LinkedBlockingQueue<PooledConnection>();



      private ConnectionPool(final ConnectionFactory factory)
      {
        this.factory = factory;
      }



      /**
       * Release all connections: do we want to block?
       */
      @Override
      public void close()
      {
        // No need for synchronization as this can only be called with the
        // policy's exclusive lock.
        poolIsClosed = true;

        PooledConnection pooledConnection;
        while ((pooledConnection = pooledConnections.poll()) != null)
        {
          pooledConnection.connection.close();
        }

        // Since we have the exclusive lock, there should be no more connections
        // in use.
        if (availableConnections.availablePermits() != poolSize)
        {
          throw new IllegalStateException(
              "Pool has remaining connections open after close");
        }
      }



      /**
       * {@inheritDoc}
       */
      @Override
      public Connection getConnection() throws DirectoryException
      {
        // This should only be called with the policy's shared lock.
        if (poolIsClosed)
        {
          throw new IllegalStateException("pool is closed");
        }

        availableConnections.acquireUninterruptibly();

        // There is either a pooled connection or we are allowed to create
        // one.
        PooledConnection pooledConnection = pooledConnections.poll();
        if (pooledConnection == null)
        {
          try
          {
            final Connection connection = factory.getConnection();
            pooledConnection = new PooledConnection(connection);
          }
          catch (final DirectoryException e)
          {
            availableConnections.release();
            throw e;
          }
        }

        return pooledConnection;
      }



      /**
       * {@inheritDoc}
       */
      @Override
      public String toString()
      {
        final StringBuilder builder = new StringBuilder();
        builder.append("ConnectionPool(");
        builder.append(factory);
        builder.append(", poolSize=");
        builder.append(poolSize);
        builder.append(", inPool=");
        builder.append(pooledConnections.size());
        builder.append(", available=");
        builder.append(availableConnections.availablePermits());
        builder.append(')');
        return builder.toString();
      }
    }



    /**
     * A simplistic two-way fail-over connection factory implementation.
     */
    private final class FailoverConnectionFactory implements ConnectionFactory,
        Closeable
    {
      private final LoadBalancer primary;
      private final LoadBalancer secondary;



      private FailoverConnectionFactory(final LoadBalancer primary,
          final LoadBalancer secondary)
      {
        this.primary = primary;
        this.secondary = secondary;
      }



      /**
       * Close underlying load-balancers.
       */
      @Override
      public void close()
      {
        primary.close();
        if (secondary != null)
        {
          secondary.close();
        }
      }



      /**
       * {@inheritDoc}
       */
      @Override
      public Connection getConnection() throws DirectoryException
      {
        if (secondary == null)
        {
          // No fail-over so just use the primary.
          return primary.getConnection();
        }
        else
        {
          try
          {
            return primary.getConnection();
          }
          catch (final DirectoryException e)
          {
            return secondary.getConnection();
          }
        }
      }



      /**
       * {@inheritDoc}
       */
      @Override
      public String toString()
      {
        final StringBuilder builder = new StringBuilder();
        builder.append("FailoverConnectionFactory(");
        builder.append(primary);
        builder.append(", ");
        builder.append(secondary);
        builder.append(')');
        return builder.toString();
      }

    }



    /**
     * A simplistic load-balancer connection factory implementation using
     * approximately round-robin balancing.
     */
    private final class LoadBalancer implements ConnectionFactory, Closeable
    {
      private final ConnectionPool[] factories;
      private final AtomicInteger nextIndex = new AtomicInteger();
      private final int maxIndex;



      private LoadBalancer(final ConnectionPool[] factories)
      {
        this.factories = factories;
        this.maxIndex = factories.length;
      }



      /**
       * Close underlying connection pools.
       */
      @Override
      public void close()
      {
        for (final ConnectionPool pool : factories)
        {
          pool.close();
        }
      }



      /**
       * {@inheritDoc}
       */
      @Override
      public Connection getConnection() throws DirectoryException
      {
        final int startIndex = getStartIndex();
        int index = startIndex;
        for (;;)
        {
          final ConnectionFactory factory = factories[index];

          try
          {
            return factory.getConnection();
          }
          catch (final DirectoryException e)
          {
            // Try the next index.
            if (++index == maxIndex)
            {
              index = 0;
            }

            // If all the factories have been tried then give up and throw the
            // exception.
            if (index == startIndex)
            {
              throw e;
            }
          }
        }
      }



      /**
       * {@inheritDoc}
       */
      @Override
      public String toString()
      {
        final StringBuilder builder = new StringBuilder();
        builder.append("LoadBalancer(");
        builder.append(nextIndex);
        for (final ConnectionFactory factory : factories)
        {
          builder.append(", ");
          builder.append(factory);
        }
        builder.append(')');
        return builder.toString();
      }



      // Determine the start index.
      private int getStartIndex()
      {
        // A round robin pool of one connection factories is unlikely in
        // practice and requires special treatment.
        if (maxIndex == 1)
        {
          return 0;
        }

        // Determine the next factory to use: avoid blocking algorithm.
        int oldNextIndex;
        int newNextIndex;
        do
        {
          oldNextIndex = nextIndex.get();
          newNextIndex = oldNextIndex + 1;
          if (newNextIndex == maxIndex)
          {
            newNextIndex = 0;
          }
        }
        while (!nextIndex.compareAndSet(oldNextIndex, newNextIndex));

        // There's a potential, but benign, race condition here: other threads
        // could jump in and rotate through the list before we return the
        // connection factory.
        return newNextIndex;
      }

    }



    /**
     * LDAP PTA policy state implementation.
     */
@@ -329,8 +814,6 @@
              {
                switch (e.getResultCode())
                {
                // FIXME: specify possible result codes. What about authz
                // errors?
                case NO_SUCH_OBJECT:
                case CLIENT_SIDE_NO_RESULTS_RETURNED:
                  // Ignore and try next base DN.
@@ -389,7 +872,6 @@
          {
            switch (e.getResultCode())
            {
            // FIXME: specify possible result codes.
            case NO_SUCH_OBJECT:
            case INVALID_CREDENTIALS:
              return false;
@@ -430,9 +912,8 @@
    // Current configuration.
    private LDAPPassThroughAuthenticationPolicyCfg configuration;

    // FIXME: initialize connection factories.
    private ConnectionFactory searchFactory = null;
    private ConnectionFactory bindFactory = null;
    private FailoverConnectionFactory searchFactory = null;
    private FailoverConnectionFactory bindFactory = null;



@@ -529,7 +1010,18 @@
      exclusiveLock.lock();
      try
      {
        // TODO: close all connections.
        if (searchFactory != null)
        {
          searchFactory.close();
          searchFactory = null;
        }

        if (bindFactory != null)
        {
          bindFactory.close();
          bindFactory = null;
        }

      }
      finally
      {
@@ -544,11 +1036,54 @@
    {
      this.configuration = configuration;

      // TODO: implement FO/LB/CP + authenticated search factory.
      final String hostPort = configuration.getPrimaryRemoteLDAPServer()
          .first();
      searchFactory = newLDAPConnectionFactory(hostPort);
      bindFactory = newLDAPConnectionFactory(hostPort);
      // Create load-balancers for primary servers.
      final LoadBalancer primarySearchLoadBalancer;
      final LoadBalancer primaryBindLoadBalancer;

      Set<String> servers = configuration.getPrimaryRemoteLDAPServer();
      ConnectionPool[] searchPool = new ConnectionPool[servers.size()];
      ConnectionPool[] bindPool = new ConnectionPool[servers.size()];
      int index = 0;
      for (final String hostPort : servers)
      {
        final ConnectionFactory factory = newLDAPConnectionFactory(hostPort);
        searchPool[index] = new ConnectionPool(
            new AuthenticatedConnectionFactory(factory));
        bindPool[index++] = new ConnectionPool(factory);
      }
      primarySearchLoadBalancer = new LoadBalancer(searchPool);
      primaryBindLoadBalancer = new LoadBalancer(bindPool);

      // Create load-balancers for secondary servers.
      final LoadBalancer secondarySearchLoadBalancer;
      final LoadBalancer secondaryBindLoadBalancer;

      servers = configuration.getSecondaryRemoteLDAPServer();
      if (servers.isEmpty())
      {
        secondarySearchLoadBalancer = null;
        secondaryBindLoadBalancer = null;
      }
      else
      {
        searchPool = new ConnectionPool[servers.size()];
        bindPool = new ConnectionPool[servers.size()];
        index = 0;
        for (final String hostPort : servers)
        {
          final ConnectionFactory factory = newLDAPConnectionFactory(hostPort);
          searchPool[index] = new ConnectionPool(
              new AuthenticatedConnectionFactory(factory));
          bindPool[index++] = new ConnectionPool(factory);
        }
        secondarySearchLoadBalancer = new LoadBalancer(searchPool);
        secondaryBindLoadBalancer = new LoadBalancer(bindPool);
      }

      searchFactory = new FailoverConnectionFactory(primarySearchLoadBalancer,
          secondarySearchLoadBalancer);
      bindFactory = new FailoverConnectionFactory(primaryBindLoadBalancer,
          secondaryBindLoadBalancer);
    }


@@ -589,6 +1124,33 @@


  /**
   * Determines whether or no a result code is expected to trigger the
   * associated connection to be closed immediately.
   *
   * @param resultCode
   *          The result code.
   * @return {@code true} if the result code is expected to trigger the
   *         associated connection to be closed immediately.
   */
  static boolean isFatalResultCode(final ResultCode resultCode)
  {
    switch (resultCode)
    {
    case BUSY:
    case UNAVAILABLE:
    case PROTOCOL_ERROR:
    case OTHER:
    case UNWILLING_TO_PERFORM:
    case OPERATIONS_ERROR:
      return true;
    default:
      return false;
    }
  }



  /**
   * Public default constructor used by the admin framework. This will use the
   * default LDAP connection factory provider.
   */

 opends/tests/unit-tests-testng/src/server/org/opends/server/extensions/LDAPPassThroughAuthenticationPolicyTestCase.java

@@ -28,6 +28,7 @@



import static org.opends.server.extensions.LDAPPassThroughAuthenticationPolicyFactory.isFatalResultCode;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertFalse;
import static org.testng.Assert.assertTrue;
@@ -309,7 +310,7 @@
    public void close()
    {
      CloseEvent event = new CloseEvent(getConnectionEvent);
      mockProvider.assertNextEventExpected(event);
      mockProvider.assertExpectedEventWasReceived(event);
    }


@@ -322,10 +323,10 @@
    {
      SearchEvent event = new SearchEvent(getConnectionEvent,
          baseDN.toString(), scope, filter.toString());
      Object result = mockProvider.assertNextEventExpected(event);
      if (result instanceof ByteString)
      Object result = mockProvider.assertExpectedEventWasReceived(event);
      if (result instanceof String)
      {
        return (ByteString) result;
        return ByteString.valueOf((String) result);
      }
      else
      {
@@ -343,7 +344,7 @@
    {
      SimpleBindEvent event = new SimpleBindEvent(getConnectionEvent,
          username.toString(), password.toString());
      DirectoryException e = mockProvider.assertNextEventExpected(event);
      DirectoryException e = mockProvider.assertExpectedEventWasReceived(event);
      if (e != null) throw e;
    }

@@ -376,7 +377,7 @@
      GetConnectionEvent event = new GetConnectionEvent(
          getLDAPConnectionFactoryEvent);

      DirectoryException e = mockProvider.assertNextEventExpected(event);
      DirectoryException e = mockProvider.assertExpectedEventWasReceived(event);
      if (e != null)
      {
        throw e;
@@ -624,14 +625,14 @@
    {
      GetLDAPConnectionFactoryEvent event = new GetLDAPConnectionFactoryEvent(
          host + ":" + port, options);
      assertNextEventExpected(event);
      assertExpectedEventWasReceived(event);
      return new MockFactory(this, event);
    }



    @SuppressWarnings("unchecked")
    <T> T assertNextEventExpected(Event<T> actualEvent)
    <T> T assertExpectedEventWasReceived(Event<T> actualEvent)
    {
      Event<?> expectedEvent = expectedEvents.poll();
      if (expectedEvent == null)
@@ -647,7 +648,7 @@



    MockProvider withExpectedEvent(Event<?> expectedEvent)
    MockProvider expectEvent(Event<?> expectedEvent)
    {
      expectedEvents.add(expectedEvent);
      return this;
@@ -655,7 +656,7 @@



    void assertNoMoreEvents()
    void assertAllExpectedEventsReceived()
    {
      assertTrue(expectedEvents.isEmpty());
    }
@@ -928,7 +929,7 @@
   * @throws Exception
   *           If an unexpected exception occurred.
   */
  @Test(enabled = false, dataProvider = "testConnectionFailureDuringSearchGetConnectionData")
  @Test(enabled = true, dataProvider = "testConnectionFailureDuringSearchGetConnectionData")
  public void testConnectionFailureDuringSearchGetConnection(
      ResultCode connectResultCode) throws Exception
  {
@@ -943,8 +944,8 @@
        phost1, cfg);
    GetConnectionEvent ceSearch = new GetConnectionEvent(fe, connectResultCode);

    MockProvider provider = new MockProvider().withExpectedEvent(fe)
        .withExpectedEvent(ceSearch);
    MockProvider provider = new MockProvider().expectEvent(fe).expectEvent(
        ceSearch);

    // Obtain policy and state.
    LDAPPassThroughAuthenticationPolicyFactory factory = new LDAPPassThroughAuthenticationPolicyFactory(
@@ -964,12 +965,12 @@
    catch (DirectoryException e)
    {
      // No valid connections available so this should always fail with
      // UNAVAILABLE.
      assertEquals(e.getResultCode(), ResultCode.UNAVAILABLE);
      // INVALID_CREDENTIALS.
      assertEquals(e.getResultCode(), ResultCode.INVALID_CREDENTIALS);
    }

    // Tear down and check final state.
    provider.assertNoMoreEvents();
    provider.assertAllExpectedEventsReceived();
    state.finalizeStateAfterBind();
    policy.finalizeAuthenticationPolicy();
  }
@@ -1008,7 +1009,7 @@
   * @throws Exception
   *           If an unexpected exception occurred.
   */
  @Test(enabled = false, dataProvider = "testConnectionFailureDuringSearchBindData")
  @Test(enabled = true, dataProvider = "testConnectionFailureDuringSearchBindData")
  public void testConnectionFailureDuringSearchBind(ResultCode bindResultCode)
      throws Exception
  {
@@ -1024,11 +1025,11 @@
    GetConnectionEvent ceSearch = new GetConnectionEvent(fe);

    MockProvider provider = new MockProvider()
        .withExpectedEvent(fe)
        .withExpectedEvent(ceSearch)
        .withExpectedEvent(
        .expectEvent(fe)
        .expectEvent(ceSearch)
        .expectEvent(
            new SimpleBindEvent(ceSearch, searchBindDNString, "searchPassword",
                bindResultCode)).withExpectedEvent(new CloseEvent(ceSearch));
                bindResultCode)).expectEvent(new CloseEvent(ceSearch));

    // Obtain policy and state.
    LDAPPassThroughAuthenticationPolicyFactory factory = new LDAPPassThroughAuthenticationPolicyFactory(
@@ -1048,12 +1049,12 @@
    catch (DirectoryException e)
    {
      // No valid connections available so this should always fail with
      // UNAVAILABLE.
      assertEquals(e.getResultCode(), ResultCode.UNAVAILABLE);
      // INVALID_CREDENTIALS.
      assertEquals(e.getResultCode(), ResultCode.INVALID_CREDENTIALS);
    }

    // Tear down and check final state.
    provider.assertNoMoreEvents();
    provider.assertAllExpectedEventsReceived();
    state.finalizeStateAfterBind();
    policy.finalizeAuthenticationPolicy();
  }
@@ -1071,6 +1072,8 @@
    // @formatter:off
    return new Object[][] {
        { ResultCode.NO_SUCH_OBJECT },
        { ResultCode.CLIENT_SIDE_NO_RESULTS_RETURNED },
        { ResultCode.CLIENT_SIDE_MORE_RESULTS_TO_RETURN },
        { ResultCode.UNAVAILABLE }
    };
    // @formatter:on
@@ -1089,7 +1092,7 @@
   * @throws Exception
   *           If an unexpected exception occurred.
   */
  @Test(enabled = false, dataProvider = "testConnectionFailureDuringSearchData")
  @Test(enabled = true, dataProvider = "testConnectionFailureDuringSearchData")
  public void testConnectionFailureDuringSearch(ResultCode searchResultCode)
      throws Exception
  {
@@ -1105,18 +1108,18 @@
    GetConnectionEvent ceSearch = new GetConnectionEvent(fe);

    MockProvider provider = new MockProvider()
        .withExpectedEvent(fe)
        .withExpectedEvent(ceSearch)
        .withExpectedEvent(
        .expectEvent(fe)
        .expectEvent(ceSearch)
        .expectEvent(
            new SimpleBindEvent(ceSearch, searchBindDNString, "searchPassword",
                ResultCode.SUCCESS))
        .withExpectedEvent(
        .expectEvent(
            new SearchEvent(ceSearch, "o=ad", SearchScope.WHOLE_SUBTREE,
                "(uid=aduser)", searchResultCode));
    if (isFatalResultCode(searchResultCode))
    {
      // The connection will fail and be closed immediately.
      provider.withExpectedEvent(new CloseEvent(ceSearch));
      provider.expectEvent(new CloseEvent(ceSearch));
    }

    // Obtain policy and state.
@@ -1137,23 +1140,23 @@
    catch (DirectoryException e)
    {
      // No valid connections available so this should always fail with
      // UNAVAILABLE.
      assertEquals(e.getResultCode(), ResultCode.UNAVAILABLE);
      // INVALID_CREDENTIALS.
      assertEquals(e.getResultCode(), ResultCode.INVALID_CREDENTIALS);
    }

    // There should be no more pending events.
    provider.assertNoMoreEvents();
    provider.assertAllExpectedEventsReceived();
    state.finalizeStateAfterBind();

    // Cached connections should be closed when the policy is finalized.
    if (!isFatalResultCode(searchResultCode))
    {
      provider.withExpectedEvent(new CloseEvent(ceSearch));
      provider.expectEvent(new CloseEvent(ceSearch));
    }

    // Tear down and check final state.
    policy.finalizeAuthenticationPolicy();
    provider.assertNoMoreEvents();
    provider.assertAllExpectedEventsReceived();
  }


@@ -1168,7 +1171,7 @@
  {
    // @formatter:off
    return new Object[][] {
        /* policy, connection failure, invalid credentials */
        /* policy, bind result code */
        { MappingPolicy.UNMAPPED, ResultCode.SUCCESS },
        { MappingPolicy.UNMAPPED, ResultCode.INVALID_CREDENTIALS },
        { MappingPolicy.UNMAPPED, ResultCode.UNAVAILABLE },
@@ -1201,7 +1204,7 @@
   * @throws Exception
   *           If an unexpected exception occurred.
   */
  @Test(enabled = false, dataProvider = "testMappingPolicyAuthenticationData")
  @Test(enabled = true, dataProvider = "testMappingPolicyAuthenticationData")
  public void testMappingPolicyAuthentication(MappingPolicy mappingPolicy,
      ResultCode bindResultCode) throws Exception
  {
@@ -1216,31 +1219,34 @@
    // Create the provider and its list of expected events.
    GetLDAPConnectionFactoryEvent fe = new GetLDAPConnectionFactoryEvent(
        phost1, cfg);
    MockProvider provider = new MockProvider().withExpectedEvent(fe);
    MockProvider provider = new MockProvider().expectEvent(fe);

    // Add search events if doing a mapped search.
    GetConnectionEvent ceSearch = null;
    if (mappingPolicy == MappingPolicy.MAPPED_SEARCH)
    {
      ceSearch = new GetConnectionEvent(fe);
      provider.withExpectedEvent(
          new SimpleBindEvent(ceSearch, searchBindDNString, "searchPassword",
              ResultCode.SUCCESS)).withExpectedEvent(
          new SearchEvent(ceSearch, "o=ad", SearchScope.WHOLE_SUBTREE,
              "(uid=aduser)", adDNString));
      provider
          .expectEvent(ceSearch)
          .expectEvent(
              new SimpleBindEvent(ceSearch, searchBindDNString,
                  "searchPassword", ResultCode.SUCCESS))
          .expectEvent(
              new SearchEvent(ceSearch, "o=ad", SearchScope.WHOLE_SUBTREE,
                  "(uid=aduser)", adDNString));
      // Connection should be cached until the policy is finalized.
    }

    // Add bind events.
    GetConnectionEvent ceBind = new GetConnectionEvent(fe);
    provider.withExpectedEvent(ceBind).withExpectedEvent(
    provider.expectEvent(ceBind).expectEvent(
        new SimpleBindEvent(ceBind,
            mappingPolicy == MappingPolicy.UNMAPPED ? opendjDNString
                : adDNString, userPassword, bindResultCode));
    if (isFatalResultCode(bindResultCode))
    {
      // The connection will fail and be closed immediately.
      provider.withExpectedEvent(new CloseEvent(ceBind));
      provider.expectEvent(new CloseEvent(ceBind));
    }

    // Connection should be cached until the policy is finalized or until the
@@ -1272,57 +1278,30 @@
      }
      catch (DirectoryException e)
      {
        // No authentication related error codes should be mapped to
        // UNAVAILABLE.
        assertEquals(e.getResultCode(), ResultCode.UNAVAILABLE);
        // No valid connections available so this should always fail with
        // INVALID_CREDENTIALS.
        assertEquals(e.getResultCode(), ResultCode.INVALID_CREDENTIALS);
      }
      break;
    }

    // There should be no more pending events.
    provider.assertNoMoreEvents();
    provider.assertAllExpectedEventsReceived();
    state.finalizeStateAfterBind();

    // Cached connections should be closed when the policy is finalized.
    if (ceSearch != null)
    {
      provider.withExpectedEvent(new CloseEvent(ceSearch));
      provider.expectEvent(new CloseEvent(ceSearch));
    }
    if (!isFatalResultCode(bindResultCode))
    {
      provider.withExpectedEvent(new CloseEvent(ceBind));
      provider.expectEvent(new CloseEvent(ceBind));
    }

    // Tear down and check final state.
    policy.finalizeAuthenticationPolicy();
    provider.assertNoMoreEvents();
  }



  /**
   * Determines whether or no a result code is expected to trigger the
   * associated connection to be closed immediately.
   *
   * @param resultCode
   *          The result code.
   * @return {@code true} if the result code is expected to trigger the
   *         associated connection to be closed immediately.
   */
  private boolean isFatalResultCode(ResultCode resultCode)
  {
    switch (resultCode)
    {
    case BUSY:
    case UNAVAILABLE:
    case PROTOCOL_ERROR:
    case OTHER:
    case UNWILLING_TO_PERFORM:
    case OPERATIONS_ERROR:
      return true;
    default:
      return false;
    }
    provider.assertAllExpectedEventsReceived();
  }