Issue OPENDJ-262: Implement pass through authentication (PTA)

Matthew Swift

16.36.2011 6992706b3b92b889db4b3b603107a6ee9fd09f17

Issue OPENDJ-262: Implement pass through authentication (PTA)

* implement automatic retry of searches and binds
* implement periodic monitoring of offline servers in order to determine when they come back online.

 opends/src/server/org/opends/server/extensions/LDAPPassThroughAuthenticationPolicyFactory.java

@@ -37,8 +37,7 @@
import java.io.IOException;
import java.net.*;
import java.util.*;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.Semaphore;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
@@ -49,10 +48,7 @@
import org.opends.messages.Message;
import org.opends.server.admin.server.ConfigurationChangeListener;
import org.opends.server.admin.std.server.*;
import org.opends.server.api.AuthenticationPolicy;
import org.opends.server.api.AuthenticationPolicyFactory;
import org.opends.server.api.AuthenticationPolicyState;
import org.opends.server.api.TrustManagerProvider;
import org.opends.server.api.*;
import org.opends.server.config.ConfigException;
import org.opends.server.core.DirectoryServer;
import org.opends.server.loggers.debug.DebugLogger;
@@ -72,11 +68,353 @@
    AuthenticationPolicyFactory<LDAPPassThroughAuthenticationPolicyCfg>
{

  // TODO: retry operations transparently until all connections exhausted.
  // TODO: handle password policy response controls? AD?
  // TODO: periodically ping offline servers in order to detect when they come
  // back.
  // TODO: provide alternative cfg for search password.
  // TODO: custom aliveness pings
  // TODO: manage account lockout
  // TODO: cache password

  /**
   * A simplistic load-balancer connection factory implementation using
   * approximately round-robin balancing.
   */
  static abstract class AbstractLoadBalancer implements ConnectionFactory,
      Runnable
  {
    /**
     * A connection which automatically retries operations on other servers.
     */
    private final class FailoverConnection implements Connection
    {
      private Connection connection;
      private MonitoredConnectionFactory factory;
      private final int startIndex;
      private int nextIndex;



      private FailoverConnection(final int startIndex)
          throws DirectoryException
      {
        this.startIndex = nextIndex = startIndex;

        DirectoryException lastException = null;
        do
        {
          factory = factories[nextIndex];
          if (factory.isAvailable())
          {
            try
            {
              if (factory.isAvailable)
              {
                connection = factory.getConnection();
                incrementNextIndex();
                return;
              }
            }
            catch (final DirectoryException e)
            {
              // Ignore this error and try the next factory.
              if (debugEnabled())
              {
                TRACER.debugCaught(DebugLogLevel.ERROR, e);
              }
              lastException = e;
            }
          }
          incrementNextIndex();
        }
        while (nextIndex != startIndex);

        // All the factories have been tried so give up and throw the exception.
        throw lastException;
      }



      /**
       * {@inheritDoc}
       */
      @Override
      public void close()
      {
        connection.close();
      }



      /**
       * {@inheritDoc}
       */
      @Override
      public ByteString search(final DN baseDN, final SearchScope scope,
          final SearchFilter filter) throws DirectoryException
      {
        for (;;)
        {
          try
          {
            return connection.search(baseDN, scope, filter);
          }
          catch (final DirectoryException e)
          {
            if (debugEnabled())
            {
              TRACER.debugCaught(DebugLogLevel.ERROR, e);
            }
            handleDirectoryException(e);
          }
        }
      }



      /**
       * {@inheritDoc}
       */
      @Override
      public void simpleBind(final ByteString username,
          final ByteString password) throws DirectoryException
      {
        for (;;)
        {
          try
          {
            connection.simpleBind(username, password);
            return;
          }
          catch (final DirectoryException e)
          {
            if (debugEnabled())
            {
              TRACER.debugCaught(DebugLogLevel.ERROR, e);
            }
            handleDirectoryException(e);
          }
        }
      }



      private void handleDirectoryException(final DirectoryException e)
          throws DirectoryException
      {
        // If the error does not indicate that the connection has failed, then
        // pass this back to the caller.
        if (!isFatalResultCode(e.getResultCode()))
        {
          throw e;
        }

        // The associated server is unavailable, so close the connection and
        // try the next connection factory.
        connection.close();
        factory.isAvailable = false;

        while (nextIndex != startIndex)
        {
          factory = factories[nextIndex];
          if (factory.isAvailable())
          {
            try
            {
              if (factory.isAvailable)
              {
                connection = factory.getConnection();
                incrementNextIndex();
                return;
              }
            }
            catch (final DirectoryException de)
            {
              // Ignore this error and try the next factory.
              if (debugEnabled())
              {
                TRACER.debugCaught(DebugLogLevel.ERROR, de);
              }
            }
          }
          incrementNextIndex();
        }

        // All the factories have been tried so give up and throw the exception.
        throw e;
      }



      private void incrementNextIndex()
      {
        // Try the next index.
        if (++nextIndex == maxIndex)
        {
          nextIndex = 0;
        }
      }

    }



    /**
     * A connection factory which caches its online/offline state in order to
     * avoid unnecessary connection attempts when it is known to be offline.
     */
    private final class MonitoredConnectionFactory implements ConnectionFactory
    {
      private final ConnectionFactory factory;
      private volatile boolean isAvailable = true;



      private MonitoredConnectionFactory(final ConnectionFactory factory)
      {
        this.factory = factory;
      }



      /**
       * {@inheritDoc}
       */
      @Override
      public void close()
      {
        factory.close();
      }



      /**
       * {@inheritDoc}
       */
      @Override
      public Connection getConnection() throws DirectoryException
      {
        try
        {
          final Connection connection = factory.getConnection();
          isAvailable = true;
          return connection;
        }
        catch (final DirectoryException e)
        {
          if (debugEnabled())
          {
            TRACER.debugCaught(DebugLogLevel.ERROR, e);
          }
          isAvailable = false;
          throw e;
        }
      }



      private boolean isAvailable()
      {
        return isAvailable;
      }
    }



    private final MonitoredConnectionFactory[] factories;
    private final int maxIndex;
    private final ScheduledFuture<?> monitorFuture;



    /**
     * Creates a new abstract load-balancer.
     *
     * @param factories
     *          The list of underlying connection factories.
     * @param scheduler
     *          The monitoring scheduler.
     */
    AbstractLoadBalancer(final ConnectionFactory[] factories,
        final ScheduledExecutorService scheduler)
    {
      this.factories = new MonitoredConnectionFactory[factories.length];
      this.maxIndex = factories.length;

      for (int i = 0; i < maxIndex; i++)
      {
        this.factories[i] = new MonitoredConnectionFactory(factories[i]);
      }

      this.monitorFuture = scheduler.scheduleWithFixedDelay(this, 5, 5,
          TimeUnit.SECONDS);
    }



    /**
     * Close underlying connection pools.
     */
    @Override
    public final void close()
    {
      monitorFuture.cancel(true);

      for (final ConnectionFactory factory : factories)
      {
        factory.close();
      }
    }



    /**
     * {@inheritDoc}
     */
    @Override
    public final Connection getConnection() throws DirectoryException
    {
      final int startIndex = getStartIndex();
      return new FailoverConnection(startIndex);
    }



    /**
     * Try to connect to any offline connection factories.
     */
    @Override
    public void run()
    {
      for (final MonitoredConnectionFactory factory : factories)
      {
        if (!factory.isAvailable())
        {
          try
          {
            factory.getConnection().close();
          }
          catch (final DirectoryException e)
          {
            if (debugEnabled())
            {
              TRACER.debugCaught(DebugLogLevel.ERROR, e);
            }
          }
        }
      }
    }



    /**
     * Return the start which should be used for the next connection attempt.
     *
     * @return The start which should be used for the next connection attempt.
     */
    abstract int getStartIndex();

  }



  /**
   * A factory which returns pre-authenticated connections for searches.
@@ -245,7 +583,7 @@
   * <p>
   * Package private for testing.
   */
  static final class ConnectionPool implements ConnectionFactory, Closeable
  static final class ConnectionPool implements ConnectionFactory
  {

    /**
@@ -445,13 +783,8 @@
   * <p>
   * Package private for testing.
   */
  static final class FailoverConnectionFactory implements ConnectionFactory,
      Closeable
  static final class FailoverLoadBalancer extends AbstractLoadBalancer
  {
    private final ConnectionFactory primary;
    private final ConnectionFactory secondary;



    /**
     * Creates a new fail-over connection factory which will always try the
@@ -461,27 +794,14 @@
     *          The primary connection factory.
     * @param secondary
     *          The secondary connection factory.
     * @param scheduler
     *          The monitoring scheduler.
     */
    FailoverConnectionFactory(final ConnectionFactory primary,
        final ConnectionFactory secondary)
    FailoverLoadBalancer(final ConnectionFactory primary,
        final ConnectionFactory secondary,
        final ScheduledExecutorService scheduler)
    {
      this.primary = primary;
      this.secondary = secondary;
    }



    /**
     * Close underlying load-balancers.
     */
    @Override
    public void close()
    {
      primary.close();
      if (secondary != null)
      {
        secondary.close();
      }
      super(new ConnectionFactory[] { primary, secondary }, scheduler);
    }


@@ -490,24 +810,10 @@
     * {@inheritDoc}
     */
    @Override
    public Connection getConnection() throws DirectoryException
    int getStartIndex()
    {
      if (secondary == null)
      {
        // No fail-over so just use the primary.
        return primary.getConnection();
      }
      else
      {
        try
        {
          return primary.getConnection();
        }
        catch (final DirectoryException e)
        {
          return secondary.getConnection();
        }
      }
      // Always start with the primaries.
      return 0;
    }

  }
@@ -1116,9 +1422,9 @@

  /**
   * An interface for obtaining a connection factory for LDAP connections to a
   * named LDAP server.
   * named LDAP server and the monitoring scheduler.
   */
  static interface LDAPConnectionFactoryProvider
  static interface Provider
  {
    /**
     * Returns a connection factory which can be used for obtaining connections
@@ -1135,6 +1441,17 @@
     */
    ConnectionFactory getLDAPConnectionFactory(String host, int port,
        LDAPPassThroughAuthenticationPolicyCfg cfg);



    /**
     * Returns the scheduler which should be used to periodically ping
     * connection factories to determine when they are online.
     *
     * @return The scheduler which should be used to periodically ping
     *         connection factories to determine when they are online.
     */
    ScheduledExecutorService getScheduledExecutorService();
  }


@@ -1143,9 +1460,8 @@
   * A simplistic load-balancer connection factory implementation using
   * approximately round-robin balancing.
   */
  static final class LoadBalancer implements ConnectionFactory, Closeable
  static final class RoundRobinLoadBalancer extends AbstractLoadBalancer
  {
    private final ConnectionFactory[] factories;
    private final AtomicInteger nextIndex = new AtomicInteger();
    private final int maxIndex;

@@ -1157,67 +1473,23 @@
     *
     * @param factories
     *          The list of underlying connection factories.
     * @param scheduler
     *          The monitoring scheduler.
     */
    LoadBalancer(final ConnectionFactory[] factories)
    RoundRobinLoadBalancer(final ConnectionFactory[] factories,
        final ScheduledExecutorService scheduler)
    {
      this.factories = factories;
      super(factories, scheduler);
      this.maxIndex = factories.length;
    }



    /**
     * Close underlying connection pools.
     */
    @Override
    public void close()
    {
      for (final ConnectionFactory factory : factories)
      {
        factory.close();
      }
    }



    /**
     * {@inheritDoc}
     */
    @Override
    public Connection getConnection() throws DirectoryException
    {
      final int startIndex = getStartIndex();
      int index = startIndex;
      for (;;)
      {
        final ConnectionFactory factory = factories[index];

        try
        {
          return factory.getConnection();
        }
        catch (final DirectoryException e)
        {
          // Try the next index.
          if (++index == maxIndex)
          {
            index = 0;
          }

          // If all the factories have been tried then give up and throw the
          // exception.
          if (index == startIndex)
          {
            throw e;
          }
        }
      }
    }



    // Determine the start index.
    private int getStartIndex()
    int getStartIndex()
    {
      // A round robin pool of one connection factories is unlikely in
      // practice and requires special treatment.
@@ -1512,8 +1784,8 @@
    // Current configuration.
    private LDAPPassThroughAuthenticationPolicyCfg cfg;

    private FailoverConnectionFactory searchFactory = null;
    private FailoverConnectionFactory bindFactory = null;
    private ConnectionFactory searchFactory = null;
    private ConnectionFactory bindFactory = null;



@@ -1642,8 +1914,10 @@
      // authenticated user.

      // Create load-balancers for primary servers.
      final LoadBalancer primarySearchLoadBalancer;
      final LoadBalancer primaryBindLoadBalancer;
      final RoundRobinLoadBalancer primarySearchLoadBalancer;
      final RoundRobinLoadBalancer primaryBindLoadBalancer;
      final ScheduledExecutorService scheduler = provider
          .getScheduledExecutorService();

      Set<String> servers = cfg.getPrimaryRemoteLDAPServer();
      ConnectionPool[] searchPool = new ConnectionPool[servers.size()];
@@ -1658,18 +1932,16 @@
                cfg.getMappedSearchBindPassword()));
        bindPool[index++] = new ConnectionPool(factory);
      }
      primarySearchLoadBalancer = new LoadBalancer(searchPool);
      primaryBindLoadBalancer = new LoadBalancer(bindPool);
      primarySearchLoadBalancer = new RoundRobinLoadBalancer(searchPool,
          scheduler);
      primaryBindLoadBalancer = new RoundRobinLoadBalancer(bindPool, scheduler);

      // Create load-balancers for secondary servers.
      final LoadBalancer secondarySearchLoadBalancer;
      final LoadBalancer secondaryBindLoadBalancer;

      servers = cfg.getSecondaryRemoteLDAPServer();
      if (servers.isEmpty())
      {
        secondarySearchLoadBalancer = null;
        secondaryBindLoadBalancer = null;
        searchFactory = primarySearchLoadBalancer;
        bindFactory = primaryBindLoadBalancer;
      }
      else
      {
@@ -1685,14 +1957,15 @@
                  cfg.getMappedSearchBindPassword()));
          bindPool[index++] = new ConnectionPool(factory);
        }
        secondarySearchLoadBalancer = new LoadBalancer(searchPool);
        secondaryBindLoadBalancer = new LoadBalancer(bindPool);
        final RoundRobinLoadBalancer secondarySearchLoadBalancer =
          new RoundRobinLoadBalancer(searchPool, scheduler);
        final RoundRobinLoadBalancer secondaryBindLoadBalancer =
          new RoundRobinLoadBalancer(bindPool, scheduler);
        searchFactory = new FailoverLoadBalancer(primarySearchLoadBalancer,
            secondarySearchLoadBalancer, scheduler);
        bindFactory = new FailoverLoadBalancer(primaryBindLoadBalancer,
            secondaryBindLoadBalancer, scheduler);
      }

      searchFactory = new FailoverConnectionFactory(primarySearchLoadBalancer,
          secondarySearchLoadBalancer);
      bindFactory = new FailoverConnectionFactory(primaryBindLoadBalancer,
          secondaryBindLoadBalancer);
    }


@@ -1725,15 +1998,32 @@
  }

  // The provider which should be used by policies to create LDAP connections.
  private final LDAPConnectionFactoryProvider provider;
  private final Provider provider;

  /**
   * The default LDAP connection factory provider.
   */
  private static final LDAPConnectionFactoryProvider DEFAULT_PROVIDER =
    new LDAPConnectionFactoryProvider()
  private static final Provider DEFAULT_PROVIDER = new Provider()
  {

    // Global scheduler used for periodically monitoring connection factories in
    // order to detect when they are online.
    private final ScheduledExecutorService scheduler = Executors
        .newScheduledThreadPool(2, new ThreadFactory()
        {

          @Override
          public Thread newThread(final Runnable r)
          {
            final Thread t = new DirectoryThread(r,
                "LDAP PTA connection monitor thread");
            t.setDaemon(true);
            return t;
          }
        });



    @Override
    public ConnectionFactory getLDAPConnectionFactory(final String host,
        final int port, final LDAPPassThroughAuthenticationPolicyCfg cfg)
@@ -1741,6 +2031,14 @@
      return new LDAPConnectionFactory(host, port, cfg);
    }



    @Override
    public ScheduledExecutorService getScheduledExecutorService()
    {
      return scheduler;
    }

  };


@@ -1843,8 +2141,7 @@
   *          The LDAP connection factory provider implementation which LDAP PTA
   *          authentication policies will use.
   */
  LDAPPassThroughAuthenticationPolicyFactory(
      final LDAPConnectionFactoryProvider provider)
  LDAPPassThroughAuthenticationPolicyFactory(final Provider provider)
  {
    this.provider = provider;
  }

 opends/tests/unit-tests-testng/src/server/org/opends/server/extensions/LDAPPassThroughAuthenticationPolicyTestCase.java

@@ -36,8 +36,7 @@
import java.net.ServerSocket;
import java.net.Socket;
import java.util.*;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.*;

import org.opends.messages.Message;
import org.opends.server.TestCaseUtils;
@@ -385,6 +384,16 @@
    /**
     * {@inheritDoc}
     */
    public void close()
    {
      // Nothing to do.
    }



    /**
     * {@inheritDoc}
     */
    @Override
    public Connection getConnection() throws DirectoryException
    {
@@ -403,16 +412,6 @@
      }
    }



    /**
     * {@inheritDoc}
     */
    public void close()
    {
      // Nothing to do.
    }

  }


@@ -666,10 +665,236 @@


  static final class MockProvider implements
      LDAPPassThroughAuthenticationPolicyFactory.LDAPConnectionFactoryProvider
      LDAPPassThroughAuthenticationPolicyFactory.Provider
  {

    private final class MockScheduledFuture implements ScheduledFuture<Void>
    {
      private final Runnable runnable;



      MockScheduledFuture(final Runnable runnable)
      {
        this.runnable = runnable;
      }



      public boolean cancel(final boolean mayInterruptIfRunning)
      {
        monitorRunnables.remove(this);
        return true;
      }



      public int compareTo(final Delayed o)
      {
        return 0;
      }



      public Void get() throws InterruptedException, ExecutionException
      {
        return null;
      }



      public Void get(final long timeout, final TimeUnit unit)
          throws InterruptedException, ExecutionException, TimeoutException
      {
        return null;
      }



      public long getDelay(final TimeUnit unit)
      {
        return 0;
      }



      public boolean isCancelled()
      {
        return false;
      }



      public boolean isDone()
      {
        return false;
      }



      Runnable getRunnable()
      {
        return runnable;
      }
    }



    private final Queue<Event<?>> expectedEvents = new LinkedList<Event<?>>();
    private final List<MockScheduledFuture> monitorRunnables = new LinkedList<MockScheduledFuture>();

    // All methods unused excepted scheduleWithFixedDelay.
    private final ScheduledExecutorService mockScheduler = new ScheduledExecutorService()
    {

      @Override
      public boolean awaitTermination(final long timeout, final TimeUnit unit)
          throws InterruptedException
      {
        throw new UnsupportedOperationException();
      }



      @Override
      public void execute(final Runnable command)
      {
        throw new UnsupportedOperationException();
      }



      @Override
      public <T> List<Future<T>> invokeAll(
          final Collection<? extends Callable<T>> tasks)
          throws InterruptedException
      {
        throw new UnsupportedOperationException();
      }



      @Override
      public <T> List<Future<T>> invokeAll(
          final Collection<? extends Callable<T>> tasks, final long timeout,
          final TimeUnit unit) throws InterruptedException
      {
        throw new UnsupportedOperationException();
      }



      @Override
      public <T> T invokeAny(final Collection<? extends Callable<T>> tasks)
          throws InterruptedException, ExecutionException
      {
        throw new UnsupportedOperationException();
      }



      @Override
      public <T> T invokeAny(final Collection<? extends Callable<T>> tasks,
          final long timeout, final TimeUnit unit) throws InterruptedException,
          ExecutionException, TimeoutException
      {
        throw new UnsupportedOperationException();
      }



      @Override
      public boolean isShutdown()
      {
        return false;
      }



      @Override
      public boolean isTerminated()
      {
        return false;
      }



      @Override
      public <V> ScheduledFuture<V> schedule(final Callable<V> callable,
          final long delay, final TimeUnit unit)
      {
        throw new UnsupportedOperationException();
      }



      @Override
      public ScheduledFuture<?> schedule(final Runnable command,
          final long delay, final TimeUnit unit)
      {
        throw new UnsupportedOperationException();
      }



      @Override
      public ScheduledFuture<?> scheduleAtFixedRate(final Runnable command,
          final long initialDelay, final long period, final TimeUnit unit)
      {
        throw new UnsupportedOperationException();
      }



      @Override
      public ScheduledFuture<?> scheduleWithFixedDelay(final Runnable command,
          final long initialDelay, final long delay, final TimeUnit unit)
      {
        final MockScheduledFuture future = new MockScheduledFuture(command);
        monitorRunnables.add(future);
        return future;
      }



      @Override
      public void shutdown()
      {
        throw new UnsupportedOperationException();
      }



      @Override
      public List<Runnable> shutdownNow()
      {
        throw new UnsupportedOperationException();
      }



      @Override
      public <T> Future<T> submit(final Callable<T> task)
      {
        throw new UnsupportedOperationException();
      }



      @Override
      public Future<?> submit(final Runnable task)
      {
        throw new UnsupportedOperationException();
      }



      @Override
      public <T> Future<T> submit(final Runnable task, final T result)
      {
        throw new UnsupportedOperationException();
      }
    };



@@ -688,6 +913,16 @@



    /**
     * {@inheritDoc}
     */
    public ScheduledExecutorService getScheduledExecutorService()
    {
      return mockScheduler;
    }



    void assertAllExpectedEventsReceived()
    {
      assertTrue(expectedEvents.isEmpty());
@@ -717,6 +952,17 @@
      expectedEvents.add(expectedEvent);
      return this;
    }



    void runMonitorTasks()
    {
      for (final MockScheduledFuture task : monitorRunnables)
      {
        task.getRunnable().run();
      }
    }

  }


@@ -1517,6 +1763,468 @@


  /**
   * Tests fail-over between 2 primary servers then to the secondary data
   * center.
   *
   * @throws Exception
   *           If an unexpected exception occurred.
   */
  @Test(enabled = true)
  public void testFailOverOnConnect() throws Exception
  {
    // Mock configuration.
    final LDAPPassThroughAuthenticationPolicyCfg cfg = mockCfg()
        .withPrimaryServer(phost1).withPrimaryServer(phost2)
        .withSecondaryServer(shost1)
        .withMappingPolicy(MappingPolicy.MAPPED_SEARCH)
        .withMappedAttribute("uid").withBaseDN("o=ad");

    // Create all the events.
    final MockProvider provider = new MockProvider();

    // First of all the connection factories are created.
    final GetLDAPConnectionFactoryEvent fe1 = new GetLDAPConnectionFactoryEvent(
        phost1, cfg);
    final GetLDAPConnectionFactoryEvent fe2 = new GetLDAPConnectionFactoryEvent(
        phost2, cfg);
    final GetLDAPConnectionFactoryEvent fe3 = new GetLDAPConnectionFactoryEvent(
        shost1, cfg);
    provider.expectEvent(fe1).expectEvent(fe2).expectEvent(fe3);

    // Get connection for phost1, then search, then bind.
    final GetConnectionEvent ceSearch1 = new GetConnectionEvent(fe1,
        ResultCode.CLIENT_SIDE_CONNECT_ERROR);
    final GetConnectionEvent ceSearch2 = new GetConnectionEvent(fe2,
        ResultCode.CLIENT_SIDE_CONNECT_ERROR);
    final GetConnectionEvent ceSearch3 = new GetConnectionEvent(fe3);

    final GetConnectionEvent ceBind1 = new GetConnectionEvent(fe1,
        ResultCode.CLIENT_SIDE_CONNECT_ERROR);
    final GetConnectionEvent ceBind2 = new GetConnectionEvent(fe2,
        ResultCode.CLIENT_SIDE_CONNECT_ERROR);
    final GetConnectionEvent ceBind3 = new GetConnectionEvent(fe3);

    provider
        .expectEvent(ceSearch1)
        .expectEvent(ceSearch2)
        .expectEvent(ceSearch3)
        .expectEvent(
            new SimpleBindEvent(ceSearch3, searchBindDNString,
                "searchPassword", ResultCode.SUCCESS))
        .expectEvent(
            new SearchEvent(ceSearch3, "o=ad", SearchScope.WHOLE_SUBTREE,
                "(uid=aduser)", adDNString))
        .expectEvent(ceBind1)
        .expectEvent(ceBind2)
        .expectEvent(ceBind3)
        .expectEvent(
            new SimpleBindEvent(ceBind3, adDNString, userPassword,
                ResultCode.SUCCESS));

    // Repeat again using cached connection to shost1: search, then bind.

    // phost1 and phost2 will have been marked as failed and won't be tried.
    provider.expectEvent(
        new SearchEvent(ceSearch3, "o=ad", SearchScope.WHOLE_SUBTREE,
            "(uid=aduser)", adDNString)).expectEvent(
        new SimpleBindEvent(ceBind3, adDNString, userPassword,
            ResultCode.SUCCESS));

    // Now simulate monitor thread run in which phost2 is determined to be
    // available again.
    final GetConnectionEvent ceSearch2ok = new GetConnectionEvent(fe2);
    final GetConnectionEvent ceBind2ok = new GetConnectionEvent(fe2);
    provider
        .expectEvent(ceSearch1)
        .expectEvent(ceSearch2ok)
        .expectEvent(
            new SimpleBindEvent(ceSearch2ok, searchBindDNString,
                "searchPassword", ResultCode.SUCCESS)).expectEvent(ceBind1)
        .expectEvent(ceBind2ok);

    provider.expectEvent(
        new SearchEvent(ceSearch2ok, "o=ad", SearchScope.WHOLE_SUBTREE,
            "(uid=aduser)", adDNString)).expectEvent(
        new SimpleBindEvent(ceBind2ok, adDNString, userPassword,
            ResultCode.SUCCESS));

    // Connections should be cached until the policy is finalized.

    // Obtain policy and state.
    final LDAPPassThroughAuthenticationPolicyFactory factory = new LDAPPassThroughAuthenticationPolicyFactory(
        provider);
    assertTrue(factory.isConfigurationAcceptable(cfg, null));
    final AuthenticationPolicy policy = factory.createAuthenticationPolicy(cfg);

    // Authenticate 3 times test above fail-over.
    for (int i = 0; i < 3; i++)
    {
      final AuthenticationPolicyState state = policy
          .createAuthenticationPolicyState(userEntry);
      assertEquals(state.getAuthenticationPolicy(), policy);

      // Run monitor which should try to connect to phost1&2 and determine that
      // phost2 is available again.
      if (i == 2)
      {
        provider.runMonitorTasks();
      }

      // Perform authentication.
      assertTrue(state.passwordMatches(ByteString.valueOf(userPassword)));

      state.finalizeStateAfterBind();
    }

    // Cached connections should be closed when the policy is finalized
    // (primaries first, then secondaries).
    provider.expectEvent(new CloseEvent(ceSearch2ok));
    provider.expectEvent(new CloseEvent(ceSearch3));
    provider.expectEvent(new CloseEvent(ceBind2ok));
    provider.expectEvent(new CloseEvent(ceBind3));

    // Tear down and check final state.
    policy.finalizeAuthenticationPolicy();
    provider.assertAllExpectedEventsReceived();
  }



  /**
   * Tests that searches which fail in one LB pool are automatically retried in
   * the secondary LB pool.
   *
   * @throws Exception
   *           If an unexpected exception occurred.
   */
  @Test(enabled = true)
  public void testFBRetrySearchOnFailure() throws Exception
  {
    // Mock configuration.
    final LDAPPassThroughAuthenticationPolicyCfg cfg = mockCfg()
        .withPrimaryServer(phost1).withSecondaryServer(shost1)
        .withMappingPolicy(MappingPolicy.MAPPED_SEARCH)
        .withMappedAttribute("uid").withBaseDN("o=ad");

    // Create all the events.
    final MockProvider provider = new MockProvider();

    // First of all the connection factories are created.
    final GetLDAPConnectionFactoryEvent fe1 = new GetLDAPConnectionFactoryEvent(
        phost1, cfg);
    final GetLDAPConnectionFactoryEvent fe2 = new GetLDAPConnectionFactoryEvent(
        shost1, cfg);
    provider.expectEvent(fe1).expectEvent(fe2);

    // Get connection for phost1, then search (fail), and retry on shost1
    final GetConnectionEvent ceSearch1 = new GetConnectionEvent(fe1);
    final GetConnectionEvent ceSearch2 = new GetConnectionEvent(fe2);
    final GetConnectionEvent ceBind1 = new GetConnectionEvent(fe1);

    provider
        .expectEvent(ceSearch1)
        .expectEvent(
            new SimpleBindEvent(ceSearch1, searchBindDNString,
                "searchPassword", ResultCode.SUCCESS))
        .expectEvent(
            new SearchEvent(ceSearch1, "o=ad", SearchScope.WHOLE_SUBTREE,
                "(uid=aduser)", ResultCode.UNAVAILABLE))
        .expectEvent(new CloseEvent(ceSearch1))
        .expectEvent(ceSearch2)
        .expectEvent(
            new SimpleBindEvent(ceSearch2, searchBindDNString,
                "searchPassword", ResultCode.SUCCESS))
        .expectEvent(
            new SearchEvent(ceSearch2, "o=ad", SearchScope.WHOLE_SUBTREE,
                "(uid=aduser)", adDNString))
        .expectEvent(ceBind1)
        .expectEvent(
            new SimpleBindEvent(ceBind1, adDNString, userPassword,
                ResultCode.SUCCESS));

    // Now simulate shost1 going down as well.

    // phost1 will have been marked as failed and won't be retried.
    provider.expectEvent(
        new SearchEvent(ceSearch2, "o=ad", SearchScope.WHOLE_SUBTREE,
            "(uid=aduser)", ResultCode.UNAVAILABLE)).expectEvent(
        new CloseEvent(ceSearch2));

    // Now simulate phost1 coming back and fail back to it.

    // Now simulate monitor thread run in which phost1 and shost1 are determined
    // to be available again.
    provider
        .expectEvent(ceSearch1)
        .expectEvent(
            new SimpleBindEvent(ceSearch1, searchBindDNString,
                "searchPassword", ResultCode.SUCCESS))
        .expectEvent(ceSearch2)
        .expectEvent(
            new SimpleBindEvent(ceSearch2, searchBindDNString,
                "searchPassword", ResultCode.SUCCESS));

    provider.expectEvent(
        new SearchEvent(ceSearch1, "o=ad", SearchScope.WHOLE_SUBTREE,
            "(uid=aduser)", adDNString)).expectEvent(
        new SimpleBindEvent(ceBind1, adDNString, userPassword,
            ResultCode.SUCCESS));

    // Connections should be cached until the policy is finalized.

    // Obtain policy and state.
    final LDAPPassThroughAuthenticationPolicyFactory factory = new LDAPPassThroughAuthenticationPolicyFactory(
        provider);
    assertTrue(factory.isConfigurationAcceptable(cfg, null));
    final AuthenticationPolicy policy = factory.createAuthenticationPolicy(cfg);

    // Authenticate 3 times, second should fail.
    for (int i = 0; i < 3; i++)
    {
      final AuthenticationPolicyState state = policy
          .createAuthenticationPolicyState(userEntry);
      assertEquals(state.getAuthenticationPolicy(), policy);

      // Perform authentication.
      switch (i)
      {
      case 0:
        // First attempt should succeed.
        assertTrue(state.passwordMatches(ByteString.valueOf(userPassword)));
        break;
      case 1:
        // Second attempt should fail.
        try
        {
          state.passwordMatches(ByteString.valueOf(userPassword));
          fail("password match unexpectedly succeeded");
        }
        catch (final DirectoryException e)
        {
          // No valid connections available so this should always fail with
          // INVALID_CREDENTIALS.
          assertEquals(e.getResultCode(), ResultCode.INVALID_CREDENTIALS,
              e.getMessage());
        }
        break;
      case 2:
        // Third attempt should succeed, once the monitor has run.
        provider.runMonitorTasks();
        assertTrue(state.passwordMatches(ByteString.valueOf(userPassword)));
        break;
      }

      state.finalizeStateAfterBind();
    }

    // Cached connections should be closed when the policy is finalized.
    provider.expectEvent(new CloseEvent(ceSearch1));
    provider.expectEvent(new CloseEvent(ceSearch2));
    provider.expectEvent(new CloseEvent(ceBind1));

    // Tear down and check final state.
    policy.finalizeAuthenticationPolicy();
    provider.assertAllExpectedEventsReceived();
  }



  /**
   * Tests configuration validation.
   *
   * @param cfg
   *          The configuration to be tested.
   * @param isValid
   *          Whether or not the provided configuration is valid.
   * @throws Exception
   *           If an unexpected exception occurred.
   */
  @Test(enabled = true, dataProvider = "testIsConfigurationAcceptableData")
  public void testIsConfigurationAcceptable(
      final LDAPPassThroughAuthenticationPolicyCfg cfg, final boolean isValid)
      throws Exception
  {
    final LDAPPassThroughAuthenticationPolicyFactory factory = new LDAPPassThroughAuthenticationPolicyFactory();
    assertEquals(
        factory.isConfigurationAcceptable(cfg, new LinkedList<Message>()),
        isValid);
  }



  /**
   * Returns test data for {@link #testIsConfigurationAcceptable}.
   *
   * @return Test data for {@link #testIsConfigurationAcceptable}.
   */
  @DataProvider
  public Object[][] testIsConfigurationAcceptableData()
  {
    // @formatter:off
    return new Object[][] {
        /* cfg, isValid */
        { mockCfg().withPrimaryServer("test:1"), true },
        { mockCfg().withPrimaryServer("test:65535"), true },
        { mockCfg().withPrimaryServer("test:0"), false },
        { mockCfg().withPrimaryServer("test:65536"), false },
        { mockCfg().withPrimaryServer("test:1000000"), false },
        { mockCfg().withSecondaryServer("test:1"), true },
        { mockCfg().withSecondaryServer("test:65535"), true },
        { mockCfg().withSecondaryServer("test:0"), false },
        { mockCfg().withSecondaryServer("test:65536"), false },
        { mockCfg().withSecondaryServer("test:1000000"), false },
    };
    // @formatter:on
  }



  /**
   * Tests that searches which fail on one server are automatically retried on
   * another within the same LB.
   *
   * @throws Exception
   *           If an unexpected exception occurred.
   */
  @Test(enabled = true)
  public void testLBRetrySearchOnFailure() throws Exception
  {
    // Mock configuration.
    final LDAPPassThroughAuthenticationPolicyCfg cfg = mockCfg()
        .withPrimaryServer(phost1).withPrimaryServer(phost2)
        .withMappingPolicy(MappingPolicy.MAPPED_SEARCH)
        .withMappedAttribute("uid").withBaseDN("o=ad");

    // Create all the events.
    final MockProvider provider = new MockProvider();

    // First of all the connection factories are created.
    final GetLDAPConnectionFactoryEvent fe1 = new GetLDAPConnectionFactoryEvent(
        phost1, cfg);
    final GetLDAPConnectionFactoryEvent fe2 = new GetLDAPConnectionFactoryEvent(
        phost2, cfg);
    provider.expectEvent(fe1).expectEvent(fe2);

    // Get connection for phost1, then search (fail), and retry on phost2
    final GetConnectionEvent ceSearch1 = new GetConnectionEvent(fe1);
    final GetConnectionEvent ceSearch2 = new GetConnectionEvent(fe2);
    final GetConnectionEvent ceBind1 = new GetConnectionEvent(fe1);
    final GetConnectionEvent ceBind2 = new GetConnectionEvent(fe2);

    provider
        .expectEvent(ceSearch1)
        .expectEvent(
            new SimpleBindEvent(ceSearch1, searchBindDNString,
                "searchPassword", ResultCode.SUCCESS))
        .expectEvent(
            new SearchEvent(ceSearch1, "o=ad", SearchScope.WHOLE_SUBTREE,
                "(uid=aduser)", ResultCode.UNAVAILABLE))
        .expectEvent(new CloseEvent(ceSearch1))
        .expectEvent(ceSearch2)
        .expectEvent(
            new SimpleBindEvent(ceSearch2, searchBindDNString,
                "searchPassword", ResultCode.SUCCESS))
        .expectEvent(
            new SearchEvent(ceSearch2, "o=ad", SearchScope.WHOLE_SUBTREE,
                "(uid=aduser)", adDNString))
        .expectEvent(ceBind1)
        .expectEvent(
            new SimpleBindEvent(ceBind1, adDNString, userPassword,
                ResultCode.SUCCESS));

    // Now simulate phost2 going down as well.

    // phost1 will have been marked as failed and won't be retried.
    provider.expectEvent(
        new SearchEvent(ceSearch2, "o=ad", SearchScope.WHOLE_SUBTREE,
            "(uid=aduser)", ResultCode.UNAVAILABLE)).expectEvent(
        new CloseEvent(ceSearch2));

    // Now simulate phost1 coming back and fail back to it.

    // Now simulate monitor thread run in which phost1 and shost1 are determined
    // to be available again.
    provider
        .expectEvent(ceSearch1)
        .expectEvent(
            new SimpleBindEvent(ceSearch1, searchBindDNString,
                "searchPassword", ResultCode.SUCCESS))
        .expectEvent(ceSearch2)
        .expectEvent(
            new SimpleBindEvent(ceSearch2, searchBindDNString,
                "searchPassword", ResultCode.SUCCESS));

    // Note that the bind will be load-balanced.
    provider
        .expectEvent(
            new SearchEvent(ceSearch1, "o=ad", SearchScope.WHOLE_SUBTREE,
                "(uid=aduser)", adDNString))
        .expectEvent(ceBind2)
        .expectEvent(
            new SimpleBindEvent(ceBind2, adDNString, userPassword,
                ResultCode.SUCCESS));

    // Connections should be cached until the policy is finalized.

    // Obtain policy and state.
    final LDAPPassThroughAuthenticationPolicyFactory factory = new LDAPPassThroughAuthenticationPolicyFactory(
        provider);
    assertTrue(factory.isConfigurationAcceptable(cfg, null));
    final AuthenticationPolicy policy = factory.createAuthenticationPolicy(cfg);

    // Authenticate 3 times, second should fail.
    for (int i = 0; i < 3; i++)
    {
      final AuthenticationPolicyState state = policy
          .createAuthenticationPolicyState(userEntry);
      assertEquals(state.getAuthenticationPolicy(), policy);

      // Perform authentication.
      switch (i)
      {
      case 0:
        // First attempt should succeed.
        assertTrue(state.passwordMatches(ByteString.valueOf(userPassword)));
        break;
      case 1:
        // Second attempt should fail.
        try
        {
          state.passwordMatches(ByteString.valueOf(userPassword));
          fail("password match unexpectedly succeeded");
        }
        catch (final DirectoryException e)
        {
          // No valid connections available so this should always fail with
          // INVALID_CREDENTIALS.
          assertEquals(e.getResultCode(), ResultCode.INVALID_CREDENTIALS,
              e.getMessage());
        }
        break;
      case 2:
        // Third attempt should succeed, once the monitor has run.
        provider.runMonitorTasks();
        assertTrue(state.passwordMatches(ByteString.valueOf(userPassword)));
        break;
      }

      state.finalizeStateAfterBind();
    }

    // Cached connections should be closed when the policy is finalized.
    provider.expectEvent(new CloseEvent(ceSearch1));
    provider.expectEvent(new CloseEvent(ceSearch2));
    provider.expectEvent(new CloseEvent(ceBind1));
    provider.expectEvent(new CloseEvent(ceBind2));

    // Tear down and check final state.
    policy.finalizeAuthenticationPolicy();
    provider.assertAllExpectedEventsReceived();
  }



  /**
   * Tests valid bind which times out at the client. These should trigger a
   * CLIENT_SIDE_TIMEOUT result code.
   *
@@ -2339,6 +3047,139 @@


  /**
   * Tests load balancing across 3 servers.
   *
   * @throws Exception
   *           If an unexpected exception occurred.
   */
  @Test(enabled = true)
  public void testLoadBalancing() throws Exception
  {
    // Mock configuration.
    final LDAPPassThroughAuthenticationPolicyCfg cfg = mockCfg()
        .withPrimaryServer(phost1).withPrimaryServer(phost2)
        .withPrimaryServer(phost3)
        .withMappingPolicy(MappingPolicy.MAPPED_SEARCH)
        .withMappedAttribute("uid").withBaseDN("o=ad");

    // Create all the events.
    final MockProvider provider = new MockProvider();

    // First of all the connection factories are created.
    final GetLDAPConnectionFactoryEvent fe1 = new GetLDAPConnectionFactoryEvent(
        phost1, cfg);
    final GetLDAPConnectionFactoryEvent fe2 = new GetLDAPConnectionFactoryEvent(
        phost2, cfg);
    final GetLDAPConnectionFactoryEvent fe3 = new GetLDAPConnectionFactoryEvent(
        phost3, cfg);
    provider.expectEvent(fe1).expectEvent(fe2).expectEvent(fe3);

    // Get connection for phost1, then search, then bind.
    final GetConnectionEvent ceSearch1 = new GetConnectionEvent(fe1);
    final GetConnectionEvent ceBind1 = new GetConnectionEvent(fe1);
    provider
        .expectEvent(ceSearch1)
        .expectEvent(
            new SimpleBindEvent(ceSearch1, searchBindDNString,
                "searchPassword", ResultCode.SUCCESS))
        .expectEvent(
            new SearchEvent(ceSearch1, "o=ad", SearchScope.WHOLE_SUBTREE,
                "(uid=aduser)", adDNString))
        .expectEvent(ceBind1)
        .expectEvent(
            new SimpleBindEvent(ceBind1, adDNString, userPassword,
                ResultCode.SUCCESS));

    // Get connection for phost2, then search, then bind.
    final GetConnectionEvent ceSearch2 = new GetConnectionEvent(fe2);
    final GetConnectionEvent ceBind2 = new GetConnectionEvent(fe2);
    provider
        .expectEvent(ceSearch2)
        .expectEvent(
            new SimpleBindEvent(ceSearch2, searchBindDNString,
                "searchPassword", ResultCode.SUCCESS))
        .expectEvent(
            new SearchEvent(ceSearch2, "o=ad", SearchScope.WHOLE_SUBTREE,
                "(uid=aduser)", adDNString))
        .expectEvent(ceBind2)
        .expectEvent(
            new SimpleBindEvent(ceBind2, adDNString, userPassword,
                ResultCode.SUCCESS));

    // Get connection for phost3, then search, then bind.
    final GetConnectionEvent ceSearch3 = new GetConnectionEvent(fe3);
    final GetConnectionEvent ceBind3 = new GetConnectionEvent(fe3);
    provider
        .expectEvent(ceSearch3)
        .expectEvent(
            new SimpleBindEvent(ceSearch3, searchBindDNString,
                "searchPassword", ResultCode.SUCCESS))
        .expectEvent(
            new SearchEvent(ceSearch3, "o=ad", SearchScope.WHOLE_SUBTREE,
                "(uid=aduser)", adDNString))
        .expectEvent(ceBind3)
        .expectEvent(
            new SimpleBindEvent(ceBind3, adDNString, userPassword,
                ResultCode.SUCCESS));

    // Repeat again using cached connection to phost1: search, then bind.
    provider.expectEvent(
        new SearchEvent(ceSearch1, "o=ad", SearchScope.WHOLE_SUBTREE,
            "(uid=aduser)", adDNString)).expectEvent(
        new SimpleBindEvent(ceBind1, adDNString, userPassword,
            ResultCode.SUCCESS));

    // Repeat again using cached connection to phost2: search, then bind.
    provider.expectEvent(
        new SearchEvent(ceSearch2, "o=ad", SearchScope.WHOLE_SUBTREE,
            "(uid=aduser)", adDNString)).expectEvent(
        new SimpleBindEvent(ceBind2, adDNString, userPassword,
            ResultCode.SUCCESS));

    // Repeat again using cached connection to phost3: search, then bind.
    provider.expectEvent(
        new SearchEvent(ceSearch3, "o=ad", SearchScope.WHOLE_SUBTREE,
            "(uid=aduser)", adDNString)).expectEvent(
        new SimpleBindEvent(ceBind3, adDNString, userPassword,
            ResultCode.SUCCESS));

    // Connections should be cached until the policy is finalized.

    // Obtain policy and state.
    final LDAPPassThroughAuthenticationPolicyFactory factory = new LDAPPassThroughAuthenticationPolicyFactory(
        provider);
    assertTrue(factory.isConfigurationAcceptable(cfg, null));
    final AuthenticationPolicy policy = factory.createAuthenticationPolicy(cfg);

    // Cycle twice through the LB pool.
    for (int i = 0; i < 6; i++)
    {
      final AuthenticationPolicyState state = policy
          .createAuthenticationPolicyState(userEntry);
      assertEquals(state.getAuthenticationPolicy(), policy);

      // Perform authentication.
      assertTrue(state.passwordMatches(ByteString.valueOf(userPassword)));

      state.finalizeStateAfterBind();
    }

    // Cached connections should be closed when the policy is finalized.
    provider.expectEvent(new CloseEvent(ceSearch1));
    provider.expectEvent(new CloseEvent(ceSearch2));
    provider.expectEvent(new CloseEvent(ceSearch3));
    provider.expectEvent(new CloseEvent(ceBind1));
    provider.expectEvent(new CloseEvent(ceBind2));
    provider.expectEvent(new CloseEvent(ceBind3));

    // Tear down and check final state.
    policy.finalizeAuthenticationPolicy();
    provider.assertAllExpectedEventsReceived();
  }



  /**
   * Tests the different mapping policies: connection attempts will succeed, as
   * will any searches, but the final user bind may or may not succeed depending
   * on the provided result code.
@@ -2486,123 +3327,63 @@


  /**
   * Tests configuration validation.
   *
   * @param cfg
   *          The configuration to be tested.
   * @param isValid
   *          Whether or not the provided configuration is valid.
   * @throws Exception
   *           If an unexpected exception occurred.
   */
  @Test(enabled = true, dataProvider = "testIsConfigurationAcceptableData")
  public void testIsConfigurationAcceptable(
      final LDAPPassThroughAuthenticationPolicyCfg cfg, final boolean isValid)
      throws Exception
  {
    final LDAPPassThroughAuthenticationPolicyFactory factory = new LDAPPassThroughAuthenticationPolicyFactory();
    assertEquals(
        factory.isConfigurationAcceptable(cfg, new LinkedList<Message>()),
        isValid);
  }



  /**
   * Returns test data for {@link #testIsConfigurationAcceptable}.
   *
   * @return Test data for {@link #testIsConfigurationAcceptable}.
   */
  @DataProvider
  public Object[][] testIsConfigurationAcceptableData()
  {
    // @formatter:off
    return new Object[][] {
        /* cfg, isValid */
        { mockCfg().withPrimaryServer("test:1"), true },
        { mockCfg().withPrimaryServer("test:65535"), true },
        { mockCfg().withPrimaryServer("test:0"), false },
        { mockCfg().withPrimaryServer("test:65536"), false },
        { mockCfg().withPrimaryServer("test:1000000"), false },
        { mockCfg().withSecondaryServer("test:1"), true },
        { mockCfg().withSecondaryServer("test:65535"), true },
        { mockCfg().withSecondaryServer("test:0"), false },
        { mockCfg().withSecondaryServer("test:65536"), false },
        { mockCfg().withSecondaryServer("test:1000000"), false },
    };
    // @formatter:on
  }



  /**
   * Tests that mapped PTA performs searches across multiple base DNs if
   * configured.
   * Tests that mapped PTA fails when no match attribute values are found.
   *
   * @throws Exception
   *           If an unexpected exception occurred.
   */
  @Test(enabled = true)
  public void testMultipleSearchBaseDNs() throws Exception
  public void testMissingMappingAttributes() throws Exception
  {
    // Mock configuration.
    final LDAPPassThroughAuthenticationPolicyCfg cfg = mockCfg()
        .withPrimaryServer(phost1)
        .withMappingPolicy(MappingPolicy.MAPPED_SEARCH)
        .withMappedAttribute("uid").withBaseDN("o=first")
        .withBaseDN("o=second").withBaseDN("o=third");
        .withMappedAttribute("uid1").withBaseDN("o=ad");

    // Create the provider and its list of expected events.
    final GetLDAPConnectionFactoryEvent fe = new GetLDAPConnectionFactoryEvent(
        phost1, cfg);
    final MockProvider provider = new MockProvider().expectEvent(fe);

    // Add search events.
    GetConnectionEvent ceSearch = new GetConnectionEvent(fe);
    provider
        .expectEvent(ceSearch)
        .expectEvent(
            new SimpleBindEvent(ceSearch, searchBindDNString, "searchPassword",
                ResultCode.SUCCESS))
        .expectEvent(
            new SearchEvent(ceSearch, "o=first", SearchScope.WHOLE_SUBTREE,
                "(uid=aduser)", ResultCode.CLIENT_SIDE_NO_RESULTS_RETURNED))
        .expectEvent(
            new SearchEvent(ceSearch, "o=second", SearchScope.WHOLE_SUBTREE,
                "(uid=aduser)", ResultCode.CLIENT_SIDE_NO_RESULTS_RETURNED))
        .expectEvent(
            new SearchEvent(ceSearch, "o=third", SearchScope.WHOLE_SUBTREE,
                "(uid=aduser)", adDNString));
    // Connection should be cached until the policy is finalized.

    // Add bind events.
    final GetConnectionEvent ceBind = new GetConnectionEvent(fe);
    provider.expectEvent(ceBind).expectEvent(
        new SimpleBindEvent(ceBind, adDNString, userPassword,
            ResultCode.SUCCESS));

    // Connection should be cached until the policy is finalized.

    // Obtain policy and state.
    final Entry testUser = TestCaseUtils.makeEntry(
        /* @formatter:off */
        "dn: " + opendjDNString,
        "objectClass: top",
        "objectClass: person",
        "sn: user",
        "cn: test user",
        "aduser: " + adDNString
        /* @formatter:on */
    );

    final LDAPPassThroughAuthenticationPolicyFactory factory = new LDAPPassThroughAuthenticationPolicyFactory(
        provider);
    assertTrue(factory.isConfigurationAcceptable(cfg, null));
    final AuthenticationPolicy policy = factory.createAuthenticationPolicy(cfg);
    final AuthenticationPolicyState state = policy
        .createAuthenticationPolicyState(userEntry);
        .createAuthenticationPolicyState(testUser);
    assertEquals(state.getAuthenticationPolicy(), policy);

    // Perform authentication.
    assertTrue(state.passwordMatches(ByteString.valueOf(userPassword)));
    try
    {
      state.passwordMatches(ByteString.valueOf(userPassword));
      fail("password match unexpectedly succeeded");
    }
    catch (final DirectoryException e)
    {
      // No mapping attributes so this should always fail with
      // INVALID_CREDENTIALS.
      assertEquals(e.getResultCode(), ResultCode.INVALID_CREDENTIALS,
          e.getMessage());
    }

    // There should be no more pending events.
    provider.assertAllExpectedEventsReceived();
    state.finalizeStateAfterBind();

    // Cached connections should be closed when the policy is finalized.
    provider.expectEvent(new CloseEvent(ceSearch));
    provider.expectEvent(new CloseEvent(ceBind));

    // Tear down and check final state.
    policy.finalizeAuthenticationPolicy();
    provider.assertAllExpectedEventsReceived();
@@ -2633,7 +3414,7 @@
    final MockProvider provider = new MockProvider().expectEvent(fe);

    // Add search events.
    GetConnectionEvent ceSearch = new GetConnectionEvent(fe);
    final GetConnectionEvent ceSearch = new GetConnectionEvent(fe);
    provider
        .expectEvent(ceSearch)
        .expectEvent(
@@ -2653,7 +3434,7 @@
    // Connection should be cached until the policy is finalized.

    // Obtain policy and state.
    Entry testUser = TestCaseUtils.makeEntry(
    final Entry testUser = TestCaseUtils.makeEntry(
        /* @formatter:off */
        "dn: " + opendjDNString,
        "objectClass: top",
@@ -2715,7 +3496,7 @@
    final MockProvider provider = new MockProvider().expectEvent(fe);

    // Add search events.
    GetConnectionEvent ceSearch = new GetConnectionEvent(fe);
    final GetConnectionEvent ceSearch = new GetConnectionEvent(fe);
    provider
        .expectEvent(ceSearch)
        .expectEvent(
@@ -2735,7 +3516,7 @@
    // Connection should be cached until the policy is finalized.

    // Obtain policy and state.
    Entry testUser = TestCaseUtils.makeEntry(
    final Entry testUser = TestCaseUtils.makeEntry(
        /* @formatter:off */
        "dn: " + opendjDNString,
        "objectClass: top",
@@ -2776,594 +3557,72 @@


  /**
   * Tests that mapped PTA fails when no match attribute values are found.
   * Tests that mapped PTA performs searches across multiple base DNs if
   * configured.
   *
   * @throws Exception
   *           If an unexpected exception occurred.
   */
  @Test(enabled = true)
  public void testMissingMappingAttributes() throws Exception
  public void testMultipleSearchBaseDNs() throws Exception
  {
    // Mock configuration.
    final LDAPPassThroughAuthenticationPolicyCfg cfg = mockCfg()
        .withPrimaryServer(phost1)
        .withMappingPolicy(MappingPolicy.MAPPED_SEARCH)
        .withMappedAttribute("uid1").withBaseDN("o=ad");
        .withMappedAttribute("uid").withBaseDN("o=first")
        .withBaseDN("o=second").withBaseDN("o=third");

    // Create the provider and its list of expected events.
    final GetLDAPConnectionFactoryEvent fe = new GetLDAPConnectionFactoryEvent(
        phost1, cfg);
    final MockProvider provider = new MockProvider().expectEvent(fe);

    // Obtain policy and state.
    Entry testUser = TestCaseUtils.makeEntry(
        /* @formatter:off */
        "dn: " + opendjDNString,
        "objectClass: top",
        "objectClass: person",
        "sn: user",
        "cn: test user",
        "aduser: " + adDNString
        /* @formatter:on */
    );
    // Add search events.
    final GetConnectionEvent ceSearch = new GetConnectionEvent(fe);
    provider
        .expectEvent(ceSearch)
        .expectEvent(
            new SimpleBindEvent(ceSearch, searchBindDNString, "searchPassword",
                ResultCode.SUCCESS))
        .expectEvent(
            new SearchEvent(ceSearch, "o=first", SearchScope.WHOLE_SUBTREE,
                "(uid=aduser)", ResultCode.CLIENT_SIDE_NO_RESULTS_RETURNED))
        .expectEvent(
            new SearchEvent(ceSearch, "o=second", SearchScope.WHOLE_SUBTREE,
                "(uid=aduser)", ResultCode.CLIENT_SIDE_NO_RESULTS_RETURNED))
        .expectEvent(
            new SearchEvent(ceSearch, "o=third", SearchScope.WHOLE_SUBTREE,
                "(uid=aduser)", adDNString));
    // Connection should be cached until the policy is finalized.

    // Add bind events.
    final GetConnectionEvent ceBind = new GetConnectionEvent(fe);
    provider.expectEvent(ceBind).expectEvent(
        new SimpleBindEvent(ceBind, adDNString, userPassword,
            ResultCode.SUCCESS));

    // Connection should be cached until the policy is finalized.

    // Obtain policy and state.
    final LDAPPassThroughAuthenticationPolicyFactory factory = new LDAPPassThroughAuthenticationPolicyFactory(
        provider);
    assertTrue(factory.isConfigurationAcceptable(cfg, null));
    final AuthenticationPolicy policy = factory.createAuthenticationPolicy(cfg);
    final AuthenticationPolicyState state = policy
        .createAuthenticationPolicyState(testUser);
        .createAuthenticationPolicyState(userEntry);
    assertEquals(state.getAuthenticationPolicy(), policy);

    // Perform authentication.
    try
    {
      state.passwordMatches(ByteString.valueOf(userPassword));
      fail("password match unexpectedly succeeded");
    }
    catch (final DirectoryException e)
    {
      // No mapping attributes so this should always fail with
      // INVALID_CREDENTIALS.
      assertEquals(e.getResultCode(), ResultCode.INVALID_CREDENTIALS,
          e.getMessage());
    }
    assertTrue(state.passwordMatches(ByteString.valueOf(userPassword)));

    // There should be no more pending events.
    provider.assertAllExpectedEventsReceived();
    state.finalizeStateAfterBind();

    // Tear down and check final state.
    policy.finalizeAuthenticationPolicy();
    provider.assertAllExpectedEventsReceived();
  }



  /**
   * Tests load balancing across 3 servers.
   *
   * @throws Exception
   *           If an unexpected exception occurred.
   */
  @Test(enabled = true)
  public void testLoadBalancing() throws Exception
  {
    // Mock configuration.
    final LDAPPassThroughAuthenticationPolicyCfg cfg = mockCfg()
        .withPrimaryServer(phost1).withPrimaryServer(phost2)
        .withPrimaryServer(phost3)
        .withMappingPolicy(MappingPolicy.MAPPED_SEARCH)
        .withMappedAttribute("uid").withBaseDN("o=ad");

    // Create all the events.
    final MockProvider provider = new MockProvider();

    // First of all the connection factories are created.
    final GetLDAPConnectionFactoryEvent fe1 = new GetLDAPConnectionFactoryEvent(
        phost1, cfg);
    final GetLDAPConnectionFactoryEvent fe2 = new GetLDAPConnectionFactoryEvent(
        phost2, cfg);
    final GetLDAPConnectionFactoryEvent fe3 = new GetLDAPConnectionFactoryEvent(
        phost3, cfg);
    provider.expectEvent(fe1).expectEvent(fe2).expectEvent(fe3);

    // Get connection for phost1, then search, then bind.
    final GetConnectionEvent ceSearch1 = new GetConnectionEvent(fe1);
    final GetConnectionEvent ceBind1 = new GetConnectionEvent(fe1);
    provider
        .expectEvent(ceSearch1)
        .expectEvent(
            new SimpleBindEvent(ceSearch1, searchBindDNString,
                "searchPassword", ResultCode.SUCCESS))
        .expectEvent(
            new SearchEvent(ceSearch1, "o=ad", SearchScope.WHOLE_SUBTREE,
                "(uid=aduser)", adDNString))
        .expectEvent(ceBind1)
        .expectEvent(
            new SimpleBindEvent(ceBind1, adDNString, userPassword,
                ResultCode.SUCCESS));

    // Get connection for phost2, then search, then bind.
    final GetConnectionEvent ceSearch2 = new GetConnectionEvent(fe2);
    final GetConnectionEvent ceBind2 = new GetConnectionEvent(fe2);
    provider
        .expectEvent(ceSearch2)
        .expectEvent(
            new SimpleBindEvent(ceSearch2, searchBindDNString,
                "searchPassword", ResultCode.SUCCESS))
        .expectEvent(
            new SearchEvent(ceSearch2, "o=ad", SearchScope.WHOLE_SUBTREE,
                "(uid=aduser)", adDNString))
        .expectEvent(ceBind2)
        .expectEvent(
            new SimpleBindEvent(ceBind2, adDNString, userPassword,
                ResultCode.SUCCESS));

    // Get connection for phost3, then search, then bind.
    final GetConnectionEvent ceSearch3 = new GetConnectionEvent(fe3);
    final GetConnectionEvent ceBind3 = new GetConnectionEvent(fe3);
    provider
        .expectEvent(ceSearch3)
        .expectEvent(
            new SimpleBindEvent(ceSearch3, searchBindDNString,
                "searchPassword", ResultCode.SUCCESS))
        .expectEvent(
            new SearchEvent(ceSearch3, "o=ad", SearchScope.WHOLE_SUBTREE,
                "(uid=aduser)", adDNString))
        .expectEvent(ceBind3)
        .expectEvent(
            new SimpleBindEvent(ceBind3, adDNString, userPassword,
                ResultCode.SUCCESS));

    // Repeat again using cached connection to phost1: search, then bind.
    provider.expectEvent(
        new SearchEvent(ceSearch1, "o=ad", SearchScope.WHOLE_SUBTREE,
            "(uid=aduser)", adDNString)).expectEvent(
        new SimpleBindEvent(ceBind1, adDNString, userPassword,
            ResultCode.SUCCESS));

    // Repeat again using cached connection to phost2: search, then bind.
    provider.expectEvent(
        new SearchEvent(ceSearch2, "o=ad", SearchScope.WHOLE_SUBTREE,
            "(uid=aduser)", adDNString)).expectEvent(
        new SimpleBindEvent(ceBind2, adDNString, userPassword,
            ResultCode.SUCCESS));

    // Repeat again using cached connection to phost3: search, then bind.
    provider.expectEvent(
        new SearchEvent(ceSearch3, "o=ad", SearchScope.WHOLE_SUBTREE,
            "(uid=aduser)", adDNString)).expectEvent(
        new SimpleBindEvent(ceBind3, adDNString, userPassword,
            ResultCode.SUCCESS));

    // Connections should be cached until the policy is finalized.

    // Obtain policy and state.
    final LDAPPassThroughAuthenticationPolicyFactory factory = new LDAPPassThroughAuthenticationPolicyFactory(
        provider);
    assertTrue(factory.isConfigurationAcceptable(cfg, null));
    final AuthenticationPolicy policy = factory.createAuthenticationPolicy(cfg);

    // Cycle twice through the LB pool.
    for (int i = 0; i < 6; i++)
    {
      final AuthenticationPolicyState state = policy
          .createAuthenticationPolicyState(userEntry);
      assertEquals(state.getAuthenticationPolicy(), policy);

      // Perform authentication.
      assertTrue(state.passwordMatches(ByteString.valueOf(userPassword)));

      state.finalizeStateAfterBind();
    }

    // Cached connections should be closed when the policy is finalized.
    provider.expectEvent(new CloseEvent(ceSearch1));
    provider.expectEvent(new CloseEvent(ceSearch2));
    provider.expectEvent(new CloseEvent(ceSearch3));
    provider.expectEvent(new CloseEvent(ceBind1));
    provider.expectEvent(new CloseEvent(ceBind2));
    provider.expectEvent(new CloseEvent(ceBind3));

    // Tear down and check final state.
    policy.finalizeAuthenticationPolicy();
    provider.assertAllExpectedEventsReceived();
  }



  /**
   * Tests fail-over between 2 primary servers then to the secondary data
   * center.
   *
   * @throws Exception
   *           If an unexpected exception occurred.
   */
  @Test(enabled = true)
  public void testFailOverOnConnect() throws Exception
  {
    // Mock configuration.
    final LDAPPassThroughAuthenticationPolicyCfg cfg = mockCfg()
        .withPrimaryServer(phost1).withPrimaryServer(phost2)
        .withSecondaryServer(shost1)
        .withMappingPolicy(MappingPolicy.MAPPED_SEARCH)
        .withMappedAttribute("uid").withBaseDN("o=ad");

    // Create all the events.
    final MockProvider provider = new MockProvider();

    // First of all the connection factories are created.
    final GetLDAPConnectionFactoryEvent fe1 = new GetLDAPConnectionFactoryEvent(
        phost1, cfg);
    final GetLDAPConnectionFactoryEvent fe2 = new GetLDAPConnectionFactoryEvent(
        phost2, cfg);
    final GetLDAPConnectionFactoryEvent fe3 = new GetLDAPConnectionFactoryEvent(
        shost1, cfg);
    provider.expectEvent(fe1).expectEvent(fe2).expectEvent(fe3);

    // Get connection for phost1, then search, then bind.
    final GetConnectionEvent ceSearch1 = new GetConnectionEvent(fe1,
        ResultCode.CLIENT_SIDE_CONNECT_ERROR);
    final GetConnectionEvent ceSearch2 = new GetConnectionEvent(fe2,
        ResultCode.CLIENT_SIDE_CONNECT_ERROR);
    final GetConnectionEvent ceSearch3 = new GetConnectionEvent(fe3);

    final GetConnectionEvent ceBind1 = new GetConnectionEvent(fe1,
        ResultCode.CLIENT_SIDE_CONNECT_ERROR);
    final GetConnectionEvent ceBind2 = new GetConnectionEvent(fe2,
        ResultCode.CLIENT_SIDE_CONNECT_ERROR);
    final GetConnectionEvent ceBind3 = new GetConnectionEvent(fe3);

    provider
        .expectEvent(ceSearch1)
        .expectEvent(ceSearch2)
        .expectEvent(ceSearch3)
        .expectEvent(
            new SimpleBindEvent(ceSearch3, searchBindDNString,
                "searchPassword", ResultCode.SUCCESS))
        .expectEvent(
            new SearchEvent(ceSearch3, "o=ad", SearchScope.WHOLE_SUBTREE,
                "(uid=aduser)", adDNString))
        .expectEvent(ceBind1)
        .expectEvent(ceBind2)
        .expectEvent(ceBind3)
        .expectEvent(
            new SimpleBindEvent(ceBind3, adDNString, userPassword,
                ResultCode.SUCCESS));

    // Repeat again using cached connection to shost1: search, then bind.

    // Note that LB will cause phost2 to be tried first, hence ceSearch2 then
    // ceSearch1.
    provider
        .expectEvent(ceSearch2)
        .expectEvent(ceSearch1)
        .expectEvent(
            new SearchEvent(ceSearch3, "o=ad", SearchScope.WHOLE_SUBTREE,
                "(uid=aduser)", adDNString))
        .expectEvent(ceBind2)
        .expectEvent(ceBind1)
        .expectEvent(
            new SimpleBindEvent(ceBind3, adDNString, userPassword,
                ResultCode.SUCCESS));

    // Now simulate phost2 coming back and fail back to it
    final GetConnectionEvent ceSearch2ok = new GetConnectionEvent(fe2);
    final GetConnectionEvent ceBind2ok = new GetConnectionEvent(fe2);
    provider
        .expectEvent(ceSearch1)
        .expectEvent(ceSearch2ok)
        .expectEvent(
            new SimpleBindEvent(ceSearch2ok, searchBindDNString,
                "searchPassword", ResultCode.SUCCESS))
        .expectEvent(
            new SearchEvent(ceSearch2ok, "o=ad", SearchScope.WHOLE_SUBTREE,
                "(uid=aduser)", adDNString))
        .expectEvent(ceBind1)
        .expectEvent(ceBind2ok)
        .expectEvent(
            new SimpleBindEvent(ceBind2ok, adDNString, userPassword,
                ResultCode.SUCCESS));

    // Connections should be cached until the policy is finalized.

    // Obtain policy and state.
    final LDAPPassThroughAuthenticationPolicyFactory factory = new LDAPPassThroughAuthenticationPolicyFactory(
        provider);
    assertTrue(factory.isConfigurationAcceptable(cfg, null));
    final AuthenticationPolicy policy = factory.createAuthenticationPolicy(cfg);

    // Authenticate 3 times test above fail-over.
    for (int i = 0; i < 3; i++)
    {
      final AuthenticationPolicyState state = policy
          .createAuthenticationPolicyState(userEntry);
      assertEquals(state.getAuthenticationPolicy(), policy);

      // Perform authentication.
      assertTrue(state.passwordMatches(ByteString.valueOf(userPassword)));

      state.finalizeStateAfterBind();
    }

    // Cached connections should be closed when the policy is finalized
    // (primaries first, then secondaries).
    provider.expectEvent(new CloseEvent(ceSearch2ok));
    provider.expectEvent(new CloseEvent(ceSearch3));
    provider.expectEvent(new CloseEvent(ceBind2ok));
    provider.expectEvent(new CloseEvent(ceBind3));

    // Tear down and check final state.
    policy.finalizeAuthenticationPolicy();
    provider.assertAllExpectedEventsReceived();
  }



  /**
   * Tests that searches which fail on one server are automatically retried on
   * another within the same LB.
   *
   * @throws Exception
   *           If an unexpected exception occurred.
   */
  @Test(enabled = false)
  public void testLBRetrySearchOnFailure() throws Exception
  {
    // Mock configuration.
    final LDAPPassThroughAuthenticationPolicyCfg cfg = mockCfg()
        .withPrimaryServer(phost1).withPrimaryServer(phost2)
        .withMappingPolicy(MappingPolicy.MAPPED_SEARCH)
        .withMappedAttribute("uid").withBaseDN("o=ad");

    // Create all the events.
    final MockProvider provider = new MockProvider();

    // First of all the connection factories are created.
    final GetLDAPConnectionFactoryEvent fe1 = new GetLDAPConnectionFactoryEvent(
        phost1, cfg);
    final GetLDAPConnectionFactoryEvent fe2 = new GetLDAPConnectionFactoryEvent(
        phost2, cfg);
    provider.expectEvent(fe1).expectEvent(fe2);

    // Get connection for phost1, then search (fail), and retry on phost2
    final GetConnectionEvent ceSearch1 = new GetConnectionEvent(fe1);
    final GetConnectionEvent ceSearch2 = new GetConnectionEvent(fe2);

    final GetConnectionEvent ceBind1 = new GetConnectionEvent(fe1);
    final GetConnectionEvent ceBind2 = new GetConnectionEvent(fe2);

    provider
        .expectEvent(ceSearch1)
        .expectEvent(
            new SimpleBindEvent(ceSearch1, searchBindDNString,
                "searchPassword", ResultCode.SUCCESS))
        .expectEvent(
            new SearchEvent(ceSearch1, "o=ad", SearchScope.WHOLE_SUBTREE,
                "(uid=aduser)", ResultCode.UNAVAILABLE))
        .expectEvent(new CloseEvent(ceSearch1))
        .expectEvent(ceSearch2)
        .expectEvent(
            new SimpleBindEvent(ceSearch2, searchBindDNString,
                "searchPassword", ResultCode.SUCCESS))
        .expectEvent(
            new SearchEvent(ceSearch2, "o=ad", SearchScope.WHOLE_SUBTREE,
                "(uid=aduser)", adDNString))
        .expectEvent(ceBind1)
        .expectEvent(
            new SimpleBindEvent(ceBind1, adDNString, userPassword,
                ResultCode.SUCCESS));

    // Now simulate phost2 going down as well.

    // Note that LB will cause phost2 to be tried first, hence ceSearch2 then
    // ceSearch1.
    provider
        .expectEvent(ceSearch2)
        .expectEvent(
            new SearchEvent(ceSearch2, "o=ad", SearchScope.WHOLE_SUBTREE,
                "(uid=aduser)", ResultCode.UNAVAILABLE))
        .expectEvent(new CloseEvent(ceSearch2))
        .expectEvent(ceSearch1)
        .expectEvent(
            new SimpleBindEvent(ceSearch1, searchBindDNString,
                "searchPassword", ResultCode.SUCCESS))
        .expectEvent(
            new SearchEvent(ceSearch1, "o=ad", SearchScope.WHOLE_SUBTREE,
                "(uid=aduser)", ResultCode.UNAVAILABLE))
        .expectEvent(new CloseEvent(ceSearch1));

    // Now simulate phost1 coming back and fail back to it.
    provider
        .expectEvent(ceSearch1)
        .expectEvent(
            new SimpleBindEvent(ceSearch1, searchBindDNString,
                "searchPassword", ResultCode.SUCCESS))
        .expectEvent(
            new SearchEvent(ceSearch1, "o=ad", SearchScope.WHOLE_SUBTREE,
                "(uid=aduser)", adDNString));

    // Connections should be cached until the policy is finalized.

    // Obtain policy and state.
    final LDAPPassThroughAuthenticationPolicyFactory factory = new LDAPPassThroughAuthenticationPolicyFactory(
        provider);
    assertTrue(factory.isConfigurationAcceptable(cfg, null));
    final AuthenticationPolicy policy = factory.createAuthenticationPolicy(cfg);

    // Authenticate 3 times, second should fail.
    for (int i = 0; i < 3; i++)
    {
      final AuthenticationPolicyState state = policy
          .createAuthenticationPolicyState(userEntry);
      assertEquals(state.getAuthenticationPolicy(), policy);

      // Perform authentication.
      if (i != 1)
      {
        // First and third attempt should succeed.
        assertTrue(state.passwordMatches(ByteString.valueOf(userPassword)));
      }
      else
      {
        // Second attempt should fail.
        try
        {
          state.passwordMatches(ByteString.valueOf(userPassword));
          fail("password match unexpectedly succeeded");
        }
        catch (final DirectoryException e)
        {
          // No valid connections available so this should always fail with
          // INVALID_CREDENTIALS.
          assertEquals(e.getResultCode(), ResultCode.INVALID_CREDENTIALS,
              e.getMessage());
        }
      }

      state.finalizeStateAfterBind();
    }

    // Cached connections should be closed when the policy is finalized.
    provider.expectEvent(new CloseEvent(ceSearch1));
    provider.expectEvent(new CloseEvent(ceBind1));
    provider.expectEvent(new CloseEvent(ceBind2));

    // Tear down and check final state.
    policy.finalizeAuthenticationPolicy();
    provider.assertAllExpectedEventsReceived();
  }



  /**
   * Tests that searches which fail in one LB pool are automatically retried in
   * the secondary LB pool.
   *
   * @throws Exception
   *           If an unexpected exception occurred.
   */
  @Test(enabled = false)
  public void testFBRetrySearchOnFailure() throws Exception
  {
    // Mock configuration.
    final LDAPPassThroughAuthenticationPolicyCfg cfg = mockCfg()
        .withPrimaryServer(phost1).withSecondaryServer(shost1)
        .withMappingPolicy(MappingPolicy.MAPPED_SEARCH)
        .withMappedAttribute("uid").withBaseDN("o=ad");

    // Create all the events.
    final MockProvider provider = new MockProvider();

    // First of all the connection factories are created.
    final GetLDAPConnectionFactoryEvent fe1 = new GetLDAPConnectionFactoryEvent(
        phost1, cfg);
    final GetLDAPConnectionFactoryEvent fe2 = new GetLDAPConnectionFactoryEvent(
        shost1, cfg);
    provider.expectEvent(fe1).expectEvent(fe2);

    // Get connection for phost1, then search (fail), and retry on shost1
    final GetConnectionEvent ceSearch1 = new GetConnectionEvent(fe1);
    final GetConnectionEvent ceSearch2 = new GetConnectionEvent(fe2);

    final GetConnectionEvent ceBind1 = new GetConnectionEvent(fe1);
    final GetConnectionEvent ceBind2 = new GetConnectionEvent(fe2);

    provider
        .expectEvent(ceSearch1)
        .expectEvent(
            new SimpleBindEvent(ceSearch1, searchBindDNString,
                "searchPassword", ResultCode.SUCCESS))
        .expectEvent(
            new SearchEvent(ceSearch1, "o=ad", SearchScope.WHOLE_SUBTREE,
                "(uid=aduser)", ResultCode.UNAVAILABLE))
        .expectEvent(new CloseEvent(ceSearch1))
        .expectEvent(ceSearch2)
        .expectEvent(
            new SimpleBindEvent(ceSearch2, searchBindDNString,
                "searchPassword", ResultCode.SUCCESS))
        .expectEvent(
            new SearchEvent(ceSearch2, "o=ad", SearchScope.WHOLE_SUBTREE,
                "(uid=aduser)", adDNString))
        .expectEvent(ceBind1)
        .expectEvent(
            new SimpleBindEvent(ceBind1, adDNString, userPassword,
                ResultCode.SUCCESS));

    // Now simulate shost1 going down as well.

    // Note that FO will retry phost1 again (unlike LB case).
    provider
        .expectEvent(ceSearch1)
        .expectEvent(
            new SimpleBindEvent(ceSearch1, searchBindDNString,
                "searchPassword", ResultCode.SUCCESS))
        .expectEvent(
            new SearchEvent(ceSearch1, "o=ad", SearchScope.WHOLE_SUBTREE,
                "(uid=aduser)", ResultCode.UNAVAILABLE))
        .expectEvent(new CloseEvent(ceSearch1))
        .expectEvent(ceSearch2)
        .expectEvent(
            new SearchEvent(ceSearch2, "o=ad", SearchScope.WHOLE_SUBTREE,
                "(uid=aduser)", ResultCode.UNAVAILABLE))
        .expectEvent(new CloseEvent(ceSearch2));

    // Now simulate phost1 coming back and fail back to it.
    provider
        .expectEvent(ceSearch1)
        .expectEvent(
            new SimpleBindEvent(ceSearch1, searchBindDNString,
                "searchPassword", ResultCode.SUCCESS))
        .expectEvent(
            new SearchEvent(ceSearch1, "o=ad", SearchScope.WHOLE_SUBTREE,
                "(uid=aduser)", adDNString));

    // Connections should be cached until the policy is finalized.

    // Obtain policy and state.
    final LDAPPassThroughAuthenticationPolicyFactory factory = new LDAPPassThroughAuthenticationPolicyFactory(
        provider);
    assertTrue(factory.isConfigurationAcceptable(cfg, null));
    final AuthenticationPolicy policy = factory.createAuthenticationPolicy(cfg);

    // Authenticate 3 times, second should fail.
    for (int i = 0; i < 3; i++)
    {
      final AuthenticationPolicyState state = policy
          .createAuthenticationPolicyState(userEntry);
      assertEquals(state.getAuthenticationPolicy(), policy);

      // Perform authentication.
      if (i != 1)
      {
        // First and third attempt should succeed.
        assertTrue(state.passwordMatches(ByteString.valueOf(userPassword)));
      }
      else
      {
        // Second attempt should fail.
        try
        {
          state.passwordMatches(ByteString.valueOf(userPassword));
          fail("password match unexpectedly succeeded");
        }
        catch (final DirectoryException e)
        {
          // No valid connections available so this should always fail with
          // INVALID_CREDENTIALS.
          assertEquals(e.getResultCode(), ResultCode.INVALID_CREDENTIALS,
              e.getMessage());
        }
      }

      state.finalizeStateAfterBind();
    }

    // Cached connections should be closed when the policy is finalized.
    provider.expectEvent(new CloseEvent(ceSearch1));
    provider.expectEvent(new CloseEvent(ceBind1));
    provider.expectEvent(new CloseEvent(ceBind2));
    provider.expectEvent(new CloseEvent(ceSearch));
    provider.expectEvent(new CloseEvent(ceBind));

    // Tear down and check final state.
    policy.finalizeAuthenticationPolicy();