/*
|
* The contents of this file are subject to the terms of the Common Development and
|
* Distribution License (the License). You may not use this file except in compliance with the
|
* License.
|
*
|
* You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the
|
* specific language governing permission and limitations under the License.
|
*
|
* When distributing Covered Software, include this CDDL Header Notice in each file and include
|
* the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL
|
* Header, with the fields enclosed by brackets [] replaced by your own identifying
|
* information: "Portions Copyright [year] [name of copyright owner]".
|
*
|
* Copyright 2006-2010 Sun Microsystems, Inc.
|
* Portions Copyright 2013-2016 ForgeRock AS.
|
*/
|
package org.opends.server.extensions;
|
|
import static org.opends.messages.ConfigMessages.*;
|
import static org.opends.messages.CoreMessages.*;
|
|
import java.util.ArrayList;
|
import java.util.List;
|
import java.util.concurrent.LinkedBlockingQueue;
|
import java.util.concurrent.TimeUnit;
|
import java.util.concurrent.atomic.AtomicLong;
|
import java.util.concurrent.locks.ReentrantReadWriteLock;
|
import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
|
import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
|
|
import org.forgerock.i18n.LocalizableMessage;
|
import org.forgerock.i18n.slf4j.LocalizedLogger;
|
import org.forgerock.opendj.config.server.ConfigException;
|
import org.forgerock.opendj.ldap.ResultCode;
|
import org.forgerock.opendj.config.server.ConfigurationChangeListener;
|
import org.forgerock.opendj.server.config.server.TraditionalWorkQueueCfg;
|
import org.opends.server.api.WorkQueue;
|
import org.opends.server.core.DirectoryServer;
|
import org.opends.server.monitors.TraditionalWorkQueueMonitor;
|
import org.opends.server.types.CancelRequest;
|
import org.forgerock.opendj.config.server.ConfigChangeResult;
|
import org.opends.server.types.DirectoryException;
|
import org.opends.server.types.InitializationException;
|
import org.opends.server.types.Operation;
|
|
/**
|
* This class defines a data structure for storing and interacting with the
|
* Directory Server work queue.
|
*/
|
public class TraditionalWorkQueue extends WorkQueue<TraditionalWorkQueueCfg>
|
implements ConfigurationChangeListener<TraditionalWorkQueueCfg>
|
{
|
private static final LocalizedLogger logger = LocalizedLogger.getLoggerForThisClass();
|
|
/**
|
* The maximum number of times to retry getting the next operation from the
|
* queue if an unexpected failure occurs.
|
*/
|
private static final int MAX_RETRY_COUNT = 5;
|
|
/** The set of worker threads that will be used to process this work queue. */
|
private final ArrayList<TraditionalWorkerThread> workerThreads = new ArrayList<>();
|
|
/** The number of operations that have been submitted to the work queue for processing. */
|
private AtomicLong opsSubmitted;
|
|
/**
|
* The number of times that an attempt to submit a new request has been
|
* rejected because the work queue is already at its maximum capacity.
|
*/
|
private AtomicLong queueFullRejects;
|
|
/**
|
* Indicates whether one or more of the worker threads needs to be killed at
|
* the next convenient opportunity.
|
*/
|
private boolean killThreads;
|
|
/** Indicates whether the Directory Server is shutting down. */
|
private boolean shutdownRequested;
|
|
/** The thread number used for the last worker thread that was created. */
|
private int lastThreadNumber;
|
|
/**
|
* The maximum number of pending requests that this work queue will allow
|
* before it will start rejecting them.
|
*/
|
private int maxCapacity;
|
|
/**
|
* The number of worker threads that should be active (or will be shortly if a
|
* configuration change has not been completely applied).
|
*/
|
private int numWorkerThreads;
|
|
/**
|
* The queue overflow policy: true indicates that operations will be blocked
|
* until the queue has available capacity, otherwise operations will be
|
* rejected.
|
* <p>
|
* This is hard-coded to true for now because a reject on full policy does not
|
* seem to have a valid use case.
|
* </p>
|
*/
|
private final boolean isBlocking = true;
|
|
/** The queue that will be used to actually hold the pending operations. */
|
private LinkedBlockingQueue<Operation> opQueue;
|
|
/** The lock used to provide threadsafe access for the queue, used for non-config changes. */
|
private final ReadLock queueReadLock;
|
|
/** The lock used to provide threadsafe access for the queue, used for config changes. */
|
private final WriteLock queueWriteLock;
|
{
|
ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
|
queueReadLock = lock.readLock();
|
queueWriteLock = lock.writeLock();
|
}
|
|
/**
|
* Creates a new instance of this work queue. All initialization should be
|
* performed in the <CODE>initializeWorkQueue</CODE> method.
|
*/
|
public TraditionalWorkQueue()
|
{
|
// No implementation should be performed here.
|
}
|
|
@Override
|
public void initializeWorkQueue(TraditionalWorkQueueCfg configuration)
|
throws ConfigException, InitializationException
|
{
|
queueWriteLock.lock();
|
try
|
{
|
shutdownRequested = false;
|
killThreads = false;
|
opsSubmitted = new AtomicLong(0);
|
queueFullRejects = new AtomicLong(0);
|
|
// Register to be notified of any configuration changes.
|
configuration.addTraditionalChangeListener(this);
|
|
// Get the necessary configuration from the provided entry.
|
numWorkerThreads =
|
computeNumWorkerThreads(configuration.getNumWorkerThreads());
|
maxCapacity = configuration.getMaxWorkQueueCapacity();
|
|
// Create the actual work queue.
|
if (maxCapacity > 0)
|
{
|
opQueue = new LinkedBlockingQueue<>(maxCapacity);
|
}
|
else
|
{
|
// This will never be the case, since the configuration definition
|
// ensures that the capacity is always finite.
|
opQueue = new LinkedBlockingQueue<>();
|
}
|
|
// Create the set of worker threads that should be used to service the
|
// work queue.
|
for (lastThreadNumber = 0; lastThreadNumber < numWorkerThreads;
|
lastThreadNumber++)
|
{
|
TraditionalWorkerThread t = new TraditionalWorkerThread(this,
|
lastThreadNumber);
|
t.start();
|
workerThreads.add(t);
|
}
|
|
// Create and register a monitor provider for the work queue.
|
try
|
{
|
TraditionalWorkQueueMonitor monitor = new TraditionalWorkQueueMonitor(
|
this);
|
monitor.initializeMonitorProvider(null);
|
DirectoryServer.registerMonitorProvider(monitor);
|
}
|
catch (Exception e)
|
{
|
logger.traceException(e);
|
logger.error(ERR_CONFIG_WORK_QUEUE_CANNOT_CREATE_MONITOR, TraditionalWorkQueueMonitor.class, e);
|
}
|
}
|
finally
|
{
|
queueWriteLock.unlock();
|
}
|
}
|
|
@Override
|
public void finalizeWorkQueue(LocalizableMessage reason)
|
{
|
queueWriteLock.lock();
|
try
|
{
|
shutdownRequested = true;
|
}
|
finally
|
{
|
queueWriteLock.unlock();
|
}
|
|
// From now on no more operations can be enqueued or dequeued.
|
|
// Send responses to any operations in the pending queue to indicate that
|
// they won't be processed because the server is shutting down.
|
CancelRequest cancelRequest = new CancelRequest(true, reason);
|
ArrayList<Operation> pendingOperations = new ArrayList<>();
|
opQueue.removeAll(pendingOperations);
|
for (Operation o : pendingOperations)
|
{
|
try
|
{
|
// The operation has no chance of responding to the cancel
|
// request so avoid waiting for a cancel response.
|
if (o.getCancelResult() == null)
|
{
|
o.abort(cancelRequest);
|
}
|
}
|
catch (Exception e)
|
{
|
logger.traceException(e);
|
logger.warn(WARN_QUEUE_UNABLE_TO_CANCEL, o, e);
|
}
|
}
|
|
// Notify all the worker threads of the shutdown.
|
for (TraditionalWorkerThread t : workerThreads)
|
{
|
try
|
{
|
t.shutDown();
|
}
|
catch (Exception e)
|
{
|
logger.traceException(e);
|
logger.warn(WARN_QUEUE_UNABLE_TO_NOTIFY_THREAD, t.getName(), e);
|
}
|
}
|
}
|
|
/**
|
* Indicates whether this work queue has received a request to shut down.
|
*
|
* @return <CODE>true</CODE> if the work queue has recieved a request to shut
|
* down, or <CODE>false</CODE> if not.
|
*/
|
public boolean shutdownRequested()
|
{
|
queueReadLock.lock();
|
try
|
{
|
return shutdownRequested;
|
}
|
finally
|
{
|
queueReadLock.unlock();
|
}
|
}
|
|
/**
|
* Submits an operation to be processed by one of the worker threads
|
* associated with this work queue.
|
*
|
* @param operation
|
* The operation to be processed.
|
* @throws DirectoryException
|
* If the provided operation is not accepted for some reason (e.g.,
|
* if the server is shutting down or the pending operation queue is
|
* already at its maximum capacity).
|
*/
|
@Override
|
public void submitOperation(Operation operation) throws DirectoryException
|
{
|
submitOperation(operation, isBlocking);
|
}
|
|
@Override
|
public boolean trySubmitOperation(Operation operation)
|
throws DirectoryException
|
{
|
try
|
{
|
submitOperation(operation, false);
|
return true;
|
}
|
catch (DirectoryException e)
|
{
|
if (ResultCode.BUSY == e.getResultCode())
|
{
|
return false;
|
}
|
throw e;
|
}
|
}
|
|
private void submitOperation(Operation operation,
|
boolean blockEnqueuingWhenFull) throws DirectoryException
|
{
|
queueReadLock.lock();
|
try
|
{
|
if (shutdownRequested)
|
{
|
LocalizableMessage message = WARN_OP_REJECTED_BY_SHUTDOWN.get();
|
throw new DirectoryException(ResultCode.UNAVAILABLE, message);
|
}
|
|
if (blockEnqueuingWhenFull)
|
{
|
try
|
{
|
// If the queue is full and there is an administrative change taking
|
// place then starvation could arise: this thread will hold the read
|
// lock, the admin thread will be waiting on the write lock, and the
|
// worker threads may be queued behind the admin thread. Since the
|
// worker threads cannot run, the queue will never empty and allow
|
// this thread to proceed. To help things out we can periodically
|
// yield the read lock when the queue is full.
|
while (!opQueue.offer(operation, 1, TimeUnit.SECONDS))
|
{
|
queueReadLock.unlock();
|
Thread.yield();
|
queueReadLock.lock();
|
|
if (shutdownRequested)
|
{
|
LocalizableMessage message = WARN_OP_REJECTED_BY_SHUTDOWN.get();
|
throw new DirectoryException(ResultCode.UNAVAILABLE, message);
|
}
|
}
|
}
|
catch (InterruptedException e)
|
{
|
// We cannot handle the interruption here. Reject the request and
|
// re-interrupt this thread.
|
Thread.currentThread().interrupt();
|
|
queueFullRejects.incrementAndGet();
|
|
LocalizableMessage message = WARN_OP_REJECTED_BY_QUEUE_INTERRUPT.get();
|
throw new DirectoryException(ResultCode.BUSY, message);
|
}
|
}
|
else
|
{
|
if (!opQueue.offer(operation))
|
{
|
queueFullRejects.incrementAndGet();
|
|
LocalizableMessage message = WARN_OP_REJECTED_BY_QUEUE_FULL.get(maxCapacity);
|
throw new DirectoryException(ResultCode.BUSY, message);
|
}
|
}
|
|
opsSubmitted.incrementAndGet();
|
}
|
finally
|
{
|
queueReadLock.unlock();
|
}
|
}
|
|
/**
|
* Retrieves the next operation that should be processed by one of the worker
|
* threads, blocking if necessary until a new request arrives. This method
|
* should only be called by a worker thread associated with this work queue.
|
*
|
* @param workerThread
|
* The worker thread that is requesting the operation.
|
* @return The next operation that should be processed, or <CODE>null</CODE>
|
* if the server is shutting down and no more operations will be
|
* processed.
|
*/
|
public Operation nextOperation(TraditionalWorkerThread workerThread)
|
{
|
return retryNextOperation(workerThread, 0);
|
}
|
|
/**
|
* Retrieves the next operation that should be processed by one of the worker
|
* threads following a previous failure attempt. A maximum of five consecutive
|
* failures will be allowed before returning <CODE>null</CODE>, which will
|
* cause the associated thread to exit.
|
*
|
* @param workerThread
|
* The worker thread that is requesting the operation.
|
* @param numFailures
|
* The number of consecutive failures that the worker thread has
|
* experienced so far. If this gets too high, then this method will
|
* return <CODE>null</CODE> rather than retrying.
|
* @return The next operation that should be processed, or <CODE>null</CODE>
|
* if the server is shutting down and no more operations will be
|
* processed, or if there have been too many consecutive failures.
|
*/
|
private Operation retryNextOperation(TraditionalWorkerThread workerThread,
|
int numFailures)
|
{
|
// See if we should kill off this thread. This could be necessary if the
|
// number of worker threads has been decreased with the server online. If
|
// so, then return null and the thread will exit.
|
queueReadLock.lock();
|
try
|
{
|
if (shutdownRequested)
|
{
|
return null;
|
}
|
|
if (killThreads && tryKillThisWorkerThread(workerThread))
|
{
|
return null;
|
}
|
|
if (numFailures > MAX_RETRY_COUNT)
|
{
|
logger.error(ERR_CONFIG_WORK_QUEUE_TOO_MANY_FAILURES, Thread
|
.currentThread().getName(), numFailures, MAX_RETRY_COUNT);
|
|
return null;
|
}
|
|
while (true)
|
{
|
Operation nextOperation = opQueue.poll(5, TimeUnit.SECONDS);
|
if (nextOperation != null)
|
{
|
return nextOperation;
|
}
|
|
// There was no work to do in the specified length of time. Release the
|
// read lock allowing shutdown or config changes to proceed and then see
|
// if we should give up or check again.
|
queueReadLock.unlock();
|
Thread.yield();
|
queueReadLock.lock();
|
|
if (shutdownRequested)
|
{
|
return null;
|
}
|
|
if (killThreads && tryKillThisWorkerThread(workerThread))
|
{
|
return null;
|
}
|
}
|
}
|
catch (InterruptedException ie)
|
{
|
// This is somewhat expected so don't log.
|
// assert debugException(CLASS_NAME, "retryNextOperation", ie);
|
|
// If this occurs, then the worker thread must have been interrupted for
|
// some reason. This could be because the Directory Server is shutting
|
// down, in which case we should return null.
|
if (shutdownRequested)
|
{
|
return null;
|
}
|
|
// If we've gotten here, then the worker thread was interrupted for some
|
// other reason. This should not happen, and we need to log a message.
|
logger.warn(WARN_WORKER_INTERRUPTED_WITHOUT_SHUTDOWN, Thread.currentThread().getName(), ie);
|
}
|
catch (Exception e)
|
{
|
logger.traceException(e);
|
|
// This should not happen. The only recourse we have is to log a message
|
// and try again.
|
logger.warn(WARN_WORKER_WAITING_UNCAUGHT_EXCEPTION, Thread.currentThread().getName(), e);
|
}
|
finally
|
{
|
queueReadLock.unlock();
|
}
|
|
// An exception has occurred - retry.
|
return retryNextOperation(workerThread, numFailures + 1);
|
}
|
|
/**
|
* Kills this worker thread if needed. This method assumes that the read lock
|
* is already taken and ensure that it is taken on exit.
|
*
|
* @param workerThread
|
* The worker thread associated with this thread.
|
* @return {@code true} if this thread was killed or is about to be killed as
|
* a result of shutdown.
|
*/
|
private boolean tryKillThisWorkerThread(TraditionalWorkerThread workerThread)
|
{
|
queueReadLock.unlock();
|
queueWriteLock.lock();
|
try
|
{
|
if (shutdownRequested)
|
{
|
// Shutdown may have been requested between unlock/lock. This thread is
|
// about to shutdown anyway, so return true.
|
return true;
|
}
|
|
int currentThreads = workerThreads.size();
|
if (currentThreads > numWorkerThreads)
|
{
|
if (workerThreads.remove(Thread.currentThread()))
|
{
|
currentThreads--;
|
}
|
|
if (currentThreads <= numWorkerThreads)
|
{
|
killThreads = false;
|
}
|
|
workerThread.setStoppedByReducedThreadNumber();
|
return true;
|
}
|
}
|
finally
|
{
|
queueWriteLock.unlock();
|
queueReadLock.lock();
|
|
if (shutdownRequested)
|
{
|
// Shutdown may have been requested between unlock/lock. This thread is
|
// about to shutdown anyway, so return true.
|
return true;
|
}
|
}
|
return false;
|
}
|
|
/**
|
* Retrieves the total number of operations that have been successfully
|
* submitted to this work queue for processing since server startup. This does
|
* not include operations that have been rejected for some reason like the
|
* queue already at its maximum capacity.
|
*
|
* @return The total number of operations that have been successfully
|
* submitted to this work queue since startup.
|
*/
|
public long getOpsSubmitted()
|
{
|
return opsSubmitted.longValue();
|
}
|
|
/**
|
* Retrieves the total number of operations that have been rejected because
|
* the work queue was already at its maximum capacity.
|
*
|
* @return The total number of operations that have been rejected because the
|
* work queue was already at its maximum capacity.
|
*/
|
public long getOpsRejectedDueToQueueFull()
|
{
|
return queueFullRejects.longValue();
|
}
|
|
/**
|
* Retrieves the number of pending operations in the queue that have not yet
|
* been picked up for processing. Note that this method is not a constant-time
|
* operation and can be relatively inefficient, so it should be used
|
* sparingly.
|
*
|
* @return The number of pending operations in the queue that have not yet
|
* been picked up for processing.
|
*/
|
public int size()
|
{
|
queueReadLock.lock();
|
try
|
{
|
return opQueue.size();
|
}
|
finally
|
{
|
queueReadLock.unlock();
|
}
|
}
|
|
@Override
|
public boolean isConfigurationChangeAcceptable(
|
TraditionalWorkQueueCfg configuration, List<LocalizableMessage> unacceptableReasons)
|
{
|
return true;
|
}
|
|
@Override
|
public ConfigChangeResult applyConfigurationChange(
|
TraditionalWorkQueueCfg configuration)
|
{
|
int newNumThreads =
|
computeNumWorkerThreads(configuration.getNumWorkerThreads());
|
int newMaxCapacity = configuration.getMaxWorkQueueCapacity();
|
|
// Apply a change to the number of worker threads if appropriate.
|
int currentThreads = workerThreads.size();
|
if (newNumThreads != currentThreads)
|
{
|
queueWriteLock.lock();
|
try
|
{
|
int threadsToAdd = newNumThreads - currentThreads;
|
if (threadsToAdd > 0)
|
{
|
for (int i = 0; i < threadsToAdd; i++)
|
{
|
TraditionalWorkerThread t = new TraditionalWorkerThread(this,
|
lastThreadNumber++);
|
workerThreads.add(t);
|
t.start();
|
}
|
|
killThreads = false;
|
}
|
else
|
{
|
killThreads = true;
|
}
|
|
numWorkerThreads = newNumThreads;
|
}
|
catch (Exception e)
|
{
|
logger.traceException(e);
|
}
|
finally
|
{
|
queueWriteLock.unlock();
|
}
|
}
|
|
// Apply a change to the maximum capacity if appropriate. Since we can't
|
// change capacity on the fly, then we'll have to create a new queue and
|
// transfer any remaining items into it. Any thread that is waiting on the
|
// original queue will time out after at most a few seconds and further
|
// checks will be against the new queue.
|
if (newMaxCapacity != maxCapacity)
|
{
|
// First switch the queue with the exclusive lock.
|
queueWriteLock.lock();
|
LinkedBlockingQueue<Operation> oldOpQueue;
|
try
|
{
|
LinkedBlockingQueue<Operation> newOpQueue = null;
|
if (newMaxCapacity > 0)
|
{
|
newOpQueue = new LinkedBlockingQueue<>(newMaxCapacity);
|
}
|
else
|
{
|
newOpQueue = new LinkedBlockingQueue<>();
|
}
|
|
oldOpQueue = opQueue;
|
opQueue = newOpQueue;
|
|
maxCapacity = newMaxCapacity;
|
}
|
finally
|
{
|
queueWriteLock.unlock();
|
}
|
|
// Now resubmit any pending requests - we'll need the shared lock.
|
Operation pendingOperation = null;
|
queueReadLock.lock();
|
try
|
{
|
// We have to be careful when adding any existing pending operations
|
// because the new capacity could be less than what was already
|
// backlogged in the previous queue. If that happens, we may have to
|
// loop a few times to get everything in there.
|
while ((pendingOperation = oldOpQueue.poll()) != null)
|
{
|
opQueue.put(pendingOperation);
|
}
|
}
|
catch (InterruptedException e)
|
{
|
// We cannot handle the interruption here. Cancel pending requests and
|
// re-interrupt this thread.
|
Thread.currentThread().interrupt();
|
|
LocalizableMessage message = WARN_OP_REJECTED_BY_QUEUE_INTERRUPT.get();
|
CancelRequest cancelRequest = new CancelRequest(true, message);
|
if (pendingOperation != null)
|
{
|
pendingOperation.abort(cancelRequest);
|
}
|
while ((pendingOperation = oldOpQueue.poll()) != null)
|
{
|
pendingOperation.abort(cancelRequest);
|
}
|
}
|
finally
|
{
|
queueReadLock.unlock();
|
}
|
}
|
|
return new ConfigChangeResult();
|
}
|
|
@Override
|
public boolean isIdle()
|
{
|
queueReadLock.lock();
|
try
|
{
|
if (!opQueue.isEmpty())
|
{
|
return false;
|
}
|
|
for (TraditionalWorkerThread t : workerThreads)
|
{
|
if (t.isActive())
|
{
|
return false;
|
}
|
}
|
|
return true;
|
}
|
finally
|
{
|
queueReadLock.unlock();
|
}
|
}
|
|
/**
|
* Return the number of worker threads used by this WorkQueue.
|
*
|
* @return the number of worker threads used by this WorkQueue
|
*/
|
@Override
|
public int getNumWorkerThreads()
|
{
|
return this.numWorkerThreads;
|
}
|
}
|
