From 15557ccb22969fce85143dec0b84b7503cfdfa4c Mon Sep 17 00:00:00 2001
From: Matthew Swift <matthew.swift@forgerock.com>
Date: Thu, 13 Dec 2012 17:32:24 +0000
Subject: [PATCH] Fix OPENDJ-652: Connections from Solaris 10 ldapclient can cause LDAPS request handler to spin
---
opendj-sdk/opends/src/server/org/opends/server/extensions/TLSByteChannel.java | 123 +++++++++++++++++++---------------------
1 files changed, 59 insertions(+), 64 deletions(-)
diff --git a/opendj-sdk/opends/src/server/org/opends/server/extensions/TLSByteChannel.java b/opendj-sdk/opends/src/server/org/opends/server/extensions/TLSByteChannel.java
index 6e35881..1525187 100644
--- a/opendj-sdk/opends/src/server/org/opends/server/extensions/TLSByteChannel.java
+++ b/opendj-sdk/opends/src/server/org/opends/server/extensions/TLSByteChannel.java
@@ -192,6 +192,18 @@
+ // It seems that the SSL engine does not remember if an error has already
+ // occurred so we must cache it here and rethrow. See OPENDJ-652.
+ private void abortOnSSLException() throws IOException
+ {
+ if (sslException != null)
+ {
+ throw sslException;
+ }
+ }
+
+
+
private void doHandshake(final boolean isReading) throws IOException
{
// This lock is probably unnecessary since tasks can be run in parallel,
@@ -244,33 +256,26 @@
{
// Read SSL packets until some unwrapped data is produced or no more
// data is available on the underlying channel.
- boolean needRead = true;
- int read = 0;
while (true)
{
- // Read wrapped data if needed.
- if (needRead)
- {
- recvWrappedBuffer.compact(); // Prepare for append.
- read = channel.read(recvWrappedBuffer);
- recvWrappedBuffer.flip(); // Restore for read.
- if (read < 0)
- {
- // Peer abort?
- sslEngine.closeInbound();
- return -1;
- }
- }
- else
- {
- needRead = true;
- }
-
- // Unwrap.
+ // Unwrap any remaining data in the buffer.
+ abortOnSSLException();
recvUnwrappedBuffer.compact(); // Prepare for append.
- final SSLEngineResult result = sslEngine.unwrap(recvWrappedBuffer,
- recvUnwrappedBuffer);
- recvUnwrappedBuffer.flip(); // Restore for read.
+ final SSLEngineResult result;
+ try
+ {
+ result = sslEngine.unwrap(recvWrappedBuffer, recvUnwrappedBuffer);
+ }
+ catch (final SSLException e)
+ {
+ // Save the error - see abortOnSSLException().
+ sslException = e;
+ throw e;
+ }
+ finally
+ {
+ recvUnwrappedBuffer.flip(); // Restore for read.
+ }
switch (result.getStatus())
{
@@ -283,7 +288,6 @@
newRecvUnwrappedBuffer.put(recvUnwrappedBuffer);
newRecvUnwrappedBuffer.flip();
recvUnwrappedBuffer = newRecvUnwrappedBuffer;
- needRead = false;
break; // Retry unwrap.
case BUFFER_UNDERFLOW:
// Not enough data was read. This either means that the inbound
@@ -291,29 +295,27 @@
final int newPktSize = sslEngine.getSession().getPacketBufferSize();
if (newPktSize > recvWrappedBuffer.capacity())
{
- // Increase the buffer size and reread.
+ // Increase the buffer size.
final ByteBuffer newRecvWrappedBuffer = ByteBuffer
.allocate(newPktSize);
newRecvWrappedBuffer.put(recvWrappedBuffer);
newRecvWrappedBuffer.flip();
recvWrappedBuffer = newRecvWrappedBuffer;
- break;
}
- else if (read == 0)
+ // Read wrapped data from underlying channel.
+ recvWrappedBuffer.compact(); // Prepare for append.
+ final int read = channel.read(recvWrappedBuffer);
+ recvWrappedBuffer.flip(); // Restore for read.
+ if (read <= 0)
{
- // Not enough data is available to read a complete SSL packet.
- return 0;
+ // Not enough data is available to read a complete SSL packet, or
+ // channel closed.
+ return read;
}
- else
- {
- // The channel read may only partially fill the buffer due to
- // buffering in the underlying channel layer, so try reading again
- // until we are sure that we are unable to proceed.
- break;
- }
+ // Loop and unwrap.
+ break;
case CLOSED:
// Peer sent SSL close notification.
- sslEngine.closeInbound();
return -1;
default: // OK
if (recvUnwrappedBuffer.hasRemaining())
@@ -325,30 +327,9 @@
{
// No application data was read, but if we are handshaking then
// try to continue.
- if (result.getHandshakeStatus() == HandshakeStatus.NEED_UNWRAP
- && !recvWrappedBuffer.hasRemaining() && read == 0)
- {
- // Not enough data is available to continue handshake.
- return 0;
- }
- else
- {
- // Continue handshake.
- doHandshake(true /* isReading */);
- }
+ doHandshake(true /* isReading */);
}
- else if (read == 0)
- {
- // No data available and not handshaking.
- return 0;
- }
- else
- {
- // The channel read may only partially fill the buffer due to
- // buffering in the underlying channel layer, so try reading again
- // until we are sure that we are unable to proceed.
- break;
- }
+ break;
}
}
}
@@ -366,8 +347,19 @@
// Repeat while there is overflow.
while (true)
{
- final SSLEngineResult result = sslEngine.wrap(unwrappedData,
- sendWrappedBuffer);
+ abortOnSSLException();
+ final SSLEngineResult result;
+ try
+ {
+ result = sslEngine.wrap(unwrappedData, sendWrappedBuffer);
+ }
+ catch (SSLException e)
+ {
+ // Save the error - see abortOnSSLException().
+ sslException = e;
+ throw e;
+ }
+
switch (result.getStatus())
{
case BUFFER_OVERFLOW:
@@ -381,7 +373,9 @@
break; // Retry.
case BUFFER_UNDERFLOW:
// This should not happen for sends.
- throw new SSLException("Got unexpected underflow while wrapping");
+ sslException =
+ new SSLException("Got unexpected underflow while wrapping");
+ throw sslException;
case CLOSED:
throw new ClosedChannelException();
default: // OK
@@ -444,6 +438,7 @@
private final ByteChannel channel;
private final SSLEngine sslEngine;
+ private volatile SSLException sslException = null;
private ByteBuffer recvWrappedBuffer;
private ByteBuffer recvUnwrappedBuffer;
private ByteBuffer sendWrappedBuffer;
--
Gitblit v1.10.0