From 2c5d31b11d6cb549c0ebfb34897d34ce5abd7c72 Mon Sep 17 00:00:00 2001
From: Valery Kharseko <vharseko@3a-systems.ru>
Date: Tue, 01 Sep 2026 08:53:57 +0000
Subject: [PATCH] [#872] Bound the connect of the JDBC pool and report a connect it cannot make (#876)
---
opendj-server-legacy/src/test/java/org/opends/server/backends/jdbc/TestCase.java | 36
opendj-server-legacy/src/test/java/org/opends/server/backends/jdbc/CachedConnectionTestCase.java | 1466 ++++++++++++++++++++++++++++++++
opendj-server-legacy/src/main/java/org/opends/server/backends/jdbc/CachedConnection.java | 1144 ++++++++++++++++++++++++
3 files changed, 2,613 insertions(+), 33 deletions(-)
diff --git a/opendj-server-legacy/src/main/java/org/opends/server/backends/jdbc/CachedConnection.java b/opendj-server-legacy/src/main/java/org/opends/server/backends/jdbc/CachedConnection.java
index 188f7ba..480c3e4 100644
--- a/opendj-server-legacy/src/main/java/org/opends/server/backends/jdbc/CachedConnection.java
+++ b/opendj-server-legacy/src/main/java/org/opends/server/backends/jdbc/CachedConnection.java
@@ -23,9 +23,22 @@
import java.sql.*;
import java.time.Duration;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Deque;
+import java.util.HashSet;
+import java.util.IdentityHashMap;
+import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.Properties;
+import java.util.Set;
import java.util.concurrent.*;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
public class CachedConnection implements Connection {
private static final LocalizedLogger logger = LocalizedLogger.getLoggerForThisClass();
@@ -33,6 +46,77 @@
static final String TTL_PROPERTY = "org.openidentityplatform.opendj.jdbc.ttl";
static final long DEFAULT_TTL_MS = 15000;
+ /**
+ * Bounds the connect and the login of one attempt to establish a connection, in seconds; 0 for
+ * no bound of its own - the deadline of {@value #POOL_TIMEOUT_PROPERTY} still bounds the
+ * attempt, since it stands for the whole borrow. Setting both to 0 is what leaves a connect
+ * unbounded.
+ */
+ static final String CONNECT_TIMEOUT_PROPERTY = "org.openidentityplatform.opendj.jdbc.connect.timeout";
+ static final long DEFAULT_CONNECT_TIMEOUT_SECONDS = 30;
+
+ /**
+ * Bounds a whole borrow - every connect attempt and every wait for a pooled connection - in
+ * seconds; 0 for no bound. Not to the millisecond: the connection in hand is validated
+ * whatever the deadline says, and an attempt is never given less than a second, so a borrow
+ * can return a validation and a last attempt past it.
+ */
+ static final String POOL_TIMEOUT_PROPERTY = "org.openidentityplatform.opendj.jdbc.pool.timeout";
+ static final long DEFAULT_POOL_TIMEOUT_SECONDS = 60;
+
+ /** Bound of the validation of a pooled connection: isValid(0) means "no timeout" in the JDBC contract. */
+ static final int VALIDATION_TIMEOUT_SECONDS = 5;
+
+ /** 08001, sqlclient_unable_to_establish_sqlconnection: the state of a connect that did not happen. */
+ private static final String CONNECT_FAILED_SQL_STATE = "08001";
+ /** 53300, too_many_connections: how the standard - and postgresql - reports a server taking no further connection. */
+ private static final String CONNECTION_LIMIT_SQL_STATE = "53300";
+ /** 57P03, cannot_connect_now: postgresql starting up, shutting down or in recovery. */
+ private static final String NOT_ACCEPTING_YET_SQL_STATE = "57P03";
+
+ static final long MAX_BACKOFF_MS = 1000;
+ static final long STALL_WARNING_AFTER_MS = 1000;
+ static final long STALL_WARNING_INTERVAL_MS = 10000;
+
+ /** How many links of the cause and getNextException() chains of a failure are looked at. */
+ private static final int MAX_CHAIN_LENGTH = 32;
+
+ /** What a connection string is cut down to where this cannot tell its credentials from the rest of it. */
+ static final String CREDENTIALS_HIDDEN = "<credentials hidden>";
+ /**
+ * A password standing where neither the userinfo nor the parameters of a url are looked for.
+ * The name goes by more than one spelling: mysql numbers the factors of a multi-factor login
+ * (password1, password2), and a wallet or a key store carries one under a name of its own
+ * (oracle.net.wallet_password, javax.net.ssl.keyStorePassword).
+ * The value of one ends at a separator of a connection string or at the first space: a driver
+ * is free to name a parameter in the middle of a sentence ("password=hunter2 for user u at
+ * h:5432"), and a value class running to the end of the string would take the host, the port
+ * and the cause of the failure into the blank along with the password.
+ */
+ private static final Pattern SECRET_PARAMETER =
+ Pattern.compile("(?i)([\\w.]*(password|passwd|pwd)\\d*)\\s*=([^\\s,)&;?]*)");
+
+ /** The parameters of a connection string worth keeping in a message: which database, not who connects. */
+ private static final Set<String> IDENTIFYING_PARAMETERS = Collections.unmodifiableSet(new HashSet<>(
+ Arrays.asList("databasename", "database", "instancename", "currentschema", "servicename")));
+
+ // setNetworkTimeout() takes the executor its timeout handling runs on: the drivers it is used
+ // with here only set a socket option in it, so it costs a call rather than a thread.
+ private static final Executor DIRECT_EXECUTOR = Runnable::run;
+
+ // Throttled per connection string: two JDBC backends stalling at once have a stall of their
+ // own to report, and a single timestamp would let one of them starve the other. Keyed by the
+ // safe form of it, the way warnedOnce below is: a static field of this class outlives every
+ // borrow, and the password of the backend has no business in one.
+ private static final Map<String, AtomicLong> lastStallWarning = new ConcurrentHashMap<>();
+ private static final AtomicLong lastReadBoundWarning = new AtomicLong();
+
+ // What has been reported once already. Every one of these reports a setting rather than an
+ // event - a property that is not a number, a url no bound of this class can reach, a driver
+ // whose property names are not known here - so it does not become truer by being repeated,
+ // and every operation of the backend comes through here.
+ static final Set<String> warnedOnce = ConcurrentHashMap.newKeySet();
+
final Connection parent;
static LoadingCache<String, BlockingQueue<CachedConnection>> cached = Caffeine.newBuilder()
@@ -55,59 +139,1042 @@
* {@value #TTL_PROPERTY} system property. An invalid value is ignored in favor of the default.
*/
private static long getCacheTtlMillis() {
- final String ttl = System.getProperty(TTL_PROPERTY);
- if (ttl != null) {
+ return getNonNegativeProperty(TTL_PROPERTY, DEFAULT_TTL_MS, "ms");
+ }
+
+ /**
+ * Returns the value of a numeric system property, ignoring a value that is not a non-negative
+ * number in favor of the default. The unit is the one the property is read in, so that the
+ * value the message names is not mistaken for another.
+ */
+ private static long getNonNegativeProperty(String name, long defaultValue, String unit) {
+ final String value = System.getProperty(name);
+ if (value != null) {
try {
- final long millis = Long.parseLong(ttl.trim());
- if (millis >= 0) {
- return millis;
+ final long parsed = Long.parseLong(value.trim());
+ if (parsed >= 0) {
+ return parsed;
}
} catch (NumberFormatException ignored) {
}
- logger.warn(LocalizableMessage.raw("Ignoring invalid value \"%s\" of the %s property, using %d ms",
- ttl, TTL_PROPERTY, DEFAULT_TTL_MS));
+ // reported once for this value: both properties are read on every borrow, so a
+ // "30s" of a typo would otherwise put two lines in the log per backend operation
+ warnOnce(name + "=" + value, "Ignoring invalid value \"%s\" of the %s property, using %d %s",
+ value, name, defaultValue, unit);
}
- return DEFAULT_TTL_MS;
+ return defaultValue;
+ }
+
+ /** Reports something about a setting once for the life of the jvm, however many borrows meet it. */
+ private static void warnOnce(String key, String format, Object... args) {
+ if (warnedOnce.add(key)) {
+ logger.warn(LocalizableMessage.raw(format, args));
+ }
+ }
+
+ /**
+ * The drivers this backend is used with, recognized by the prefix of the connection string,
+ * together with the properties that bound one attempt to establish a connection. Not one of
+ * them bounds the attempt with a single property: the one named first covers the socket
+ * connect, and the login behind it - the reads of the prelogin handshake, of TLS and of
+ * authentication, the phase a proxy at its connection limit or a moved VIP leaves unanswered -
+ * needs the read bound behind it. That holds for the SQL Server driver too, whose loginTimeout
+ * leaves the read of the prelogin answer unbounded - and for pgjdbc, whose loginTimeout is not
+ * a bound of the socket at all: Driver.connect hands the login to a daemon thread of its own
+ * and abandons it at the timeout, so an unbounded read there leaks a thread and a socket per
+ * borrow instead of failing one (CachedConnectionTestCase covers every one of them against a
+ * socket that never answers).
+ */
+ enum ConnectDialect {
+ /**
+ * postgresql: every property of the three takes seconds. connectTimeout covers the socket
+ * connect and socketTimeout the reads of the login: pgjdbc puts an SO_TIMEOUT on the login
+ * socket only where socketTimeout is set (ConnectionFactoryImpl.tryConnect, both before and
+ * after enableSSL), and it defaults to none. loginTimeout is kept on top of the two for a
+ * url naming more than one host, where each of them costs a login of its own - the connect
+ * is one budget for all of them, taken from the single System.nanoTime() in front of the
+ * loop over the hosts - but it is not a bound this class could rely on alone: Driver.connect
+ * runs the login on a daemon thread, gives up on the thread rather than on the login, and
+ * the thread stays parked in the read for as long as the read lasts.
+ */
+ POSTGRES("jdbc:postgresql:", '?',
+ new String[]{"connectTimeout", "loginTimeout"}, 1, 0,
+ new String[]{"socketTimeout"}, 1, true,
+ new int[]{}, new int[]{}),
+ /** mysql: both properties take milliseconds; socketTimeout is a socket read timeout that outlives the login. */
+ MYSQL("jdbc:mysql:", '?',
+ new String[]{"connectTimeout"}, 1000, 0,
+ new String[]{"socketTimeout"}, 1000, true,
+ new int[]{1040, 1203}, new int[]{1053}),
+ /** oracle: both properties take milliseconds; ReadTimeout is a socket read timeout that outlives the login. */
+ ORACLE("jdbc:oracle:", '?',
+ new String[]{"oracle.net.CONNECT_TIMEOUT"}, 1000, 0,
+ // the read bound goes by two names the driver reads: the property set here and the
+ // property of oracle net it stands for, inside a tns descriptor by the last segment of
+ // either. A bound under one of them is a bound of the administrator, so ours is not set
+ // on top of it - and neither of theirs is lifted with ours once the login is through.
+ // Only the first of the two is a name the driver also reads out of the system
+ // properties (SYSTEM_PROPERTY_NAMES below): oracle.net.READ_TIMEOUT reaches the socket
+ // from the connection properties alone, so a -D of it bounds nothing and must not be
+ // taken for a bound of theirs.
+ // RECV_TIMEOUT is not one of them: it is a parameter of sqlnet.ora and of the listener,
+ // and the name does not appear in ojdbc8 at all, so a descriptor carrying one would
+ // have taken our bound off a connection that never had one of its own.
+ new String[]{"oracle.jdbc.ReadTimeout", "oracle.net.READ_TIMEOUT"}, 1000, true,
+ // ORA-01033 and ORA-01034: the instance is starting up or not there yet; ORA-01089:
+ // it is shutting down. ORA-12514 is left out of these on purpose - a listener that
+ // does not know the service is also what a service name of a typo looks like, forever
+ new int[]{20, 12516, 12518, 12519, 12520}, new int[]{1033, 1034, 1089}),
+ /**
+ * ms sql server: loginTimeout takes seconds, socketTimeout milliseconds; the latter is a
+ * socket read timeout that outlives the login. loginTimeout is the one property of the
+ * four with a range of its own - SQLServerDriverIntProperty.LOGIN_TIMEOUT is validated
+ * against [0, 65535], and a value beyond it fails every connect the driver is asked for.
+ */
+ MICROSOFT("jdbc:sqlserver:", ';',
+ new String[]{"loginTimeout"}, 1, 65535,
+ new String[]{"socketTimeout"}, 1000, true,
+ // 921 and 922: the database has not been recovered yet, or is being recovered; 927: it
+ // is in the middle of a restore; 40613: azure sql reporting it not available for now
+ new int[]{17809, 10928, 10929}, new int[]{921, 922, 927, 40613});
+
+ final String urlPrefix;
+ /** the character that separates the parameters of this dialect from the url in front of them */
+ final char parameterSeparator;
+ /** the properties bounding the connect: the socket connect, and whatever the driver wraps it in */
+ final String[] connectProperties;
+ final int connectUnitsPerSecond;
+ /** the largest value the driver accepts for a connect property, 0 for a driver that takes any */
+ final long maxConnectSeconds;
+ /** the read bound of the login: the first name is the one set here, the rest are the names it also goes by */
+ final String[] readProperties;
+ final int readUnitsPerSecond;
+ /** whether the read bound of the login stays in force for every statement issued afterwards */
+ final boolean readBoundOutlivesLogin;
+ /** the vendor codes of this dialect for "no further connection is accepted" */
+ final int[] connectionLimitCodes;
+ /** the vendor codes of this dialect for "not accepting connections yet": a database on its way up */
+ final int[] notAcceptingYetCodes;
+
+ ConnectDialect(String urlPrefix, char parameterSeparator,
+ String[] connectProperties, int connectUnitsPerSecond, long maxConnectSeconds,
+ String[] readProperties, int readUnitsPerSecond, boolean readBoundOutlivesLogin,
+ int[] connectionLimitCodes, int[] notAcceptingYetCodes) {
+ this.urlPrefix = urlPrefix;
+ this.parameterSeparator = parameterSeparator;
+ this.connectProperties = connectProperties;
+ this.connectUnitsPerSecond = connectUnitsPerSecond;
+ this.maxConnectSeconds = maxConnectSeconds;
+ this.readProperties = readProperties;
+ this.readUnitsPerSecond = readUnitsPerSecond;
+ this.readBoundOutlivesLogin = readBoundOutlivesLogin;
+ this.connectionLimitCodes = connectionLimitCodes;
+ this.notAcceptingYetCodes = notAcceptingYetCodes;
+ }
+
+ /** The dialect of a connection string, or null for a driver whose property names are not known here. */
+ static ConnectDialect of(String connectionString) {
+ final String url = connectionString.toLowerCase(Locale.ROOT);
+ for (final ConnectDialect dialect : values()) {
+ if (url.startsWith(dialect.urlPrefix)) {
+ return dialect;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Fills in the properties bounding one connect attempt, leaving out what the administrator
+ * bounded themselves - an explicit setting of theirs keeps precedence, on the SQL Server,
+ * mysql and oracle drivers because a supplied property outranks the url, and on postgresql
+ * because the url outranks the property. A driver with a range of its own for its connect
+ * property is not handed a value beyond it: a bound it rejects is no bound at all, it is a
+ * connect that never happens.
+ * Returns whether a read bound outliving the login was set and has to be lifted once the
+ * connection is established.
+ */
+ boolean bound(String connectionString, Properties properties, long timeoutSeconds) {
+ final long connectSeconds = maxConnectSeconds > 0
+ ? Math.min(timeoutSeconds, maxConnectSeconds) : timeoutSeconds;
+ // The connect side is one budget rather than a set of independent knobs, so a bound of
+ // the administrator under any of its names leaves all of them alone. On postgresql
+ // connectTimeout bounds the socket connect and loginTimeout the login behind it:
+ // filling in the one they left out caps the one they set, and a "?connectTimeout=300"
+ // answered with a loginTimeout of ours is a login pgjdbc gives up on at 30 s - Driver
+ // .connect branches into its own thread as soon as loginTimeout is anything but 0.
+ if (!declared(connectionString, connectProperties)) {
+ for (final String property : connectProperties) {
+ properties.setProperty(property, Long.toString(connectSeconds * connectUnitsPerSecond));
+ }
+ }
+ reportBoundTurnedOffInUrl(connectionString);
+ if (!declared(connectionString, readProperties)) {
+ properties.setProperty(readProperties[0], Long.toString(timeoutSeconds * readUnitsPerSecond));
+ return readBoundOutlivesLogin;
+ }
+ return false;
+ }
+
+ /**
+ * Reports a url that turns a bound off where nothing this class supplies can put one back.
+ * A parameter of a postgresql url outranks the property this class hands the driver, so a
+ * "socketTimeout=0" there is not a default to be replaced - it is the administrator asking
+ * for an unbounded read, and a borrow that meets a database accepting the connection and
+ * answering nothing is then parked with no deadline able to reach it.
+ */
+ private void reportBoundTurnedOffInUrl(String connectionString) {
+ if (!urlOutranksProperties()) {
+ return;
+ }
+ // Every one of them, and keyed by the property rather than by the url alone: a url
+ // turns off the read bound and the login bound both ("?socketTimeout=0&loginTimeout=0",
+ // where the second is the per-host budget of a failover url), and safeUrl() keeps none
+ // of the timeout parameters - so a single key would report the first offender and
+ // leave the administrator to find the rest of them on their own.
+ for (final String[] properties : new String[][]{readProperties, connectProperties}) {
+ for (final String property : properties) {
+ final String value = parameterValue(connectionString, property);
+ if (value != null && !isBound(value)) {
+ warnOnce(safeUrl(connectionString) + "|unbounded|" + property,
+ "%s sets \"%s=%s\": a parameter of a postgresql url outranks the property this backend"
+ + " supplies, so that phase of a connect carries no bound. An operation reaching a"
+ + " database that accepts the connection and does not answer stays parked",
+ safeUrl(connectionString), property, value);
+ }
+ }
+ }
+ }
+
+ /** Whether a vendor code of this dialect is one that waiting for the database can clear. */
+ boolean isWorthRetrying(int errorCode) {
+ return contains(connectionLimitCodes, errorCode) || contains(notAcceptingYetCodes, errorCode);
+ }
+
+ private static boolean contains(int[] codes, int code) {
+ for (final int candidate : codes) {
+ if (candidate == code) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ // Whether the administrator bounded one of these properties themselves. The dialects
+ // separate their parameters differently - "?a=1&b=2" (postgresql, mysql), ";a=1;b=2" (sql
+ // server), "(A=1)" inside the descriptor of an oracle tns url, where the property also goes
+ // by the last segment of its name alone - so a parameter is recognized by the delimiter in
+ // front of it and the "=" behind it rather than by parsing the url syntax of every driver.
+ // The connection string is not the only channel of theirs: the oracle driver reads some of
+ // its properties out of the system properties as well, which is how a whole jvm is bounded
+ // with -Doracle.jdbc.ReadTimeout, and a property supplied to a driver outranks the system
+ // property without a word - and would then be lifted after the login as if it were ours,
+ // leaving a connection with no read bound where the administrator had set one.
+ private boolean declared(String connectionString, String... properties) {
+ for (final String property : properties) {
+ if (declaredInUrl(connectionString, property) || setAsSystemProperty(property)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /** Whether the connection string bounds this property, under its own name or the last segment of it. */
+ private boolean declaredInUrl(String connectionString, String property) {
+ if (containsParameter(connectionString, property)) {
+ return true;
+ }
+ final int dot = property.lastIndexOf('.');
+ return dot >= 0 && containsParameter(connectionString, property.substring(dot + 1));
+ }
+
+ // Which names a driver reads out of the system properties, listed rather than told from
+ // the shape of the name: ojdbc8 resolves oracle.jdbc.ReadTimeout and
+ // oracle.net.CONNECT_TIMEOUT in three tiers (the properties it was supplied, then
+ // System.getProperty, then the properties of the data source), while
+ // oracle.net.READ_TIMEOUT - a dotted name of the same driver - is read out of the
+ // connection properties alone: the six classes carrying the literal hand it to
+ // Properties.get, and none of them to System.getProperty. Taking a -D of it for a bound of
+ // the administrator would leave the login with no read bound at all - theirs not read by
+ // the driver and ours not set, because we believed theirs was in force.
+ private static final Set<String> SYSTEM_PROPERTY_NAMES = Collections.unmodifiableSet(new HashSet<>(
+ Arrays.asList("oracle.jdbc.ReadTimeout", "oracle.net.CONNECT_TIMEOUT")));
+
+ private static boolean setAsSystemProperty(String property) {
+ return SYSTEM_PROPERTY_NAMES.contains(property) && isBound(System.getProperty(property));
+ }
+
+ // pgjdbc parses the url over the properties it was handed - Driver.connect copies them
+ // into a flat map and parseURL then writes the parameters of the url on top - so a value
+ // standing in a postgresql url is the value the driver uses, and the one supplied here
+ // never reaches the socket. A zero there is not a default of the driver to be replaced: it
+ // cannot be replaced, and setting ours on top of it would leave this class lifting a read
+ // bound the login never had. The other three let a supplied property win, so a zero of
+ // theirs is ours to override.
+ private boolean urlOutranksProperties() {
+ return this == POSTGRES;
+ }
+
+ /** Whether this property is bounded by the connection string, as the driver of this dialect reads it. */
+ private boolean containsParameter(String connectionString, String property) {
+ final String value = parameterValue(connectionString, property);
+ return value != null && (urlOutranksProperties() || isBound(value));
+ }
+
+ // Matched the way the driver of this dialect matches it: pgjdbc and Connector/J look their
+ // properties up by their exact name - PropertyKey.fromValue answers null for a name of
+ // another case and the parameter is then a parameter of nobody, so "?SocketTimeout=" must
+ // not be taken for a bound of the administrator - while the SQL Server driver
+ // (getNormalizedPropertyName) and the keywords of an oracle descriptor match either way.
+ private String parameterValue(String connectionString, String property) {
+ final boolean exact = this == POSTGRES || this == MYSQL;
+ final String url = exact ? connectionString : connectionString.toLowerCase(Locale.ROOT);
+ final String name = exact ? property : property.toLowerCase(Locale.ROOT);
+ String value = null;
+ for (int i = url.indexOf(name); i >= 0; i = url.indexOf(name, i + name.length())) {
+ final int end = i + name.length();
+ if (i > 0 && "?&;(,".indexOf(url.charAt(i - 1)) >= 0 && end < url.length() && url.charAt(end) == '=') {
+ // the last of them: a driver parsing a url into a map lets the last assignment stand
+ value = valueOf(url, end + 1);
+ }
+ }
+ return value;
+ }
+
+ /** The value of the parameter that starts here: up to the delimiter in front of the next one. */
+ private static String valueOf(String url, int from) {
+ int end = from;
+ while (end < url.length() && "&;),?".indexOf(url.charAt(end)) < 0) {
+ end++;
+ }
+ return url.substring(from, end);
+ }
+
+ /**
+ * Whether a value of the administrator bounds anything. Every one of these drivers reads 0
+ * as "wait as long as it takes", so a property set to it is not a bound of theirs to stay
+ * out of the way of - it is the default this class exists to replace, and one of ours goes
+ * on top of it wherever a supplied property outranks the url. Where it does not, on
+ * postgresql, the zero stands and is reported instead of being written over. A value that
+ * is no number is left to the driver it belongs to.
+ */
+ private static boolean isBound(String value) {
+ if (value == null || value.trim().isEmpty()) {
+ return false;
+ }
+ try {
+ return Double.parseDouble(value.trim()) != 0; // pgjdbc takes a float for its loginTimeout
+ } catch (NumberFormatException notANumber) {
+ return true;
+ }
+ }
}
final String connectionString;
+ /**
+ * Whether this connection may go back into the pool once it is closed. A connection carrying a
+ * read bound that could not be lifted serves the borrower waiting for it and is closed
+ * afterwards: left in the pool it would fail every statement slower than that bound - an
+ * import batch among them - for every borrow the pool hands it to.
+ */
+ private final boolean poolable;
+
public CachedConnection(String connectionString, Connection parent) {
+ this(connectionString, parent, true);
+ }
+
+ CachedConnection(String connectionString, Connection parent, boolean poolable) {
this.connectionString = connectionString;
this.parent = parent;
+ this.poolable = poolable;
}
+ /**
+ * Borrows a connection: a usable one out of the pool, or a newly established one. Bounded in
+ * both phases - every operation of this backend, the open of a backend and the import
+ * included, comes through here, and an unbounded borrow turns a database that listens but does
+ * not answer into a hang rather than into an error the caller can report.
+ */
static Connection getConnection(String connectionString) throws Exception {
- return getConnection(connectionString, 0);
- }
-
- static Connection getConnection(String connectionString, final int waitTime) throws Exception {
- CachedConnection con = cached.get(connectionString).poll(waitTime, TimeUnit.MILLISECONDS);
-
- while (con != null) {
- if (!con.isValid(0)) {
- try {
- con.parent.close();
- } catch (SQLException e) {
- con = null;
+ final ConnectDialect dialect = ConnectDialect.of(connectionString);
+ reportUnknownDialect(connectionString, dialect);
+ final long connectTimeoutSeconds = Math.min(
+ getNonNegativeProperty(CONNECT_TIMEOUT_PROPERTY, DEFAULT_CONNECT_TIMEOUT_SECONDS, "s"),
+ Integer.MAX_VALUE / 1000);
+ final long poolTimeoutSeconds = getNonNegativeProperty(POOL_TIMEOUT_PROPERTY, DEFAULT_POOL_TIMEOUT_SECONDS, "s");
+ final long startedAt = System.currentTimeMillis();
+ final long deadline = (poolTimeoutSeconds == 0 || poolTimeoutSeconds >= Long.MAX_VALUE / 1000)
+ ? Long.MAX_VALUE : startedAt + poolTimeoutSeconds * 1000;
+ long waitMs = 0;
+ long backoffMs = 0;
+ int attempts = 0;
+ while (true) {
+ final CachedConnection pooled = poll(connectionString, waitMs, deadline);
+ if (pooled != null) {
+ return pooled;
+ }
+ attempts++;
+ try {
+ return connect(connectionString, dialect, attemptSeconds(connectTimeoutSeconds, deadline));
+ } catch (SQLException e) {
+ // A database that takes no connection for the moment is the failure worth waiting
+ // out: it is at its connection limit, and one of ours is going to come back to the
+ // pool - or it is on its way up, and the state clears itself in seconds. Everything
+ // else - a password that is not accepted, a database that is down, a driver that is
+ // not on the classpath - is reported to the caller instead of being retried behind
+ // its back.
+ if (!isWorthRetrying(e, dialect)) {
+ throw reported(e, connectionString);
}
- con = cached.get(connectionString).poll();
- } else {
- return con;
+ final long remaining = deadline - System.currentTimeMillis();
+ if (remaining <= 0) {
+ // 08001, the state of a connect that did not happen, rather than none at all:
+ // this is the failure of a borrow, and a caller reading the state of what it
+ // caught would otherwise see null where the driver's own exception carried one
+ final SQLTimeoutException timeout = new SQLTimeoutException("no connection to "
+ + safeUrl(connectionString) + " could be borrowed within " + poolTimeoutSeconds + "s ("
+ + attempts + " attempts): the database took no connection for the moment and none was"
+ + " returned to the pool, last error: " + redact(e.getMessage(), connectionString),
+ CONNECT_FAILED_SQL_STATE);
+ timeout.initCause(reported(e, connectionString));
+ throw timeout;
+ }
+ backoffMs = Math.min(backoffMs == 0 ? 1 : backoffMs * 2, MAX_BACKOFF_MS);
+ waitMs = Math.min(backoffMs, remaining);
+ warnStall(connectionString, attempts, startedAt, e);
+ } catch (RuntimeException e) {
+ // a driver reporting a connect it will not make as an unchecked failure carries the
+ // connection string of the backend in its message as readily as a SQLException does
+ throw reportedUnchecked(e, connectionString);
}
}
- Connection conNew = null;
+ }
+
+ /**
+ * Reports a connection string this class knows no bound for. The properties bounding a connect
+ * are the ones of a driver, so a driver outside the four leaves every attempt unbounded - and
+ * the deadline of the borrow cannot reach into a connect that is already under way, since the
+ * driver is the only thing holding the socket.
+ */
+ private static void reportUnknownDialect(String connectionString, ConnectDialect dialect) {
+ if (dialect != null) {
+ return;
+ }
+ final StringBuilder known = new StringBuilder();
+ for (final ConnectDialect candidate : ConnectDialect.values()) {
+ known.append(known.length() > 0 ? ", " : "").append(candidate.urlPrefix);
+ }
+ warnOnce(safeUrl(connectionString) + "|unknown-dialect",
+ "%s names a driver whose timeout properties are not known to this backend (%s are): a connect to a"
+ + " database that accepts it and does not answer is left without a bound, and the %s property"
+ + " cannot end it",
+ safeUrl(connectionString), known, POOL_TIMEOUT_PROPERTY);
+ }
+
+ /**
+ * The bound of one connect attempt. The deadline of the borrow bounds it as well - the
+ * {@value #POOL_TIMEOUT_PROPERTY} property stands for the whole borrow, and an attempt of its
+ * own left to run out would overrun it by a full connect timeout. That holds for an attempt
+ * the {@value #CONNECT_TIMEOUT_PROPERTY} property gives no bound of its own, too: turning the
+ * per-attempt bound off must not turn the bound of the borrow off with it. Never 0 for an
+ * attempt that is bounded at all: 0 is the value that stands for no bound. And never past what
+ * an int of milliseconds takes - the pool timeout has no upper bound of its own, while the SQL
+ * Server driver rejects a socketTimeout beyond Integer.MAX_VALUE outright, failing every
+ * connect of that backend with the name of a property nobody typed.
+ */
+ static long attemptSeconds(long connectTimeoutSeconds, long deadline) {
+ if (deadline == Long.MAX_VALUE) {
+ // 0 stands for an attempt with no bound of its own and stays 0; anything else is
+ // clamped here as well, so that the range holds whichever branch answers
+ return connectTimeoutSeconds == 0 ? 0 : Math.min(connectTimeoutSeconds, Integer.MAX_VALUE / 1000);
+ }
+ final long remainingSeconds = (deadline - System.currentTimeMillis() + 999) / 1000;
+ final long bound = connectTimeoutSeconds == 0
+ ? remainingSeconds : Math.min(connectTimeoutSeconds, remainingSeconds);
+ return Math.max(1, Math.min(bound, Integer.MAX_VALUE / 1000));
+ }
+
+ /**
+ * Takes a usable connection out of the pool, waiting up to waitMs for one to be returned to it.
+ * The validation of a connection costs a round trip, and the pool has no upper bound on the
+ * number of them it holds, so draining a pool the database no longer answers is given the
+ * deadline of the borrow as well: past it, establishing a connection is the faster answer.
+ * The connection in hand is always validated first, whatever the deadline says - a database at
+ * its connection limit has no other source of connections than the ones coming back, and one
+ * returned to the pool a moment before the deadline is the very connection this borrow waited
+ * for. Only a connection the database no longer answers is closed here.
+ */
+ private static CachedConnection poll(String connectionString, long waitMs, long deadline) throws InterruptedException {
+ CachedConnection con = cached.get(connectionString).poll(waitMs, TimeUnit.MILLISECONDS);
+ while (con != null) {
+ if (isUsable(con)) {
+ return con;
+ }
+ closeQuietly(con.parent);
+ if (System.currentTimeMillis() >= deadline) {
+ return null;
+ }
+ con = cached.get(connectionString).poll();
+ }
+ return null;
+ }
+
+ private static boolean isUsable(CachedConnection con) {
+ // The validation needs a bound of its own: isValid(0) means "no timeout" in the JDBC
+ // contract, and a connection whose socket is half-open answers it no sooner than it
+ // answers anything else. isValid(n) is not that bound on every driver either - the SQL
+ // Server driver turns it into a query timeout (setQueryTimeout, then "SELECT 1"), which
+ // needs an answer from the server to fire at all - so the socket is bounded here, for the
+ // validation only.
+ final int restore = boundValidation(con.parent);
+ if (restore == VALIDATION_BOUND_FAILED) {
+ // the bound of the validation is not in force, and the driver may well have applied it
+ // before failing: validating here would be the unbounded isValid() this exists to
+ // avoid, and pooling it would hand out a connection carrying a bound of ours
+ return false;
+ }
+ boolean usable;
try {
- conNew = DriverManager.getConnection(connectionString);
+ usable = con.isValid(VALIDATION_TIMEOUT_SECONDS);
+ } catch (SQLException | RuntimeException e) { // a driver reporting the validation as an error: discard it
+ // an unchecked failure out of a driver would unwind through poll(), which stands
+ // outside every try of the borrow, and leave this connection dequeued and unclosed
+ usable = false;
+ }
+ if (!usable) {
+ // On its way out, and the driver knows it: Connector/J answers a failed validation by
+ // aborting the connection and the SQL Server driver by terminating it, so putting the
+ // previous bound back would fail as well - and warn about a bound of a connection that
+ // is about to be closed, over a reaped idle connection that is nobody's problem.
+ return false;
+ }
+ if (restore >= 0 && !setNetworkTimeout(con.parent, restore,
+ "the connection is closed rather than pooled")) {
+ return false; // it would carry the bound of the validation into every statement
+ }
+ return true;
+ }
+
+ /** A connection left alone by {@link #boundValidation}: no bound of ours to put back afterwards. */
+ private static final int VALIDATION_BOUND_LEFT_ALONE = -1;
+ /** A connection {@link #boundValidation} could not bound, which may still carry the bound it failed to report. */
+ private static final int VALIDATION_BOUND_FAILED = -2;
+
+ /**
+ * Bounds the socket of a pooled connection for the length of its validation, returning the
+ * network timeout to put back afterwards - or {@link #VALIDATION_BOUND_LEFT_ALONE} for a
+ * connection left alone, either because the driver does not take a network timeout or because
+ * it is bounded at least as tightly already, by a read timeout of the connection string that
+ * is not ours to widen.
+ * A driver that takes the call and then fails inside it is told apart from both: it is free to
+ * have applied the bound before failing, and a connection put back into the pool carrying five
+ * seconds of ours fails every statement slower than that for the rest of its life.
+ */
+ private static int boundValidation(Connection con) {
+ final int bound = VALIDATION_TIMEOUT_SECONDS * 1000;
+ final int previous;
+ try {
+ previous = con.getNetworkTimeout();
+ } catch (SQLException | RuntimeException e) { // a driver that does not take one: nothing was changed
+ return VALIDATION_BOUND_LEFT_ALONE;
+ }
+ if (previous > 0 && previous <= bound) {
+ return VALIDATION_BOUND_LEFT_ALONE;
+ }
+ try {
+ con.setNetworkTimeout(DIRECT_EXECUTOR, bound);
+ } catch (SQLException | RuntimeException e) {
+ return VALIDATION_BOUND_FAILED;
+ }
+ // A driver answering a negative timeout is outside the contract of getNetworkTimeout(),
+ // where 0 stands for no limit and nothing below it stands for anything. Handed back as it
+ // is, it would be one of the two sentinels above: the bound just set would be read as a
+ // bound that was never set, and the connection would go into the pool carrying five
+ // seconds of ours into every statement of whoever borrows it next.
+ return previous < 0 ? 0 : previous;
+ }
+
+ static CachedConnection connect(String connectionString, ConnectDialect dialect, long connectTimeoutSeconds)
+ throws SQLException {
+ // A driver is free to write into the map it is handed, so it gets one of its own.
+ final Properties properties = new Properties();
+ final boolean readBoundSet = dialect != null && connectTimeoutSeconds > 0
+ && dialect.bound(connectionString, properties, connectTimeoutSeconds);
+ final Connection conNew = DriverManager.getConnection(connectionString, properties);
+ boolean poolable = true;
+ try {
+ // still under the read bound: both of these are round trips of their own
conNew.setAutoCommit(false);
conNew.setTransactionIsolation(TRANSACTION_READ_COMMITTED);
- return new CachedConnection(connectionString, conNew);
- } catch (SQLException e) { // max_connection server error: try recursion for reuse connection
- if (conNew != null) { // the connection was established but not set up: nothing else would close it
- try {
- conNew.close();
- } catch (SQLException e2) {}
+ if (readBoundSet) {
+ // a driver that will not take the bound back has warned about it already: the
+ // connection serves the borrower that is waiting for it and is closed rather than
+ // pooled, so the bound of the login does not outlive it in the pool
+ poolable = relaxReadBound(conNew, connectTimeoutSeconds);
}
- return getConnection(connectionString, (waitTime == 0) ? 1 : waitTime * 2);
+ } catch (SQLException | RuntimeException e) { // nothing holds this connection yet: it would leak
+ closeQuietly(conNew);
+ throw e;
+ }
+ return new CachedConnection(connectionString, conNew, poolable);
+ }
+
+ // The second bound of the login is a socket read timeout on mysql, oracle and sql server, in
+ // force for the whole life of the connection: left in place it would break every statement
+ // slower than it - an import batch, the statistics of a freshly loaded table - so it is lifted
+ // as soon as the login is through, restoring the behaviour of a connection this class
+ // established before. A read bound the connection string sets itself is never touched here:
+ // it is not set at all, so nothing of the administrator's is lifted along with it. Returns
+ // whether the bound is gone - a connection still carrying it must not be pooled.
+ // Named by the bound the login was given rather than by the property it came from: with
+ // CONNECT_TIMEOUT_PROPERTY at 0 the attempt takes its bound from what is left of the deadline
+ // of the borrow, so naming that property would point at the one setting that is not in force.
+ private static boolean relaxReadBound(Connection con, long boundSeconds) {
+ return setNetworkTimeout(con, 0, "statements taking longer than the " + boundSeconds
+ + "s the login of this connection was bounded by fail on it, and it is closed rather than pooled");
+ }
+
+ /**
+ * Puts a network timeout on a connection, reporting a driver that will not take one. The
+ * consequence is the caller's to name: the same failure ends a freshly established connection
+ * carrying the read bound of its login and a pooled one whose bound could not be put back.
+ */
+ private static boolean setNetworkTimeout(Connection con, int millis, String consequence) {
+ try {
+ con.setNetworkTimeout(DIRECT_EXECUTOR, millis);
+ return true;
+ } catch (SQLException | RuntimeException e) {
+ // Throttled rather than reported once for the life of the JVM: every connection this
+ // happens to carries a read bound it was never meant to keep, and a statement dying of
+ // it hours later needs a warning of its own to be traced back to here.
+ final long now = System.currentTimeMillis();
+ final long last = lastReadBoundWarning.get();
+ if (now - last >= STALL_WARNING_INTERVAL_MS && lastReadBoundWarning.compareAndSet(last, now)) {
+ logger.warn(LocalizableMessage.raw(
+ "The read bound of a JDBC connection could not be set to %d ms (%s): %s",
+ millis, e.getMessage(), consequence));
+ }
+ return false;
+ }
+ }
+
+ /**
+ * Whether the database took no connection for the moment, rather than refusing one for good:
+ * it is at its connection limit - one of our own connections is on its way back to the pool -
+ * or it is not accepting connections yet, the state a database on its way up reports while it
+ * recovers - the one JDBCStorage.open() has no second attempt of its own for, so a backend
+ * that meets it stays locked down until the server is restarted. Both clear themselves in
+ * seconds; every other failure is the caller's to see.
+ */
+ static boolean isWorthRetrying(SQLException e, ConnectDialect dialect) {
+ // a failure of the driver is often wrapped, and a SQLException carries two chains of its
+ // own: the causes behind it and the further exceptions of getNextException()
+ final Deque<Throwable> pending = new ArrayDeque<>();
+ final Set<Throwable> visited = Collections.newSetFromMap(new IdentityHashMap<Throwable, Boolean>());
+ enqueue(pending, visited, e);
+ for (int links = 0; !pending.isEmpty() && links < MAX_CHAIN_LENGTH; links++) {
+ final Throwable t = pending.poll();
+ if (t instanceof SQLException) {
+ final SQLException sql = (SQLException) t;
+ final String sqlState = sql.getSQLState();
+ if (CONNECTION_LIMIT_SQL_STATE.equals(sqlState) || NOT_ACCEPTING_YET_SQL_STATE.equals(sqlState)
+ || (dialect != null && dialect.isWorthRetrying(sql.getErrorCode()))) {
+ return true;
+ }
+ enqueue(pending, visited, sql.getNextException());
+ }
+ enqueue(pending, visited, t.getCause());
+ }
+ return false;
+ }
+
+ // A stall has to reach the server log: without it a database accepting no further connection
+ // is indistinguishable from a hang. Throttled, since every operation of the backend borrows
+ // through here and would otherwise log a copy of its own.
+ private static void warnStall(String connectionString, int attempts, long startedAt, SQLException cause) {
+ final long now = System.currentTimeMillis();
+ if (now - startedAt < STALL_WARNING_AFTER_MS) {
+ return;
+ }
+ final AtomicLong lastOfThisUrl =
+ lastStallWarning.computeIfAbsent(safeUrl(connectionString), url -> new AtomicLong());
+ final long last = lastOfThisUrl.get();
+ if (now - last >= STALL_WARNING_INTERVAL_MS && lastOfThisUrl.compareAndSet(last, now)) {
+ logger.warn(LocalizableMessage.raw("%s", stallMessage(connectionString, attempts, now - startedAt, cause)));
+ }
+ }
+
+ /**
+ * The stall as it reaches the log. Built apart from the logging of it so that the rule it has
+ * to keep - neither the connection string nor the message of the driver reaches a log as it
+ * stands - is a rule a test can hold it to.
+ */
+ static String stallMessage(String connectionString, int attempts, long waitedMs, SQLException cause) {
+ return String.format("%s takes no further connection: waiting %d ms for a pooled one so far (%d attempts),"
+ + " last error: %s", safeUrl(connectionString), waitedMs, attempts,
+ redact(cause.getMessage(), connectionString));
+ }
+
+ /**
+ * The failure of a connect as it may leave this class: the exception itself where nothing of it
+ * names the credentials of the backend, and a redacted rebuild of its whole chain where
+ * something does. Rebuilt rather than wrapped: a wrapper keeps its cause, and everything that
+ * prints a failure prints the causes along with it - a debug build of
+ * stackTraceToSingleLineString walks them, the config manager traces them, and
+ * RootContainer.open() makes the message of the cause the message of what it throws - so a
+ * link left as it stands would carry the password past the wrapper. The SQLState and the
+ * vendor code of every link survive it: they are what tells a caller what happened.
+ */
+ static SQLException reported(SQLException e, String connectionString) {
+ return holdsCredentials(e, connectionString)
+ ? redactedCopy(e, connectionString, new int[] { MAX_CHAIN_LENGTH })
+ : e;
+ }
+
+ /** The same of an unchecked failure: a driver is free to report a connect it will not make as one. */
+ static Exception reportedUnchecked(RuntimeException e, String connectionString) {
+ if (!holdsCredentials(e, connectionString)) {
+ return e;
+ }
+ final SQLException redacted = new SQLNonTransientConnectionException(e.getClass().getName()
+ + (e.getMessage() == null ? "" : ": " + redact(e.getMessage(), connectionString)),
+ CONNECT_FAILED_SQL_STATE);
+ redacted.setStackTrace(e.getStackTrace());
+ return redacted;
+ }
+
+ /**
+ * Whether anything in the chain of a failure names what a connection string keeps out of the
+ * log. A chain longer than this walk is given answers "yes": what is reported unredacted is
+ * what this class has looked at whole, and a link it never reached is not that. The cost of
+ * being wrong that way is a chain rebuilt - bounded in its turn - while the cost of being
+ * wrong the other way is the password of the backend in the server error log.
+ */
+ private static boolean holdsCredentials(Throwable failure, String connectionString) {
+ final Deque<Throwable> pending = new ArrayDeque<>();
+ final Set<Throwable> visited = Collections.newSetFromMap(new IdentityHashMap<Throwable, Boolean>());
+ enqueue(pending, visited, failure);
+ for (int links = 0; !pending.isEmpty(); links++) {
+ if (links >= MAX_CHAIN_LENGTH) {
+ return true;
+ }
+ final Throwable t = pending.poll();
+ final String message = t.getMessage();
+ if (message != null && !message.equals(redact(message, connectionString))) {
+ return true;
+ }
+ if (t instanceof SQLException) {
+ enqueue(pending, visited, ((SQLException) t).getNextException());
+ }
+ enqueue(pending, visited, t.getCause());
+ }
+ return false;
+ }
+
+ // By identity rather than by equals(): a link of a chain carries a cause and a next exception
+ // both, and a driver is free to make the two the same failure. Enqueued twice, a chain of
+ // those fans out into a copy of itself at every step and spends the budget of a walk on links
+ // it has already looked at - five levels of one are enough to exhaust MAX_CHAIN_LENGTH.
+ private static void enqueue(Deque<Throwable> pending, Set<Throwable> visited, Throwable t) {
+ if (t != null && visited.add(t)) {
+ pending.add(t);
+ }
+ }
+
+ // The budget counts the links this rebuilds, the way holdsCredentials() counts the ones it
+ // visits - not how deep it has gone. A link of a chain carries a cause and a next exception
+ // both, and a driver is free to make them the same failure, so a bound on depth alone leaves
+ // room for a chain that fans out into two copies of itself at every step.
+ private static SQLException redactedCopy(SQLException e, String connectionString, int[] budget) {
+ budget[0]--;
+ final SQLException copy =
+ new SQLException(redact(e.getMessage(), connectionString), e.getSQLState(), e.getErrorCode());
+ copy.setStackTrace(e.getStackTrace());
+ if (e.getNextException() != null) {
+ copy.setNextException(budget[0] > 0
+ ? redactedCopy(e.getNextException(), connectionString, budget) : droppedTail());
+ }
+ if (e.getCause() != null) {
+ copy.initCause(budget[0] > 0 ? redactedLink(e.getCause(), connectionString, budget) : droppedTail());
+ }
+ return copy;
+ }
+
+ // What stands where the budget ran out. Without it the same failure logs its root cause when
+ // the url of the backend has no password in it and loses it without a word when it has, which
+ // is a report of a connect nobody can read against a report of one they can.
+ private static SQLException droppedTail() {
+ return new SQLException("the rest of this failure was left out: a chain of more than "
+ + MAX_CHAIN_LENGTH + " links is rebuilt only that far");
+ }
+
+ // A link that is no SQLException keeps its class name in the message: its type is not one this
+ // can rebuild, and the name of the failure is what a reader of the log is after.
+ private static Throwable redactedLink(Throwable t, String connectionString, int[] budget) {
+ if (t instanceof SQLException) {
+ return redactedCopy((SQLException) t, connectionString, budget);
+ }
+ budget[0]--;
+ final Throwable copy = new Throwable(t.getClass().getName()
+ + (t.getMessage() == null ? "" : ": " + redact(t.getMessage(), connectionString)));
+ copy.setStackTrace(t.getStackTrace());
+ if (t.getCause() != null) {
+ copy.initCause(budget[0] > 0 ? redactedLink(t.getCause(), connectionString, budget) : droppedTail());
+ }
+ return copy;
+ }
+
+ /**
+ * A message of a driver as it may be logged. A driver is free to put the connection string it
+ * was handed into it - the jdk itself does, "No suitable driver found for " + url, which is
+ * what the ordinary oracle misconfiguration of a driver jar left out of lib/extensions arrives
+ * as - and that connection string is where the credentials of this backend live.
+ * What it cannot answer for is a driver quoting back a part of a url it failed to parse:
+ * a whole credential is replaced, a fragment of one is not.
+ */
+ static String redact(String message, String connectionString) {
+ if (message == null || message.isEmpty()) {
+ return message;
+ }
+ String redacted = message.replace(connectionString, safeUrl(connectionString));
+ // The parameter before the values: a password blanked here is one the loop below no longer
+ // finds, while the other way round a "<credentials hidden>" standing behind a "password="
+ // would be cut in half by a pattern that ends its value at the first space.
+ redacted = SECRET_PARAMETER.matcher(redacted).replaceAll("$1=***");
+ for (final String secret : secretsOf(connectionString)) {
+ redacted = secretPattern(secret).matcher(redacted)
+ .replaceAll(Matcher.quoteReplacement(CREDENTIALS_HIDDEN));
+ }
+ return redacted;
+ }
+
+ /**
+ * A secret as it is looked for in a message: the value itself, wherever it does not stand
+ * inside a longer run of letters and digits. A password is free to be one character long, and
+ * a bare one of those is a substring of half the lines a driver writes - a password of "1"
+ * takes "ORA-12541: TNS:no listener" apart into a line nobody can read, and it makes every
+ * failure of that backend one this class believes names the credentials, so the whole chain is
+ * rebuilt as well. A redaction that destroys the diagnostic it protects is the worse of the
+ * two failures. What a driver quotes back is a credential standing on its own - between the
+ * delimiters of a url, or in a sentence of its own - and that is still replaced.
+ */
+ private static Pattern secretPattern(String secret) {
+ final String before = isAlphanumeric(secret.charAt(0)) ? "(?<![A-Za-z0-9])" : "";
+ final String after = isAlphanumeric(secret.charAt(secret.length() - 1)) ? "(?![A-Za-z0-9])" : "";
+ return Pattern.compile(before + Pattern.quote(secret) + after);
+ }
+
+ private static boolean isAlphanumeric(char c) {
+ return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9');
+ }
+
+ /** What of a connection string must not stand in a message: the credentials safeUrl() takes out of it. */
+ private static List<String> secretsOf(String connectionString) {
+ final List<String> secrets = new ArrayList<>();
+ final ConnectDialect dialect = ConnectDialect.of(connectionString);
+ final String separators = dialect == null ? "?;" : String.valueOf(dialect.parameterSeparator);
+ final int scheme = connectionString.indexOf(':', "jdbc:".length()) + 1;
+ if (scheme <= 0) {
+ return secrets;
+ }
+ final int authority = startOfAuthority(connectionString, scheme);
+ if (authority < 0) {
+ final int at = connectionString.indexOf('@', scheme);
+ if (at > scheme) {
+ addSecret(secrets, connectionString.substring(credentialsStart(connectionString, scheme, at), at));
+ }
+ } else {
+ final int end = endOfAuthority(connectionString, authority, separators);
+ for (final String host : connectionString.substring(authority, end).split(",", -1)) {
+ final int at = host.lastIndexOf('@');
+ if (at > 0) {
+ addSecret(secrets, host.substring(0, at));
+ }
+ }
+ }
+ final Matcher parameter = SECRET_PARAMETER.matcher(connectionString);
+ while (parameter.find()) {
+ addSecret(secrets, parameter.group(3));
+ }
+ return secrets;
+ }
+
+ // The credentials of one host, and the password inside them without the user name in front of
+ // it: a driver quoting a url back names either.
+ private static void addSecret(List<String> secrets, String credentials) {
+ if (credentials.isEmpty()) {
+ return;
+ }
+ secrets.add(credentials);
+ final int password = indexOfAny(credentials, ":/", 0);
+ if (password >= 0 && password + 1 < credentials.length()) {
+ secrets.add(credentials.substring(password + 1));
+ }
+ }
+
+ // The connection string carries the credentials of the backend - JDBCStorage hands the whole
+ // db-directory of the configuration to this class, so the url is the only place they live -
+ // and it is never logged as it stands. Three shapes hold them and all three are taken off: the
+ // "user/password@" in front of an oracle descriptor; the userinfo of an authority, one per
+ // host of it, since a url of Connector/J gives every host credentials of its own
+ // ("//u:p@h1:3306,u2:p2@h2:3306"); and the parameters behind their first separator,
+ // "?user=...&password=..." on postgresql, mysql and oracle, ";password=..." on sql server.
+ // What is left is looked over once more: the key-value host syntax of Connector/J puts a
+ // password inside the authority itself ("//address=(host=h)(user=u)(password=p)"), where
+ // neither of the first two shapes stands, so a "password=" of any case is blanked out wherever
+ // it is left standing.
+ //
+ // A password is free to hold either of the delimiters, so neither of them is looked for in the
+ // whole string. The credentials of an oracle url stand between the subprotocol and the first
+ // "@", which is the delimiter of its descriptor, so a "?" of one is part of the password
+ // rather than the start of the parameters. Everywhere else they stand inside the authority,
+ // between "//" and the path behind it, so a "?" of a password is inside them and an "@" of a
+ // parameter value ("?user=u@example.com") is not mistaken for the end of them: the host
+ // survives in the message either way.
+ //
+ // And a url none of this took apart is not logged past its subprotocol. An "@" left standing
+ // anywhere but where the credentials of an oracle url ended is one this did not recognize - a
+ // password holding a "/" inside an authority, a quoted one holding an "@" - and the host of a
+ // stall report is worth less than a password in the server log.
+ static String safeUrl(String connectionString) {
+ final ConnectDialect dialect = ConnectDialect.of(connectionString);
+ final String separators = dialect == null ? "?;" : String.valueOf(dialect.parameterSeparator);
+ final int scheme = connectionString.indexOf(':', "jdbc:".length()) + 1; // the end of "jdbc:<subprotocol>:"
+ if (scheme <= 0) {
+ return CREDENTIALS_HIDDEN;
+ }
+ final String stripped = stripCredentials(connectionString, scheme, separators);
+ final int parameters = indexOfAny(stripped, separators, scheme);
+ final String url = parameters < 0 ? stripped : stripped.substring(0, parameters);
+ final String redacted = SECRET_PARAMETER.matcher(url).replaceAll("$1=***");
+ // an "@" standing anywhere but where the credentials of an oracle url ended is one this
+ // did not recognize - a password holding a "/" inside an authority, a quoted one holding
+ // an "@" - and the host of a stall report is worth less than a password in the server log
+ if (redacted.lastIndexOf('@') > endOfRecognizedCredentials(connectionString, scheme)) {
+ return redacted.substring(0, scheme) + CREDENTIALS_HIDDEN;
+ }
+ return parameters < 0 ? redacted
+ : redacted + identifyingParameters(stripped.substring(parameters), dialect);
+ }
+
+ /**
+ * The parameters worth keeping in a message: the ones naming the database rather than whoever
+ * connects to it. Two backends of one host answer to the same url up to their parameters, and
+ * a stall report that cannot tell them apart is a stall report of neither. Everything else is
+ * dropped rather than looked at - a name this does not know is a name free to carry a secret.
+ */
+ private static String identifyingParameters(String parameters, ConnectDialect dialect) {
+ final char separator = dialect == null ? ';' : dialect.parameterSeparator;
+ final StringBuilder kept = new StringBuilder();
+ for (final String parameter : parameters.split("[?&;]")) {
+ final int equals = parameter.indexOf('=');
+ if (equals > 0
+ && IDENTIFYING_PARAMETERS.contains(parameter.substring(0, equals).toLowerCase(Locale.ROOT))) {
+ kept.append(kept.length() == 0 || separator != '?' ? separator : '&').append(parameter);
+ }
+ }
+ return kept.toString();
+ }
+
+ private static String stripCredentials(String url, int scheme, String separators) {
+ final int authority = startOfAuthority(url, scheme);
+ if (authority < 0) {
+ // no authority: the credentials of an oracle url stand between the subprotocol and the
+ // first "@", which is the delimiter of the descriptor behind it - a password holding
+ // an "@" of its own has to be quoted for the driver itself
+ final int at = url.indexOf('@', scheme);
+ return at < 0 ? url : url.substring(0, credentialsStart(url, scheme, at)) + url.substring(at);
+ }
+ final int end = endOfAuthority(url, authority, separators);
+ return url.substring(0, authority) + withoutUserinfo(url.substring(authority, end)) + url.substring(end);
+ }
+
+ /**
+ * Where the credentials of a url that names no authority start: behind the token naming the
+ * kind of driver, which stands in front of them ("jdbc:oracle:thin:user/pw@...") and is worth
+ * keeping - thin against oci is a first question of an oracle connect. The token is the one
+ * right behind the subprotocol rather than the last one in front of the "@", since a password
+ * is free to hold a ":" of its own.
+ */
+ private static int credentialsStart(String url, int scheme, int at) {
+ final int driverType = url.indexOf(':', scheme);
+ return driverType >= 0 && driverType < at ? driverType + 1 : scheme;
+ }
+
+ /**
+ * The last position a stripped url may still carry an "@" at: where the credentials of an
+ * oracle url ended, since the "@" is the delimiter of the descriptor behind them and stays.
+ * An authority keeps none of its own - every userinfo of it is taken off, delimiter included.
+ */
+ private static int endOfRecognizedCredentials(String url, int scheme) {
+ final int at = url.indexOf('@', scheme);
+ return startOfAuthority(url, scheme) < 0 && at > scheme ? credentialsStart(url, scheme, at) : scheme;
+ }
+
+ /**
+ * Where the hosts of a url of this shape start, or -1 for a url that names no authority. The
+ * subprotocol is free to name the kind of connection in front of it - "jdbc:mysql:replication://"
+ * - so the "//" is looked for rather than expected right behind the subprotocol. An "@" in
+ * front of it belongs to an oracle url ("jdbc:oracle:thin:user/pw@//host"), whose credentials
+ * stand where an authority has no place for them.
+ */
+ private static int startOfAuthority(String url, int scheme) {
+ final int slashes = url.indexOf("//", scheme);
+ if (slashes < 0 || url.lastIndexOf('@', slashes) >= scheme) {
+ return -1;
+ }
+ // ... and so does a "/" in front of them: it is what separates the credentials of an
+ // oracle url ("thin:user/pw@..."), so a password holding a "//" of its own would start an
+ // authority inside itself. The "@" ending the credentials stands behind that point, the
+ // userinfo taken off is a piece of the password rather than the whole of it, and the "@"
+ // the guard of safeUrl() looks for is gone with it - leaving the user name and the head of
+ // the password in the message of a stall.
+ final int slash = url.indexOf('/', scheme);
+ return slash >= 0 && slash < slashes ? -1 : slashes + 2;
+ }
+
+ /**
+ * Where the hosts of an authority end: at the path behind them - a password holds a "?" more
+ * readily than a "/" - or at the first parameter of a url that has no path.
+ */
+ private static int endOfAuthority(String url, int authority, String separators) {
+ final int path = url.indexOf('/', authority);
+ if (path >= 0) {
+ return path;
+ }
+ final int parameter = indexOfAny(url, separators, authority);
+ return parameter < 0 ? url.length() : parameter;
+ }
+
+ /** The hosts of an authority, each of them without the credentials a url may give it. */
+ private static String withoutUserinfo(String authority) {
+ final StringBuilder hosts = new StringBuilder();
+ final String[] split = authority.split(",", -1);
+ for (int i = 0; i < split.length; i++) {
+ if (i > 0) { // by the position rather than by what is in hand: a first host may be empty
+ hosts.append(',');
+ }
+ final int at = split[i].lastIndexOf('@');
+ hosts.append(at < 0 ? split[i] : split[i].substring(at + 1));
+ }
+ return hosts.toString();
+ }
+
+ private static int indexOfAny(String url, String separators, int from) {
+ int found = -1;
+ for (int i = 0; i < separators.length(); i++) {
+ final int at = url.indexOf(separators.charAt(i), from);
+ if (at >= 0 && (found < 0 || at < found)) {
+ found = at;
+ }
+ }
+ return found;
+ }
+
+ private static void closeQuietly(Connection con) {
+ try {
+ con.close();
+ } catch (SQLException e) {
+ // ignore: it is on its way out anyway
}
}
@@ -153,7 +1220,18 @@
@Override
public void close() throws SQLException {
- rollback();
+ try {
+ rollback();
+ } catch (SQLException e) {
+ // A connection that cannot be rolled back must not be handed to the next borrower -
+ // and must not be dropped on the floor either: nothing else holds it any more.
+ closeQuietly(parent);
+ throw e;
+ }
+ if (!poolable) {
+ closeQuietly(parent);
+ return;
+ }
cached.get(connectionString).add(this);
}
diff --git a/opendj-server-legacy/src/test/java/org/opends/server/backends/jdbc/CachedConnectionTestCase.java b/opendj-server-legacy/src/test/java/org/opends/server/backends/jdbc/CachedConnectionTestCase.java
new file mode 100644
index 0000000..0caad17
--- /dev/null
+++ b/opendj-server-legacy/src/test/java/org/opends/server/backends/jdbc/CachedConnectionTestCase.java
@@ -0,0 +1,1466 @@
+/*
+ * The contents of this file are subject to the terms of the Common Development and
+ * Distribution License (the License). You may not use this file except in compliance with the
+ * License.
+ *
+ * You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the
+ * specific language governing permission and limitations under the License.
+ *
+ * When distributing Covered Software, include this CDDL Header Notice in each file and include
+ * the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL
+ * Header, with the fields enclosed by brackets [] replaced by your own identifying
+ * information: "Portions Copyright [year] [name of copyright owner]".
+ *
+ * Copyright 2026 3A Systems, LLC.
+ */
+package org.opends.server.backends.jdbc;
+
+import org.opends.server.DirectoryServerTestCase;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import java.net.InetAddress;
+import java.net.ServerSocket;
+import java.sql.Connection;
+import java.sql.Driver;
+import java.sql.DriverManager;
+import java.sql.DriverPropertyInfo;
+import java.sql.SQLException;
+import java.sql.SQLTimeoutException;
+import java.util.ArrayDeque;
+import java.util.Collections;
+import java.util.Deque;
+import java.util.IdentityHashMap;
+import java.util.Properties;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executor;
+import java.util.concurrent.FutureTask;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.logging.Logger;
+
+import org.mockito.InOrder;
+
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.anyInt;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.eq;
+import static org.mockito.Mockito.inOrder;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertNull;
+import static org.testng.Assert.assertSame;
+import static org.testng.Assert.assertTrue;
+import static org.testng.Assert.fail;
+
+/**
+ * The pool every operation of the JDBC backend borrows from must bound both of its phases and
+ * report a connect it cannot make, rather than retrying it out of sight of the caller (#872).
+ * Needs no database: the dialects are exercised against a socket that never answers and against a
+ * driver of this test, so a regression fails the build wherever it runs.
+ */
+@SuppressWarnings("javadoc")
+@Test(groups = { "precommit", "jdbc" }, sequential = true)
+public class CachedConnectionTestCase extends DirectoryServerTestCase {
+
+ /** A connect attempt of a bounded dialect must give up in about this long, plus room for a slow machine. */
+ private static final long BOUND_SECONDS = 2;
+ /**
+ * Room for a slow machine on top of a bound, and no more than that. A minute of it turned
+ * every assertion below into "it does not run forever": a connect that has to be reported at
+ * once and a borrow that has to give up at its two second deadline both passed at 59 s.
+ */
+ private static final long BOUND_MARGIN_MS = 10000;
+
+ private final StubDriver stub = new StubDriver();
+
+ @BeforeClass
+ public void registerStubDriver() throws Exception {
+ DriverManager.registerDriver(stub);
+ }
+
+ @AfterClass
+ public void deregisterStubDriver() throws Exception {
+ DriverManager.deregisterDriver(stub);
+ }
+
+ @AfterMethod
+ public void clearProperties() {
+ System.clearProperty(CachedConnection.CONNECT_TIMEOUT_PROPERTY);
+ System.clearProperty(CachedConnection.POOL_TIMEOUT_PROPERTY);
+ // what has been reported once is remembered for the life of the jvm: left standing, the key
+ // of one test is what the next one finds when it asserts that it reported something itself
+ CachedConnection.warnedOnce.clear();
+ }
+
+ /**
+ * A driver that is not on the classpath - the JDBC backend needs one dropped into
+ * lib/extensions by hand - is a configuration error the caller has to see. Retried, it is
+ * indistinguishable from a database that hangs.
+ */
+ @Test(timeOut = 120000)
+ public void testMissingDriverIsReportedAtOnce() throws Exception {
+ final long startedAt = System.currentTimeMillis();
+ try {
+ CachedConnection.getConnection("jdbc:nosuchengine://127.0.0.1:5432/opendj");
+ fail("a connection string no registered driver accepts must be reported");
+ } catch (SQLException expected) {
+ assertTrue(expected.getMessage().contains("No suitable driver"), expected.getMessage());
+ }
+ assertElapsedWithinBound(startedAt, 0);
+ }
+
+ /**
+ * ... and the report of it carries no password. This is the path the credentials leave by: the
+ * jdk itself builds "No suitable driver found for " + url, a missing driver jar is the ordinary
+ * oracle misconfiguration, and what this class throws reaches the server error log in full -
+ * JDBCStorage.open() hands it to RootContainer, which makes the message of the cause the
+ * message of what it throws, and BackendConfigManager logs that at ERROR and answers a config
+ * change with it. Every link of the chain is asserted, not only the message on top: everything
+ * that prints a failure prints its causes along with it.
+ */
+ @Test(timeOut = 120000)
+ public void testAReportedConnectCarriesNoCredentials() throws Exception {
+ final String url = "jdbc:nosuchengine://opendj:S3cretOfTheBackend@127.0.0.1:5432/opendj";
+ try {
+ CachedConnection.getConnection(url);
+ fail("a connection string no registered driver accepts must be reported");
+ } catch (SQLException expected) {
+ assertNoCredentials(expected);
+ assertTrue(expected.getMessage().contains("No suitable driver"),
+ "the failure has to stay recognizable: " + expected.getMessage());
+ assertTrue(expected.getMessage().contains("127.0.0.1:5432"),
+ "the host is what a report of a connect is read for: " + expected.getMessage());
+ }
+ }
+
+ /**
+ * A url this class knows no timeout properties for is reported once. The properties bounding a
+ * connect are the ones of a driver, so a driver outside the four - an admin-added mariadb, an
+ * h2 - leaves every attempt unbounded, and the deadline of the borrow cannot reach into a
+ * connect already under way: the driver is the only thing holding the socket. Silently, that
+ * is #872 again, for a backend nobody thinks of as unbounded.
+ */
+ @Test(timeOut = 120000)
+ public void testAUrlThisBackendCannotBoundIsReportedOnce() throws Exception {
+ final String url = "jdbc:nosuchengine-unbounded://127.0.0.1:5432/opendj";
+ final String key = CachedConnection.safeUrl(url) + "|unknown-dialect";
+ // the set is the gate warnOnce() logs behind, so it has to start empty for this to be a
+ // test of what this borrow reported rather than of what some earlier one left behind
+ CachedConnection.warnedOnce.clear();
+ borrowExpectingFailure(url);
+ assertEquals(CachedConnection.warnedOnce, Collections.singleton(key),
+ "a connection string no bound of this class can reach was not reported");
+
+ // ... and once: every operation of the backend borrows through here, so a report per borrow
+ // is one nobody reads. A second borrow that would log again is one that adds a key again.
+ CachedConnection.warnedOnce.remove(key);
+ borrowExpectingFailure(url);
+ assertEquals(CachedConnection.warnedOnce, Collections.singleton(key),
+ "the report is not the one warnOnce() gates: " + CachedConnection.warnedOnce);
+ borrowExpectingFailure(url);
+ assertEquals(CachedConnection.warnedOnce, Collections.singleton(key),
+ "the same url was reported a second time");
+ }
+
+ /** A borrow that has to fail: what the test is after is what was logged on the way. */
+ private static void borrowExpectingFailure(String url) throws Exception {
+ try {
+ CachedConnection.getConnection(url);
+ fail("a connection string no registered driver accepts must be reported");
+ } catch (SQLException expected) {
+ // the point of the test is what was logged on the way, not what came back
+ }
+ }
+
+ /**
+ * ... and so is a postgresql url that turns the read bound off: a parameter of one outranks the
+ * property this class supplies, so a "socketTimeout=0" there cannot be replaced. Nothing is
+ * left to end a borrow that reaches a database accepting the connection and answering nothing,
+ * and an administrator who wrote that zero has to be able to find it in the log.
+ */
+ @Test
+ public void testAReadBoundTurnedOffInAPostgresUrlIsReportedOnce() throws Exception {
+ final String url = "jdbc:postgresql://reported:5432/db?socketTimeout=0";
+ final String key = CachedConnection.safeUrl(url) + "|unbounded|socketTimeout";
+ CachedConnection.warnedOnce.clear();
+ assertFalse(CachedConnection.ConnectDialect.POSTGRES.bound(url, new Properties(), 7));
+ assertEquals(CachedConnection.warnedOnce, Collections.singleton(key),
+ "a url leaving the reads of its login unbounded was not reported");
+
+ assertFalse(CachedConnection.ConnectDialect.POSTGRES.bound(url, new Properties(), 7));
+ assertEquals(CachedConnection.warnedOnce, Collections.singleton(key),
+ "the same url was reported a second time");
+ }
+
+ /**
+ * ... and every parameter of it that is turned off, not only the first one. A url is free to
+ * turn off the read bound and the login bound both, and the login bound is the per-host budget
+ * of a failover url - safeUrl() keeps neither parameter, so a key of the url alone would name
+ * the first offender, remember the url as reported, and leave the rest of them unmentionable.
+ */
+ @Test
+ public void testEveryBoundTurnedOffInAPostgresUrlIsReported() throws Exception {
+ final String url = "jdbc:postgresql://reported-twice:5432/db?socketTimeout=0&loginTimeout=0";
+ final String safe = CachedConnection.safeUrl(url);
+ CachedConnection.warnedOnce.clear();
+
+ CachedConnection.ConnectDialect.POSTGRES.bound(url, new Properties(), 7);
+
+ assertTrue(CachedConnection.warnedOnce.contains(safe + "|unbounded|socketTimeout"),
+ "the read bound left at 0 was not reported: " + CachedConnection.warnedOnce);
+ assertTrue(CachedConnection.warnedOnce.contains(safe + "|unbounded|loginTimeout"),
+ "the login bound left at 0 was not reported: " + CachedConnection.warnedOnce);
+ }
+
+ /**
+ * Nothing in the chain of a failure names the password of the backend, however deep it stands.
+ * Walked by identity rather than link by link: a driver is free to make the cause and the next
+ * exception of a link the same failure, which is the very shape the sibling test builds, and a
+ * helper looping on it would hang the run it is checking for exactly that.
+ */
+ private static void assertNoCredentials(Throwable failure) {
+ final Set<Throwable> seen = Collections.newSetFromMap(new IdentityHashMap<Throwable, Boolean>());
+ final Deque<Throwable> pending = new ArrayDeque<>();
+ enqueue(pending, seen, failure);
+ while (!pending.isEmpty()) {
+ final Throwable t = pending.poll();
+ assertFalse(String.valueOf(t.getMessage()).contains("S3cretOfTheBackend"),
+ "the password of the backend reached a message: " + t);
+ if (t instanceof SQLException) {
+ enqueue(pending, seen, ((SQLException) t).getNextException());
+ }
+ enqueue(pending, seen, t.getCause());
+ }
+ }
+
+ private static void enqueue(Deque<Throwable> pending, Set<Throwable> seen, Throwable t) {
+ if (t != null && seen.add(t)) {
+ pending.add(t);
+ }
+ }
+
+ /**
+ * A link of a chain carries a cause and a next exception both, and a driver is free to make the
+ * two the same failure. Rebuilt under a bound on the depth alone, a chain of those is copied
+ * twice over at every step - 2^32 links for one reaching the bound, which is a report of a
+ * failed connect that never comes back. What bounds the redaction is the number of links it
+ * rebuilds, the way the walk looking for credentials is bounded by the ones it visits.
+ */
+ @Test(timeOut = 60000)
+ public void testAFailureWhoseCauseIsItsNextExceptionIsRedactedInBoundedTime() throws Exception {
+ final String url = "jdbc:postgresql://opendj:S3cretOfTheBackend@127.0.0.1:5432/opendj";
+ SQLException chain = new SQLException("connect to " + url + " failed", "08006", 1);
+ for (int i = 0; i < 64; i++) {
+ final SQLException link = new SQLException("link " + i + " of " + url, "08006", i);
+ link.setNextException(chain);
+ link.initCause(chain);
+ chain = link;
+ }
+ final SQLException reported = CachedConnection.reported(chain, url);
+ assertNoCredentials(reported);
+ assertEquals(reported.getSQLState(), "08006", "the SQLState of a link has to survive its redaction");
+ }
+
+ /**
+ * ... and the chain of one is walked link by link rather than copy by copy. Enqueued twice, a
+ * failure whose cause is its own next exception fans out into a level twice the size of the one
+ * above it, so five levels of it are enough to spend the whole budget of the walk: the link
+ * that names the url stands at level six of seven and was never reached - and a walk that ends
+ * without finding credentials is one that reports the failure as it stands, password and all.
+ */
+ @Test(timeOut = 60000)
+ public void testACredentialBehindADuplicatedChainIsStillRedacted() throws Exception {
+ final String url = "jdbc:postgresql://opendj:S3cretOfTheBackend@127.0.0.1:5432/opendj";
+ SQLException chain = new SQLException("connect to " + url + " failed", "08006", 1);
+ for (int i = 0; i < 6; i++) {
+ final SQLException link = new SQLException("wrapper " + i, "08006", i);
+ link.setNextException(chain);
+ link.initCause(chain);
+ chain = link;
+ }
+
+ assertNoCredentials(CachedConnection.reported(chain, url));
+ }
+
+ /**
+ * The tail a rebuild has no budget left for is named rather than dropped. Without it the same
+ * failure keeps its root cause where the url of the backend carries no password and loses it
+ * without a word where it does - and the root cause is what a report of a failed connect is
+ * read for.
+ */
+ @Test(timeOut = 60000)
+ public void testTheTailOfALongChainIsNamedRatherThanDropped() throws Exception {
+ final String url = "jdbc:postgresql://opendj:S3cretOfTheBackend@127.0.0.1:5432/opendj";
+ SQLException chain = new SQLException("Connection to 127.0.0.1:5432 refused", "08006", 1);
+ for (int i = 0; i < 40; i++) {
+ final SQLException link = new SQLException("wrapper " + i + " of " + url, "08006", i);
+ link.initCause(chain);
+ chain = link;
+ }
+
+ final SQLException reported = CachedConnection.reported(chain, url);
+
+ assertNoCredentials(reported);
+ Throwable last = reported;
+ while (last.getCause() != null) {
+ last = last.getCause();
+ }
+ assertTrue(String.valueOf(last.getMessage()).contains("left out"),
+ "the tail a rebuild had no budget for has to say so: " + last.getMessage());
+ }
+
+ /**
+ * A database that is not listening at all: every dialect reports it instead of retrying the
+ * refused connect until the caller gives up on the operation.
+ */
+ @Test(timeOut = 120000)
+ public void testRefusedConnectIsReportedAtOnce() throws Exception {
+ final int closedPort = closedPort();
+ System.setProperty(CachedConnection.CONNECT_TIMEOUT_PROPERTY, Long.toString(BOUND_SECONDS));
+ for (final String url : urlsOf(closedPort)) {
+ final long startedAt = System.currentTimeMillis();
+ try {
+ CachedConnection.getConnection(url);
+ fail("a refused connect must be reported: " + CachedConnection.safeUrl(url));
+ } catch (SQLException expected) {
+ // the failure of the moment, reported rather than retried
+ }
+ assertElapsedWithinBound(startedAt, BOUND_SECONDS * 1000);
+ }
+ }
+
+ /**
+ * The failure this bound exists for: a database that completes the TCP connection and then
+ * says nothing - a moved VIP, a proxy at its connection limit, a host that lost its answer -
+ * leaving the login of the driver, and with it the operation, without an end. The connection
+ * of the accept queue is never answered here, so every dialect has to give up on its own.
+ */
+ @Test(timeOut = 300000)
+ public void testLoginIsBoundedWhenTheDatabaseNeverAnswers() throws Exception {
+ System.setProperty(CachedConnection.CONNECT_TIMEOUT_PROPERTY, Long.toString(BOUND_SECONDS));
+ // a socket that is bound and never accepted: the kernel completes the handshake, so the
+ // connect of the driver succeeds and every read of the login that follows hangs
+ try (final ServerSocket blackhole = new ServerSocket(0, 50, InetAddress.getLoopbackAddress())) {
+ for (final String url : urlsOf(blackhole.getLocalPort())) {
+ // borrowed on a thread of its own: a bound that a driver does not honour has to
+ // fail this test at once, and not by hanging the run it is part of
+ final FutureTask<Connection> borrow = new FutureTask<>(() -> CachedConnection.getConnection(url));
+ final Thread thread = new Thread(borrow, "borrow-" + CachedConnection.safeUrl(url));
+ thread.setDaemon(true);
+ thread.start();
+ try {
+ final Connection con = borrow.get(BOUND_SECONDS * 1000 + BOUND_MARGIN_MS, TimeUnit.MILLISECONDS);
+ fail("a database that never answers must not hand out a connection: " + con);
+ } catch (TimeoutException e) {
+ fail("the login of " + CachedConnection.safeUrl(url) + " is not bounded: it never gave up");
+ } catch (ExecutionException expected) {
+ assertTrue(expected.getCause() instanceof SQLException, String.valueOf(expected.getCause()));
+ }
+ }
+ }
+ }
+
+ /**
+ * The deadline of the borrow bounds the attempt inside it as well: the pool timeout stands for
+ * the whole borrow, and an attempt left to run out its own bound would overrun it by that bound.
+ */
+ @Test(timeOut = 120000)
+ public void testTheAttemptIsBoundedByTheDeadlineOfTheBorrow() throws Exception {
+ System.setProperty(CachedConnection.CONNECT_TIMEOUT_PROPERTY, "600");
+ System.setProperty(CachedConnection.POOL_TIMEOUT_PROPERTY, "2");
+ try (final ServerSocket blackhole = new ServerSocket(0, 50, InetAddress.getLoopbackAddress())) {
+ final String url = "jdbc:postgresql://127.0.0.1:" + blackhole.getLocalPort() + "/opendj?user=opendj&password=opendj";
+ final long startedAt = System.currentTimeMillis();
+ try {
+ CachedConnection.getConnection(url);
+ fail("a database that never answers must not hand out a connection");
+ } catch (SQLException expected) {
+ // reported, and within the borrow it was given rather than the 600 s of the attempt
+ }
+ final long elapsed = System.currentTimeMillis() - startedAt;
+ assertTrue(elapsed < 2000 + BOUND_MARGIN_MS,
+ "the attempt outlived the deadline of the borrow: " + elapsed + " ms");
+ }
+ }
+
+ /** Pool exhaustion stays a retry - one of our own connections is on its way back to the pool. */
+ @Test(timeOut = 120000)
+ public void testConnectionLimitIsRetried() throws Exception {
+ final String url = StubDriver.PREFIX + "retried";
+ stub.failWith(tooManyConnections(), 2);
+ System.setProperty(CachedConnection.POOL_TIMEOUT_PROPERTY, "30");
+
+ final Connection con = CachedConnection.getConnection(url);
+
+ assertNotNull(con);
+ assertEquals(stub.attempts.get(), 3, "the connect must be retried while the database is at its limit");
+ }
+
+ /** ... but under a deadline: the retry used to double its wait from 1 ms with no end to it. */
+ @Test(timeOut = 120000)
+ public void testConnectionLimitGivesUpAtTheDeadline() throws Exception {
+ final String url = StubDriver.PREFIX + "deadline";
+ stub.failWith(tooManyConnections(), StubDriver.ALWAYS);
+ System.setProperty(CachedConnection.POOL_TIMEOUT_PROPERTY, "2");
+
+ final long startedAt = System.currentTimeMillis();
+ try {
+ CachedConnection.getConnection(url);
+ fail("a database that stays at its connection limit must be reported, not waited out forever");
+ } catch (SQLTimeoutException expected) {
+ assertTrue(expected.getMessage().contains("2s"), expected.getMessage());
+ assertEquals(((SQLException) expected.getCause()).getSQLState(), "53300");
+ // the state of a connect that did not happen, rather than none at all: this is the
+ // failure of a borrow, and monitoring reading the state off what it caught would
+ // otherwise see null where the driver's own exception carried one
+ assertEquals(expected.getSQLState(), "08001");
+ }
+ final long elapsed = System.currentTimeMillis() - startedAt;
+ assertTrue(elapsed >= 2000, "gave up after " + elapsed + " ms, before the deadline it was given");
+ assertElapsedWithinBound(startedAt, 2000);
+ assertTrue(stub.attempts.get() > 1, "the connect must be retried while the deadline lasts");
+ }
+
+ /**
+ * A database on its way up - starting, recovering, shutting down - says so, and says it for
+ * seconds: the backend it belongs to would otherwise stay locked down until the next restart
+ * of the server, since nothing above JDBCStorage.open() attempts it a second time.
+ */
+ @Test(timeOut = 120000)
+ public void testDatabaseOnItsWayUpIsRetried() throws Exception {
+ final String url = StubDriver.PREFIX + "starting-up";
+ stub.failWith(new SQLException("the database system is starting up", "57P03"), 2);
+ System.setProperty(CachedConnection.POOL_TIMEOUT_PROPERTY, "30");
+
+ final Connection con = CachedConnection.getConnection(url);
+
+ assertNotNull(con);
+ assertEquals(stub.attempts.get(), 3, "a database that is starting up must be waited out");
+ }
+
+ /** ... and it is recognized however the driver wrapped it: a SQLException carries two chains. */
+ @Test(timeOut = 120000)
+ public void testTheWholeChainOfTheFailureIsLookedAt() throws Exception {
+ final String url = StubDriver.PREFIX + "wrapped";
+ final SQLException wrapped = new SQLException("could not connect to the server", "08006");
+ wrapped.setNextException(tooManyConnections());
+ stub.failWith(wrapped, 1);
+ System.setProperty(CachedConnection.POOL_TIMEOUT_PROPERTY, "30");
+
+ assertNotNull(CachedConnection.getConnection(url));
+ assertEquals(stub.attempts.get(), 2, "the failure behind the one reported must be looked at");
+ }
+
+ /**
+ * The rest of the insufficient_resources class is not worth waiting out: a server out of disk
+ * is not made whole by a connection of ours coming back to the pool.
+ */
+ @Test(timeOut = 120000)
+ public void testDiskFullIsNotRetried() throws Exception {
+ final String url = StubDriver.PREFIX + "disk-full";
+ stub.failWith(new SQLException("could not extend file: No space left on device", "53100"), StubDriver.ALWAYS);
+ System.setProperty(CachedConnection.POOL_TIMEOUT_PROPERTY, "30");
+
+ try {
+ CachedConnection.getConnection(url);
+ fail("a database out of disk must be reported to the caller");
+ } catch (SQLException expected) {
+ assertEquals(expected.getSQLState(), "53100");
+ }
+ assertEquals(stub.attempts.get(), 1, "a failure that waiting cannot clear must be attempted once");
+ }
+
+ /**
+ * Every other failure is the caller's to report. A password the database does not accept is
+ * never going to be accepted by waiting, and the retry that swallowed it left the operation
+ * hanging with nothing in the log.
+ */
+ @Test(timeOut = 120000)
+ public void testRejectedLoginIsNotRetried() throws Exception {
+ final String url = StubDriver.PREFIX + "rejected";
+ stub.failWith(new SQLException("password authentication failed", "28P01"), StubDriver.ALWAYS);
+ System.setProperty(CachedConnection.POOL_TIMEOUT_PROPERTY, "30");
+
+ try {
+ CachedConnection.getConnection(url);
+ fail("a rejected login must be reported to the caller");
+ } catch (SQLException expected) {
+ assertEquals(expected.getSQLState(), "28P01");
+ }
+ assertEquals(stub.attempts.get(), 1, "a rejected login must be attempted once");
+ }
+
+ /** A connection the setup of which failed belongs to nobody: it has to be closed, not leaked. */
+ @Test(timeOut = 120000)
+ public void testConnectionIsClosedWhenItsSetupFails() throws Exception {
+ final String url = StubDriver.PREFIX + "setup-failure";
+ final Connection broken = mock(Connection.class);
+ doThrow(new SQLException("read only")).when(broken).setAutoCommit(false);
+ stub.answerWith(broken);
+
+ try {
+ CachedConnection.getConnection(url);
+ fail("a connection that cannot be set up must be reported");
+ } catch (SQLException expected) {
+ assertEquals(expected.getMessage(), "read only");
+ }
+ verify(broken).close();
+ }
+
+ /** The same, for a driver whose failure in the setup is not a SQLException but an unchecked one. */
+ @Test(timeOut = 120000)
+ public void testConnectionIsClosedWhenItsSetupFailsWithAnUncheckedError() throws Exception {
+ final String url = StubDriver.PREFIX + "setup-unchecked";
+ final Connection broken = mock(Connection.class);
+ doThrow(new IllegalStateException("driver internal")).when(broken).setTransactionIsolation(anyInt());
+ stub.answerWith(broken);
+
+ try {
+ CachedConnection.getConnection(url);
+ fail("a connection that cannot be set up must be reported");
+ } catch (IllegalStateException expected) {
+ assertEquals(expected.getMessage(), "driver internal");
+ }
+ verify(broken).close();
+ }
+
+ /**
+ * isValid(n) is not a bound at the socket on every driver - the SQL Server driver turns it
+ * into a query timeout, which needs an answer from the server to fire - and the read bound of
+ * the login was lifted the moment the connection was established, so the socket carries the
+ * bound of the validation, for the length of the validation only.
+ */
+ @Test(timeOut = 120000)
+ public void testValidationOfAPooledConnectionIsBoundedAtTheSocket() throws Exception {
+ final String url = StubDriver.PREFIX + "validation-bound";
+ final Connection pooled = mock(Connection.class);
+ when(pooled.isValid(anyInt())).thenReturn(true);
+ when(pooled.getNetworkTimeout()).thenReturn(0);
+ CachedConnection.cached.get(url).add(new CachedConnection(url, pooled));
+
+ final Connection borrowed = CachedConnection.getConnection(url);
+
+ assertSame(((CachedConnection) borrowed).parent, pooled);
+ final InOrder inOrder = inOrder(pooled);
+ inOrder.verify(pooled).setNetworkTimeout(any(Executor.class), eq(CachedConnection.VALIDATION_TIMEOUT_SECONDS * 1000));
+ inOrder.verify(pooled).isValid(CachedConnection.VALIDATION_TIMEOUT_SECONDS);
+ inOrder.verify(pooled).setNetworkTimeout(any(Executor.class), eq(0));
+ }
+
+ /**
+ * A driver that takes the call bounding the validation and then fails inside it is told apart
+ * from one that never takes a network timeout at all: it is free to have applied the bound
+ * before failing, so the connection is discarded rather than validated unbounded and handed
+ * out - pooled, it would carry five seconds of ours into every statement for the rest of its
+ * life, and the import batch of a backend open is the first thing to die of that.
+ */
+ @Test(timeOut = 120000)
+ public void testAPooledConnectionThatCannotBeBoundedForItsValidationIsDiscarded() throws Exception {
+ final String url = StubDriver.PREFIX + "validation-bound-fails";
+ final Connection pooled = mock(Connection.class);
+ when(pooled.getNetworkTimeout()).thenReturn(0);
+ when(pooled.isValid(anyInt())).thenReturn(true);
+ doThrow(new SQLException("the driver took the bound and then failed"))
+ .when(pooled).setNetworkTimeout(any(Executor.class), anyInt());
+ CachedConnection.cached.get(url).add(new CachedConnection(url, pooled));
+ final Connection fresh = mock(Connection.class);
+ stub.answerWith(fresh);
+
+ final Connection borrowed = CachedConnection.getConnection(url);
+
+ assertSame(((CachedConnection) borrowed).parent, fresh, "a connection this could not bound was handed out");
+ verify(pooled, never()).isValid(anyInt());
+ verify(pooled).close();
+ }
+
+ /**
+ * A driver answering a negative network timeout is outside the contract of the call - 0 is no
+ * limit and nothing below it stands for anything - and taken back as it is, it is one of the
+ * two sentinels this class tells its own outcomes apart by: the bound of the validation would
+ * be read as a bound that was never set, and the connection would go back into the pool still
+ * carrying five seconds of ours into every statement of the next borrower.
+ */
+ @Test(timeOut = 120000)
+ public void testAPooledConnectionWhoseDriverAnswersANegativeBoundIsPutBackUnbounded() throws Exception {
+ final String url = StubDriver.PREFIX + "validation-negative-bound";
+ final Connection pooled = mock(Connection.class);
+ when(pooled.isValid(anyInt())).thenReturn(true);
+ when(pooled.getNetworkTimeout()).thenReturn(-1);
+ CachedConnection.cached.get(url).add(new CachedConnection(url, pooled));
+
+ final Connection borrowed = CachedConnection.getConnection(url);
+
+ assertSame(((CachedConnection) borrowed).parent, pooled);
+ final InOrder inOrder = inOrder(pooled);
+ inOrder.verify(pooled)
+ .setNetworkTimeout(any(Executor.class), eq(CachedConnection.VALIDATION_TIMEOUT_SECONDS * 1000));
+ inOrder.verify(pooled).isValid(CachedConnection.VALIDATION_TIMEOUT_SECONDS);
+ inOrder.verify(pooled).setNetworkTimeout(any(Executor.class), eq(0));
+ }
+
+ /** A read bound of the connection string is tighter than ours and stays untouched. */
+ @Test(timeOut = 120000)
+ public void testValidationLeavesTheBoundOfTheConnectionStringAlone() throws Exception {
+ final String url = StubDriver.PREFIX + "validation-tighter";
+ final Connection pooled = mock(Connection.class);
+ when(pooled.isValid(anyInt())).thenReturn(true);
+ when(pooled.getNetworkTimeout()).thenReturn(2000);
+ CachedConnection.cached.get(url).add(new CachedConnection(url, pooled));
+
+ assertNotNull(CachedConnection.getConnection(url));
+
+ verify(pooled, never()).setNetworkTimeout(any(Executor.class), anyInt());
+ }
+
+ /**
+ * The pool has no upper bound on the number of connections it holds, and a validation is a
+ * round trip: after a failover that left them half-open, draining the pool must not outlive
+ * the deadline of the borrow - establishing a connection is the faster answer past it.
+ */
+ @Test(timeOut = 120000)
+ public void testDrainOfThePoolStopsAtTheDeadline() throws Exception {
+ final String url = StubDriver.PREFIX + "drain-deadline";
+ final int pooled = 8;
+ final AtomicInteger validated = new AtomicInteger();
+ for (int i = 0; i < pooled; i++) {
+ final Connection stale = mock(Connection.class);
+ when(stale.isValid(anyInt())).thenAnswer(invocation -> {
+ validated.incrementAndGet();
+ Thread.sleep(500); // a database that no longer answers: every validation waits out its bound
+ return false;
+ });
+ CachedConnection.cached.get(url).add(new CachedConnection(url, stale));
+ }
+ final Connection fresh = mock(Connection.class);
+ stub.answerWith(fresh);
+ System.setProperty(CachedConnection.POOL_TIMEOUT_PROPERTY, "1");
+
+ final Connection borrowed = CachedConnection.getConnection(url);
+
+ assertSame(((CachedConnection) borrowed).parent, fresh);
+ assertTrue(validated.get() > 0 && validated.get() < pooled,
+ "the drain has to start and to stop at the deadline: " + validated.get() + " of " + pooled);
+ }
+
+ /** A pooled connection that no longer validates is closed and replaced, not handed out. */
+ @Test(timeOut = 120000)
+ public void testBrokenPooledConnectionIsDiscarded() throws Exception {
+ final String url = StubDriver.PREFIX + "broken-pooled";
+ final Connection stale = mock(Connection.class);
+ when(stale.isValid(anyInt())).thenReturn(false);
+ CachedConnection.cached.get(url).add(new CachedConnection(url, stale));
+ final Connection fresh = mock(Connection.class);
+ when(fresh.isValid(anyInt())).thenReturn(true);
+ stub.answerWith(fresh);
+
+ final Connection borrowed = CachedConnection.getConnection(url);
+
+ assertSame(((CachedConnection) borrowed).parent, fresh);
+ verify(stale).close();
+ // the validation of a pooled connection needs a bound of its own as well
+ verify(stale, never()).isValid(0);
+ verify(stale).isValid(CachedConnection.VALIDATION_TIMEOUT_SECONDS);
+ }
+
+ /**
+ * The same for a driver whose answer to the validation is an unchecked failure: it unwinds
+ * through poll(), which stands outside every try of the borrow, so the connection it was
+ * raised over is already out of the pool and would be held by nobody.
+ */
+ @Test(timeOut = 120000)
+ public void testAPooledConnectionWhoseValidationThrowsIsDiscarded() throws Exception {
+ final String url = StubDriver.PREFIX + "validation-unchecked";
+ final Connection broken = mock(Connection.class);
+ when(broken.isValid(anyInt())).thenThrow(new IllegalStateException("driver internal"));
+ CachedConnection.cached.get(url).add(new CachedConnection(url, broken));
+ final Connection fresh = mock(Connection.class);
+ stub.answerWith(fresh);
+
+ final Connection borrowed = CachedConnection.getConnection(url);
+
+ assertSame(((CachedConnection) borrowed).parent, fresh);
+ verify(broken).close();
+ }
+
+ /** A connection that cannot be rolled back must not go back into the pool - nor be dropped. */
+ @Test(timeOut = 120000)
+ public void testConnectionThatCannotBeRolledBackIsClosed() throws Exception {
+ final String url = StubDriver.PREFIX + "rollback-failure";
+ final Connection parent = mock(Connection.class);
+ doThrow(new SQLException("connection is closed")).when(parent).rollback();
+
+ try {
+ new CachedConnection(url, parent).close();
+ fail("a failed rollback must be reported");
+ } catch (SQLException expected) {
+ assertEquals(expected.getMessage(), "connection is closed");
+ }
+ verify(parent).close();
+ assertTrue(CachedConnection.cached.get(url).isEmpty(), "a connection that cannot be rolled back was pooled");
+ }
+
+ @Test
+ public void testDialectIsRecognizedByTheConnectionString() throws Exception {
+ assertEquals(CachedConnection.ConnectDialect.of("jdbc:postgresql://h:5432/db"), CachedConnection.ConnectDialect.POSTGRES);
+ assertEquals(CachedConnection.ConnectDialect.of("jdbc:mysql://h:3306/db"), CachedConnection.ConnectDialect.MYSQL);
+ assertEquals(CachedConnection.ConnectDialect.of("jdbc:oracle:thin:@//h:1521/svc"), CachedConnection.ConnectDialect.ORACLE);
+ assertEquals(CachedConnection.ConnectDialect.of("jdbc:sqlserver://h:1433;databaseName=db"), CachedConnection.ConnectDialect.MICROSOFT);
+ assertNull(CachedConnection.ConnectDialect.of("jdbc:h2:mem:db"), "an unknown engine must not be fed the properties of another");
+ }
+
+ /** Both phases are bounded, in the units of the driver: the connect alone leaves the login open. */
+ @Test
+ public void testBothPhasesOfTheLoginAreBounded() throws Exception {
+ // pgjdbc puts an SO_TIMEOUT on the login socket only where socketTimeout is set: without it
+ // loginTimeout bounds the caller alone, and the thread the driver runs the login on stays
+ // parked in the read it abandoned
+ final Properties postgres = new Properties();
+ assertTrue(CachedConnection.ConnectDialect.POSTGRES.bound("jdbc:postgresql://h:5432/db", postgres, 7),
+ "the read bound of postgresql outlives the login and has to be lifted");
+ assertEquals(postgres.getProperty("connectTimeout"), "7");
+ assertEquals(postgres.getProperty("socketTimeout"), "7");
+ assertEquals(postgres.getProperty("loginTimeout"), "7", "the bound of a url naming more than one host");
+
+ final Properties mysql = new Properties();
+ assertTrue(CachedConnection.ConnectDialect.MYSQL.bound("jdbc:mysql://h:3306/db", mysql, 7),
+ "the read bound of mysql outlives the login and has to be lifted");
+ assertEquals(mysql.getProperty("connectTimeout"), "7000");
+ assertEquals(mysql.getProperty("socketTimeout"), "7000");
+
+ final Properties oracle = new Properties();
+ assertTrue(CachedConnection.ConnectDialect.ORACLE.bound("jdbc:oracle:thin:@//h:1521/svc", oracle, 7));
+ assertEquals(oracle.getProperty("oracle.net.CONNECT_TIMEOUT"), "7000");
+ assertEquals(oracle.getProperty("oracle.jdbc.ReadTimeout"), "7000");
+
+ // the loginTimeout of the sql server driver leaves the read of the prelogin answer open
+ final Properties microsoft = new Properties();
+ assertTrue(CachedConnection.ConnectDialect.MICROSOFT.bound("jdbc:sqlserver://h:1433;databaseName=db", microsoft, 7));
+ assertEquals(microsoft.getProperty("loginTimeout"), "7");
+ assertEquals(microsoft.getProperty("socketTimeout"), "7000");
+ }
+
+ /**
+ * A driver with a range of its own for its connect property is never handed a value beyond it:
+ * SQLServerDriverIntProperty.LOGIN_TIMEOUT is validated against [0, 65535], so a bound past
+ * that would not widen the connect, it would fail every one of them.
+ */
+ @Test
+ public void testConnectBoundStaysInTheRangeTheDriverTakes() throws Exception {
+ final Properties microsoft = new Properties();
+ CachedConnection.ConnectDialect.MICROSOFT.bound("jdbc:sqlserver://h:1433;databaseName=db", microsoft, 100000);
+ assertEquals(microsoft.getProperty("loginTimeout"), "65535");
+ assertEquals(microsoft.getProperty("socketTimeout"), "100000000", "the read bound takes any value");
+ }
+
+ /**
+ * A bound the administrator put into the connection string by hand - the only workaround this
+ * backend had - keeps precedence, property by property.
+ */
+ @Test
+ public void testConnectionStringKeepsPrecedence() throws Exception {
+ final Properties postgres = new Properties();
+ CachedConnection.ConnectDialect.POSTGRES.bound(
+ "jdbc:postgresql://h:5432/db?user=u&password=p&loginTimeout=30&socketTimeout=300", postgres, 7);
+ assertNull(postgres.getProperty("loginTimeout"), "the setting of the connection string was overridden");
+ assertNull(postgres.getProperty("socketTimeout"), "the setting of the connection string was overridden");
+ assertNull(postgres.getProperty("connectTimeout"),
+ "the connect side is one budget: a bound of the administrator under either of its names is theirs");
+
+ // the sql server driver gives a supplied property precedence over the one of the url
+ final Properties microsoft = new Properties();
+ CachedConnection.ConnectDialect.MICROSOFT.bound("jdbc:sqlserver://h:1433;loginTimeout=45;databaseName=db", microsoft, 7);
+ assertNull(microsoft.getProperty("loginTimeout"), "the setting of the connection string was overridden");
+ assertEquals(microsoft.getProperty("socketTimeout"), "7000");
+
+ // inside the descriptor of an oracle tns url the property goes by the last segment of its name
+ final Properties oracle = new Properties();
+ CachedConnection.ConnectDialect.ORACLE.bound(
+ "jdbc:oracle:thin:@(DESCRIPTION=(CONNECT_TIMEOUT=3)(ADDRESS=(HOST=h)(PORT=1521)))", oracle, 7);
+ assertNull(oracle.getProperty("oracle.net.CONNECT_TIMEOUT"));
+ assertEquals(oracle.getProperty("oracle.jdbc.ReadTimeout"), "7000");
+
+ // inside the descriptor the read bound goes by READ_TIMEOUT, and one of the administrator
+ // is never lifted after the login, because ours is not set on top of it
+ final Properties read = new Properties();
+ assertFalse(CachedConnection.ConnectDialect.ORACLE.bound(
+ "jdbc:oracle:thin:@(DESCRIPTION=(READ_TIMEOUT=30)(ADDRESS=(HOST=h)(PORT=1521)))", read, 7),
+ "a read bound of the connection string must not be lifted once the login is through");
+ assertNull(read.getProperty("oracle.jdbc.ReadTimeout"));
+
+ // ... while RECV_TIMEOUT is a parameter of sqlnet.ora and of the listener that ojdbc8 does
+ // not read - the name appears nowhere in the driver - so a descriptor carrying one is not
+ // a read bound of the connection and must not take ours off it
+ final Properties recv = new Properties();
+ assertTrue(CachedConnection.ConnectDialect.ORACLE.bound(
+ "jdbc:oracle:thin:@(DESCRIPTION=(RECV_TIMEOUT=30)(ADDRESS=(HOST=h)(PORT=1521)))", recv, 7),
+ "a parameter no driver reads left the login of this connection unbounded");
+ assertEquals(recv.getProperty("oracle.jdbc.ReadTimeout"), "7000");
+
+ // a name that only appears as the tail of another parameter is not a setting of its own
+ final Properties mysql = new Properties();
+ CachedConnection.ConnectDialect.MYSQL.bound("jdbc:mysql://h:3306/db?xconnectTimeout=1&socketTimeoutX=2", mysql, 7);
+ assertEquals(mysql.getProperty("connectTimeout"), "7000");
+ assertEquals(mysql.getProperty("socketTimeout"), "7000");
+ }
+
+ /**
+ * A parameter is recognized the way the driver of its dialect recognizes it: pgjdbc and
+ * Connector/J look their properties up by their exact name, so a name of another case is a
+ * parameter of nobody - neither side bounds anything by it - and must not pass for a bound the
+ * administrator set, while the other two match either way.
+ */
+ @Test
+ public void testTheCaseOfAParameterIsTheOneOfItsDriver() throws Exception {
+ final Properties postgres = new Properties();
+ CachedConnection.ConnectDialect.POSTGRES.bound("jdbc:postgresql://h:5432/db?ConnectTimeout=5", postgres, 7);
+ assertEquals(postgres.getProperty("connectTimeout"), "7", "pgjdbc ignores a parameter of another case");
+
+ // PropertyKey.fromValue("SocketTimeout") answers null, and Connector/J then reads no bound
+ // out of the url either: a mis-cased parameter left the borrower parked on a host that
+ // completes the handshake and says nothing
+ final Properties mysql = new Properties();
+ CachedConnection.ConnectDialect.MYSQL.bound("jdbc:mysql://h:3306/db?SocketTimeout=1", mysql, 7);
+ assertEquals(mysql.getProperty("socketTimeout"), "7000", "Connector/J ignores a parameter of another case");
+
+ final Properties microsoft = new Properties();
+ CachedConnection.ConnectDialect.MICROSOFT.bound("jdbc:sqlserver://h:1433;LoginTimeout=45", microsoft, 7);
+ assertNull(microsoft.getProperty("loginTimeout"), "the sql server driver normalizes the name of a property");
+
+ // the keywords of an oracle descriptor are matched without regard to case as well
+ final Properties oracle = new Properties();
+ CachedConnection.ConnectDialect.ORACLE.bound(
+ "jdbc:oracle:thin:@(description=(connect_timeout=3)(address=(host=h)(port=1521)))", oracle, 7);
+ assertNull(oracle.getProperty("oracle.net.CONNECT_TIMEOUT"), "an oracle descriptor is read without case");
+ }
+
+ /**
+ * The connection string is not the only channel of the administrator: the oracle driver reads
+ * some of its properties out of the system properties as well, which is how a whole jvm is
+ * bounded with -Doracle.jdbc.ReadTimeout. A property supplied to the driver outranks that one
+ * without a word, and this class would then lift it once the login is through as if it were
+ * its own - leaving a connection with no read bound at all where the administrator set one.
+ */
+ @Test
+ public void testASystemPropertyOfTheAdministratorKeepsPrecedence() throws Exception {
+ System.setProperty("oracle.jdbc.ReadTimeout", "30000");
+ try {
+ final Properties oracle = new Properties();
+ assertFalse(CachedConnection.ConnectDialect.ORACLE.bound("jdbc:oracle:thin:@//h:1521/svc", oracle, 7),
+ "a read bound of the administrator must not be lifted once the login is through");
+ assertNull(oracle.getProperty("oracle.jdbc.ReadTimeout"), "the setting of the administrator was overridden");
+ assertEquals(oracle.getProperty("oracle.net.CONNECT_TIMEOUT"), "7000",
+ "the property it leaves open must still be bounded");
+ } finally {
+ System.clearProperty("oracle.jdbc.ReadTimeout");
+ }
+
+ // the connect property of the same driver, over the same channel
+ System.setProperty("oracle.net.CONNECT_TIMEOUT", "30000");
+ try {
+ final Properties oracle = new Properties();
+ CachedConnection.ConnectDialect.ORACLE.bound("jdbc:oracle:thin:@//h:1521/svc", oracle, 7);
+ assertNull(oracle.getProperty("oracle.net.CONNECT_TIMEOUT"), "the setting of the administrator was overridden");
+ } finally {
+ System.clearProperty("oracle.net.CONNECT_TIMEOUT");
+ }
+
+ // a plain name is common enough to be somebody else's system property: only a name a
+ // driver of these actually reads out of them is one of the administrator's
+ System.setProperty("socketTimeout", "30000");
+ try {
+ final Properties mysql = new Properties();
+ assertTrue(CachedConnection.ConnectDialect.MYSQL.bound("jdbc:mysql://h:3306/db", mysql, 7));
+ assertEquals(mysql.getProperty("socketTimeout"), "7000");
+ } finally {
+ System.clearProperty("socketTimeout");
+ }
+ }
+
+ /**
+ * ... and a dotted name is not a name a driver reads out of the system properties by the shape
+ * of it. ojdbc8 resolves oracle.jdbc.ReadTimeout and oracle.net.CONNECT_TIMEOUT in three tiers
+ * (the properties it was supplied, then System.getProperty, then the properties of the data
+ * source), while oracle.net.READ_TIMEOUT - a dotted name of the same driver, and the name the
+ * socket option is finally read under - reaches the socket from the connection properties
+ * alone: the classes carrying the literal hand it to Properties.get, none of them to
+ * System.getProperty. Taken for a bound of the administrator, a -D of it leaves the login with
+ * no read bound whatever: theirs is not read and ours is not set.
+ */
+ @Test
+ public void testASystemPropertyNoDriverReadsIsNoBound() throws Exception {
+ System.setProperty("oracle.net.READ_TIMEOUT", "30000");
+ try {
+ final Properties oracle = new Properties();
+ assertTrue(CachedConnection.ConnectDialect.ORACLE.bound("jdbc:oracle:thin:@//h:1521/svc", oracle, 7),
+ "a -D the driver never reads left this login with no read bound at all");
+ assertEquals(oracle.getProperty("oracle.jdbc.ReadTimeout"), "7000");
+ } finally {
+ System.clearProperty("oracle.net.READ_TIMEOUT");
+ }
+
+ // ... while the same name written into the connection string is one the driver does read
+ final Properties declared = new Properties();
+ assertFalse(CachedConnection.ConnectDialect.ORACLE.bound(
+ "jdbc:oracle:thin:@(DESCRIPTION=(READ_TIMEOUT=30)(ADDRESS=(HOST=h)(PORT=1521)))", declared, 7));
+ assertNull(declared.getProperty("oracle.jdbc.ReadTimeout"));
+ }
+
+ /**
+ * A property the administrator set to 0 is not a bound of theirs: every one of these drivers
+ * reads 0 as "wait as long as it takes", which is the default this class exists to replace -
+ * and on the three whose driver lets a supplied property win, ours is set on top of it.
+ * Postgresql is the one where it cannot be, and is covered on its own below.
+ */
+ @Test
+ public void testAZeroIsNotABoundOfTheAdministrator() throws Exception {
+ final Properties mysql = new Properties();
+ CachedConnection.ConnectDialect.MYSQL.bound("jdbc:mysql://h:3306/db?connectTimeout=0&socketTimeout=0", mysql, 7);
+ assertEquals(mysql.getProperty("connectTimeout"), "7000");
+ assertEquals(mysql.getProperty("socketTimeout"), "7000");
+
+ // ... and neither is a property left without a value
+ final Properties microsoft = new Properties();
+ CachedConnection.ConnectDialect.MICROSOFT.bound("jdbc:sqlserver://h:1433;socketTimeout=;databaseName=db", microsoft, 7);
+ assertEquals(microsoft.getProperty("socketTimeout"), "7000");
+
+ // the same of a system property, and of the descriptor of an oracle url
+ System.setProperty("oracle.jdbc.ReadTimeout", "0");
+ try {
+ final Properties oracle = new Properties();
+ assertTrue(CachedConnection.ConnectDialect.ORACLE.bound(
+ "jdbc:oracle:thin:@(DESCRIPTION=(READ_TIMEOUT=0)(ADDRESS=(HOST=h)(PORT=1521)))", oracle, 7));
+ assertEquals(oracle.getProperty("oracle.jdbc.ReadTimeout"), "7000");
+ } finally {
+ System.clearProperty("oracle.jdbc.ReadTimeout");
+ }
+
+ // a value that is no number is left to the driver it belongs to: it is not this class's to read
+ final Properties unreadable = new Properties();
+ assertFalse(CachedConnection.ConnectDialect.MYSQL.bound(
+ "jdbc:mysql://h:3306/db?socketTimeout=PT30S", unreadable, 7));
+ assertNull(unreadable.getProperty("socketTimeout"));
+ }
+
+ /**
+ * On postgresql a parameter of the url outranks the property this class supplies: Driver
+ * .connect copies what it was handed into a flat map and parseURL then writes the parameters of
+ * the url on top of it. So a "socketTimeout=0" there cannot be replaced, and setting ours
+ * regardless would leave this class believing it bounded a login that carries no bound - and
+ * lifting a read bound after it that was never in force. The effective values are read back
+ * through the parser of the driver itself, since asserting on the map handed to it is
+ * asserting on the half of the story this bug lived in.
+ */
+ @Test
+ public void testAParameterOfAPostgresUrlOutranksTheBoundOfThisClass() throws Exception {
+ final String url = "jdbc:postgresql://h:5432/db?connectTimeout=0&socketTimeout=0&loginTimeout=0";
+ final Properties supplied = new Properties();
+ assertFalse(CachedConnection.ConnectDialect.POSTGRES.bound(url, supplied, 7),
+ "a read bound that never reaches the driver must not be reported as one to lift");
+ assertNull(supplied.getProperty("socketTimeout"));
+ assertNull(supplied.getProperty("connectTimeout"));
+ assertNull(supplied.getProperty("loginTimeout"));
+
+ final Properties effective = org.postgresql.Driver.parseURL(url, supplied);
+ assertEquals(effective.getProperty("socketTimeout"), "0", "the url is what the driver ends up reading");
+ assertEquals(effective.getProperty("connectTimeout"), "0");
+ assertEquals(effective.getProperty("loginTimeout"), "0");
+ }
+
+ /**
+ * The connect side of a dialect is one budget rather than a set of independent knobs, so a
+ * bound of the administrator under any of its names leaves all of them alone. On postgresql
+ * connectTimeout bounds the socket connect and loginTimeout the login behind it: filling in the
+ * one they left out caps the one they set, since Driver.connect hands the login to a thread of
+ * its own as soon as loginTimeout is anything but 0 and gives up on it there.
+ */
+ @Test
+ public void testAConnectBudgetOfTheAdministratorIsNotCappedByThisClass() throws Exception {
+ final String url = "jdbc:postgresql://h:5432/db?connectTimeout=300";
+ final Properties supplied = new Properties();
+ assertTrue(CachedConnection.ConnectDialect.POSTGRES.bound(url, supplied, 30),
+ "the read bound is a budget of its own and is still set");
+ assertNull(supplied.getProperty("loginTimeout"),
+ "a loginTimeout of ours caps the connectTimeout the administrator set");
+ assertNull(supplied.getProperty("connectTimeout"));
+ assertEquals(supplied.getProperty("socketTimeout"), "30");
+
+ final Properties effective = org.postgresql.Driver.parseURL(url, supplied);
+ assertEquals(effective.getProperty("connectTimeout"), "300", "the budget of the administrator, in full");
+ assertNull(effective.getProperty("loginTimeout"), "nothing of ours hands this login to a thread to abandon");
+ }
+
+ /**
+ * The bound handed to a driver stays inside the range an int of milliseconds takes. Where the
+ * per-attempt property is off, the attempt takes what is left of the deadline of the borrow,
+ * and the pool timeout has no upper bound of its own - while the SQL Server driver rejects a
+ * socketTimeout past Integer.MAX_VALUE outright, failing every connect of that backend with
+ * the name of a property nobody typed.
+ */
+ @Test(timeOut = 120000)
+ public void testTheBoundHandedToADriverStaysInTheRangeAnIntTakes() throws Exception {
+ final long deadline = System.currentTimeMillis() + 3000000L * 1000; // 34 days: past 2^31 ms
+ assertEquals(CachedConnection.attemptSeconds(0, deadline), Integer.MAX_VALUE / 1000);
+
+ System.setProperty(CachedConnection.CONNECT_TIMEOUT_PROPERTY, "0");
+ System.setProperty(CachedConnection.POOL_TIMEOUT_PROPERTY, "3000000");
+ // a socket that answers the connect and closes it, rather than a port bound and released:
+ // with every bound of this borrow turned off, anything taking that port in between would
+ // leave the test hanging on the timeOut instead of failing
+ try (final ServerSocket rejecting = rejectingSocket()) {
+ final String url = "jdbc:sqlserver://127.0.0.1:" + rejecting.getLocalPort()
+ + ";databaseName=opendj;user=opendj;password=opendj;encrypt=false";
+ final long startedAt = System.currentTimeMillis();
+ try {
+ CachedConnection.getConnection(url);
+ fail("a connect the database closes must be reported");
+ } catch (SQLException expected) {
+ assertFalse(expected.getMessage().contains("socketTimeout"),
+ "the driver was handed a bound it does not take: " + expected.getMessage());
+ }
+ assertElapsedWithinBound(startedAt, 0);
+ }
+ }
+
+ /** The clamp of the range holds where the borrow has no deadline to take it from either. */
+ @Test
+ public void testTheBoundOfAnAttemptStaysInRangeWithoutADeadline() throws Exception {
+ assertEquals(CachedConnection.attemptSeconds(Long.MAX_VALUE, Long.MAX_VALUE), Integer.MAX_VALUE / 1000);
+ assertEquals(CachedConnection.attemptSeconds(0, Long.MAX_VALUE), 0, "0 stands for an attempt with no bound");
+ assertEquals(CachedConnection.attemptSeconds(30, Long.MAX_VALUE), 30);
+ }
+
+ /** The connection string holds the credentials of the backend: a stall report must not carry them. */
+ @Test
+ public void testLoggedConnectionStringCarriesNoCredentials() throws Exception {
+ assertEquals(CachedConnection.safeUrl("jdbc:postgresql://h:5432/db?user=u&password=secret"), "jdbc:postgresql://h:5432/db");
+ // the parameter naming the database stays: two backends of one sql server host answer to
+ // the same url up to it, and a stall report that cannot tell them apart is one of neither
+ assertEquals(CachedConnection.safeUrl("jdbc:sqlserver://h:1433;databaseName=db;password=secret"),
+ "jdbc:sqlserver://h:1433;databaseName=db");
+ // ... and the token naming the kind of oracle driver stays as well: thin against oci is a
+ // first question of an oracle connect, and it stands in front of the credentials
+ assertEquals(CachedConnection.safeUrl("jdbc:oracle:thin:scott/secret@//h:1521/svc"), "jdbc:oracle:thin:@//h:1521/svc");
+ assertEquals(CachedConnection.safeUrl("jdbc:mysql://u:secret@h:3306/db"), "jdbc:mysql://h:3306/db");
+ assertEquals(CachedConnection.safeUrl("jdbc:oracle:thin:@//h:1521/svc"), "jdbc:oracle:thin:@//h:1521/svc");
+ // a password holding the parameter separator of another dialect: ";" separates nothing on
+ // an oracle url, so the credentials are cut in front of the "@" rather than inside them
+ assertEquals(CachedConnection.safeUrl("jdbc:oracle:thin:scott/pa;ss@//h:1521/svc"), "jdbc:oracle:thin:@//h:1521/svc");
+ // ... and one holding a ":" is cut in front of it too: the token of the driver is the one
+ // behind the subprotocol, not the last one standing in front of the "@"
+ assertEquals(CachedConnection.safeUrl("jdbc:oracle:thin:scott/pa:ss@//h:1521/svc"), "jdbc:oracle:thin:@//h:1521/svc");
+ // ... and an "@" that stands inside a parameter is not the end of credentials: the host survives
+ assertEquals(CachedConnection.safeUrl("jdbc:postgresql://h:5432/db?user=u@example.com&password=secret"),
+ "jdbc:postgresql://h:5432/db");
+ // a password holding the parameter separator of its own dialect: on an oracle url the
+ // parameters stand behind the descriptor, so a "?" in front of the "@" is part of the
+ // password and cutting there would leave the start of it in the log
+ assertEquals(CachedConnection.safeUrl("jdbc:oracle:thin:scott/pa?ss@//h:1521/svc"), "jdbc:oracle:thin:@//h:1521/svc");
+ // the same inside an authority, where the credentials end at the path rather than at a "?"
+ assertEquals(CachedConnection.safeUrl("jdbc:mysql://u:sec?ret@h:3306/db"), "jdbc:mysql://h:3306/db");
+ // a descriptor of an oracle url carries no credentials and survives whole
+ assertEquals(CachedConnection.safeUrl("jdbc:oracle:thin:@(DESCRIPTION=(ADDRESS=(HOST=h)(PORT=1521)))"),
+ "jdbc:oracle:thin:@(DESCRIPTION=(ADDRESS=(HOST=h)(PORT=1521)))");
+ // a first host with nothing in front of the comma keeps the comma: what is left is the
+ // hosts of a url, not one host of it
+ assertEquals(CachedConnection.safeUrl("jdbc:mysql://,h2:3306/db"), "jdbc:mysql://,h2:3306/db");
+ // a password under a name of its own, and one numbered by the factor it belongs to
+ assertEquals(CachedConnection.safeUrl("jdbc:mysql://(host=h,user=u,password2=secret)/db"),
+ "jdbc:mysql://(host=h,user=u,password2=***)/db");
+
+ // a url of Connector/J gives every host of it credentials of its own, and every one of
+ // them goes: the second used to stay in the message with the password of the failover host
+ assertEquals(CachedConnection.safeUrl("jdbc:mysql://u:p@h1:3306,u2:p2@h2:3306/db"),
+ "jdbc:mysql://h1:3306,h2:3306/db");
+ // ... including a url whose subprotocol names the kind of connection in front of the hosts
+ assertEquals(CachedConnection.safeUrl("jdbc:mysql:replication://master:p1@h1:3306,slave:p2@h2:3306/db"),
+ "jdbc:mysql:replication://h1:3306,h2:3306/db");
+ // the key-value host syntax of Connector/J puts the credentials inside the authority, where
+ // neither the userinfo nor the parameters of a url stand
+ assertEquals(CachedConnection.safeUrl("jdbc:mysql://address=(host=h)(port=3306)(user=u)(password=secret)/db"),
+ "jdbc:mysql://address=(host=h)(port=3306)(user=u)(password=***)/db");
+ assertEquals(CachedConnection.safeUrl("jdbc:mysql://(host=h,port=3306,user=u,password=secret)/db"),
+ "jdbc:mysql://(host=h,port=3306,user=u,password=***)/db");
+ assertEquals(CachedConnection.safeUrl("jdbc:sqlserver://h:1433;databaseName=db;PWD=secret"),
+ "jdbc:sqlserver://h:1433;databaseName=db");
+
+ // a shape none of this took apart is not logged past its subprotocol: a password holding a
+ // "/" ends the authority in front of the "@" that would have given the credentials away
+ assertEquals(CachedConnection.safeUrl("jdbc:mysql://u:pa/ss@h:3306/db"),
+ "jdbc:mysql:" + CachedConnection.CREDENTIALS_HIDDEN);
+ // ... and so does a password that holds an "@" and was quoted for the driver
+ assertEquals(CachedConnection.safeUrl("jdbc:oracle:thin:scott/\"pa@ss\"@//h:1521/svc"),
+ "jdbc:oracle:" + CachedConnection.CREDENTIALS_HIDDEN);
+ // a string that is no connection string of any driver carries nothing to the log either
+ assertEquals(CachedConnection.safeUrl("h:5432/db?password=secret"), CachedConnection.CREDENTIALS_HIDDEN);
+ }
+
+ /**
+ * A driver is free to quote the connection string it was handed back into the message of its
+ * failure, and that message travels: RootContainer makes it the message of what it throws,
+ * BackendConfigManager logs it at ERROR and answers a config change with it. So the message is
+ * redacted rather than the two call sites that happen to log a url.
+ */
+ @Test
+ public void testAMessageOfADriverCarriesNoCredentials() throws Exception {
+ final String url = "jdbc:postgresql://opendj:S3cret@h:5432/db";
+ assertEquals(CachedConnection.redact("No suitable driver found for " + url, url),
+ "No suitable driver found for jdbc:postgresql://h:5432/db");
+ // a driver naming the credentials alone, without the url around them
+ assertEquals(CachedConnection.redact("authentication of opendj:S3cret failed", url),
+ "authentication of " + CachedConnection.CREDENTIALS_HIDDEN + " failed");
+ // ... and naming the password alone
+ assertEquals(CachedConnection.redact("the password S3cret was not accepted", url),
+ "the password " + CachedConnection.CREDENTIALS_HIDDEN + " was not accepted");
+ // the credentials of an oracle url stand in front of its descriptor
+ final String oracle = "jdbc:oracle:thin:scott/S3cret@//h:1521/svc";
+ assertEquals(CachedConnection.redact("IO Error connecting to " + oracle, oracle),
+ "IO Error connecting to jdbc:oracle:thin:@//h:1521/svc");
+ // a password of a parameter is blanked wherever the message carries it
+ assertEquals(CachedConnection.redact("bad url jdbc:sqlserver://h:1433;password=S3cret",
+ "jdbc:sqlserver://h:1433;password=S3cret"), "bad url jdbc:sqlserver://h:1433");
+ // a message naming nothing of the connection string is left as it stands
+ assertEquals(CachedConnection.redact("Connection to h:5432 refused", url), "Connection to h:5432 refused");
+ assertNull(CachedConnection.redact(null, url));
+
+ // the stall report is the other way a driver's message reaches the log, and it carries the
+ // url of the backend alongside it
+ final String stall = CachedConnection.stallMessage(url, 3, 4000,
+ new SQLException("FATAL: too many connections for " + url));
+ assertFalse(stall.contains("S3cret"), stall);
+ assertTrue(stall.contains("jdbc:postgresql://h:5432/db"), stall);
+ assertTrue(stall.contains("4000 ms") && stall.contains("(3 attempts)"), stall);
+ }
+
+ /**
+ * A password is free to be one character long, and a bare one of those stands inside half the
+ * lines a driver writes. Replaced wherever it is found, it takes the diagnostic apart along
+ * with the credential - and, since the walk looking for credentials asks the redaction whether
+ * it changed anything, it makes every failure of that backend one whose chain is rebuilt.
+ */
+ @Test
+ public void testAShortPasswordDoesNotRewriteTheMessageOfADriver() throws Exception {
+ final String url = "jdbc:oracle:thin:opendj/1@//h:1521/svc";
+ // the "1" of an ORA number is part of a number, not a credential of anybody
+ assertEquals(CachedConnection.redact("ORA-12541: TNS:no listener", url), "ORA-12541: TNS:no listener");
+ // ... while the same password quoted back on its own is still taken out
+ assertEquals(CachedConnection.redact("the password 1 was not accepted", url),
+ "the password " + CachedConnection.CREDENTIALS_HIDDEN + " was not accepted");
+ assertEquals(CachedConnection.redact("IO Error connecting to " + url, url),
+ "IO Error connecting to jdbc:oracle:thin:@//h:1521/svc");
+ }
+
+ /**
+ * A driver is free to name a parameter in the middle of a sentence. The value of one ends at
+ * the first space: reaching to the end of the string, it would take the host, the port and the
+ * cause of the failure into the blank along with the password - the very information the
+ * redaction of a connection string goes out of its way to keep.
+ */
+ @Test
+ public void testAPasswordParameterDoesNotSwallowTheRestOfTheMessage() throws Exception {
+ final String url = "jdbc:postgresql://h:5432/db?user=u&password=hunter2";
+ assertEquals(CachedConnection.redact("Connection refused: password=hunter2 for user u at h:5432", url),
+ "Connection refused: password=*** for user u at h:5432");
+ }
+
+ /**
+ * The credentials of an oracle url are separated by a "/", so a password holding a "//" of
+ * its own used to start an authority inside itself: what was taken off as a userinfo was the
+ * tail of the password, the "@" the last guard of safeUrl() looks for went with it, and the
+ * user name and the head of the password stayed in the message of a stall.
+ */
+ @Test
+ public void testAPasswordHoldingASlashPairIsNotLeftInTheLog() throws Exception {
+ final String easyConnect = "jdbc:oracle:thin:opendj/pa//ss@//h:1521/svc";
+ assertEquals(CachedConnection.safeUrl(easyConnect), "jdbc:oracle:thin:@//h:1521/svc");
+ // ... and the same password in front of a host that names no "//" of its own
+ final String sid = "jdbc:oracle:thin:opendj/pa//ss@h:1521:svc";
+ assertEquals(CachedConnection.safeUrl(sid), "jdbc:oracle:thin:@h:1521:svc");
+ // the message of a driver quoting the url back carries no more of it than the log does
+ assertEquals(CachedConnection.redact("IO Error connecting to " + easyConnect, easyConnect),
+ "IO Error connecting to jdbc:oracle:thin:@//h:1521/svc");
+ }
+
+ /**
+ * pgjdbc enforces loginTimeout out of process: Driver.connect hands the login to a daemon
+ * thread of its own and gives up on the thread rather than on the login. Against the database
+ * this bound exists for - one that completes the handshake and then says nothing - an
+ * unbounded read there leaves that thread, and the socket it holds, behind on every borrow;
+ * a few operations a second are enough to run the server out of threads and file descriptors.
+ */
+ @Test(timeOut = 300000)
+ public void testTheLoginThreadOfPostgresDoesNotOutliveTheBorrow() throws Exception {
+ System.setProperty(CachedConnection.CONNECT_TIMEOUT_PROPERTY, Long.toString(BOUND_SECONDS));
+ // counted as a delta of this borrow rather than as a count of the jvm: three tests of this
+ // file open a pgjdbc login against a socket that never answers, and the order they run in
+ // is not contractual - a thread left by any of them would be reported here
+ final int before = loginThreadsOfPostgres();
+ try (final ServerSocket blackhole = new ServerSocket(0, 50, InetAddress.getLoopbackAddress())) {
+ final String url = "jdbc:postgresql://127.0.0.1:" + blackhole.getLocalPort() + "/opendj?user=opendj&password=opendj";
+ try {
+ CachedConnection.getConnection(url);
+ fail("a database that never answers must not hand out a connection");
+ } catch (SQLException expected) {
+ // reported to the caller, as the bound of the attempt promises
+ }
+ final long giveUpAt = System.currentTimeMillis() + BOUND_SECONDS * 1000 + BOUND_MARGIN_MS;
+ while (loginThreadsOfPostgres() > before && System.currentTimeMillis() < giveUpAt) {
+ Thread.sleep(100);
+ }
+ assertTrue(loginThreadsOfPostgres() <= before,
+ "the login thread pgjdbc abandoned outlived the borrow: the read of the login is not bounded");
+ }
+ }
+
+ private static int loginThreadsOfPostgres() {
+ int alive = 0;
+ for (final Thread thread : Thread.getAllStackTraces().keySet()) {
+ if (thread.isAlive() && thread.getName().startsWith("PostgreSQL JDBC driver connection thread")) {
+ alive++;
+ }
+ }
+ return alive;
+ }
+
+ /**
+ * The deadline of the borrow stands for the whole borrow, so it bounds the attempt inside it
+ * even where the per-attempt property gives it no bound of its own: turning that property off
+ * must not turn the bound of the borrow off with it.
+ */
+ @Test(timeOut = 300000)
+ public void testTheDeadlineBoundsAnAttemptTheConnectPropertyDoesNot() throws Exception {
+ System.setProperty(CachedConnection.CONNECT_TIMEOUT_PROPERTY, "0");
+ System.setProperty(CachedConnection.POOL_TIMEOUT_PROPERTY, "2");
+ try (final ServerSocket blackhole = new ServerSocket(0, 50, InetAddress.getLoopbackAddress())) {
+ final String url = "jdbc:postgresql://127.0.0.1:" + blackhole.getLocalPort() + "/opendj?user=opendj&password=opendj";
+ final long startedAt = System.currentTimeMillis();
+ try {
+ CachedConnection.getConnection(url);
+ fail("a database that never answers must not hand out a connection");
+ } catch (SQLException expected) {
+ // bounded by what is left of the deadline of the borrow
+ }
+ // the wait is the point of this one as much as its end is: a borrow against a socket that
+ // never answers cannot be over before the deadline unless something else ended it
+ final long elapsed = System.currentTimeMillis() - startedAt;
+ assertTrue(elapsed >= 1000, "the borrow was over after " + elapsed + " ms, before its deadline");
+ assertElapsedWithinBound(startedAt, 2000);
+ }
+ }
+
+ /**
+ * The deadline stops the drain of the pool; it does not throw away the connection in hand. A
+ * database at its connection limit has no other source of connections than the ones coming
+ * back to the pool, and closing one unvalidated takes it out of that source for good - while
+ * the borrow that closed it fails with a timeout anyway.
+ */
+ @Test(timeOut = 120000)
+ public void testAPooledConnectionIsNotDiscardedUnvalidatedAtTheDeadline() throws Exception {
+ final String url = StubDriver.PREFIX + "unvalidated-at-deadline";
+ final Connection stale = mock(Connection.class);
+ when(stale.isValid(anyInt())).thenAnswer(invocation -> {
+ Thread.sleep(1500); // a database that no longer answers: the validation waits out its bound
+ return false;
+ });
+ final Connection good = mock(Connection.class);
+ when(good.isValid(anyInt())).thenReturn(true);
+ CachedConnection.cached.get(url).add(new CachedConnection(url, stale));
+ CachedConnection.cached.get(url).add(new CachedConnection(url, good));
+ final Connection fresh = mock(Connection.class);
+ stub.answerWith(fresh);
+ System.setProperty(CachedConnection.POOL_TIMEOUT_PROPERTY, "1");
+
+ final Connection borrowed = CachedConnection.getConnection(url);
+
+ assertSame(((CachedConnection) borrowed).parent, fresh, "the drain must stop at the deadline");
+ verify(stale).close();
+ verify(good, never()).close();
+ assertFalse(CachedConnection.cached.get(url).isEmpty(), "a connection the deadline was reached in front of was lost");
+ }
+
+ /**
+ * A connection the validation of which failed is on its way out, and its driver knows it:
+ * Connector/J answers a failed validation by aborting the connection and the SQL Server driver
+ * by terminating it. Putting the previous bound back on it fails, and warns about statements
+ * of a connection that is being closed - over an idle connection the server reaped, which is
+ * nobody's problem.
+ */
+ @Test(timeOut = 120000)
+ public void testAConnectionOnItsWayOutIsNotGivenItsBoundBack() throws Exception {
+ final String url = StubDriver.PREFIX + "reaped-idle";
+ final Connection reaped = mock(Connection.class);
+ when(reaped.getNetworkTimeout()).thenReturn(0);
+ when(reaped.isValid(anyInt())).thenReturn(false);
+ CachedConnection.cached.get(url).add(new CachedConnection(url, reaped));
+ final Connection fresh = mock(Connection.class);
+ stub.answerWith(fresh);
+
+ assertSame(((CachedConnection) CachedConnection.getConnection(url)).parent, fresh);
+
+ verify(reaped).setNetworkTimeout(any(Executor.class), eq(CachedConnection.VALIDATION_TIMEOUT_SECONDS * 1000));
+ verify(reaped, never()).setNetworkTimeout(any(Executor.class), eq(0));
+ verify(reaped).close();
+ }
+
+ /**
+ * The read bound of the login is lifted once the login is through, because left in place it
+ * fails every statement slower than it. A driver that will not take it back leaves a
+ * connection that must not be pooled: it would carry that bound into every borrow the pool
+ * hands it to, an import batch among them.
+ */
+ @Test(timeOut = 120000)
+ public void testAConnectionStillCarryingTheBoundOfItsLoginIsNotPooled() throws Exception {
+ final String url = StubDriver.PREFIX + "unliftable-bound";
+ final Connection parent = mock(Connection.class);
+ doThrow(new SQLException("setNetworkTimeout is not supported"))
+ .when(parent).setNetworkTimeout(any(Executor.class), eq(0));
+ stub.answerWith(parent);
+
+ final CachedConnection borrowed = CachedConnection.connect(url, CachedConnection.ConnectDialect.MYSQL, 30);
+ borrowed.close();
+
+ verify(parent).close();
+ assertTrue(CachedConnection.cached.get(url).isEmpty(),
+ "a connection still carrying the read bound of its login went back into the pool");
+ }
+
+ private static SQLException tooManyConnections() {
+ // 53300, too_many_connections, of the insufficient_resources class
+ return new SQLException("sorry, too many clients already", "53300");
+ }
+
+ /** A connection string of every dialect pointing at one host and port. */
+ private static String[] urlsOf(int port) {
+ return new String[]{
+ "jdbc:postgresql://127.0.0.1:" + port + "/opendj?user=opendj&password=opendj",
+ "jdbc:mysql://127.0.0.1:" + port + "/opendj?user=opendj&password=opendj",
+ "jdbc:oracle:thin:opendj/opendj@//127.0.0.1:" + port + "/free",
+ "jdbc:sqlserver://127.0.0.1:" + port + ";databaseName=opendj;user=opendj;password=opendj;encrypt=false"
+ };
+ }
+
+ private static int closedPort() throws Exception {
+ try (final ServerSocket socket = new ServerSocket(0, 1, InetAddress.getLoopbackAddress())) {
+ return socket.getLocalPort();
+ } // closed again: nothing listens there any more
+ }
+
+ /**
+ * A socket that answers a connect and closes it at once. A port bound and released is the
+ * shape of a refused connect a test would reach for, but it races whatever else on the host
+ * may take that port; this one is the failure it stands for and belongs to nobody else.
+ */
+ private static ServerSocket rejectingSocket() throws Exception {
+ final ServerSocket socket = new ServerSocket(0, 50, InetAddress.getLoopbackAddress());
+ final Thread accepting = new Thread(() -> {
+ while (!socket.isClosed()) {
+ try {
+ socket.accept().close();
+ } catch (Exception closed) {
+ return;
+ }
+ }
+ }, "opendj-test-rejecting-socket");
+ accepting.setDaemon(true);
+ accepting.start();
+ return socket;
+ }
+
+ // The margin grows with the bound instead of dwarfing it: what this has to catch is a bound
+ // that is not in force, and a bound of 2 s that is not in force is not a wait of 12 s - it is
+ // the 30 s of the connect property, the 600 s of a driver, or no end at all.
+ private static void assertElapsedWithinBound(long startedAt, long boundMs) {
+ final long elapsed = System.currentTimeMillis() - startedAt;
+ final long margin = Math.max(BOUND_MARGIN_MS, boundMs);
+ assertTrue(elapsed < boundMs + margin,
+ "gave up only after " + elapsed + " ms, past the " + boundMs + " ms it was bounded by");
+ }
+
+ /**
+ * Stands in for a database whose answer to a connect is the point of the test: the vendor
+ * codes and SQL states below are what the retry has to tell apart, and no engine is needed to
+ * produce them.
+ */
+ private static final class StubDriver implements Driver {
+ static final String PREFIX = "jdbc:opendj-stub:";
+ static final int ALWAYS = -1;
+
+ final AtomicInteger attempts = new AtomicInteger();
+ private volatile SQLException failure;
+ private volatile int failuresLeft;
+ private volatile Connection answer;
+
+ void failWith(SQLException failure, int times) {
+ this.failure = failure;
+ this.failuresLeft = times;
+ this.answer = null;
+ this.attempts.set(0);
+ }
+
+ void answerWith(Connection answer) {
+ this.failure = null;
+ this.failuresLeft = 0;
+ this.answer = answer;
+ this.attempts.set(0);
+ }
+
+ @Override
+ public Connection connect(String url, Properties info) throws SQLException {
+ if (!acceptsURL(url)) {
+ return null;
+ }
+ attempts.incrementAndGet();
+ if (failuresLeft != 0) {
+ if (failuresLeft > 0) {
+ failuresLeft--;
+ }
+ throw failure;
+ }
+ if (answer != null) {
+ return answer;
+ }
+ final Connection con = mock(Connection.class);
+ when(con.isValid(anyInt())).thenReturn(true);
+ return con;
+ }
+
+ @Override
+ public boolean acceptsURL(String url) {
+ return url != null && url.startsWith(PREFIX);
+ }
+
+ @Override
+ public DriverPropertyInfo[] getPropertyInfo(String url, Properties info) {
+ return new DriverPropertyInfo[0];
+ }
+
+ @Override
+ public int getMajorVersion() {
+ return 1;
+ }
+
+ @Override
+ public int getMinorVersion() {
+ return 0;
+ }
+
+ @Override
+ public boolean jdbcCompliant() {
+ return false;
+ }
+
+ @Override
+ public Logger getParentLogger() {
+ return Logger.getLogger(StubDriver.class.getName());
+ }
+ }
+}
diff --git a/opendj-server-legacy/src/test/java/org/opends/server/backends/jdbc/TestCase.java b/opendj-server-legacy/src/test/java/org/opends/server/backends/jdbc/TestCase.java
index b650301..9d7cdac 100644
--- a/opendj-server-legacy/src/test/java/org/opends/server/backends/jdbc/TestCase.java
+++ b/opendj-server-legacy/src/test/java/org/opends/server/backends/jdbc/TestCase.java
@@ -45,6 +45,7 @@
import java.util.Collections;
import java.util.List;
import java.util.NoSuchElementException;
+import java.util.Properties;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -56,6 +57,7 @@
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertFalse;
import static org.testng.Assert.assertNotEquals;
+import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertNull;
import static org.testng.Assert.assertTrue;
import static org.testng.Assert.fail;
@@ -140,6 +142,40 @@
protected abstract String getJdbcUrl();
+ /**
+ * The second property bounding a login is a socket read timeout on mysql, oracle and sql
+ * server: in force for the whole life of the connection it would fail every statement slower
+ * than it - an import batch, the statistics of a freshly loaded table - so it has to be lifted
+ * as soon as the login is through (#872).
+ */
+ @Test(timeOut = 120000)
+ public void testLoginBoundDoesNotOutliveTheLogin() throws Exception {
+ final String url = createBackendCfg().getDBDirectory();
+ final CachedConnection.ConnectDialect dialect = CachedConnection.ConnectDialect.of(url);
+ assertNotNull(dialect, "the dialect of the container is one this backend bounds: " + CachedConnection.safeUrl(url));
+ System.setProperty(CachedConnection.CONNECT_TIMEOUT_PROPERTY, "2");
+ try {
+ // the bound this lifts has to be in force first, or the assertion below holds of a
+ // connection that never carried one: established here with the very properties the
+ // borrow uses, and read back off the socket of this driver
+ final Properties bounding = new Properties();
+ assertTrue(dialect.bound(url, bounding, 2),
+ "the read bound of the login is not set for this dialect, so there is nothing to lift");
+ try (final Connection bounded = DriverManager.getConnection(url, bounding)) {
+ assertEquals(bounded.getNetworkTimeout(), 2000,
+ "the property this dialect names does not bound the socket of its login");
+ }
+
+ // a pooled connection would be handed back without being established again
+ CachedConnection.cached.invalidate(url);
+ try (final Connection con = CachedConnection.getConnection(url)) {
+ assertEquals(con.getNetworkTimeout(), 0, "the read bound of the login is still in force");
+ }
+ } finally {
+ System.clearProperty(CachedConnection.CONNECT_TIMEOUT_PROPERTY);
+ }
+ }
+
private static ByteString key(int i) {
return ByteString.valueOfUtf8(String.format("key%02d", i));
}
--
Gitblit v1.10.0