/* * The contents of this file are subject to the terms of the Common Development and * Distribution License (the License). You may not use this file except in compliance with the * License. * * You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the * specific language governing permission and limitations under the License. * * When distributing Covered Software, include this CDDL Header Notice in each file and include * the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL * Header, with the fields enclosed by brackets [] replaced by your own identifying * information: "Portions Copyright [year] [name of copyright owner]". * * Copyright 2024-2026 3A Systems, LLC. */ package org.opends.server.backends.jdbc; import com.github.benmanes.caffeine.cache.Caffeine; import com.github.benmanes.caffeine.cache.LoadingCache; import com.github.benmanes.caffeine.cache.RemovalCause; import org.forgerock.i18n.LocalizableMessage; import org.forgerock.i18n.slf4j.LocalizedLogger; import java.sql.*; import java.time.Duration; import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Deque; import java.util.HashSet; import java.util.IdentityHashMap; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Properties; import java.util.Set; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicLong; import java.util.regex.Matcher; import java.util.regex.Pattern; public class CachedConnection implements Connection { private static final LocalizedLogger logger = LocalizedLogger.getLoggerForThisClass(); // What has been reported once already. Every one of these reports a setting rather than an // event - a property that is not a number, a url no bound of this class can reach, a driver // whose property names are not known here - so it does not become truer by being repeated, // and every operation of the backend comes through here. // Declared above every field whose initializer can reach warnOnce(): class variable // initializers run in textual order (JLS 12.4.2), so a set declared below aliveBypassNanos // would still be null the moment a property this class reports on carries a value worth // warning about - a window longer than the ttl, or one that is not a number - and the report // would leave the class uninitializable rather than merely configured oddly. static final Set warnedOnce = ConcurrentHashMap.newKeySet(); static final String TTL_PROPERTY = "org.openidentityplatform.opendj.jdbc.ttl"; static final long DEFAULT_TTL_MS = 15000; /** * How long a pooled connection is handed out without being validated after it was last proven * alive, in ms; 0 validates every borrow, the way this pool did before the window existed. *

* The validation of a connection is a round trip of its own - an empty query on postgresql, a * ping on mysql, a round trip of its own on oracle and sql server - and every operation of * this backend pays it next to the single statement the operation came for. It earns that on a * connection that has been sitting in the pool, which the database or a firewall may have * dropped in the meantime; it earns nothing on one that answered a moment ago, which is most * of them under load. So a connection proven alive within this window is trusted rather than * validated, the way the aliveBypassWindow of HikariCP does it. */ static final String ALIVE_BYPASS_PROPERTY = "org.openidentityplatform.opendj.jdbc.alive.bypass"; static final long DEFAULT_ALIVE_BYPASS_MS = 500; /** * The longest window this class uses, whatever {@value #ALIVE_BYPASS_PROPERTY} and the * {@value #TTL_PROPERTY} it is clamped to say. The clamp to the ttl alone does not bound it: * the ttl has no upper bound of its own, and with both set high enough the conversion to * nanoseconds saturates - the window then outlasts every reading it is compared against, and * no connection of the pool is ever validated again. An hour is already far past what this * window is about, which is a connection that answered a moment ago. *

* A compile-time constant, so that it holds its value wherever it is read from: the initializer * of {@link #aliveBypassNanos} reaches it, and a field initialized in declaration order would * still be 0 there if it were ever moved below (JLS 12.4.2). */ static final long MAX_ALIVE_BYPASS_MS = 60 * 60 * 1000L; // Read once, at class initialization: every operation of this backend borrows a connection, // and the borrow is not the place to parse a system property. Not final so that a test can // vary the window without a class loader of its own, and volatile because a non-final static // long is written neither atomically nor visibly to the threads reading it (JLS 17.7) - every // worker of the backend and every replay thread reads this one. static volatile long aliveBypassNanos = TimeUnit.MILLISECONDS.toNanos(getAliveBypassMillis()); /** * Bounds the connect and the login of one attempt to establish a connection, in seconds; 0 for * no bound of its own - the deadline of {@value #POOL_TIMEOUT_PROPERTY} still bounds the * attempt, since it stands for the whole borrow. Setting both to 0 is what leaves a connect * unbounded. */ static final String CONNECT_TIMEOUT_PROPERTY = "org.openidentityplatform.opendj.jdbc.connect.timeout"; static final long DEFAULT_CONNECT_TIMEOUT_SECONDS = 30; /** * Bounds a whole borrow - every connect attempt and every wait for a pooled connection - in * seconds; 0 for no bound. Not to the millisecond: the connection in hand is validated * whatever the deadline says, and an attempt is never given less than a second, so a borrow * can return a validation and a last attempt past it. */ static final String POOL_TIMEOUT_PROPERTY = "org.openidentityplatform.opendj.jdbc.pool.timeout"; static final long DEFAULT_POOL_TIMEOUT_SECONDS = 60; /** Bound of the validation of a pooled connection: isValid(0) means "no timeout" in the JDBC contract. */ static final int VALIDATION_TIMEOUT_SECONDS = 5; /** 08001, sqlclient_unable_to_establish_sqlconnection: the state of a connect that did not happen. */ private static final String CONNECT_FAILED_SQL_STATE = "08001"; /** 53300, too_many_connections: how the standard - and postgresql - reports a server taking no further connection. */ private static final String CONNECTION_LIMIT_SQL_STATE = "53300"; /** 57P03, cannot_connect_now: postgresql starting up, shutting down or in recovery. */ private static final String NOT_ACCEPTING_YET_SQL_STATE = "57P03"; static final long MAX_BACKOFF_MS = 1000; static final long STALL_WARNING_AFTER_MS = 1000; static final long STALL_WARNING_INTERVAL_MS = 10000; /** How many links of the cause and getNextException() chains of a failure are looked at. */ private static final int MAX_CHAIN_LENGTH = 32; /** What a connection string is cut down to where this cannot tell its credentials from the rest of it. */ static final String CREDENTIALS_HIDDEN = "

* Read at class initialization, like the ttl it is clamped to, so a value set after that * changes neither. */ static long getAliveBypassMillis() { long configured = getNonNegativeProperty(ALIVE_BYPASS_PROPERTY, DEFAULT_ALIVE_BYPASS_MS, "ms"); final long ttl = getCacheTtlMillis(); if (configured > ttl) { warnOnce(ALIVE_BYPASS_PROPERTY + "=" + configured + ">" + ttl, "The %s window of %d ms is longer than the %d ms of %s a pooled connection is kept for," + " and is used as %d ms: a connection trusted for longer than the pool keeps it would" + " be trusted past the point the pool closed it", ALIVE_BYPASS_PROPERTY, configured, ttl, TTL_PROPERTY, ttl); configured = ttl; } if (configured > MAX_ALIVE_BYPASS_MS) { // the ttl it was just clamped to has no upper bound of its own warnOnce(ALIVE_BYPASS_PROPERTY + ">" + MAX_ALIVE_BYPASS_MS, "The %s window of %d ms is longer than the %d ms this pool trusts a connection for at most," + " and is used as %d ms: a longer one saturates the arithmetic it is compared in and" + " leaves every connection of the pool trusted for the life of the server", ALIVE_BYPASS_PROPERTY, configured, MAX_ALIVE_BYPASS_MS, MAX_ALIVE_BYPASS_MS); return MAX_ALIVE_BYPASS_MS; } return configured; } /** * Returns the value of a numeric system property, ignoring a value that is not a non-negative * number in favor of the default. The unit is the one the property is read in, so that the * value the message names is not mistaken for another. */ private static long getNonNegativeProperty(String name, long defaultValue, String unit) { final String value = System.getProperty(name); if (value != null) { try { final long parsed = Long.parseLong(value.trim()); if (parsed >= 0) { return parsed; } } catch (NumberFormatException ignored) { } // reported once for this value: both properties are read on every borrow, so a // "30s" of a typo would otherwise put two lines in the log per backend operation warnOnce(name + "=" + value, "Ignoring invalid value \"%s\" of the %s property, using %d %s", value, name, defaultValue, unit); } return defaultValue; } /** Reports something about a setting once for the life of the jvm, however many borrows meet it. */ private static void warnOnce(String key, String format, Object... args) { if (warnedOnce.add(key)) { logger.warn(LocalizableMessage.raw(format, args)); } } /** * The drivers this backend is used with, recognized by the prefix of the connection string, * together with the properties that bound one attempt to establish a connection. Not one of * them bounds the attempt with a single property: the one named first covers the socket * connect, and the login behind it - the reads of the prelogin handshake, of TLS and of * authentication, the phase a proxy at its connection limit or a moved VIP leaves unanswered - * needs the read bound behind it. That holds for the SQL Server driver too, whose loginTimeout * leaves the read of the prelogin answer unbounded - and for pgjdbc, whose loginTimeout is not * a bound of the socket at all: Driver.connect hands the login to a daemon thread of its own * and abandons it at the timeout, so an unbounded read there leaks a thread and a socket per * borrow instead of failing one (CachedConnectionTestCase covers every one of them against a * socket that never answers). */ enum ConnectDialect { /** * postgresql: every property of the three takes seconds. connectTimeout covers the socket * connect and socketTimeout the reads of the login: pgjdbc puts an SO_TIMEOUT on the login * socket only where socketTimeout is set (ConnectionFactoryImpl.tryConnect, both before and * after enableSSL), and it defaults to none. loginTimeout is kept on top of the two for a * url naming more than one host, where each of them costs a login of its own - the connect * is one budget for all of them, taken from the single System.nanoTime() in front of the * loop over the hosts - but it is not a bound this class could rely on alone: Driver.connect * runs the login on a daemon thread, gives up on the thread rather than on the login, and * the thread stays parked in the read for as long as the read lasts. */ POSTGRES("jdbc:postgresql:", '?', new String[]{"connectTimeout", "loginTimeout"}, 1, 0, new String[]{"socketTimeout"}, 1, true, new int[]{}, new int[]{}), /** mysql: both properties take milliseconds; socketTimeout is a socket read timeout that outlives the login. */ MYSQL("jdbc:mysql:", '?', new String[]{"connectTimeout"}, 1000, 0, new String[]{"socketTimeout"}, 1000, true, new int[]{1040, 1203}, new int[]{1053}), /** oracle: both properties take milliseconds; ReadTimeout is a socket read timeout that outlives the login. */ ORACLE("jdbc:oracle:", '?', new String[]{"oracle.net.CONNECT_TIMEOUT"}, 1000, 0, // the read bound goes by two names the driver reads: the property set here and the // property of oracle net it stands for, inside a tns descriptor by the last segment of // either. A bound under one of them is a bound of the administrator, so ours is not set // on top of it - and neither of theirs is lifted with ours once the login is through. // Only the first of the two is a name the driver also reads out of the system // properties (SYSTEM_PROPERTY_NAMES below): oracle.net.READ_TIMEOUT reaches the socket // from the connection properties alone, so a -D of it bounds nothing and must not be // taken for a bound of theirs. // RECV_TIMEOUT is not one of them: it is a parameter of sqlnet.ora and of the listener, // and the name does not appear in ojdbc8 at all, so a descriptor carrying one would // have taken our bound off a connection that never had one of its own. new String[]{"oracle.jdbc.ReadTimeout", "oracle.net.READ_TIMEOUT"}, 1000, true, // ORA-01033 and ORA-01034: the instance is starting up or not there yet; ORA-01089: // it is shutting down. ORA-12514 is left out of these on purpose - a listener that // does not know the service is also what a service name of a typo looks like, forever new int[]{20, 12516, 12518, 12519, 12520}, new int[]{1033, 1034, 1089}), /** * ms sql server: loginTimeout takes seconds, socketTimeout milliseconds; the latter is a * socket read timeout that outlives the login. loginTimeout is the one property of the * four with a range of its own - SQLServerDriverIntProperty.LOGIN_TIMEOUT is validated * against [0, 65535], and a value beyond it fails every connect the driver is asked for. */ MICROSOFT("jdbc:sqlserver:", ';', new String[]{"loginTimeout"}, 1, 65535, new String[]{"socketTimeout"}, 1000, true, // 921 and 922: the database has not been recovered yet, or is being recovered; 927: it // is in the middle of a restore; 40613: azure sql reporting it not available for now new int[]{17809, 10928, 10929}, new int[]{921, 922, 927, 40613}); final String urlPrefix; /** the character that separates the parameters of this dialect from the url in front of them */ final char parameterSeparator; /** the properties bounding the connect: the socket connect, and whatever the driver wraps it in */ final String[] connectProperties; final int connectUnitsPerSecond; /** the largest value the driver accepts for a connect property, 0 for a driver that takes any */ final long maxConnectSeconds; /** the read bound of the login: the first name is the one set here, the rest are the names it also goes by */ final String[] readProperties; final int readUnitsPerSecond; /** whether the read bound of the login stays in force for every statement issued afterwards */ final boolean readBoundOutlivesLogin; /** the vendor codes of this dialect for "no further connection is accepted" */ final int[] connectionLimitCodes; /** the vendor codes of this dialect for "not accepting connections yet": a database on its way up */ final int[] notAcceptingYetCodes; ConnectDialect(String urlPrefix, char parameterSeparator, String[] connectProperties, int connectUnitsPerSecond, long maxConnectSeconds, String[] readProperties, int readUnitsPerSecond, boolean readBoundOutlivesLogin, int[] connectionLimitCodes, int[] notAcceptingYetCodes) { this.urlPrefix = urlPrefix; this.parameterSeparator = parameterSeparator; this.connectProperties = connectProperties; this.connectUnitsPerSecond = connectUnitsPerSecond; this.maxConnectSeconds = maxConnectSeconds; this.readProperties = readProperties; this.readUnitsPerSecond = readUnitsPerSecond; this.readBoundOutlivesLogin = readBoundOutlivesLogin; this.connectionLimitCodes = connectionLimitCodes; this.notAcceptingYetCodes = notAcceptingYetCodes; } /** The dialect of a connection string, or null for a driver whose property names are not known here. */ static ConnectDialect of(String connectionString) { final String url = connectionString.toLowerCase(Locale.ROOT); for (final ConnectDialect dialect : values()) { if (url.startsWith(dialect.urlPrefix)) { return dialect; } } return null; } /** * Fills in the properties bounding one connect attempt, leaving out what the administrator * bounded themselves - an explicit setting of theirs keeps precedence, on the SQL Server, * mysql and oracle drivers because a supplied property outranks the url, and on postgresql * because the url outranks the property. A driver with a range of its own for its connect * property is not handed a value beyond it: a bound it rejects is no bound at all, it is a * connect that never happens. * Returns whether a read bound outliving the login was set and has to be lifted once the * connection is established. */ boolean bound(String connectionString, Properties properties, long timeoutSeconds) { final long connectSeconds = maxConnectSeconds > 0 ? Math.min(timeoutSeconds, maxConnectSeconds) : timeoutSeconds; // The connect side is one budget rather than a set of independent knobs, so a bound of // the administrator under any of its names leaves all of them alone. On postgresql // connectTimeout bounds the socket connect and loginTimeout the login behind it: // filling in the one they left out caps the one they set, and a "?connectTimeout=300" // answered with a loginTimeout of ours is a login pgjdbc gives up on at 30 s - Driver // .connect branches into its own thread as soon as loginTimeout is anything but 0. if (!declared(connectionString, connectProperties)) { for (final String property : connectProperties) { properties.setProperty(property, Long.toString(connectSeconds * connectUnitsPerSecond)); } } reportBoundTurnedOffInUrl(connectionString); if (!declared(connectionString, readProperties)) { properties.setProperty(readProperties[0], Long.toString(timeoutSeconds * readUnitsPerSecond)); return readBoundOutlivesLogin; } return false; } /** * Reports a url that turns a bound off where nothing this class supplies can put one back. * A parameter of a postgresql url outranks the property this class hands the driver, so a * "socketTimeout=0" there is not a default to be replaced - it is the administrator asking * for an unbounded read, and a borrow that meets a database accepting the connection and * answering nothing is then parked with no deadline able to reach it. */ private void reportBoundTurnedOffInUrl(String connectionString) { if (!urlOutranksProperties()) { return; } // Every one of them, and keyed by the property rather than by the url alone: a url // turns off the read bound and the login bound both ("?socketTimeout=0&loginTimeout=0", // where the second is the per-host budget of a failover url), and safeUrl() keeps none // of the timeout parameters - so a single key would report the first offender and // leave the administrator to find the rest of them on their own. for (final String[] properties : new String[][]{readProperties, connectProperties}) { for (final String property : properties) { final String value = parameterValue(connectionString, property); if (value != null && !isBound(value)) { warnOnce(safeUrl(connectionString) + "|unbounded|" + property, "%s sets \"%s=%s\": a parameter of a postgresql url outranks the property this backend" + " supplies, so that phase of a connect carries no bound. An operation reaching a" + " database that accepts the connection and does not answer stays parked", safeUrl(connectionString), property, value); } } } } /** Whether a vendor code of this dialect is one that waiting for the database can clear. */ boolean isWorthRetrying(int errorCode) { return contains(connectionLimitCodes, errorCode) || contains(notAcceptingYetCodes, errorCode); } private static boolean contains(int[] codes, int code) { for (final int candidate : codes) { if (candidate == code) { return true; } } return false; } // Whether the administrator bounded one of these properties themselves. The dialects // separate their parameters differently - "?a=1&b=2" (postgresql, mysql), ";a=1;b=2" (sql // server), "(A=1)" inside the descriptor of an oracle tns url, where the property also goes // by the last segment of its name alone - so a parameter is recognized by the delimiter in // front of it and the "=" behind it rather than by parsing the url syntax of every driver. // The connection string is not the only channel of theirs: the oracle driver reads some of // its properties out of the system properties as well, which is how a whole jvm is bounded // with -Doracle.jdbc.ReadTimeout, and a property supplied to a driver outranks the system // property without a word - and would then be lifted after the login as if it were ours, // leaving a connection with no read bound where the administrator had set one. private boolean declared(String connectionString, String... properties) { for (final String property : properties) { if (declaredInUrl(connectionString, property) || setAsSystemProperty(property)) { return true; } } return false; } /** Whether the connection string bounds this property, under its own name or the last segment of it. */ private boolean declaredInUrl(String connectionString, String property) { if (containsParameter(connectionString, property)) { return true; } final int dot = property.lastIndexOf('.'); return dot >= 0 && containsParameter(connectionString, property.substring(dot + 1)); } // Which names a driver reads out of the system properties, listed rather than told from // the shape of the name: ojdbc8 resolves oracle.jdbc.ReadTimeout and // oracle.net.CONNECT_TIMEOUT in three tiers (the properties it was supplied, then // System.getProperty, then the properties of the data source), while // oracle.net.READ_TIMEOUT - a dotted name of the same driver - is read out of the // connection properties alone: the six classes carrying the literal hand it to // Properties.get, and none of them to System.getProperty. Taking a -D of it for a bound of // the administrator would leave the login with no read bound at all - theirs not read by // the driver and ours not set, because we believed theirs was in force. private static final Set SYSTEM_PROPERTY_NAMES = Collections.unmodifiableSet(new HashSet<>( Arrays.asList("oracle.jdbc.ReadTimeout", "oracle.net.CONNECT_TIMEOUT"))); private static boolean setAsSystemProperty(String property) { return SYSTEM_PROPERTY_NAMES.contains(property) && isBound(System.getProperty(property)); } // pgjdbc parses the url over the properties it was handed - Driver.connect copies them // into a flat map and parseURL then writes the parameters of the url on top - so a value // standing in a postgresql url is the value the driver uses, and the one supplied here // never reaches the socket. A zero there is not a default of the driver to be replaced: it // cannot be replaced, and setting ours on top of it would leave this class lifting a read // bound the login never had. The other three let a supplied property win, so a zero of // theirs is ours to override. private boolean urlOutranksProperties() { return this == POSTGRES; } /** Whether this property is bounded by the connection string, as the driver of this dialect reads it. */ private boolean containsParameter(String connectionString, String property) { final String value = parameterValue(connectionString, property); return value != null && (urlOutranksProperties() || isBound(value)); } // Matched the way the driver of this dialect matches it: pgjdbc and Connector/J look their // properties up by their exact name - PropertyKey.fromValue answers null for a name of // another case and the parameter is then a parameter of nobody, so "?SocketTimeout=" must // not be taken for a bound of the administrator - while the SQL Server driver // (getNormalizedPropertyName) and the keywords of an oracle descriptor match either way. private String parameterValue(String connectionString, String property) { final boolean exact = this == POSTGRES || this == MYSQL; final String url = exact ? connectionString : connectionString.toLowerCase(Locale.ROOT); final String name = exact ? property : property.toLowerCase(Locale.ROOT); String value = null; for (int i = url.indexOf(name); i >= 0; i = url.indexOf(name, i + name.length())) { final int end = i + name.length(); if (i > 0 && "?&;(,".indexOf(url.charAt(i - 1)) >= 0 && end < url.length() && url.charAt(end) == '=') { // the last of them: a driver parsing a url into a map lets the last assignment stand value = valueOf(url, end + 1); } } return value; } /** The value of the parameter that starts here: up to the delimiter in front of the next one. */ private static String valueOf(String url, int from) { int end = from; while (end < url.length() && "&;),?".indexOf(url.charAt(end)) < 0) { end++; } return url.substring(from, end); } /** * Whether a value of the administrator bounds anything. Every one of these drivers reads 0 * as "wait as long as it takes", so a property set to it is not a bound of theirs to stay * out of the way of - it is the default this class exists to replace, and one of ours goes * on top of it wherever a supplied property outranks the url. Where it does not, on * postgresql, the zero stands and is reported instead of being written over. A value that * is no number is left to the driver it belongs to. */ private static boolean isBound(String value) { if (value == null || value.trim().isEmpty()) { return false; } try { return Double.parseDouble(value.trim()) != 0; // pgjdbc takes a float for its loginTimeout } catch (NumberFormatException notANumber) { return true; } } } final String connectionString; /** * Whether this connection may go back into the pool once it is closed. A connection carrying a * read bound that could not be lifted serves the borrower waiting for it and is closed * afterwards: left in the pool it would fail every statement slower than that bound - an * import batch among them - for every borrow the pool hands it to. */ private final boolean poolable; /** * When this connection last answered the database, as a {@link System#nanoTime()} reading: * established - the login and the two round trips that set it up have just answered - or * validated. It is never stamped on the way back into the pool, although that is where a * connection has most recently been used: {@link #close()} ends the transaction, and pgjdbc * short-circuits both {@code rollback()} and {@code commit()} when the transaction state is * IDLE, so on a borrow that issued no statement - {@code JDBCStorage.open()}, a configuration * change that leaves the base DNs alone, an import of nothing - not a byte reaches the server * and the stamp would prove nothing, while marking a connection the database may have dropped * as the freshest one in the pool. Stamping proof rather than use makes the window mean * "validated at most once per window", which is a claim this class can always back. *

* It stands for the moment the connection was asked, not the moment its answer was * filed: {@link #distrustPool} is compared against it as an ordering of two moments, and a * proof that took a second to arrive would otherwise outlive a drop reported while it was * still in flight. Reading it early only ever ages the proof, which costs a validation and * never skips one. */ private volatile long lastKnownAliveNanos; public CachedConnection(String connectionString, Connection parent) { this(connectionString, parent, true); } CachedConnection(String connectionString, Connection parent, boolean poolable) { this.connectionString = connectionString; this.parent = parent; this.poolable = poolable; this.lastKnownAliveNanos = System.nanoTime(); } /** * Borrows a connection: a usable one out of the pool, or a newly established one. Bounded in * both phases - every operation of this backend, the open of a backend and the import * included, comes through here, and an unbounded borrow turns a database that listens but does * not answer into a hang rather than into an error the caller can report. */ static Connection getConnection(String connectionString) throws Exception { return getConnection(connectionString, true); } /** * Borrows a connection, either trusting the alive window of {@value #ALIVE_BYPASS_PROPERTY} or * validating whatever comes out of the pool. * * @param trusted false for a borrow nothing compensates a dropped connection on. What the * window trades away is the connection that breaks inside it, and {@code JDBCStorage} takes * that off the caller where it can - a write is replayed, a read tells the pool - but the * borrows that open a backend, remove its files or start an import have neither: they issue * their statements far from the borrow, and the one that opens a backend issues none at all, * so a dropped connection would surface out of the {@code rollback()} of its release. Each of * them is one borrow of a cold path, where the round trip the window saves is worth nothing. */ static Connection getConnection(String connectionString, boolean trusted) throws Exception { final ConnectDialect dialect = ConnectDialect.of(connectionString); reportUnknownDialect(connectionString, dialect); final long connectTimeoutSeconds = Math.min( getNonNegativeProperty(CONNECT_TIMEOUT_PROPERTY, DEFAULT_CONNECT_TIMEOUT_SECONDS, "s"), Integer.MAX_VALUE / 1000); final long poolTimeoutSeconds = getNonNegativeProperty(POOL_TIMEOUT_PROPERTY, DEFAULT_POOL_TIMEOUT_SECONDS, "s"); final long startedAt = System.currentTimeMillis(); final long deadline = (poolTimeoutSeconds == 0 || poolTimeoutSeconds >= Long.MAX_VALUE / 1000) ? Long.MAX_VALUE : startedAt + poolTimeoutSeconds * 1000; long waitMs = 0; long backoffMs = 0; int attempts = 0; while (true) { final CachedConnection pooled = poll(connectionString, waitMs, deadline, trusted); if (pooled != null) { return pooled; } attempts++; try { return connect(connectionString, dialect, attemptSeconds(connectTimeoutSeconds, deadline)); } catch (SQLException e) { // A database that takes no connection for the moment is the failure worth waiting // out: it is at its connection limit, and one of ours is going to come back to the // pool - or it is on its way up, and the state clears itself in seconds. Everything // else - a password that is not accepted, a database that is down, a driver that is // not on the classpath - is reported to the caller instead of being retried behind // its back. if (!isWorthRetrying(e, dialect)) { throw reported(e, connectionString); } final long remaining = deadline - System.currentTimeMillis(); if (remaining <= 0) { // 08001, the state of a connect that did not happen, rather than none at all: // this is the failure of a borrow, and a caller reading the state of what it // caught would otherwise see null where the driver's own exception carried one final SQLTimeoutException timeout = new SQLTimeoutException("no connection to " + safeUrl(connectionString) + " could be borrowed within " + poolTimeoutSeconds + "s (" + attempts + " attempts): the database took no connection for the moment and none was" + " returned to the pool, last error: " + redact(e.getMessage(), connectionString), CONNECT_FAILED_SQL_STATE); timeout.initCause(reported(e, connectionString)); throw timeout; } backoffMs = Math.min(backoffMs == 0 ? 1 : backoffMs * 2, MAX_BACKOFF_MS); waitMs = Math.min(backoffMs, remaining); warnStall(connectionString, attempts, startedAt, e); } catch (RuntimeException e) { // a driver reporting a connect it will not make as an unchecked failure carries the // connection string of the backend in its message as readily as a SQLException does throw reportedUnchecked(e, connectionString); } } } /** * Reports a connection string this class knows no bound for. The properties bounding a connect * are the ones of a driver, so a driver outside the four leaves every attempt unbounded - and * the deadline of the borrow cannot reach into a connect that is already under way, since the * driver is the only thing holding the socket. */ private static void reportUnknownDialect(String connectionString, ConnectDialect dialect) { if (dialect != null) { return; } final StringBuilder known = new StringBuilder(); for (final ConnectDialect candidate : ConnectDialect.values()) { known.append(known.length() > 0 ? ", " : "").append(candidate.urlPrefix); } warnOnce(safeUrl(connectionString) + "|unknown-dialect", "%s names a driver whose timeout properties are not known to this backend (%s are): a connect to a" + " database that accepts it and does not answer is left without a bound, and the %s property" + " cannot end it", safeUrl(connectionString), known, POOL_TIMEOUT_PROPERTY); } /** * The bound of one connect attempt. The deadline of the borrow bounds it as well - the * {@value #POOL_TIMEOUT_PROPERTY} property stands for the whole borrow, and an attempt of its * own left to run out would overrun it by a full connect timeout. That holds for an attempt * the {@value #CONNECT_TIMEOUT_PROPERTY} property gives no bound of its own, too: turning the * per-attempt bound off must not turn the bound of the borrow off with it. Never 0 for an * attempt that is bounded at all: 0 is the value that stands for no bound. And never past what * an int of milliseconds takes - the pool timeout has no upper bound of its own, while the SQL * Server driver rejects a socketTimeout beyond Integer.MAX_VALUE outright, failing every * connect of that backend with the name of a property nobody typed. */ static long attemptSeconds(long connectTimeoutSeconds, long deadline) { if (deadline == Long.MAX_VALUE) { // 0 stands for an attempt with no bound of its own and stays 0; anything else is // clamped here as well, so that the range holds whichever branch answers return connectTimeoutSeconds == 0 ? 0 : Math.min(connectTimeoutSeconds, Integer.MAX_VALUE / 1000); } final long remainingSeconds = (deadline - System.currentTimeMillis() + 999) / 1000; final long bound = connectTimeoutSeconds == 0 ? remainingSeconds : Math.min(connectTimeoutSeconds, remainingSeconds); return Math.max(1, Math.min(bound, Integer.MAX_VALUE / 1000)); } /** * Takes a usable connection out of the pool, waiting up to waitMs for one to be returned to it. * The validation of a connection costs a round trip, and the pool has no upper bound on the * number of them it holds, so draining a pool the database no longer answers is given the * deadline of the borrow as well: past it, establishing a connection is the faster answer. * The connection in hand is always looked at first - trusted or validated, see * {@link #isKnownAlive} - whatever the deadline says: a database at its connection limit has * no other source of connections than the ones coming back, and one returned to the pool a * moment before the deadline is the very connection this borrow waited for. Only a connection * the database no longer answers is closed here. */ private static CachedConnection poll(String connectionString, long waitMs, long deadline, boolean trusted) throws InterruptedException { CachedConnection con = cached.get(connectionString).pollFirst(waitMs, TimeUnit.MILLISECONDS); while (con != null) { if (isUsable(con, trusted)) { return con; } closeQuietly(con.parent); if (System.currentTimeMillis() >= deadline) { return null; } con = cached.get(connectionString).pollFirst(); } return null; } private static boolean isUsable(CachedConnection con, boolean trusted) { if (trusted && isKnownAlive(con)) { return true; } // The validation needs a bound of its own: isValid(0) means "no timeout" in the JDBC // contract, and a connection whose socket is half-open answers it no sooner than it // answers anything else. isValid(n) is not that bound on every driver either - the SQL // Server driver turns it into a query timeout (setQueryTimeout, then "SELECT 1"), which // needs an answer from the server to fire at all - so the socket is bounded here, for the // validation only. final int restore = boundValidation(con.parent); if (restore == VALIDATION_BOUND_FAILED) { // the bound of the validation is not in force, and the driver may well have applied it // before failing: validating here would be the unbounded isValid() this exists to // avoid, and pooling it would hand out a connection carrying a bound of ours return false; } // Read before the round trip rather than after it: this stamp is what distrustPool() is // compared against, as an ordering of two moments. A validation is allowed // VALIDATION_TIMEOUT_SECONDS, so a stamp filed once the answer is in can be younger than a // drop another operation reported while it was still in flight - and the connection would // then be trusted for the rest of the window by the very check that exists to stop it. final long provenAt = System.nanoTime(); boolean usable; try { usable = con.isValid(VALIDATION_TIMEOUT_SECONDS); } catch (SQLException | RuntimeException e) { // a driver reporting the validation as an error: discard it // an unchecked failure out of a driver would unwind through poll(), which stands // outside every try of the borrow, and leave this connection dequeued and unclosed usable = false; } if (!usable) { // On its way out, and the driver knows it: Connector/J answers a failed validation by // aborting the connection and the SQL Server driver by terminating it, so putting the // previous bound back would fail as well - and warn about a bound of a connection that // is about to be closed, over a reaped idle connection that is nobody's problem. return false; } if (restore >= 0 && !setNetworkTimeout(con.parent, restore, "the connection is closed rather than pooled")) { return false; // it would carry the bound of the validation into every statement } con.lastKnownAliveNanos = provenAt; return true; } /** A connection left alone by {@link #boundValidation}: no bound of ours to put back afterwards. */ private static final int VALIDATION_BOUND_LEFT_ALONE = -1; /** A connection {@link #boundValidation} could not bound, which may still carry the bound it failed to report. */ private static final int VALIDATION_BOUND_FAILED = -2; /** * Whether a connection can be handed out on the strength of the last answer it gave, without a * round trip to ask for another. Three things have to hold: the window is on, the answer is * younger than it, and nothing has reported since that the database dropped a connection of * this pool. *

* What the window trades away is the connection that breaks inside it: it is handed out, and * the failure surfaces on the statement of the caller rather than on the borrow. That is where * a connection breaking mid-operation surfaces anyway - but not every caller of this backend * reports such a failure to the client, so the trade is not the caller's alone to bear. * {@code JDBCStorage} answers it on both sides: a write is replayed on a connection the next * attempt borrows of its own, and a read as much as a write marks the pool distrusted, which * closes the window for the rest of the generation the dropped connection belonged to. */ private static boolean isKnownAlive(CachedConnection con) { final long window = aliveBypassNanos; if (window <= 0) { return false; } final long provenAt = con.lastKnownAliveNanos; if (System.nanoTime() - provenAt >= window) { // the overflow safe form of the comparison return false; } final Long distrusted = poolDistrustedAt.get(con.connectionString); if (distrusted != null && provenAt - distrusted <= 0) { // the overflow safe form of the comparison return false; } // What the validation this replaces also answered: the removalListener above closes every // connection it finds in the deque when the pool expires, and it iterates a weakly // consistent view, so a connection taken out by a borrow running at the same time can be // closed under it. Answered by the driver out of a flag of its own, not by a round trip. return !isClosed(con.parent); } /** Whether the driver reports the connection as closed; one that cannot say is not one to trust. */ private static boolean isClosed(Connection con) { try { return con.isClosed(); } catch (SQLException e) { return true; } } /** * Reports that the database dropped a connection of this pool, so that no connection proven * alive before now is handed out unvalidated again. It is called by the operation that saw the * failure: this class only ever learns of one from the statement it broke, since a borrow * inside the window asks the database nothing. */ static void distrustPool(String connectionString) { // merge(later of the two) rather than computeIfAbsent().set(): two operations reporting a // drop at once would otherwise move the distrust point backwards - the later reading is // written first and the earlier one overwrites it - and the AtomicLong of computeIfAbsent // is published holding its initial 0 before set() runs, which a borrow racing it reads as // "never". Not Math.max: nanoTime() has no defined origin, so the readings are compared by // their difference, the way every other comparison of one in this class is. poolDistrustedAt.merge(connectionString, System.nanoTime(), (reported, now) -> now - reported > 0 ? now : reported); } /** * Bounds the socket of a pooled connection for the length of its validation, returning the * network timeout to put back afterwards - or {@link #VALIDATION_BOUND_LEFT_ALONE} for a * connection left alone, either because the driver does not take a network timeout or because * it is bounded at least as tightly already, by a read timeout of the connection string that * is not ours to widen. * A driver that takes the call and then fails inside it is told apart from both: it is free to * have applied the bound before failing, and a connection put back into the pool carrying five * seconds of ours fails every statement slower than that for the rest of its life. */ private static int boundValidation(Connection con) { final int bound = VALIDATION_TIMEOUT_SECONDS * 1000; final int previous; try { previous = con.getNetworkTimeout(); } catch (SQLException | RuntimeException e) { // a driver that does not take one: nothing was changed return VALIDATION_BOUND_LEFT_ALONE; } if (previous > 0 && previous <= bound) { return VALIDATION_BOUND_LEFT_ALONE; } try { con.setNetworkTimeout(DIRECT_EXECUTOR, bound); } catch (SQLException | RuntimeException e) { return VALIDATION_BOUND_FAILED; } // A driver answering a negative timeout is outside the contract of getNetworkTimeout(), // where 0 stands for no limit and nothing below it stands for anything. Handed back as it // is, it would be one of the two sentinels above: the bound just set would be read as a // bound that was never set, and the connection would go into the pool carrying five // seconds of ours into every statement of whoever borrows it next. return previous < 0 ? 0 : previous; } static CachedConnection connect(String connectionString, ConnectDialect dialect, long connectTimeoutSeconds) throws SQLException { // A driver is free to write into the map it is handed, so it gets one of its own. final Properties properties = new Properties(); final boolean readBoundSet = dialect != null && connectTimeoutSeconds > 0 && dialect.bound(connectionString, properties, connectTimeoutSeconds); // Read before the connect rather than after it, for the reason isUsable() reads it before // the validation: the login answered somewhere inside this attempt, and a stamp taken once // it returned could outlive a drop reported while it was still going on. final long provenAt = System.nanoTime(); final Connection conNew = DriverManager.getConnection(connectionString, properties); boolean poolable = true; try { // still under the read bound: both of these are round trips of their own conNew.setAutoCommit(false); conNew.setTransactionIsolation(TRANSACTION_READ_COMMITTED); if (readBoundSet) { // a driver that will not take the bound back has warned about it already: the // connection serves the borrower that is waiting for it and is closed rather than // pooled, so the bound of the login does not outlive it in the pool poolable = relaxReadBound(conNew, connectTimeoutSeconds); } } catch (SQLException | RuntimeException e) { // nothing holds this connection yet: it would leak closeQuietly(conNew); throw e; } final CachedConnection established = new CachedConnection(connectionString, conNew, poolable); established.lastKnownAliveNanos = provenAt; return established; } // The second bound of the login is a socket read timeout on mysql, oracle and sql server, in // force for the whole life of the connection: left in place it would break every statement // slower than it - an import batch, the statistics of a freshly loaded table - so it is lifted // as soon as the login is through, restoring the behaviour of a connection this class // established before. A read bound the connection string sets itself is never touched here: // it is not set at all, so nothing of the administrator's is lifted along with it. Returns // whether the bound is gone - a connection still carrying it must not be pooled. // Named by the bound the login was given rather than by the property it came from: with // CONNECT_TIMEOUT_PROPERTY at 0 the attempt takes its bound from what is left of the deadline // of the borrow, so naming that property would point at the one setting that is not in force. private static boolean relaxReadBound(Connection con, long boundSeconds) { return setNetworkTimeout(con, 0, "statements taking longer than the " + boundSeconds + "s the login of this connection was bounded by fail on it, and it is closed rather than pooled"); } /** * Puts a network timeout on a connection, reporting a driver that will not take one. The * consequence is the caller's to name: the same failure ends a freshly established connection * carrying the read bound of its login and a pooled one whose bound could not be put back. */ private static boolean setNetworkTimeout(Connection con, int millis, String consequence) { try { con.setNetworkTimeout(DIRECT_EXECUTOR, millis); return true; } catch (SQLException | RuntimeException e) { // Throttled rather than reported once for the life of the JVM: every connection this // happens to carries a read bound it was never meant to keep, and a statement dying of // it hours later needs a warning of its own to be traced back to here. final long now = System.currentTimeMillis(); final long last = lastReadBoundWarning.get(); if (now - last >= STALL_WARNING_INTERVAL_MS && lastReadBoundWarning.compareAndSet(last, now)) { logger.warn(LocalizableMessage.raw( "The read bound of a JDBC connection could not be set to %d ms (%s): %s", millis, e.getMessage(), consequence)); } return false; } } /** * Whether the database took no connection for the moment, rather than refusing one for good: * it is at its connection limit - one of our own connections is on its way back to the pool - * or it is not accepting connections yet, the state a database on its way up reports while it * recovers - the one JDBCStorage.open() has no second attempt of its own for, so a backend * that meets it stays locked down until the server is restarted. Both clear themselves in * seconds; every other failure is the caller's to see. */ static boolean isWorthRetrying(SQLException e, ConnectDialect dialect) { // a failure of the driver is often wrapped, and a SQLException carries two chains of its // own: the causes behind it and the further exceptions of getNextException() final Deque pending = new ArrayDeque<>(); final Set visited = Collections.newSetFromMap(new IdentityHashMap()); enqueue(pending, visited, e); for (int links = 0; !pending.isEmpty() && links < MAX_CHAIN_LENGTH; links++) { final Throwable t = pending.poll(); if (t instanceof SQLException) { final SQLException sql = (SQLException) t; final String sqlState = sql.getSQLState(); if (CONNECTION_LIMIT_SQL_STATE.equals(sqlState) || NOT_ACCEPTING_YET_SQL_STATE.equals(sqlState) || (dialect != null && dialect.isWorthRetrying(sql.getErrorCode()))) { return true; } enqueue(pending, visited, sql.getNextException()); } enqueue(pending, visited, t.getCause()); } return false; } // A stall has to reach the server log: without it a database accepting no further connection // is indistinguishable from a hang. Throttled, since every operation of the backend borrows // through here and would otherwise log a copy of its own. private static void warnStall(String connectionString, int attempts, long startedAt, SQLException cause) { final long now = System.currentTimeMillis(); if (now - startedAt < STALL_WARNING_AFTER_MS) { return; } final AtomicLong lastOfThisUrl = lastStallWarning.computeIfAbsent(safeUrl(connectionString), url -> new AtomicLong()); final long last = lastOfThisUrl.get(); if (now - last >= STALL_WARNING_INTERVAL_MS && lastOfThisUrl.compareAndSet(last, now)) { logger.warn(LocalizableMessage.raw("%s", stallMessage(connectionString, attempts, now - startedAt, cause))); } } /** * The stall as it reaches the log. Built apart from the logging of it so that the rule it has * to keep - neither the connection string nor the message of the driver reaches a log as it * stands - is a rule a test can hold it to. */ static String stallMessage(String connectionString, int attempts, long waitedMs, SQLException cause) { return String.format("%s takes no further connection: waiting %d ms for a pooled one so far (%d attempts)," + " last error: %s", safeUrl(connectionString), waitedMs, attempts, redact(cause.getMessage(), connectionString)); } /** * The failure of a connect as it may leave this class: the exception itself where nothing of it * names the credentials of the backend, and a redacted rebuild of its whole chain where * something does. Rebuilt rather than wrapped: a wrapper keeps its cause, and everything that * prints a failure prints the causes along with it - a debug build of * stackTraceToSingleLineString walks them, the config manager traces them, and * RootContainer.open() makes the message of the cause the message of what it throws - so a * link left as it stands would carry the password past the wrapper. The SQLState and the * vendor code of every link survive it: they are what tells a caller what happened. */ static SQLException reported(SQLException e, String connectionString) { return holdsCredentials(e, connectionString) ? redactedCopy(e, connectionString, new int[] { MAX_CHAIN_LENGTH }) : e; } /** The same of an unchecked failure: a driver is free to report a connect it will not make as one. */ static Exception reportedUnchecked(RuntimeException e, String connectionString) { if (!holdsCredentials(e, connectionString)) { return e; } final SQLException redacted = new SQLNonTransientConnectionException(e.getClass().getName() + (e.getMessage() == null ? "" : ": " + redact(e.getMessage(), connectionString)), CONNECT_FAILED_SQL_STATE); redacted.setStackTrace(e.getStackTrace()); return redacted; } /** * Whether anything in the chain of a failure names what a connection string keeps out of the * log. A chain longer than this walk is given answers "yes": what is reported unredacted is * what this class has looked at whole, and a link it never reached is not that. The cost of * being wrong that way is a chain rebuilt - bounded in its turn - while the cost of being * wrong the other way is the password of the backend in the server error log. */ private static boolean holdsCredentials(Throwable failure, String connectionString) { final Deque pending = new ArrayDeque<>(); final Set visited = Collections.newSetFromMap(new IdentityHashMap()); enqueue(pending, visited, failure); for (int links = 0; !pending.isEmpty(); links++) { if (links >= MAX_CHAIN_LENGTH) { return true; } final Throwable t = pending.poll(); final String message = t.getMessage(); if (message != null && !message.equals(redact(message, connectionString))) { return true; } if (t instanceof SQLException) { enqueue(pending, visited, ((SQLException) t).getNextException()); } enqueue(pending, visited, t.getCause()); } return false; } // By identity rather than by equals(): a link of a chain carries a cause and a next exception // both, and a driver is free to make the two the same failure. Enqueued twice, a chain of // those fans out into a copy of itself at every step and spends the budget of a walk on links // it has already looked at - five levels of one are enough to exhaust MAX_CHAIN_LENGTH. private static void enqueue(Deque pending, Set visited, Throwable t) { if (t != null && visited.add(t)) { pending.add(t); } } // The budget counts the links this rebuilds, the way holdsCredentials() counts the ones it // visits - not how deep it has gone. A link of a chain carries a cause and a next exception // both, and a driver is free to make them the same failure, so a bound on depth alone leaves // room for a chain that fans out into two copies of itself at every step. private static SQLException redactedCopy(SQLException e, String connectionString, int[] budget) { budget[0]--; final SQLException copy = new SQLException(redact(e.getMessage(), connectionString), e.getSQLState(), e.getErrorCode()); copy.setStackTrace(e.getStackTrace()); if (e.getNextException() != null) { copy.setNextException(budget[0] > 0 ? redactedCopy(e.getNextException(), connectionString, budget) : droppedTail()); } if (e.getCause() != null) { copy.initCause(budget[0] > 0 ? redactedLink(e.getCause(), connectionString, budget) : droppedTail()); } return copy; } // What stands where the budget ran out. Without it the same failure logs its root cause when // the url of the backend has no password in it and loses it without a word when it has, which // is a report of a connect nobody can read against a report of one they can. private static SQLException droppedTail() { return new SQLException("the rest of this failure was left out: a chain of more than " + MAX_CHAIN_LENGTH + " links is rebuilt only that far"); } // A link that is no SQLException keeps its class name in the message: its type is not one this // can rebuild, and the name of the failure is what a reader of the log is after. private static Throwable redactedLink(Throwable t, String connectionString, int[] budget) { if (t instanceof SQLException) { return redactedCopy((SQLException) t, connectionString, budget); } budget[0]--; final Throwable copy = new Throwable(t.getClass().getName() + (t.getMessage() == null ? "" : ": " + redact(t.getMessage(), connectionString))); copy.setStackTrace(t.getStackTrace()); if (t.getCause() != null) { copy.initCause(budget[0] > 0 ? redactedLink(t.getCause(), connectionString, budget) : droppedTail()); } return copy; } /** * A message of a driver as it may be logged. A driver is free to put the connection string it * was handed into it - the jdk itself does, "No suitable driver found for " + url, which is * what the ordinary oracle misconfiguration of a driver jar left out of lib/extensions arrives * as - and that connection string is where the credentials of this backend live. * What it cannot answer for is a driver quoting back a part of a url it failed to parse: * a whole credential is replaced, a fragment of one is not. */ static String redact(String message, String connectionString) { if (message == null || message.isEmpty()) { return message; } String redacted = message.replace(connectionString, safeUrl(connectionString)); // The parameter before the values: a password blanked here is one the loop below no longer // finds, while the other way round a "