From a0761226696209ba2f70b752a4f40a2cedce24bd Mon Sep 17 00:00:00 2001
From: Valery Kharseko <vharseko@3a-systems.ru>
Date: Wed, 02 Sep 2026 06:10:07 +0000
Subject: [PATCH] [#890] Persist a compressed schema token before handing it out, and report the ones with no definition (#894)
---
opendj-server-legacy/src/test/java/org/opends/server/api/CompressedSchemaTestCase.java | 809 +++++++++++++++++++++++++++++++++++
opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/PersistentCompressedSchema.java | 18
opendj-server-legacy/src/main/java/org/opends/server/api/CompressedSchema.java | 371 +++++++++++++++-
opendj-server-legacy/src/test/java/org/opends/server/core/DefaultCompressedSchemaTestCase.java | 96 ++++
opendj-server-legacy/src/main/java/org/opends/server/core/DefaultCompressedSchema.java | 39 +
opendj-server-legacy/src/messages/org/opends/messages/core.properties | 6
6 files changed, 1,311 insertions(+), 28 deletions(-)
diff --git a/opendj-server-legacy/src/main/java/org/opends/server/api/CompressedSchema.java b/opendj-server-legacy/src/main/java/org/opends/server/api/CompressedSchema.java
index 74088ab..c80a949 100644
--- a/opendj-server-legacy/src/main/java/org/opends/server/api/CompressedSchema.java
+++ b/opendj-server-legacy/src/main/java/org/opends/server/api/CompressedSchema.java
@@ -18,6 +18,7 @@
package org.opends.server.api;
import static org.opends.messages.CoreMessages.*;
+import static org.opends.server.util.StaticUtils.bytesToHexNoSpace;
import java.util.AbstractMap.SimpleImmutableEntry;
import java.util.Collection;
@@ -28,11 +29,15 @@
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.NoSuchElementException;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.locks.ReentrantLock;
+import org.forgerock.i18n.LocalizableMessage;
+import org.forgerock.i18n.LocalizableMessageDescriptor;
+import org.forgerock.i18n.slf4j.LocalizedLogger;
import org.forgerock.opendj.ldap.AttributeDescription;
import org.forgerock.opendj.ldap.ByteSequenceReader;
import org.forgerock.opendj.ldap.ByteString;
@@ -59,6 +64,8 @@
mayInvoke = false)
public class CompressedSchema
{
+ private static final LocalizedLogger logger = LocalizedLogger.getLoggerForThisClass();
+
/** Encloses all the encode and decode mappings for attribute and object classes. */
private static final class Mappings
{
@@ -84,6 +91,9 @@
}
}
+ /** The most bytes {@link #encodeId(int)} ever writes a schema element ID in. */
+ private static final int MAX_ID_BYTES = 4;
+
private final ServerContext serverContext;
/** Lock serializing all mutations (id registration and schema reload). */
private final ReentrantLock exclusiveLock = new ReentrantLock();
@@ -155,7 +165,19 @@
{
for(int id=0;id<mappings.adDecodeMap.size();id++){
final AttributeDescription ad = mappings.adDecodeMap.get(id);
- loadAttributeToMaps(id, ad.getAttributeType().getNameOrOID(), ad.getOptions(), newMappings);
+ if (ad != null)
+ {
+ loadAttributeToMaps(id, ad.getAttributeType().getNameOrOID(), ad.getOptions(), newMappings);
+ }
+ else
+ {
+ // A decode map can carry a gap: it is padded with null for the ids missing from the
+ // compressed schema it was loaded from. Carry the gap over rather than dereferencing it,
+ // and carry it over as a gap - dropping it would shift the ids of the elements after it,
+ // and would let the next registration hand out an id an already written entry carries.
+ // The ids are walked in order from zero, so the new map holds exactly id elements here.
+ newMappings.adDecodeMap.add(null);
+ }
}
}
@@ -166,7 +188,16 @@
private void reloadObjectClassesMap(Mappings mappings, Mappings newMappings)
{
for(int id=0;id<mappings.ocDecodeMap.size();id++){
- loadObjectClassesToMaps(id, mappings.ocDecodeMap.get(id).values(), newMappings, false);
+ final Map<ObjectClass, String> ocMap = mappings.ocDecodeMap.get(id);
+ if (ocMap != null)
+ {
+ loadObjectClassesToMaps(id, ocMap.values(), newMappings, false);
+ }
+ else
+ {
+ // A gap, as in reloadAttributeTypeMaps().
+ newMappings.ocDecodeMap.add(null);
+ }
}
}
@@ -184,15 +215,16 @@
throws DirectoryException
{
// First decode the encoded attribute description id.
- final int adId = decodeId(reader);
+ final byte[] adIdBytes = readIdBytes(reader, ERR_COMPRESSEDSCHEMA_UNREADABLE_AD_TOKEN);
+ final int adId = decodeId(adIdBytes);
// Before returning the attribute, make sure that the attribute type is not stale.
final Mappings mappings = reloadMappingsIfSchemaChanged();
- final AttributeDescription ad = mappings.adDecodeMap.get(adId);
+ final AttributeDescription ad = decodeMapGet(mappings.adDecodeMap, adId);
if (ad == null)
{
throw new DirectoryException(DirectoryServer.getCoreConfigManager().getServerErrorResultCode(),
- ERR_COMPRESSEDSCHEMA_UNRECOGNIZED_AD_TOKEN.get(adId));
+ ERR_COMPRESSEDSCHEMA_UNRECOGNIZED_AD_TOKEN.get(tokenInMessage(adIdBytes, adId)));
}
AttributeType attrType = ad.getAttributeType();
@@ -224,6 +256,44 @@
}
/**
+ * Returns the element a token addresses, or {@code null} where it addresses none: the token can
+ * be outside the range of the decode map, or address one of the slots the map is padded with for
+ * the ids missing from the compressed schema it was loaded from. Both are reported to the caller
+ * as the unknown token they are, rather than let out of the decode path as an unchecked
+ * exception the callers of that path are not written for.
+ *
+ * @param decodeMap
+ * The decode map to look the token up in.
+ * @param id
+ * The decoded token.
+ * @return The element registered under the token, or {@code null} if there is none.
+ */
+ private static <T> T decodeMapGet(final List<T> decodeMap, final int id)
+ {
+ if (id < 0)
+ {
+ return null;
+ }
+ try
+ {
+ return decodeMap.get(id);
+ }
+ catch (final IndexOutOfBoundsException e)
+ {
+ // Caught rather than kept away by a comparison against size(): size() and get() of a
+ // CopyOnWriteArrayList read the array separately, so the comparison would not make the
+ // lookup safe anyway, and this runs for every attribute of every entry read from a backend -
+ // the common path is left with the single read it had.
+ //
+ // Traced here because the caller turns this into a DirectoryException carrying the token:
+ // the generic catch of Entry.decode(), which used to convert this exception, logged the
+ // stack, and where the token came from is worth keeping for a corrupt store.
+ logger.traceException(e);
+ return null;
+ }
+ }
+
+ /**
* Decodes an object class set from the provided byte string.
*
* @param reader
@@ -237,15 +307,16 @@
final ByteSequenceReader reader) throws DirectoryException
{
// First decode the encoded object class id.
- final int ocId = decodeId(reader);
+ final byte[] ocIdBytes = readIdBytes(reader, ERR_COMPRESSEDSCHEMA_UNREADABLE_OC_TOKEN);
+ final int ocId = decodeId(ocIdBytes);
// Before returning the object classes, make sure that none of them are stale.
final Mappings mappings = reloadMappingsIfSchemaChanged();
- Map<ObjectClass, String> ocMap = mappings.ocDecodeMap.get(ocId);
+ Map<ObjectClass, String> ocMap = decodeMapGet(mappings.ocDecodeMap, ocId);
if (ocMap == null)
{
throw new DirectoryException(DirectoryServer.getCoreConfigManager().getServerErrorResultCode(),
- ERR_COMPRESSEDSCHEMA_UNKNOWN_OC_TOKEN.get(ocId));
+ ERR_COMPRESSEDSCHEMA_UNKNOWN_OC_TOKEN.get(tokenInMessage(ocIdBytes, ocId)));
}
return ocMap;
}
@@ -297,10 +368,7 @@
id = mappings.adEncodeMap.get(ad);
if (id == null)
{
- id = mappings.adDecodeMap.size();
- mappings.adDecodeMap.add(ad);
- mappings.adEncodeMap.put(ad, id);
- storeAttribute(encodeId(id), ad.getAttributeType().getNameOrOID(), ad.getOptions());
+ id = registerAttribute(mappings, ad);
}
return id;
}
@@ -311,6 +379,93 @@
}
/**
+ * Registers a new attribute description and returns the id allocated to it. The registration is
+ * persisted before it is published, and is withdrawn if it cannot be persisted: an entry must
+ * never be written with a token whose definition did not reach the storage, because nothing
+ * stores it afterwards and the entry cannot be decoded once the server is restarted.
+ * <p>
+ * Must be called with the exclusive lock held, which is what makes the id allocated here still
+ * the last element of the decode map when it has to be withdrawn. The lock is reentrant, so
+ * that holds only while the store stays out of this compressed schema: an implementation of
+ * {@link #storeAttribute(byte[], String, Iterable)} must re-enter neither the encode, the load
+ * nor the decode path of it, which its own javadoc says as well. That it did stay out is
+ * checked by {@link #withdraw(Mappings, List, int, Object)} rather than assumed, since removing
+ * an element this registration did not append is worse than the leak it withdraws.
+ */
+ private int registerAttribute(final Mappings mappings, final AttributeDescription ad) throws DirectoryException
+ {
+ final int id = mappings.adDecodeMap.size();
+ // Appended to the decode map first: storeAttribute() is free to persist the whole content of
+ // this compressed schema rather than the single element it is handed - DefaultCompressedSchema
+ // rewrites its file from getAllAttributes() - so the element being registered has to be part
+ // of it by then. The decode map is not what an encode reaches the id through, so nothing can
+ // yet write an entry carrying it.
+ mappings.adDecodeMap.add(ad);
+ boolean registered = false;
+ try
+ {
+ storeAttribute(encodeId(id), ad.getAttributeType().getNameOrOID(), ad.getOptions());
+ // Published only once persisted: the encode map is read without the lock, so an id another
+ // thread finds there can be carried by an entry a moment later and must never be withdrawn.
+ mappings.adEncodeMap.put(ad, id);
+ registered = true;
+ }
+ finally
+ {
+ if (!registered)
+ {
+ withdraw(mappings, mappings.adDecodeMap, id, ad);
+ }
+ }
+ return id;
+ }
+
+ /**
+ * Withdraws the element a failed registration appended, so that the next attempt allocates the
+ * id again and stores it. Removed by index, and by the index of the last element: every append
+ * is made under the exclusive lock, so this is still the element appended by the registration
+ * being withdrawn, and no other id shifts.
+ * <p>
+ * That the element is still there is checked rather than assumed, because the lock is
+ * reentrant: a store re-entering the encode or the load path appends to the same decode map,
+ * and one re-entering the decode path replaces the mappings altogether. Neither is allowed by
+ * the contract of {@link #storeAttribute(byte[], String, Iterable)}, and neither is withdrawn
+ * from here - removing an element this registration did not append shifts the ids of everything
+ * after it, and the entries already written carry them. The violation is reported instead, and
+ * the element is left where it is: the registration leaks, which is what this method exists to
+ * prevent, but no id already handed out starts decoding as something else.
+ * <p>
+ * Reported rather than thrown, because this runs while the failure of the store is on its way
+ * out: that failure is what the caller has to be told.
+ *
+ * @param mappings
+ * The mappings the registration appended to.
+ * @param decodeMap
+ * The decode map of {@code mappings} the element was appended to.
+ * @param appendedAt
+ * The index the element was appended at, which is the id allocated to it.
+ * @param appended
+ * The element that was appended.
+ */
+ private void withdraw(final Mappings mappings, final List<?> decodeMap, final int appendedAt,
+ final Object appended)
+ {
+ if (mappings == this.mappings
+ && decodeMap.size() == appendedAt + 1
+ && decodeMap.get(appendedAt) == appended)
+ {
+ decodeMap.remove(appendedAt);
+ return;
+ }
+ logger.error(LocalizableMessage.raw(
+ "The registration of the compressed schema id %s could not be withdrawn after its store failed, "
+ + "because the store re-entered the compressed schema it was called from: the id is now taken "
+ + "by a definition that was never persisted. This is a defect of %s, whose store must re-enter "
+ + "neither the encode, the load nor the decode path of the compressed schema.",
+ appendedAt, getClass().getName()));
+ }
+
+ /**
* Encodes the provided set of object classes to a byte array. If the same set
* had been previously encoded, then the cached value will be used. Otherwise,
* a new value will be created.
@@ -354,10 +509,7 @@
id = mappings.ocEncodeMap.get(objectClasses);
if (id == null)
{
- id = mappings.ocDecodeMap.size();
- mappings.ocDecodeMap.add(objectClasses);
- mappings.ocEncodeMap.put(objectClasses, id);
- storeObjectClasses(encodeId(id), objectClasses.values());
+ id = registerObjectClasses(mappings, objectClasses);
}
return id;
}
@@ -368,6 +520,36 @@
}
/**
+ * Registers a new object class set and returns the id allocated to it, persisting the
+ * registration before publishing it and withdrawing it if it cannot be persisted, exactly as
+ * {@link #registerAttribute(Mappings, AttributeDescription)} does.
+ * <p>
+ * Must be called with the exclusive lock held, and under the same constraint on what
+ * {@link #storeObjectClasses(byte[], Collection)} may re-enter.
+ */
+ private int registerObjectClasses(final Mappings mappings, final Map<ObjectClass, String> objectClasses)
+ throws DirectoryException
+ {
+ final int id = mappings.ocDecodeMap.size();
+ mappings.ocDecodeMap.add(objectClasses);
+ boolean registered = false;
+ try
+ {
+ storeObjectClasses(encodeId(id), objectClasses.values());
+ mappings.ocEncodeMap.put(objectClasses, id);
+ registered = true;
+ }
+ finally
+ {
+ if (!registered)
+ {
+ withdraw(mappings, mappings.ocDecodeMap, id, objectClasses);
+ }
+ }
+ return id;
+ }
+
+ /**
* Returns a view of the encoded attributes in this compressed schema which can be used for saving
* the entire content to disk.
* <p>
@@ -385,19 +567,43 @@
return new Iterator<Entry<byte[], Entry<String, Iterable<String>>>>()
{
private int id;
- private List<AttributeDescription> adDecodeMap = getMappings().adDecodeMap;
+ private final List<AttributeDescription> adDecodeMap = getMappings().adDecodeMap;
@Override
public boolean hasNext()
{
- return id < adDecodeMap.size();
+ // Skips the gaps: a decode map padded with null for the ids missing from the
+ // compressed schema it was loaded from is still saved, and the ids around a gap are
+ // preserved by the token each element is written with. Looked up through
+ // decodeMapGet(), because withdrawing a registration shortens the decode map and a
+ // CopyOnWriteArrayList reads its array separately for size() and for get(). In tree
+ // this iteration runs under the exclusive lock - the only caller of save() is a store
+ // - but the class is extensible and a subclass can reach here from anywhere.
+ while (id < adDecodeMap.size())
+ {
+ if (decodeMapGet(adDecodeMap, id) != null)
+ {
+ return true;
+ }
+ id++;
+ }
+ return false;
}
@Override
public Entry<byte[], Entry<String, Iterable<String>>> next()
{
+ if (!hasNext())
+ {
+ throw new NoSuchElementException();
+ }
final byte[] encodedAttribute = encodeId(id);
- final AttributeDescription ad = adDecodeMap.get(id++);
+ final AttributeDescription ad = decodeMapGet(adDecodeMap, id++);
+ if (ad == null)
+ {
+ // The decode map was shortened between hasNext() and here.
+ throw new NoSuchElementException();
+ }
return new SimpleImmutableEntry<byte[], Entry<String, Iterable<String>>>(
encodedAttribute,
new SimpleImmutableEntry<String, Iterable<String>>(
@@ -437,14 +643,32 @@
@Override
public boolean hasNext()
{
- return id < ocDecodeMap.size();
+ // Skips the gaps, and looks the elements up the same way, as in getAllAttributes().
+ while (id < ocDecodeMap.size())
+ {
+ if (decodeMapGet(ocDecodeMap, id) != null)
+ {
+ return true;
+ }
+ id++;
+ }
+ return false;
}
@Override
public Entry<byte[], Collection<String>> next()
{
+ if (!hasNext())
+ {
+ throw new NoSuchElementException();
+ }
final byte[] encodedObjectClasses = encodeId(id);
- final Map<ObjectClass, String> ocMap = ocDecodeMap.get(id++);
+ final Map<ObjectClass, String> ocMap = decodeMapGet(ocDecodeMap, id++);
+ if (ocMap == null)
+ {
+ // The decode map was shortened between hasNext() and here.
+ throw new NoSuchElementException();
+ }
return new SimpleImmutableEntry<>(encodedObjectClasses, ocMap.values());
}
@@ -631,6 +855,17 @@
* assume that this method is not being called by other threads. Note that
* this method is not thread-safe with respect to
* {@link #storeObjectClasses(byte[], Collection)}.
+ * <p>
+ * Called with the exclusive lock of this compressed schema held, and that lock is reentrant, so
+ * an implementation must re-enter neither the encode, the load nor the decode path of the
+ * compressed schema it belongs to. The registration being persisted has already been appended
+ * to the decode map - so that an implementation persisting the whole content rather than the
+ * element it is handed, as {@code DefaultCompressedSchema} does, has it - and is withdrawn from
+ * there if this method throws. Encoding or loading appends to the same decode map, and the
+ * withdrawal would take back whatever was appended last; decoding rebuilds the mappings when the
+ * schema has changed, and the withdrawal would then have to take the element out of a map that
+ * has already been replaced. Neither is withdrawn: the violation is reported and the
+ * registration is left behind, holding an id no definition was persisted under.
*
* @param encodedAttribute
* The encoded attribute description.
@@ -654,6 +889,17 @@
* can assume that this method is not being called by other threads. Note that
* this method is not thread-safe with respect to
* {@link #storeAttribute(byte[], String, Iterable)}.
+ * <p>
+ * Called with the exclusive lock of this compressed schema held, and that lock is reentrant, so
+ * an implementation must re-enter neither the encode, the load nor the decode path of the
+ * compressed schema it belongs to. The registration being persisted has already been appended
+ * to the decode map - so that an implementation persisting the whole content rather than the
+ * element it is handed, as {@code DefaultCompressedSchema} does, has it - and is withdrawn from
+ * there if this method throws. Encoding or loading appends to the same decode map, and the
+ * withdrawal would take back whatever was appended last; decoding rebuilds the mappings when the
+ * schema has changed, and the withdrawal would then have to take the element out of a map that
+ * has already been replaced. Neither is withdrawn: the violation is reported and the
+ * registration is left behind, holding an id no definition was persisted under.
*
* @param encodedObjectClasses
* The encoded object classes.
@@ -686,12 +932,89 @@
return id - 1; // Subtract 1 to compensate for old behavior.
}
- private int decodeId(final ByteSequenceReader reader)
+ /**
+ * Reads the encoded schema element ID at the current position, reporting a record the ID cannot
+ * be read from rather than letting the read out of the decode path as an unchecked exception -
+ * for the same reason the lookup the ID feeds does not: the callers of a {@code @PublicAPI}
+ * decode path are written for {@link DirectoryException}.
+ *
+ * @param reader
+ * The byte string reader positioned on an encoded schema element ID.
+ * @param unreadableToken
+ * The message reporting a token this decode path cannot read.
+ * @return The encoded schema element ID, as the storage holds it.
+ * @throws DirectoryException
+ * If the record holds no readable schema element ID at the current position.
+ */
+ private static byte[] readIdBytes(final ByteSequenceReader reader,
+ final LocalizableMessageDescriptor.Arg1<Object> unreadableToken) throws DirectoryException
{
- final int length = reader.readBERLength();
+ final int length;
+ try
+ {
+ length = reader.readBERLength();
+ }
+ catch (final IndexOutOfBoundsException e)
+ {
+ // Both of the conditions readBERLength() reports this way: the record ends inside the
+ // length itself, and a length header naming more than the four bytes a length is written in.
+ throw unreadable(unreadableToken,
+ "the record ends inside the length of the token, or that length names more than four bytes", e);
+ }
+ if (length < 0 || length > MAX_ID_BYTES)
+ {
+ // The length is composed from up to four bytes unsigned, so a corrupt record can name a
+ // negative count - 0xFFFFFFFF - or more bytes than an id is ever written in. The upper
+ // bound is what encodeId() emits, and it is not only a sanity check: decodeId() folds
+ // whatever it is handed, so a token padded with leading zeros decodes to the id its
+ // canonical token addresses, and a record carrying one would read as a live definition
+ // instead of being reported. Checked before the array is allocated, too - a length of
+ // 0x7FFFFFFF is a two gigabyte allocation no reader of a corrupt record should attempt.
+ throw unreadable(unreadableToken, "the token names " + (length & 0xFFFFFFFFL)
+ + " bytes, and an id is never encoded in more than " + MAX_ID_BYTES, null);
+ }
+ if (length > reader.remaining())
+ {
+ throw unreadable(unreadableToken,
+ "the token names " + length + " bytes and the record holds " + reader.remaining(), null);
+ }
final byte[] idBytes = new byte[length];
reader.readBytes(idBytes);
- return decodeId(idBytes);
+ return idBytes;
+ }
+
+ /**
+ * Returns the exception reporting a token this decode path cannot read, tracing what the read
+ * raised where it raised anything: the callers of this path convert an exception of their own
+ * into a message, and where the record went wrong is worth keeping for a corrupt store.
+ */
+ private static DirectoryException unreadable(
+ final LocalizableMessageDescriptor.Arg1<Object> unreadableToken, final String reason,
+ final RuntimeException cause)
+ {
+ if (cause != null)
+ {
+ logger.traceException(cause);
+ }
+ return new DirectoryException(DirectoryServer.getCoreConfigManager().getServerErrorResultCode(),
+ unreadableToken.get(reason), cause);
+ }
+
+ /**
+ * Names a token in a message as the storage holds it - the key a definition is written under -
+ * together with the id it decodes to. The id on its own is one less than what was read, so a
+ * token that no definition was ever written under is reported as a value appearing nowhere in
+ * the stored data: an all-zero token reads as the id -1.
+ *
+ * @param idBytes
+ * The encoded schema element ID, as it was read.
+ * @param id
+ * The schema element ID it decoded to.
+ * @return The token as a message should name it.
+ */
+ private static String tokenInMessage(final byte[] idBytes, final int id)
+ {
+ return "0x" + bytesToHexNoSpace(idBytes) + " (id " + id + ")";
}
/**
diff --git a/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/PersistentCompressedSchema.java b/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/PersistentCompressedSchema.java
index af484ab..76fb256 100644
--- a/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/PersistentCompressedSchema.java
+++ b/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/PersistentCompressedSchema.java
@@ -13,6 +13,7 @@
*
* Copyright 2008-2009 Sun Microsystems, Inc.
* Portions Copyright 2013-2016 ForgeRock AS.
+ * Portions Copyright 2026 3A Systems, LLC.
*/
package org.opends.server.backends.pluggable;
@@ -112,7 +113,17 @@
}
catch (final IOException e)
{
- // TODO: Shouldn't happen but should log a message
+ // Reported rather than absorbed. Defensive as things stand: the writer encodes into a
+ // ByteStringBuilder, and none of the write methods of the OutputStream it hands out declares
+ // IOException, so nothing under this try can raise one - the catch compiles because the
+ // ASN1Writer interface declares it. What makes a withdrawal reachable in a running server is
+ // store()'s own catch, on a storage.write that failed. Were this one ever to fire, nothing
+ // would have reached the tree either, and a store that did not happen must not return
+ // normally: the caller would publish the token, and an entry written with a token whose
+ // definition is nowhere cannot be decoded once the server is restarted.
+ logger.traceException(e);
+ throw new DirectoryException(DirectoryServer.getCoreConfigManager().getServerErrorResultCode(),
+ ERR_COMPSCHEMA_CANNOT_STORE_EX.get(e.getMessage()), e);
}
}
@@ -133,7 +144,10 @@
}
catch (final IOException e)
{
- // TODO: Shouldn't happen but should log a message
+ // Reported rather than absorbed, as in storeAttribute().
+ logger.traceException(e);
+ throw new DirectoryException(DirectoryServer.getCoreConfigManager().getServerErrorResultCode(),
+ ERR_COMPSCHEMA_CANNOT_STORE_EX.get(e.getMessage()), e);
}
}
diff --git a/opendj-server-legacy/src/main/java/org/opends/server/core/DefaultCompressedSchema.java b/opendj-server-legacy/src/main/java/org/opends/server/core/DefaultCompressedSchema.java
index 04f975f..a2818af 100644
--- a/opendj-server-legacy/src/main/java/org/opends/server/core/DefaultCompressedSchema.java
+++ b/opendj-server-legacy/src/main/java/org/opends/server/core/DefaultCompressedSchema.java
@@ -36,6 +36,7 @@
import org.forgerock.opendj.io.ASN1Reader;
import org.forgerock.opendj.io.ASN1Writer;
import org.forgerock.opendj.ldap.ByteString;
+import org.forgerock.util.annotations.VisibleForTesting;
import org.opends.server.api.CompressedSchema;
import org.opends.server.types.DirectoryException;
@@ -154,6 +155,40 @@
}
/**
+ * Returns the counter to record once the provided token has been written: the token after the
+ * highest one written so far, rather than the number of records written. A decode map carrying a
+ * gap - the compressed schema was loaded from a storage holding no definition for some of its
+ * ids - emits fewer records than the ids it spans, and a counter taken from the count would name
+ * a token that is live. Both ends of this file read the counters as "No longer used", but a
+ * release old enough to seed from them would re-issue those tokens.
+ * <p>
+ * A gap at the end is the one case this cannot cover: where the definition that was lost is the
+ * one under the highest id, that id has no slot in the decode map at all - a load pads the map
+ * up to the ids it holds definitions for and no further - so it is never iterated here and
+ * contributes no token. The counter then names the token after the highest one that survived,
+ * and a release seeding from it re-issues an id that entries already carry. Nothing in the file
+ * says otherwise, so nothing here can fix it; a compressed schema written by this release does
+ * not carry that gap, because a registration whose store fails is withdrawn.
+ *
+ * @param counter
+ * The counter as it stands.
+ * @param encodedToken
+ * The token just written.
+ * @return The counter to record.
+ */
+ @VisibleForTesting
+ static int counterAfter(final int counter, final byte[] encodedToken)
+ {
+ int token = 0;
+ for (final byte b : encodedToken)
+ {
+ token <<= 8;
+ token |= b & 0xFF;
+ }
+ return Math.max(counter, token + 1);
+ }
+
+ /**
* Writes the compressed schema information to disk.
*
* @throws DirectoryException
@@ -189,7 +224,7 @@
writer.writeOctetString(ocName);
}
writer.writeEndSequence();
- ocCounter++;
+ ocCounter = counterAfter(ocCounter, mapEntry.getKey());
}
writer.writeEndSequence();
@@ -214,7 +249,7 @@
writer.writeOctetString(option);
}
writer.writeEndSequence();
- adCounter++;
+ adCounter = counterAfter(adCounter, mapEntry.getKey());
}
writer.writeEndSequence();
diff --git a/opendj-server-legacy/src/messages/org/opends/messages/core.properties b/opendj-server-legacy/src/messages/org/opends/messages/core.properties
index 7be7a7d..27f4069 100644
--- a/opendj-server-legacy/src/messages/org/opends/messages/core.properties
+++ b/opendj-server-legacy/src/messages/org/opends/messages/core.properties
@@ -1080,6 +1080,12 @@
WARN_COMPRESSEDSCHEMA_CANNOT_SAVE_PREVIOUS_DATA_756=Unable to keep a copy of \
the previous compressed schema token data by renaming %s to %s: %s. The \
updated token data has still been written
+ERR_COMPRESSEDSCHEMA_UNREADABLE_AD_TOKEN_757=Unable to decode the \
+ provided attribute because its attribute description token could not be read \
+ from the record: %s
+ERR_COMPRESSEDSCHEMA_UNREADABLE_OC_TOKEN_758=Unable to decode the \
+ provided object class set because its token could not be read from the \
+ record: %s
ERR_ENTRYENCODECFG_INVALID_LENGTH_623=Unable to decode the provided \
entry encode configuration element because it has an invalid length
INFO_RESULT_NO_OPERATION_624=No Operation
diff --git a/opendj-server-legacy/src/test/java/org/opends/server/api/CompressedSchemaTestCase.java b/opendj-server-legacy/src/test/java/org/opends/server/api/CompressedSchemaTestCase.java
new file mode 100644
index 0000000..4faf69d
--- /dev/null
+++ b/opendj-server-legacy/src/test/java/org/opends/server/api/CompressedSchemaTestCase.java
@@ -0,0 +1,809 @@
+/*
+ * The contents of this file are subject to the terms of the Common Development and
+ * Distribution License (the License). You may not use this file except in compliance with the
+ * License.
+ *
+ * You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the
+ * specific language governing permission and limitations under the License.
+ *
+ * When distributing Covered Software, include this CDDL Header Notice in each file and include
+ * the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL
+ * Header, with the fields enclosed by brackets [] replaced by your own identifying
+ * information: "Portions Copyright [year] [name of copyright owner]".
+ *
+ * Copyright 2026 3A Systems, LLC.
+ */
+package org.opends.server.api;
+
+import static org.opends.messages.CoreMessages.*;
+import static org.testng.Assert.*;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.forgerock.i18n.LocalizableMessage;
+import org.forgerock.opendj.ldap.ByteString;
+import org.forgerock.opendj.ldap.ByteStringBuilder;
+import org.forgerock.opendj.ldap.ByteSequenceReader;
+import org.forgerock.opendj.ldap.ResultCode;
+import org.forgerock.opendj.ldap.schema.ObjectClass;
+import org.opends.server.TestCaseUtils;
+import org.opends.server.core.DirectoryServer;
+import org.opends.server.types.Attribute;
+import org.opends.server.types.Attributes;
+import org.opends.server.types.DirectoryException;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+/**
+ * Tests that a compressed schema never hands out a token whose definition was not persisted, and
+ * that a token it holds no definition for is reported rather than let out of the decode path as an
+ * unchecked exception.
+ */
+@SuppressWarnings("javadoc")
+public class CompressedSchemaTestCase extends APITestCase
+{
+ /** A compressed schema whose store can be made to fail, recording what it did persist. */
+ private static final class TestCompressedSchema extends CompressedSchema
+ {
+ private final Map<Integer, String> storedAttributes = new LinkedHashMap<>();
+ private final Map<Integer, Collection<String>> storedObjectClasses = new LinkedHashMap<>();
+ private int attributeStoreCount;
+ private int objectClassStoreCount;
+ private boolean failStore;
+ /** Counted down when a store is entered, when the store is gated. */
+ private CountDownLatch enteredStore;
+ /** Awaited by a gated store, which holds the exclusive lock while it waits. */
+ private CountDownLatch leaveStore;
+
+ private TestCompressedSchema()
+ {
+ super(DirectoryServer.getInstance().getServerContext());
+ }
+
+ @Override
+ protected void storeAttribute(final byte[] encodedAttribute, final String attributeName,
+ final Iterable<String> attributeOptions) throws DirectoryException
+ {
+ attributeStoreCount++;
+ awaitIfGated();
+ failIfRequested();
+ storedAttributes.put(token(encodedAttribute), attributeName);
+ }
+
+ @Override
+ protected void storeObjectClasses(final byte[] encodedObjectClasses, final Collection<String> objectClassNames)
+ throws DirectoryException
+ {
+ objectClassStoreCount++;
+ awaitIfGated();
+ failIfRequested();
+ storedObjectClasses.put(token(encodedObjectClasses), new ArrayList<>(objectClassNames));
+ }
+
+ private void failIfRequested() throws DirectoryException
+ {
+ if (failStore)
+ {
+ throw new DirectoryException(ResultCode.OTHER, LocalizableMessage.raw("the store failed"));
+ }
+ }
+
+ private void awaitIfGated() throws DirectoryException
+ {
+ if (enteredStore == null)
+ {
+ return;
+ }
+ enteredStore.countDown();
+ try
+ {
+ if (!leaveStore.await(30, TimeUnit.SECONDS))
+ {
+ throw new DirectoryException(ResultCode.OTHER, LocalizableMessage.raw("the gated store timed out"));
+ }
+ }
+ catch (final InterruptedException e)
+ {
+ Thread.currentThread().interrupt();
+ throw new DirectoryException(ResultCode.OTHER, LocalizableMessage.raw("the gated store was interrupted"), e);
+ }
+ }
+
+ /** Loads a definition under the provided token, as an implementation does at startup. */
+ private void loadAttributeAt(final int id, final String attributeName)
+ {
+ loadAttribute(encodedToken(id), attributeName, Collections.<String> emptySet());
+ }
+
+ private void loadObjectClassesAt(final int id, final Collection<String> objectClassNames)
+ {
+ loadObjectClasses(encodedToken(id), objectClassNames);
+ }
+
+ /** The tokens the whole content would be saved under, as DefaultCompressedSchema saves it. */
+ private List<Integer> savedAttributeTokens()
+ {
+ final List<Integer> tokens = new ArrayList<>();
+ for (final Entry<byte[], Entry<String, Iterable<String>>> attribute : getAllAttributes())
+ {
+ tokens.add(token(attribute.getKey()));
+ }
+ return tokens;
+ }
+
+ private List<Integer> savedObjectClassTokens()
+ {
+ final List<Integer> tokens = new ArrayList<>();
+ for (final Entry<byte[], Collection<String>> objectClasses : getAllObjectClasses())
+ {
+ tokens.add(token(objectClasses.getKey()));
+ }
+ return tokens;
+ }
+ }
+
+ @BeforeClass
+ public void setUp() throws Exception
+ {
+ TestCaseUtils.startServer();
+ }
+
+ /**
+ * A registration whose store failed must be withdrawn: the next encode of the same attribute has
+ * to allocate and store the token again, rather than take the lock-free fast path and write an
+ * entry carrying a token whose definition is nowhere.
+ */
+ @Test
+ public void attributeTokenIsWithdrawnWhenItCannotBeStored() throws Exception
+ {
+ final TestCompressedSchema compressedSchema = new TestCompressedSchema();
+ final Attribute attribute = Attributes.create("description", "a value");
+
+ compressedSchema.failStore = true;
+ try
+ {
+ compressedSchema.encodeAttribute(new ByteStringBuilder(), attribute);
+ fail("the encode should have failed with the store");
+ }
+ catch (final DirectoryException expected)
+ {
+ // The operation fails, which is what the caller is told.
+ }
+ assertEquals(compressedSchema.attributeStoreCount, 1);
+ assertTrue(compressedSchema.storedAttributes.isEmpty(), "nothing was persisted");
+
+ compressedSchema.failStore = false;
+ final ByteStringBuilder builder = new ByteStringBuilder();
+ compressedSchema.encodeAttribute(builder, attribute);
+ assertEquals(compressedSchema.attributeStoreCount, 2, "the failed registration was left behind");
+
+ final int encodedToken = tokenOf(builder.toByteString());
+ assertEquals(encodedToken, 0, "the withdrawn id was not allocated again");
+ assertTrue(compressedSchema.storedAttributes.containsKey(encodedToken),
+ "the entry carries token " + encodedToken + ", which was never stored");
+ // What the withdrawal exists for: an element left behind by the failed registration would be
+ // saved here under a token whose store never returned. Asserting the store count and the token
+ // of the retry is not enough on its own - a registration that leaks its decode map element
+ // simply allocates the next id, and every other assertion of this test still holds.
+ assertEquals(compressedSchema.savedAttributeTokens(), Collections.singletonList(0),
+ "the whole content still holds the element of the failed registration");
+ final Attribute decoded = compressedSchema.decodeAttribute(builder.toByteString().asReader());
+ assertEquals(decoded.getAttributeDescription(), attribute.getAttributeDescription());
+ assertEquals(decoded.iterator().next().toString(), "a value");
+ }
+
+ /** The same for an object class set. */
+ @Test
+ public void objectClassTokenIsWithdrawnWhenItCannotBeStored() throws Exception
+ {
+ final TestCompressedSchema compressedSchema = new TestCompressedSchema();
+ final Map<ObjectClass, String> objectClasses = objectClasses("top", "person");
+
+ compressedSchema.failStore = true;
+ try
+ {
+ compressedSchema.encodeObjectClasses(new ByteStringBuilder(), objectClasses);
+ fail("the encode should have failed with the store");
+ }
+ catch (final DirectoryException expected)
+ {
+ // The operation fails, which is what the caller is told.
+ }
+ assertEquals(compressedSchema.objectClassStoreCount, 1);
+ assertTrue(compressedSchema.storedObjectClasses.isEmpty(), "nothing was persisted");
+
+ compressedSchema.failStore = false;
+ final ByteStringBuilder builder = new ByteStringBuilder();
+ compressedSchema.encodeObjectClasses(builder, objectClasses);
+ assertEquals(compressedSchema.objectClassStoreCount, 2, "the failed registration was left behind");
+
+ final int encodedToken = tokenOf(builder.toByteString());
+ assertEquals(encodedToken, 0, "the withdrawn id was not allocated again");
+ assertTrue(compressedSchema.storedObjectClasses.containsKey(encodedToken),
+ "the entry carries token " + encodedToken + ", which was never stored");
+ // As in attributeTokenIsWithdrawnWhenItCannotBeStored().
+ assertEquals(compressedSchema.savedObjectClassTokens(), Collections.singletonList(0),
+ "the whole content still holds the element of the failed registration");
+ assertEquals(compressedSchema.decodeObjectClasses(builder.toByteString().asReader()), objectClasses);
+ }
+
+ /**
+ * The withdrawal has to take the element the failed registration appended, and only that one.
+ * On a decode map holding a single element every removal looks alike - index 0 is also the last
+ * index, and the sole element is also the one that was appended - so a withdrawal taking the
+ * wrong element is only visible once something was registered before the one that fails.
+ * <p>
+ * What it costs is this defect from the other end: removing an element the registration did not
+ * append shifts the ids of everything after it, and the entries already written carry them.
+ */
+ @Test
+ public void theAttributeWithdrawalTakesTheElementItAppended() throws Exception
+ {
+ final TestCompressedSchema compressedSchema = new TestCompressedSchema();
+ final Attribute first = Attributes.create("description", "a value");
+ final Attribute second = Attributes.create("cn", "a value");
+ compressedSchema.encodeAttribute(new ByteStringBuilder(), first);
+
+ compressedSchema.failStore = true;
+ try
+ {
+ compressedSchema.encodeAttribute(new ByteStringBuilder(), second);
+ fail("the encode should have failed with the store");
+ }
+ catch (final DirectoryException expected)
+ {
+ // The operation fails, which is what the caller is told.
+ }
+
+ compressedSchema.failStore = false;
+ final ByteStringBuilder builder = new ByteStringBuilder();
+ compressedSchema.encodeAttribute(builder, second);
+ assertEquals(compressedSchema.attributeStoreCount, 3, "the failed registration was left behind");
+ assertEquals(tokenOf(builder.toByteString()), 1, "the withdrawn id was not allocated again");
+ assertEquals(compressedSchema.savedAttributeTokens(), Arrays.asList(0, 1),
+ "the whole content does not span the ids that were registered");
+ // The tokens alone do not separate a withdrawal of the last element from one of the first:
+ // both leave two elements behind, under the tokens 0 and 1. What they decode to does.
+ assertEquals(attributeNameAt(compressedSchema, 0), "description",
+ "the id registered before the failure decodes as another attribute");
+ assertEquals(attributeNameAt(compressedSchema, 1), "cn");
+ }
+
+ /** The same for an object class set. */
+ @Test
+ public void theObjectClassWithdrawalTakesTheElementItAppended() throws Exception
+ {
+ final TestCompressedSchema compressedSchema = new TestCompressedSchema();
+ final Map<ObjectClass, String> first = objectClasses("top", "person");
+ final Map<ObjectClass, String> second = objectClasses("top", "organizationalUnit");
+ compressedSchema.encodeObjectClasses(new ByteStringBuilder(), first);
+
+ compressedSchema.failStore = true;
+ try
+ {
+ compressedSchema.encodeObjectClasses(new ByteStringBuilder(), second);
+ fail("the encode should have failed with the store");
+ }
+ catch (final DirectoryException expected)
+ {
+ // The operation fails, which is what the caller is told.
+ }
+
+ compressedSchema.failStore = false;
+ final ByteStringBuilder builder = new ByteStringBuilder();
+ compressedSchema.encodeObjectClasses(builder, second);
+ assertEquals(compressedSchema.objectClassStoreCount, 3, "the failed registration was left behind");
+ assertEquals(tokenOf(builder.toByteString()), 1, "the withdrawn id was not allocated again");
+ assertEquals(compressedSchema.savedObjectClassTokens(), Arrays.asList(0, 1),
+ "the whole content does not span the ids that were registered");
+ // As in theAttributeWithdrawalTakesTheElementItAppended().
+ assertEquals(objectClassesAt(compressedSchema, 0), first,
+ "the id registered before the failure decodes as another object class set");
+ assertEquals(objectClassesAt(compressedSchema, 1), second);
+ }
+
+ /**
+ * The id of a registration reaches the encode map - the lock-free path an encode takes to it -
+ * only once the definition is persisted, so that no other thread can write an entry carrying an
+ * id that a failing store is about to withdraw.
+ */
+ @Test
+ public void aTokenIsPublishedOnlyOnceItIsStored() throws Exception
+ {
+ final TestCompressedSchema compressedSchema = new TestCompressedSchema();
+ final Attribute attribute = Attributes.create("description", "a value");
+ final ExecutorService executor = Executors.newFixedThreadPool(2);
+ try
+ {
+ // One encode is held inside the store of the registration it made, holding the exclusive lock.
+ compressedSchema.enteredStore = new CountDownLatch(1);
+ compressedSchema.leaveStore = new CountDownLatch(1);
+ final Future<Integer> registering = executor.submit(encoding(compressedSchema, attribute, null));
+ assertTrue(compressedSchema.enteredStore.await(30, TimeUnit.SECONDS), "the store was never reached");
+
+ // Another encode of the same attribute must not be handed the id being stored: it has to
+ // park on the exclusive lock until the store returns.
+ final AtomicReference<Thread> concurrentThread = new AtomicReference<>();
+ final Future<Integer> concurrent = executor.submit(encoding(compressedSchema, attribute, concurrentThread));
+ awaitParkedOnTheLock(concurrent, concurrentThread);
+ assertFalse(concurrent.isDone(), "the token was handed out before it was stored");
+
+ compressedSchema.leaveStore.countDown();
+ assertEquals(registering.get(30, TimeUnit.SECONDS), Integer.valueOf(0));
+ assertEquals(concurrent.get(30, TimeUnit.SECONDS), Integer.valueOf(0));
+ assertEquals(compressedSchema.attributeStoreCount, 1, "the same token was stored twice");
+ }
+ finally
+ {
+ executor.shutdownNow();
+ }
+ }
+
+ /** The same for an object class set, whose registration orders the two maps the same way. */
+ @Test
+ public void anObjectClassTokenIsPublishedOnlyOnceItIsStored() throws Exception
+ {
+ final TestCompressedSchema compressedSchema = new TestCompressedSchema();
+ final Map<ObjectClass, String> objectClasses = objectClasses("top", "person");
+ final ExecutorService executor = Executors.newFixedThreadPool(2);
+ try
+ {
+ compressedSchema.enteredStore = new CountDownLatch(1);
+ compressedSchema.leaveStore = new CountDownLatch(1);
+ final Future<Integer> registering = executor.submit(encoding(compressedSchema, objectClasses, null));
+ assertTrue(compressedSchema.enteredStore.await(30, TimeUnit.SECONDS), "the store was never reached");
+
+ final AtomicReference<Thread> concurrentThread = new AtomicReference<>();
+ final Future<Integer> concurrent = executor.submit(encoding(compressedSchema, objectClasses, concurrentThread));
+ awaitParkedOnTheLock(concurrent, concurrentThread);
+ assertFalse(concurrent.isDone(), "the token was handed out before it was stored");
+
+ compressedSchema.leaveStore.countDown();
+ assertEquals(registering.get(30, TimeUnit.SECONDS), Integer.valueOf(0));
+ assertEquals(concurrent.get(30, TimeUnit.SECONDS), Integer.valueOf(0));
+ assertEquals(compressedSchema.objectClassStoreCount, 1, "the same token was stored twice");
+ }
+ finally
+ {
+ executor.shutdownNow();
+ }
+ }
+
+ /**
+ * Waits for the provided encode to park on the exclusive lock, which is what it must do while
+ * another thread holds that lock inside a store. Waiting for the thread to park is what makes
+ * this prove the encode reached the lock-free read of the encode map: a latch counted down
+ * inside the task only proves the task body started, so a build that published an id before
+ * storing it would be recorded as a pass whenever the thread was slow between the two.
+ * <p>
+ * Where the thread is parked is checked as well as that it is parked. A state on its own says
+ * nothing about what the thread waits for, and a build handing out an id before storing it
+ * parks nowhere: it takes the lock-free path, completes, and leaves the whole discrimination to
+ * a non-atomic isDone() sample - so any unrelated park, sampled in the instant before the task
+ * publishes its completion, would record that build as a pass.
+ */
+ private static void awaitParkedOnTheLock(final Future<Integer> encode, final AtomicReference<Thread> runningOn)
+ throws Exception
+ {
+ final long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(30);
+ while (System.nanoTime() < deadline)
+ {
+ if (encode.isDone())
+ {
+ fail("the token was handed out before it was stored: " + encode.get());
+ }
+ final Thread thread = runningOn.get();
+ // WAITING alone: the exclusive lock is a ReentrantLock, which parks through LockSupport, so
+ // this is the state it puts a thread in. BLOCKED is monitor entry and cannot come from that
+ // lock at all - accepting it would admit only parks this test is not about.
+ if (thread != null && thread.getState() == Thread.State.WAITING && parkedOnTheLockOfAnId(thread))
+ {
+ return;
+ }
+ Thread.sleep(1);
+ }
+ fail("the concurrent encode never parked on the exclusive lock");
+ }
+
+ /**
+ * Returns whether the provided thread is parked on a lock taken on the way to an id of a
+ * compressed schema, rather than anywhere else - the executor parking its idle worker on the
+ * task queue is a park too, and so is a logger or a class initializer.
+ */
+ private static boolean parkedOnTheLockOfAnId(final Thread thread)
+ {
+ boolean parkedOnALock = false;
+ // The frames run from the park outwards, so the lock is seen before whoever is taking it.
+ for (final StackTraceElement frame : thread.getStackTrace())
+ {
+ if (frame.getClassName().startsWith("java.util.concurrent.locks."))
+ {
+ parkedOnALock = true;
+ }
+ else if (parkedOnALock && CompressedSchema.class.getName().equals(frame.getClassName()))
+ {
+ return "getAttributeId".equals(frame.getMethodName()) || "getObjectClassId".equals(frame.getMethodName());
+ }
+ }
+ return false;
+ }
+
+ private static Callable<Integer> encoding(final CompressedSchema compressedSchema, final Attribute attribute,
+ final AtomicReference<Thread> runningOn)
+ {
+ return new Callable<Integer>()
+ {
+ @Override
+ public Integer call() throws Exception
+ {
+ if (runningOn != null)
+ {
+ runningOn.set(Thread.currentThread());
+ }
+ final ByteStringBuilder builder = new ByteStringBuilder();
+ compressedSchema.encodeAttribute(builder, attribute);
+ return tokenOf(builder.toByteString());
+ }
+ };
+ }
+
+ private static Callable<Integer> encoding(final CompressedSchema compressedSchema,
+ final Map<ObjectClass, String> objectClasses, final AtomicReference<Thread> runningOn)
+ {
+ return new Callable<Integer>()
+ {
+ @Override
+ public Integer call() throws Exception
+ {
+ if (runningOn != null)
+ {
+ runningOn.set(Thread.currentThread());
+ }
+ final ByteStringBuilder builder = new ByteStringBuilder();
+ compressedSchema.encodeObjectClasses(builder, objectClasses);
+ return tokenOf(builder.toByteString());
+ }
+ };
+ }
+
+ /**
+ * A token with no definition is reported, whether it is below the range of the decode map, the
+ * first id past its end, or well beyond it - and against a populated map as well as an empty
+ * one, since it is the size of that map the lookup is measured against.
+ */
+ @Test
+ public void unknownAttributeTokenIsReported() throws Exception
+ {
+ final TestCompressedSchema empty = new TestCompressedSchema();
+ for (final int unknownToken : new int[] { -1, 0, 7 })
+ {
+ assertAttributeTokenIsReported(empty, unknownToken);
+ }
+
+ final TestCompressedSchema populated = new TestCompressedSchema();
+ populated.loadAttributeAt(0, "description");
+ populated.loadAttributeAt(1, "cn");
+ for (final int unknownToken : new int[] { -1, 2, 7 })
+ {
+ assertAttributeTokenIsReported(populated, unknownToken);
+ }
+ }
+
+ @Test
+ public void unknownObjectClassTokenIsReported() throws Exception
+ {
+ final TestCompressedSchema empty = new TestCompressedSchema();
+ for (final int unknownToken : new int[] { -1, 0, 7 })
+ {
+ assertObjectClassTokenIsReported(empty, unknownToken);
+ }
+
+ final TestCompressedSchema populated = new TestCompressedSchema();
+ populated.loadObjectClassesAt(0, Arrays.asList("top", "person"));
+ populated.loadObjectClassesAt(1, Arrays.asList("top", "organizationalUnit"));
+ for (final int unknownToken : new int[] { -1, 2, 7 })
+ {
+ assertObjectClassTokenIsReported(populated, unknownToken);
+ }
+ }
+
+ /**
+ * A record the token cannot even be read from - it ends inside the token, or the length of the
+ * token names more bytes than the record holds - is reported like a token no definition was
+ * stored for. The read is what precedes the lookup, so leaving it unguarded would let the
+ * decode path of a {@code PublicAPI} class raise NegativeArraySizeException or
+ * IndexOutOfBoundsException at a caller written for DirectoryException.
+ */
+ @Test
+ public void aTokenThatCannotBeReadIsReported() throws Exception
+ {
+ final TestCompressedSchema compressedSchema = new TestCompressedSchema();
+ for (final ByteString unreadable : unreadableTokens())
+ {
+ try
+ {
+ compressedSchema.decodeAttribute(unreadable.asReader());
+ fail("the token of " + unreadable + " cannot be read and should have been reported");
+ }
+ catch (final DirectoryException expected)
+ {
+ assertMessageIs(expected, ERR_COMPRESSEDSCHEMA_UNREADABLE_AD_TOKEN.get(unreadable),
+ "the unreadable token " + unreadable);
+ }
+
+ try
+ {
+ compressedSchema.decodeObjectClasses(unreadable.asReader());
+ fail("the token of " + unreadable + " cannot be read and should have been reported");
+ }
+ catch (final DirectoryException expected)
+ {
+ assertMessageIs(expected, ERR_COMPRESSEDSCHEMA_UNREADABLE_OC_TOKEN.get(unreadable),
+ "the unreadable token " + unreadable);
+ }
+ }
+ }
+
+ /**
+ * A token padded past the four bytes an id is ever encoded in decodes to the same id as the
+ * canonical token of that id, because the decode folds whatever it is handed. A record carrying
+ * one would therefore read as a live definition rather than be reported, which is the one shape
+ * of a corrupt token that answers with data instead of an error.
+ */
+ @Test
+ public void anOverlongTokenIsNotDecodedAsTheIdItPadsTo() throws Exception
+ {
+ final TestCompressedSchema compressedSchema = new TestCompressedSchema();
+ compressedSchema.loadAttributeAt(2, "description");
+ compressedSchema.loadObjectClassesAt(2, Arrays.asList("top", "person"));
+
+ // The canonical token of the id 2 is 0x03, and this is that value padded to five bytes.
+ final byte[] padded = new byte[] { 0x00, 0x00, 0x00, 0x00, 0x03 };
+ try
+ {
+ compressedSchema.decodeAttribute(recordWithToken(padded, true).asReader());
+ fail("the token is longer than an id is ever encoded in and should have been reported");
+ }
+ catch (final DirectoryException expected)
+ {
+ assertMessageIs(expected, ERR_COMPRESSEDSCHEMA_UNREADABLE_AD_TOKEN.get("padded"), "the overlong token");
+ }
+
+ try
+ {
+ compressedSchema.decodeObjectClasses(recordWithToken(padded, false).asReader());
+ fail("the token is longer than an id is ever encoded in and should have been reported");
+ }
+ catch (final DirectoryException expected)
+ {
+ assertMessageIs(expected, ERR_COMPRESSEDSCHEMA_UNREADABLE_OC_TOKEN.get("padded"), "the overlong token");
+ }
+ }
+
+ /** A record carrying the provided token, with a single value where an attribute is asked for. */
+ private static ByteString recordWithToken(final byte[] idBytes, final boolean withAValue)
+ {
+ final ByteStringBuilder builder = new ByteStringBuilder();
+ builder.appendBERLength(idBytes.length);
+ builder.appendBytes(idBytes);
+ if (withAValue)
+ {
+ builder.appendBERLength(1);
+ builder.appendBERLength(1);
+ builder.appendBytes(new byte[] { 'x' });
+ }
+ return builder.toByteString();
+ }
+
+ /** Records a decode path cannot read a token from, as a corrupt or truncated store holds them. */
+ private static List<ByteString> unreadableTokens()
+ {
+ return Arrays.asList(
+ // The record ends before the length of the token.
+ ByteString.empty(),
+ // The length names one byte the record does not hold.
+ ByteString.wrap(new byte[] { 0x01 }),
+ // A four byte length composing to 0xFFFFFFFF, which is -1 as an int.
+ ByteString.wrap(new byte[] { (byte) 0x84, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF }),
+ // A four byte length of 0x7FFFFFFF: two gigabytes, which must not be allocated to find
+ // out that the record does not hold them.
+ ByteString.wrap(new byte[] { (byte) 0x84, 0x7F, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF }));
+ }
+
+ private static void assertAttributeTokenIsReported(final TestCompressedSchema compressedSchema,
+ final int unknownToken) throws Exception
+ {
+ try
+ {
+ compressedSchema.decodeAttribute(encodedAttribute(unknownToken).asReader());
+ fail("the token " + unknownToken + " has no definition and should have been reported");
+ }
+ catch (final DirectoryException expected)
+ {
+ // Reported as the unknown token it is, and named as such: nothing else in this decode path
+ // is allowed to answer for a token, and the message the operator gets is what says which.
+ assertMessageIs(expected, ERR_COMPRESSEDSCHEMA_UNRECOGNIZED_AD_TOKEN.get(unknownToken),
+ "the token " + unknownToken);
+ assertTokenIsNamed(expected, unknownToken);
+ }
+ }
+
+ private static void assertObjectClassTokenIsReported(final TestCompressedSchema compressedSchema,
+ final int unknownToken) throws Exception
+ {
+ try
+ {
+ compressedSchema.decodeObjectClasses(encodedToken(unknownToken, new ByteStringBuilder()).asReader());
+ fail("the token " + unknownToken + " has no definition and should have been reported");
+ }
+ catch (final DirectoryException expected)
+ {
+ assertMessageIs(expected, ERR_COMPRESSEDSCHEMA_UNKNOWN_OC_TOKEN.get(unknownToken),
+ "the token " + unknownToken);
+ assertTokenIsNamed(expected, unknownToken);
+ }
+ }
+
+ /** Asserts that the exception carries the expected message, by resource and id rather than text. */
+ private static void assertMessageIs(final DirectoryException reported, final LocalizableMessage expected,
+ final String context)
+ {
+ final LocalizableMessage message = reported.getMessageObject();
+ assertEquals(message.resourceName() + "-" + message.ordinal(),
+ expected.resourceName() + "-" + expected.ordinal(),
+ context + " was reported as something else: " + message);
+ }
+
+ /**
+ * Asserts that the message names the token the way an operator has to read it: the key the
+ * storage holds, with the id it decodes to. Asserted on the text, because comparing two
+ * messages by resource and ordinal says nothing about the arguments they carry - the accessors
+ * take an Object, so the id this test passes and the rendering production passes do not differ
+ * at compile time either, and a rendering returning an empty string would keep the suite green.
+ */
+ private static void assertTokenIsNamed(final DirectoryException reported, final int unknownToken)
+ {
+ final String named = String.format("0x%02X (id %d)", (unknownToken + 1) & 0xFF, unknownToken);
+ final LocalizableMessage message = reported.getMessageObject();
+ assertTrue(message.toString().contains(named),
+ "the token is not named as the storage holds it, expected " + named + " in: " + message);
+ }
+
+ /**
+ * A compressed schema loaded from a storage that holds no definition for some of the tokens
+ * carries a gap. Decoding across the gap, reloading the maps for a changed schema and saving the
+ * whole content must all walk over it, and the ids around it must not shift - the entries already
+ * written carry them.
+ */
+ @Test
+ public void aGapInTheDecodeMapsIsCarriedRatherThanDereferenced() throws Exception
+ {
+ final TestCompressedSchema compressedSchema = new TestCompressedSchema();
+ compressedSchema.loadAttributeAt(2, "description");
+ compressedSchema.loadObjectClassesAt(2, Arrays.asList("top", "person"));
+
+ // The first decode also rebuilds the maps for the current schema, which is what used to walk
+ // into the gap with no null check.
+ try
+ {
+ compressedSchema.decodeAttribute(encodedAttribute(0).asReader());
+ fail("the token 0 falls in the gap and should have been reported");
+ }
+ catch (final DirectoryException expected)
+ {
+ // Reported as the unknown token it is.
+ }
+ try
+ {
+ compressedSchema.decodeObjectClasses(encodedToken(1, new ByteStringBuilder()).asReader());
+ fail("the token 1 falls in the gap and should have been reported");
+ }
+ catch (final DirectoryException expected)
+ {
+ // Reported as the unknown token it is.
+ }
+
+ // What is around the gap is still reachable under the ids it was loaded with.
+ assertEquals(compressedSchema.decodeAttribute(encodedAttribute(2).asReader())
+ .getAttributeDescription().getAttributeType().getNameOrOID(), "description");
+ assertEquals(compressedSchema.decodeObjectClasses(encodedToken(2, new ByteStringBuilder()).asReader()),
+ objectClasses("top", "person"));
+
+ // And the next registration allocates the id after the gap, not one inside it.
+ final ByteStringBuilder attributeBuilder = new ByteStringBuilder();
+ compressedSchema.encodeAttribute(attributeBuilder, Attributes.create("cn", "a value"));
+ assertEquals(tokenOf(attributeBuilder.toByteString()), 3);
+
+ final ByteStringBuilder objectClassesBuilder = new ByteStringBuilder();
+ compressedSchema.encodeObjectClasses(objectClassesBuilder, objectClasses("top", "organizationalUnit"));
+ assertEquals(tokenOf(objectClassesBuilder.toByteString()), 3);
+
+ // The whole content is still saveable, which is how DefaultCompressedSchema persists a store.
+ assertEquals(compressedSchema.savedAttributeTokens(), Arrays.asList(2, 3));
+ assertEquals(compressedSchema.savedObjectClassTokens(), Arrays.asList(2, 3));
+ }
+
+ /** The attribute the provided token decodes to, named as the schema names it. */
+ private static String attributeNameAt(final TestCompressedSchema compressedSchema, final int id) throws Exception
+ {
+ return compressedSchema.decodeAttribute(encodedAttribute(id).asReader())
+ .getAttributeDescription().getAttributeType().getNameOrOID();
+ }
+
+ /** The object class set the provided token decodes to. */
+ private static Map<ObjectClass, String> objectClassesAt(final TestCompressedSchema compressedSchema, final int id)
+ throws Exception
+ {
+ return compressedSchema.decodeObjectClasses(encodedToken(id, new ByteStringBuilder()).asReader());
+ }
+
+ private static Map<ObjectClass, String> objectClasses(final String... names)
+ {
+ final Map<ObjectClass, String> objectClasses = new LinkedHashMap<>(names.length);
+ for (final String name : names)
+ {
+ objectClasses.put(DirectoryServer.getInstance().getServerContext().getSchema().getObjectClass(name), name);
+ }
+ return objectClasses;
+ }
+
+ /** Encodes an attribute holding a single value under the provided token. */
+ private static ByteString encodedAttribute(final int id)
+ {
+ final ByteStringBuilder builder = new ByteStringBuilder();
+ encodedToken(id, builder);
+ builder.appendBERLength(1);
+ builder.appendBERLength(1);
+ builder.appendBytes(new byte[] { 'x' });
+ return builder.toByteString();
+ }
+
+ private static ByteString encodedToken(final int id, final ByteStringBuilder builder)
+ {
+ final byte[] idBytes = encodedToken(id);
+ builder.appendBERLength(idBytes.length);
+ builder.appendBytes(idBytes);
+ return builder.toByteString();
+ }
+
+ /** Encodes a token the way CompressedSchema does, one byte being enough for the tests. */
+ private static byte[] encodedToken(final int id)
+ {
+ return new byte[] { (byte) ((id + 1) & 0xFF) };
+ }
+
+ /** Decodes a token the way CompressedSchema does. */
+ private static int token(final byte[] idBytes)
+ {
+ int id = 0;
+ for (final byte b : idBytes)
+ {
+ id <<= 8;
+ id |= b & 0xFF;
+ }
+ return id - 1;
+ }
+
+ /** Reads the token an encoded attribute or object class set starts with. */
+ private static int tokenOf(final ByteString encoded)
+ {
+ final ByteSequenceReader reader = encoded.asReader();
+ final byte[] idBytes = new byte[reader.readBERLength()];
+ reader.readBytes(idBytes);
+ return token(idBytes);
+ }
+}
diff --git a/opendj-server-legacy/src/test/java/org/opends/server/core/DefaultCompressedSchemaTestCase.java b/opendj-server-legacy/src/test/java/org/opends/server/core/DefaultCompressedSchemaTestCase.java
new file mode 100644
index 0000000..c6903d3
--- /dev/null
+++ b/opendj-server-legacy/src/test/java/org/opends/server/core/DefaultCompressedSchemaTestCase.java
@@ -0,0 +1,96 @@
+/*
+ * The contents of this file are subject to the terms of the Common Development and
+ * Distribution License (the License). You may not use this file except in compliance with the
+ * License.
+ *
+ * You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the
+ * specific language governing permission and limitations under the License.
+ *
+ * When distributing Covered Software, include this CDDL Header Notice in each file and include
+ * the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL
+ * Header, with the fields enclosed by brackets [] replaced by your own identifying
+ * information: "Portions Copyright [year] [name of copyright owner]".
+ *
+ * Copyright 2026 3A Systems, LLC.
+ */
+package org.opends.server.core;
+
+import static org.testng.Assert.*;
+
+import org.testng.annotations.Test;
+
+/**
+ * Tests the counter {@link DefaultCompressedSchema} records with the content it saves. The counter
+ * is read as "No longer used" by both ends of that file, but a release old enough to seed from it
+ * must not be handed a token that is live.
+ */
+@SuppressWarnings("javadoc")
+public class DefaultCompressedSchemaTestCase extends CoreTestCase
+{
+ /** A content with no gaps records what counting the records recorded: ids 0..N-1 give N+1. */
+ @Test
+ public void theCounterOfADenseContentIsTheTokenAfterTheLastOne()
+ {
+ assertEquals(counterAfterWriting(), 1, "an empty content is left at the counter it starts at");
+ assertEquals(counterAfterWriting(1), 2);
+ assertEquals(counterAfterWriting(1, 2, 3), 4);
+ }
+
+ /**
+ * A gap inside the content emits fewer records than the ids it spans, so a counter taken from
+ * the number of records would name a token that is live.
+ */
+ @Test
+ public void theCounterOfAContentWithAGapIsTheTokenAfterTheHighestOne()
+ {
+ // The definition of the id 1 was lost: two records, spanning the tokens 1 and 3.
+ assertEquals(counterAfterWriting(1, 3), 4, "the counter names a token that is live");
+ // The same across a token of more than one byte.
+ assertEquals(counterAfterWriting(1, 2, 300), 301);
+ }
+
+ /** The counter never walks back, whatever order the content is written in. */
+ @Test
+ public void theCounterIsNotLoweredByALaterToken()
+ {
+ assertEquals(counterAfterWriting(3, 1), 4);
+ }
+
+ /**
+ * The gap at the end is the one this cannot cover, and it is the one the withdrawal of an
+ * unstored registration exists to prevent: the id whose definition was lost is not in the
+ * content at all, so nothing names it and the counter stops at the token before it.
+ */
+ @Test
+ public void aGapAtTheEndOfTheContentIsNotCoveredByTheCounter()
+ {
+ // The ids 0 and 1 survived and the definition of the id 2 was lost: the content spans the
+ // tokens 1 and 2, and the counter reads as though the id 2 were free.
+ assertEquals(counterAfterWriting(1, 2), 3, "the residual a file written by an older server can carry");
+ }
+
+ /** The counter left by writing the provided tokens, in the order they are given. */
+ private static int counterAfterWriting(final int... tokens)
+ {
+ int counter = 1;
+ for (final int token : tokens)
+ {
+ counter = DefaultCompressedSchema.counterAfter(counter, encodedToken(token));
+ }
+ return counter;
+ }
+
+ /** Encodes a token the way a compressed schema writes it: as many bytes as it needs. */
+ private static byte[] encodedToken(final int token)
+ {
+ if (token <= 0xFF)
+ {
+ return new byte[] { (byte) token };
+ }
+ if (token <= 0xFFFF)
+ {
+ return new byte[] { (byte) (token >> 8), (byte) token };
+ }
+ return new byte[] { (byte) (token >> 16), (byte) (token >> 8), (byte) token };
+ }
+}
--
Gitblit v1.10.0