/* * The contents of this file are subject to the terms of the Common Development and * Distribution License (the License). You may not use this file except in compliance with the * License. * * You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the * specific language governing permission and limitations under the License. * * When distributing Covered Software, include this CDDL Header Notice in each file and include * the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL * Header, with the fields enclosed by brackets [] replaced by your own identifying * information: "Portions Copyright [year] [name of copyright owner]". * * Copyright 2006-2010 Sun Microsystems, Inc. * Portions Copyright 2012-2016 ForgeRock AS. */ package org.opends.server.backends.pluggable; import static org.forgerock.opendj.ldap.ResultCode.UNWILLING_TO_PERFORM; import static org.forgerock.util.Reject.*; import static org.forgerock.util.Utils.*; import static org.opends.messages.BackendMessages.*; import static org.opends.server.backends.pluggable.CursorTransformer.transformKeysAndValues; import static org.opends.server.core.DirectoryServer.*; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.util.zip.DataFormatException; import java.util.zip.DeflaterOutputStream; import java.util.zip.InflaterInputStream; import java.util.zip.InflaterOutputStream; import org.forgerock.i18n.slf4j.LocalizedLogger; import org.forgerock.opendj.io.ASN1; import org.forgerock.opendj.io.ASN1Reader; import org.forgerock.opendj.ldap.ByteSequence; import org.forgerock.opendj.ldap.ByteSequenceReader; import org.forgerock.opendj.ldap.ByteString; import org.forgerock.opendj.ldap.ByteStringBuilder; import org.forgerock.opendj.ldap.DecodeException; import org.forgerock.util.Function; import org.forgerock.util.Reject; import org.opends.server.api.CompressedSchema; import org.opends.server.backends.pluggable.spi.Cursor; import org.opends.server.backends.pluggable.spi.ReadableTransaction; import org.opends.server.backends.pluggable.spi.StorageRuntimeException; import org.opends.server.backends.pluggable.spi.TreeName; import org.opends.server.backends.pluggable.spi.WriteableTransaction; import org.opends.server.core.DirectoryServer; import org.opends.server.types.CryptoManagerException; import org.opends.server.types.DirectoryException; import org.opends.server.types.Entry; import org.opends.server.types.LDAPException; /** * Represents the tree containing the LDAP entries. * The key is the entry ID and the value is the entry contents. */ class ID2Entry extends AbstractTree { private static final LocalizedLogger logger = LocalizedLogger.getLoggerForThisClass(); /** Transforms cursor keys into EntryIDs. */ private static final Function TO_ENTRY_ID = new Function() { @Override public EntryID apply(ByteString value) throws Exception { return new EntryID(value); } }; /** Transforms cursor values into Entry objects. */ private final CursorTransformer.ValueTransformer TO_ENTRY = new CursorTransformer.ValueTransformer() { @Override public Entry transform(ByteString key, ByteString value) throws Exception { return get0(value); } }; /** Parameters for compression and encryption. */ private DataConfig dataConfig; /** Cached encoding buffers. */ private static final ThreadLocal ENTRY_CODEC_CACHE = new ThreadLocal() { @Override protected EntryCodec initialValue() { return new EntryCodec(); } }; private static EntryCodec acquireEntryCodec() { EntryCodec codec = ENTRY_CODEC_CACHE.get(); if (codec.maxBufferSize != getMaxInternalBufferSize()) { // Setting has changed, so recreate the codec. codec = new EntryCodec(); ENTRY_CODEC_CACHE.set(codec); } return codec; } /** A cached set of ByteStringBuilder buffers and ASN1Writer used to encode entries. */ private static final class EntryCodec { /** * The format version used encode and decode entries in previous versions. * Not used anymore, kept for compatibility during upgrade. */ static final byte FORMAT_VERSION = 0x01; /** The ASN1 tag for the ByteString type. */ private static final byte TAG_TREE_ENTRY = 0x60; private static final int BUFFER_INIT_SIZE = 512; private static final byte PLAIN_ENTRY = 0x00; private static final byte COMPRESS_ENTRY = 0x01; private static final byte ENCRYPT_ENTRY = 0x02; /** The format version for entry encoding. */ static final byte FORMAT_VERSION_V2 = 0x02; private final ByteStringBuilder encodedBuffer = new ByteStringBuilder(); private final ByteStringBuilder entryBuffer = new ByteStringBuilder(); private final ByteStringBuilder compressedEntryBuffer = new ByteStringBuilder(); private final int maxBufferSize; private EntryCodec() { this.maxBufferSize = getMaxInternalBufferSize(); } private void release() { encodedBuffer.clearAndTruncate(maxBufferSize, BUFFER_INIT_SIZE); entryBuffer.clearAndTruncate(maxBufferSize, BUFFER_INIT_SIZE); } private Entry decode(ByteString bytes, CompressedSchema compressedSchema) throws DirectoryException, DecodeException, IOException { final byte formatVersion = bytes.byteAt(0); switch(formatVersion) { case FORMAT_VERSION: return decodeV1(bytes, compressedSchema); case FORMAT_VERSION_V2: return decodeV2(bytes, compressedSchema); default: throw DecodeException.error(ERR_INCOMPATIBLE_ENTRY_VERSION.get(formatVersion)); } } /** * Decodes an entry from the old format. *

* An entry on disk is ASN1 encoded in this format: * * 

     * ByteString ::= [APPLICATION 0] IMPLICIT SEQUENCE {
     *  uncompressedSize      INTEGER,      -- A zero value means not compressed.
     *  dataBytes             OCTET STRING  -- Optionally compressed encoding of
     *                                         the data bytes.
     * }
     *
     * ID2EntryValue ::= ByteString
     *  -- Where dataBytes contains an encoding of DirectoryServerEntry.
     *
     * DirectoryServerEntry ::= [APPLICATION 1] IMPLICIT SEQUENCE {
     *  dn                      LDAPDN,
     *  objectClasses           SET OF LDAPString,
     *  userAttributes          AttributeList,
     *  operationalAttributes   AttributeList
     * }
     *
* * @param bytes A byte array containing the encoded tree value. * @param compressedSchema The compressed schema manager to use when decoding. * @return The decoded entry. * @throws DecodeException If the data is not in the expected ASN.1 encoding * format. * @throws DirectoryException If a Directory Server error occurs. * @throws IOException if an error occurs while reading the ASN1 sequence. */ private Entry decodeV1(ByteString bytes, CompressedSchema compressedSchema) throws DirectoryException, DecodeException, IOException { // Read the ASN1 sequence. ASN1Reader reader = ASN1.getReader(bytes.subSequence(1, bytes.length())); reader.readStartSequence(); // See if it was compressed. int uncompressedSize = (int)reader.readInteger(); if(uncompressedSize > 0) { // It was compressed. reader.readOctetString(compressedEntryBuffer); OutputStream decompressor = null; try { // TODO: Should handle the case where uncompress fails decompressor = new InflaterOutputStream(entryBuffer.asOutputStream()); compressedEntryBuffer.copyTo(decompressor); } finally { closeSilently(decompressor); } // Since we are used the cached buffers (ByteStringBuilders), // the decoded attribute values will not refer back to the // original buffer. return Entry.decode(entryBuffer.asReader(), compressedSchema); } else { // Since we don't have to do any decompression, we can just decode // the entry directly. ByteString encodedEntry = reader.readOctetString(); return Entry.decode(encodedEntry.asReader(), compressedSchema); } } /** * Decodes an entry in the new extensible format. * Enties are encoded according to the sequence * {VERSION_BYTE, FLAG_BYTE, COMPACT_INTEGER_LENGTH, ID2ENTRY_VALUE} * where * * ID2ENTRY_VALUE = encoding of Entry as in decodeV1() * VERSION_BYTE = 0x2 * FLAG_BYTE = bit field of OR'ed values indicating post-encoding processing. * possible meaningful flags are COMPRESS_ENTRY and ENCRYPT_ENTRY. * COMPACT_INTEGER_LENGTH = length of ID2ENTRY_VALUE * * @param bytes A byte array containing the encoded tree value. * @param compressedSchema The compressed schema manager to use when decoding. * @return The decoded entry. * @throws DecodeException If the data is not in the expected ASN.1 encoding * format or a decryption error occurs. * @throws DirectoryException If a Directory Server error occurs. * @throws IOException if an error occurs while reading the ASN1 sequence. */ private Entry decodeV2(ByteString bytes, CompressedSchema compressedSchema) throws DirectoryException, DecodeException, IOException { ByteSequenceReader reader = bytes.asReader(); // skip version byte reader.position(1); int format = reader.readByte(); int encodedEntryLen = reader.readCompactUnsignedInt(); try { if (format == PLAIN_ENTRY) { return Entry.decode(reader, compressedSchema); } InputStream is = reader.asInputStream(); if ((format & ENCRYPT_ENTRY) == ENCRYPT_ENTRY) { is = getCryptoManager().getCipherInputStream(is); } if ((format & COMPRESS_ENTRY) == COMPRESS_ENTRY) { is = new InflaterInputStream(is); } byte[] data = new byte[encodedEntryLen]; int readBytes; int position = 0; int leftToRead = encodedEntryLen; // CipherInputStream does not read more than block size... do { if ((readBytes = is.read(data, position, leftToRead)) == -1 ) { throw DecodeException.error(ERR_CANNOT_DECODE_ENTRY.get()); } position += readBytes; leftToRead -= readBytes; } while (leftToRead > 0 && readBytes > 0); return Entry.decode(ByteString.wrap(data).asReader(), compressedSchema); } catch (CryptoManagerException cme) { logger.traceException(cme); throw DecodeException.error(cme.getMessageObject()); } } private ByteString encode(Entry entry, DataConfig dataConfig) throws DirectoryException { encodeVolatile(entry, dataConfig); return encodedBuffer.toByteString(); } private void encodeVolatile(Entry entry, DataConfig dataConfig) throws DirectoryException { entry.encode(entryBuffer, dataConfig.getEntryEncodeConfig()); OutputStream os = encodedBuffer.asOutputStream(); try { byte[] formatFlags = { FORMAT_VERSION_V2, 0}; os.write(formatFlags); encodedBuffer.appendCompactUnsigned(entryBuffer.length()); if (dataConfig.isCompressed()) { os = new DeflaterOutputStream(os); formatFlags[1] = COMPRESS_ENTRY; } if (dataConfig.isEncrypted()) { os = dataConfig.getCryptoSuite().getCipherOutputStream(os); formatFlags[1] |= ENCRYPT_ENTRY; } encodedBuffer.setByte(1, formatFlags[1]); entryBuffer.copyTo(os); os.flush(); } catch(CryptoManagerException | IOException e) { logger.traceException(e); throw new DirectoryException(UNWILLING_TO_PERFORM, ERR_CANNOT_ENCODE_ENTRY.get(e.getLocalizedMessage())); } finally { try { os.close(); } catch (IOException ioe) { throw new DirectoryException(UNWILLING_TO_PERFORM, ERR_CANNOT_ENCODE_ENTRY.get(ioe.getLocalizedMessage())); } } } } /** * Create a new ID2Entry object. * * @param name The name of the entry tree. * @param dataConfig The desired compression and encryption options for data * stored in the entry tree. * @throws StorageRuntimeException If an error occurs in the storage. */ ID2Entry(TreeName name, DataConfig dataConfig) throws StorageRuntimeException { super(name); this.dataConfig = dataConfig; } @Override void afterOpen(WriteableTransaction txn, boolean createOnDemand) throws StorageRuntimeException { // Make sure the tree is there and readable, even if the storage is READ_ONLY. // Would be nice if there were a better way... try (final Cursor cursor = txn.openCursor(getName())) { cursor.next(); } } /** * Decodes an entry from its tree representation. * * @param bytes A byte array containing the encoded tree value. * @param compressedSchema The compressed schema manager to use when decoding. * @return The decoded entry. * @throws DecodeException If the data is not in the expected ASN.1 encoding * format. * @throws LDAPException If the data is not in the expected ASN.1 encoding * format. * @throws DataFormatException If an error occurs while trying to decompress * compressed data. * @throws DirectoryException If a Directory Server error occurs. * @throws IOException if an error occurs while reading the ASN1 sequence. */ Entry entryFromDatabase(ByteString bytes, CompressedSchema compressedSchema) throws DirectoryException, DecodeException, LDAPException, DataFormatException, IOException { EntryCodec codec = acquireEntryCodec(); try { return codec.decode(bytes, compressedSchema); } finally { codec.release(); } } /** * Encodes an entry to the raw tree format, with optional compression. * * @param entry The entry to encode. * @param dataConfig Compression and cryptographic options. * @return A ByteSTring containing the encoded tree value. * * @throws DirectoryException If a problem occurs while attempting to encode * the entry. */ ByteString entryToDatabase(Entry entry, DataConfig dataConfig) throws DirectoryException { EntryCodec codec = acquireEntryCodec(); try { return codec.encode(entry, dataConfig); } finally { codec.release(); } } ByteString encode(Entry entry) throws DirectoryException { return entryToDatabase(entry, dataConfig); } /** * Write a record in the entry tree. * * @param txn a non null transaction * @param entryID The entry ID which forms the key. * @param entry The LDAP entry. * @throws StorageRuntimeException If an error occurs in the storage. * @throws DirectoryException If a problem occurs while attempting to encode * the entry. */ public void put(WriteableTransaction txn, EntryID entryID, Entry entry) throws StorageRuntimeException, DirectoryException { put(txn, entryID, encode(entry)); } public void put(WriteableTransaction txn, EntryID entryID, ByteSequence encodedEntry) throws StorageRuntimeException, DirectoryException { Reject.ifNull(txn, "txn must not be null."); txn.put(getName(), entryID.toByteString(), encodedEntry); } /** * Fetch a record from the entry tree. * * @param txn a non null transaction * @param entryID The desired entry ID which forms the key. * @return The requested entry, or null if there is no such record. * @throws DirectoryException If a problem occurs while getting the entry. * @throws StorageRuntimeException If an error occurs in the storage. */ public Entry get(ReadableTransaction txn, EntryID entryID) throws DirectoryException, StorageRuntimeException { try { return get0(txn.read(getName(), entryID.toByteString())); } catch (Exception e) { throw new DirectoryException(DirectoryServer.getServerErrorResultCode(), ERR_ENTRY_DATABASE_CORRUPT.get(entryID)); } } Cursor openCursor(ReadableTransaction txn) { return transformKeysAndValues(txn.openCursor(getName()), TO_ENTRY_ID, TO_ENTRY); } /** * Check that a record entry exists in the entry tree. * * @param txn a non null transaction * @param entryID The entry ID which forms the key. * @return True if an entry with entryID exists * @throws StorageRuntimeException If an error occurs in the storage. */ public boolean containsEntryID(ReadableTransaction txn, EntryID entryID) { checkNotNull(txn, "txn must not be null"); checkNotNull(entryID, "entryID must not be null"); try(final Cursor cursor = txn.openCursor(getName())) { return cursor.positionToKey(entryID.toByteString()); } } private Entry get0(ByteString value) throws Exception { if (value == null) { return null; } final Entry entry = entryFromDatabase(value, dataConfig.getEntryEncodeConfig().getCompressedSchema()); entry.processVirtualAttributes(); return entry; } /** * Set the desired compression and encryption options for data * stored in the entry tree. * * @param dataConfig The desired compression and encryption options for data * stored in the entry tree. */ void setDataConfig(DataConfig dataConfig) { this.dataConfig = dataConfig; } @Override public String keyToString(ByteString key) { return new EntryID(key).toString(); } @Override public String valueToString(ByteString value) { try { return "\n" + get0(value).toString(); } catch (Exception e) { return e.getMessage(); } } @Override public ByteString generateKey(String data) { EntryID entryID = new EntryID(Long.parseLong(data)); return entryID.toByteString(); } }