OPENDJ-2616 Support protection of pluggable backend data at rest

Fabio Pistolesi

22.30.2016 1a2cdfb5cf5f89348e8fee7ceeaa699d4aa54cea

 opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/ID2Entry.java

@@ -12,10 +12,11 @@
 * information: "Portions Copyright [year] [name of copyright owner]".
 *
 * Copyright 2006-2010 Sun Microsystems, Inc.
 * Portions Copyright 2012-2015 ForgeRock AS.
 * Portions Copyright 2012-2016 ForgeRock AS.
 */
package org.opends.server.backends.pluggable;

import static org.forgerock.opendj.ldap.ResultCode.UNWILLING_TO_PERFORM;
import static org.forgerock.util.Reject.*;
import static org.forgerock.util.Utils.*;
import static org.opends.messages.BackendMessages.*;
@@ -23,16 +24,18 @@
import static org.opends.server.core.DirectoryServer.*;

import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.zip.DataFormatException;
import java.util.zip.DeflaterOutputStream;
import java.util.zip.InflaterInputStream;
import java.util.zip.InflaterOutputStream;

import org.forgerock.i18n.slf4j.LocalizedLogger;
import org.forgerock.opendj.io.ASN1;
import org.forgerock.opendj.io.ASN1Reader;
import org.forgerock.opendj.io.ASN1Writer;
import org.forgerock.opendj.ldap.ByteSequence;
import org.forgerock.opendj.ldap.ByteSequenceReader;
import org.forgerock.opendj.ldap.ByteString;
import org.forgerock.opendj.ldap.ByteStringBuilder;
import org.forgerock.opendj.ldap.DecodeException;
@@ -45,6 +48,7 @@
import org.opends.server.backends.pluggable.spi.TreeName;
import org.opends.server.backends.pluggable.spi.WriteableTransaction;
import org.opends.server.core.DirectoryServer;
import org.opends.server.types.CryptoManagerException;
import org.opends.server.types.DirectoryException;
import org.opends.server.types.Entry;
import org.opends.server.types.LDAPException;
@@ -103,40 +107,87 @@
  /** A cached set of ByteStringBuilder buffers and ASN1Writer used to encode entries. */
  private static final class EntryCodec
  {
    /**
     * The format version used encode and decode entries in previous versions.
     * Not used anymore, kept for compatibility during upgrade.
     */
    static final byte FORMAT_VERSION = 0x01;

    /** The ASN1 tag for the ByteString type. */
    private static final byte TAG_TREE_ENTRY = 0x60;
    private static final int BUFFER_INIT_SIZE = 512;
    private static final byte PLAIN_ENTRY = 0x00;
    private static final byte COMPRESS_ENTRY = 0x01;
    private static final byte ENCRYPT_ENTRY = 0x02;

    /** The format version for entry encoding. */
    static final byte FORMAT_VERSION_V2 = 0x02;

    private final ByteStringBuilder encodedBuffer = new ByteStringBuilder();
    private final ByteStringBuilder entryBuffer = new ByteStringBuilder();
    private final ByteStringBuilder compressedEntryBuffer = new ByteStringBuilder();
    private final ASN1Writer writer;
    private final int maxBufferSize;

    private EntryCodec()
    {
      this.maxBufferSize = getMaxInternalBufferSize();
      this.writer = ASN1.getWriter(encodedBuffer, maxBufferSize);
    }

    private void release()
    {
      closeSilently(writer);
      encodedBuffer.clearAndTruncate(maxBufferSize, BUFFER_INIT_SIZE);
      entryBuffer.clearAndTruncate(maxBufferSize, BUFFER_INIT_SIZE);
      compressedEntryBuffer.clearAndTruncate(maxBufferSize, BUFFER_INIT_SIZE);
    }

    private Entry decode(ByteString bytes, CompressedSchema compressedSchema)
        throws DirectoryException, DecodeException, IOException
    {
      // Get the format version.
      byte formatVersion = bytes.byteAt(0);
      if(formatVersion != DnKeyFormat.FORMAT_VERSION)
      final byte formatVersion = bytes.byteAt(0);
      switch(formatVersion)
      {
      case FORMAT_VERSION:
        return decodeV1(bytes, compressedSchema);
      case FORMAT_VERSION_V2:
        return decodeV2(bytes, compressedSchema);
      default:
        throw DecodeException.error(ERR_INCOMPATIBLE_ENTRY_VERSION.get(formatVersion));
      }
    }

    /**
     * Decodes an entry from the old format.
     * <p>
     * An entry on disk is ASN1 encoded in this format:
     *
     * <pre>
     * ByteString ::= [APPLICATION 0] IMPLICIT SEQUENCE {
     *  uncompressedSize      INTEGER,      -- A zero value means not compressed.
     *  dataBytes             OCTET STRING  -- Optionally compressed encoding of
     *                                         the data bytes.
     * }
     *
     * ID2EntryValue ::= ByteString
     *  -- Where dataBytes contains an encoding of DirectoryServerEntry.
     *
     * DirectoryServerEntry ::= [APPLICATION 1] IMPLICIT SEQUENCE {
     *  dn                      LDAPDN,
     *  objectClasses           SET OF LDAPString,
     *  userAttributes          AttributeList,
     *  operationalAttributes   AttributeList
     * }
     * </pre>
     *
     * @param bytes A byte array containing the encoded tree value.
     * @param compressedSchema The compressed schema manager to use when decoding.
     * @return The decoded entry.
     * @throws DecodeException If the data is not in the expected ASN.1 encoding
     * format.
     * @throws DirectoryException If a Directory Server error occurs.
     * @throws IOException if an error occurs while reading the ASN1 sequence.
     */
    private Entry decodeV1(ByteString bytes, CompressedSchema compressedSchema)
        throws DirectoryException, DecodeException, IOException
    {
      // Read the ASN1 sequence.
      ASN1Reader reader = ASN1.getReader(bytes.subSequence(1, bytes.length()));
      reader.readStartSequence();
@@ -173,6 +224,71 @@
      }
    }

    /**
     * Decodes an entry in the new extensible format.
     * Enties are encoded according to the sequence
     *   {VERSION_BYTE, FLAG_BYTE, COMPACT_INTEGER_LENGTH, ID2ENTRY_VALUE}
     * where
     *
     * ID2ENTRY_VALUE = encoding of Entry as in decodeV1()
     * VERSION_BYTE = 0x2
     * FLAG_BYTE = bit field of OR'ed values indicating post-encoding processing.
     *     possible meaningful flags are COMPRESS_ENTRY and ENCRYPT_ENTRY.
     * COMPACT_INTEGER_LENGTH = length of ID2ENTRY_VALUE
     *
     * @param bytes A byte array containing the encoded tree value.
     * @param compressedSchema The compressed schema manager to use when decoding.
     * @return The decoded entry.
     * @throws DecodeException If the data is not in the expected ASN.1 encoding
     * format or a decryption error occurs.
     * @throws DirectoryException If a Directory Server error occurs.
     * @throws IOException if an error occurs while reading the ASN1 sequence.
     */
    private Entry decodeV2(ByteString bytes, CompressedSchema compressedSchema)
        throws DirectoryException, DecodeException, IOException
    {
      ByteSequenceReader reader = bytes.asReader();
      // skip version byte
      reader.position(1);
      int format = reader.readByte();
      int encodedEntryLen = reader.readCompactUnsignedInt();
      try
      {
        if (format == PLAIN_ENTRY)
        {
          return Entry.decode(reader, compressedSchema);
        }
        InputStream is = reader.asInputStream();
        if ((format & ENCRYPT_ENTRY) == ENCRYPT_ENTRY)
        {
          is = getCryptoManager().getCipherInputStream(is);
        }
        if ((format & COMPRESS_ENTRY) == COMPRESS_ENTRY)
        {
          is = new InflaterInputStream(is);
        }
        byte[] data = new byte[encodedEntryLen];
        int readBytes;
        int position = 0;
        int leftToRead = encodedEntryLen;
        // CipherInputStream does not read more than block size...
        do
        {
          if ((readBytes = is.read(data, position, leftToRead)) == -1 )
          {
            throw DecodeException.error(ERR_CANNOT_DECODE_ENTRY.get());
          }
          position += readBytes;
          leftToRead -= readBytes;
        } while (leftToRead > 0 && readBytes > 0);
        return Entry.decode(ByteString.wrap(data).asReader(), compressedSchema);
      }
      catch (CryptoManagerException cme)
      {
        throw DecodeException.error(cme.getMessageObject());
      }
    }

    private ByteString encode(Entry entry, DataConfig dataConfig) throws DirectoryException
    {
      encodeVolatile(entry, dataConfig);
@@ -181,44 +297,44 @@

    private void encodeVolatile(Entry entry, DataConfig dataConfig) throws DirectoryException
    {
      // Encode the entry for later use.
      entry.encode(entryBuffer, dataConfig.getEntryEncodeConfig());

      // First write the DB format version byte.
      encodedBuffer.appendByte(DnKeyFormat.FORMAT_VERSION);

      OutputStream os = encodedBuffer.asOutputStream();
      try
      {
        // Then start the ASN1 sequence.
        writer.writeStartSequence(TAG_TREE_ENTRY);

        byte[] formatFlags = { FORMAT_VERSION_V2, 0};
        os.write(formatFlags);
        encodedBuffer.appendCompactUnsigned(entryBuffer.length());
        if (dataConfig.isCompressed())
        {
          OutputStream compressor = null;
          try {
            compressor = new DeflaterOutputStream(compressedEntryBuffer.asOutputStream());
            entryBuffer.copyTo(compressor);
          }
          finally {
            closeSilently(compressor);
          }

          // Compression needed and successful.
          writer.writeInteger(entryBuffer.length());
          writer.writeOctetString(compressedEntryBuffer);
          os = new DeflaterOutputStream(os);
          formatFlags[1] = COMPRESS_ENTRY;
        }
        else
        if (dataConfig.isEncrypted())
        {
          writer.writeInteger(0);
          writer.writeOctetString(entryBuffer);
          os = dataConfig.getCryptoSuite().getCipherOutputStream(os);
          formatFlags[1] |= ENCRYPT_ENTRY;
        }
        encodedBuffer.setByte(1, formatFlags[1]);

        writer.writeEndSequence();
        entryBuffer.copyTo(os);
        os.flush();
      }
      catch(IOException ioe)
      catch(CryptoManagerException | IOException e)
      {
        // TODO: This should never happen with byte buffer.
        logger.traceException(ioe);
        logger.traceException(e);
        throw new DirectoryException(UNWILLING_TO_PERFORM, ERR_CANNOT_ENCODE_ENTRY.get(e.getLocalizedMessage()));
      }
      finally
      {
        try
        {
          os.close();
        }
        catch (IOException ioe)
        {
          throw new DirectoryException(UNWILLING_TO_PERFORM, ERR_CANNOT_ENCODE_ENTRY.get(ioe.getLocalizedMessage()));
        }
      }
    }
  }
@@ -250,26 +366,6 @@

  /**
   * Decodes an entry from its tree representation.
   * <p>
   * An entry on disk is ASN1 encoded in this format:
   *
   * <pre>
   * ByteString ::= [APPLICATION 0] IMPLICIT SEQUENCE {
   *  uncompressedSize      INTEGER,      -- A zero value means not compressed.
   *  dataBytes             OCTET STRING  -- Optionally compressed encoding of
   *                                         the data bytes.
   * }
   *
   * ID2EntryValue ::= ByteString
   *  -- Where dataBytes contains an encoding of DirectoryServerEntry.
   *
   * DirectoryServerEntry ::= [APPLICATION 1] IMPLICIT SEQUENCE {
   *  dn                      LDAPDN,
   *  objectClasses           SET OF LDAPString,
   *  userAttributes          AttributeList,
   *  operationalAttributes   AttributeList
   * }
   * </pre>
   *
   * @param bytes A byte array containing the encoded tree value.
   * @param compressedSchema The compressed schema manager to use when decoding.
@@ -283,7 +379,7 @@
   * @throws DirectoryException If a Directory Server error occurs.
   * @throws IOException if an error occurs while reading the ASN1 sequence.
   */
  static Entry entryFromDatabase(ByteString bytes,
  Entry entryFromDatabase(ByteString bytes,
      CompressedSchema compressedSchema) throws DirectoryException,
      DecodeException, LDAPException, DataFormatException, IOException
  {
@@ -308,7 +404,7 @@
   * @throws  DirectoryException  If a problem occurs while attempting to encode
   *                              the entry.
   */
  static ByteString entryToDatabase(Entry entry, DataConfig dataConfig) throws DirectoryException
  ByteString entryToDatabase(Entry entry, DataConfig dataConfig) throws DirectoryException
  {
    EntryCodec codec = acquireEntryCodec();
    try