OPENDJ-1602 (CR-5566) New pluggable storage based backend

Jean-Noel Rouvignac

17.48.2014 5031429bf032af5e3d8797210cc47b402ef831d6

OPENDJ-1602 (CR-5566) New pluggable storage based backend

Reduced the number of compilation errors in package "org.opends.server.backends.pluggable".



PersistItStorage.java: ADDED
Initial implementation of the Persisit storage

JebException.java: REMOVED - replaced by StorageRuntimeException for now.

DatabaseEnvironmentMonitor.java: ADDED
Copied from org.opends.server.monitors.DatabaseEnvironmentMonitor and stripped out all JE related code + changed client code to use it.

SuffixContainer.java, jeb.EntryContainer.java:
Removed getEntryCount() from interface because I need to add a ReadableStorage parameter which is unacceptable for this class.



pluggable.EntryContainer.java:
Worked with TreeName objects instead of plain String objects.
Replaced all former JE calls of "if (environment.getConfig().getTransactional())" by using a straight (Read|Write)Storage object which implementation may or may not be transactional.
Removed preparePrefix().

EntryIDSetSorter.java:
Used EntryContainer instead of SuffixContainer.

RootContainer.java:
Removed now useless methods + code.



*.java:
Passed (Readable|Writable)Storage parameters down method calls.
Replaced all references to JebException by StorageRuntimeException.
In several applyConfiguration*() methods, created write operations.
In applyConfiguration*() methods, created a ConfigChangeResult object at the start of the method and accumulated information on it instead of creating it at the end of the method + passed the ConfigChangeResult object down method calls.

 opendj3-server-dev/src/server/org/opends/server/backends/jeb/EntryContainer.java

@@ -2687,7 +2687,6 @@
   * @return The number of entries stored in this entry container.
   * @throws DatabaseException If an error occurs in the JE database.
   */
  @Override
  public long getEntryCount() throws DatabaseException
  {
    EntryID entryID = dn2id.get(null, baseDN, LockMode.DEFAULT);

 opendj3-server-dev/src/server/org/opends/server/backends/persistit/PersistItStorage.java

New file
@@ -0,0 +1,431 @@
package org.opends.server.backends.persistit;

import java.io.File;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;

import org.forgerock.opendj.ldap.ByteSequence;
import org.forgerock.opendj.ldap.ByteString;
import org.opends.server.admin.std.server.LocalDBBackendCfg;
import org.opends.server.backends.pluggable.BackendImpl.Cursor;
import org.opends.server.backends.pluggable.BackendImpl.Importer;
import org.opends.server.backends.pluggable.BackendImpl.ReadOperation;
import org.opends.server.backends.pluggable.BackendImpl.Storage;
import org.opends.server.backends.pluggable.BackendImpl.StorageRuntimeException;
import org.opends.server.backends.pluggable.BackendImpl.TreeName;
import org.opends.server.backends.pluggable.BackendImpl.WriteOperation;
import org.opends.server.backends.pluggable.BackendImpl.WriteableStorage;
import org.opends.server.types.DN;

import com.persistit.Exchange;
import com.persistit.Key;
import com.persistit.Persistit;
import com.persistit.Transaction;
import com.persistit.Tree;
import com.persistit.TreeBuilder;
import com.persistit.Value;
import com.persistit.Volume;
import com.persistit.exception.PersistitException;
import com.persistit.exception.RollbackException;

@SuppressWarnings("javadoc")
public final class PersistItStorage implements Storage {
    private final class ImporterImpl implements Importer {
        private final Map<TreeName, Tree> trees = new HashMap<TreeName, Tree>();
        private final TreeBuilder importer = new TreeBuilder(db);
        private final Key importKey = new Key(db);
        private final Value importValue = new Value(db);

        @Override
        public void createTree(TreeName treeName) {
            try {
                // FIXME: how do we set the comparator?
                final Tree tree = getVolume(treeName).getTree(treeName.toString(), true);
                trees.put(treeName, tree);
            } catch (PersistitException e) {
                throw new StorageRuntimeException(e);
            }
        }

        @Override
        public void put(TreeName treeName, ByteSequence key, ByteSequence value) {
            try {
                final Tree tree = trees.get(treeName);
                byte[] keyBytes = key.toByteArray();
                importKey.clear().appendByteArray(keyBytes, 0, keyBytes.length);
                importValue.clear().putByteArray(value.toByteArray());
                importer.store(tree, importKey, importValue);
            } catch (Exception e) {
                throw new StorageRuntimeException(e);
            }
        }

        @Override
        public void close() {
            try {
                importer.merge();
            } catch (Exception e) {
                throw new StorageRuntimeException(e);
            } finally {
                PersistItStorage.this.close();
            }
        }
    }

    private final class StorageImpl implements WriteableStorage {
        private final Map<TreeName, Exchange> exchanges = new HashMap<TreeName, Exchange>();

        private void release() {
            for (Exchange ex : exchanges.values()) {
                db.releaseExchange(ex);
            }
        }

        private Exchange getExchange(TreeName treeName) throws PersistitException {
            Exchange exchange = exchanges.get(treeName);
            if (exchange == null) {
                exchange = getExchange0(treeName, false);
                exchanges.put(treeName, exchange);
            }
            return exchange;
        }

        @Override
        public ByteString get(TreeName treeName, ByteSequence key) {
            try {
                final Exchange ex = getExchange(treeName);
                ex.getKey().clear().append(key.toByteArray());
                ex.fetch();
                final Value value = ex.getValue();
                if (value.isDefined()) {
                    return ByteString.wrap(value.getByteArray());
                }
                return null;
            } catch (PersistitException e) {
                throw new StorageRuntimeException(e);
            }
        }

        @Override
        public ByteString getRMW(TreeName treeName, ByteSequence key) {
            return get(treeName, key);
        }

        @Override
        public void put(TreeName treeName, ByteSequence key, ByteSequence value) {
            try {
                final Exchange ex = getExchange(treeName);
                ex.getKey().clear().append(key.toByteArray());
                ex.getValue().clear().putByteArray(value.toByteArray());
                ex.store();
            } catch (Exception e) {
                throw new StorageRuntimeException(e);
            }
        }

        @Override
        public boolean putIfAbsent(TreeName treeName, ByteSequence key, ByteSequence value) {
            try {
                final Exchange ex = getExchange(treeName);
                ex.getKey().clear().append(key.toByteArray());
                ex.fetch();
                // FIXME poor man's CAS: this will not work under high volume,
                // but PersistIt does not provide APIs for this use case.
                if (ex.isValueDefined()) {
                    return false;
                }
                ex.getValue().clear().putByteArray(value.toByteArray());
                ex.store();
                return true;
            } catch (Exception e) {
                throw new StorageRuntimeException(e);
            }
        }

        @Override
        public boolean remove(TreeName treeName, ByteSequence key) {
            try {
                final Exchange ex = getExchange(treeName);
                ex.getKey().clear().append(key.toByteArray());
                return ex.remove();
            } catch (PersistitException e) {
                throw new StorageRuntimeException(e);
            }
        }

        @Override
        public Cursor openCursor(TreeName treeName) {
            try {
                return new CursorImpl(getExchange(treeName));
            } catch (PersistitException e) {
                throw new StorageRuntimeException(e);
            }
        }

        @Override
        public void openTree(TreeName treeName) {
            Exchange ex = null;
            try {
                ex = getExchange0(treeName, true);
            } catch (PersistitException e) {
                throw new StorageRuntimeException(e);
            } finally {
                db.releaseExchange(ex);
            }
        }
    }

    private final class CursorImpl implements Cursor {

        private final Exchange ex;
        private boolean useCurrentKeyForNext = false;

        public CursorImpl(Exchange exchange) {
            this.ex = exchange;
        }

        @Override
        public boolean positionToKey(ByteSequence key) {
            ex.getKey().clear().append(key.toByteArray());
            try {
                ex.fetch();
                useCurrentKeyForNext = ex.getValue().isDefined();
                return useCurrentKeyForNext;
            } catch (PersistitException e) {
                useCurrentKeyForNext = false;
                throw new StorageRuntimeException(e);
            }
        }

        @Override
        public boolean positionToKeyOrNext(ByteSequence key) {
            ex.getKey().clear().append(key.toByteArray());
            try {
                ex.fetch();
                if (ex.getValue().isDefined()) {
                    useCurrentKeyForNext = true;
                } else {
                    // provided key does not exist, look for next key
                    useCurrentKeyForNext = ex.next();
                }
                return useCurrentKeyForNext;
            } catch (PersistitException e) {
                useCurrentKeyForNext = false;
                throw new StorageRuntimeException(e);
            }
        }

        @Override
        public boolean positionToLastKey() {
            try {
                ex.getKey().to(Key.AFTER);
                useCurrentKeyForNext = ex.previous() && ex.getValue().isDefined();
                return useCurrentKeyForNext;
            } catch (PersistitException e) {
                useCurrentKeyForNext = false;
                throw new StorageRuntimeException(e);
            }
        }

        @Override
        public boolean next() {
            if (useCurrentKeyForNext) {
                useCurrentKeyForNext = false;
                return true;
            }
            try {
                return ex.next();
            } catch (PersistitException e) {
                throw new StorageRuntimeException(e);
            }
        }

        @Override
        public boolean previous() {
            try {
                return ex.previous();
            } catch (PersistitException e) {
                throw new StorageRuntimeException(e);
            }
        }

        @Override
        public ByteString getKey() {
            return ByteString.wrap(ex.getKey().decodeByteArray());
        }

        @Override
        public ByteString getValue() {
            return ByteString.wrap(ex.getValue().getByteArray());
        }

        @Override
        public void close() {
            // Exchange is released by StorageImpl.release()
            // once the Read/Write Operation is closed
        }
    }

    private final File backendDirectory;
    private final LocalDBBackendCfg config;
    private Persistit db;
    private final ConcurrentMap<TreeName, Volume> volumes = new ConcurrentHashMap<TreeName, Volume>();
    private Properties properties;

    public PersistItStorage(File backendDirectory, LocalDBBackendCfg config) {
        this.backendDirectory = backendDirectory;
        this.config = config;
    }

    private Volume getVolume(TreeName treeName) {
        return volumes.get(treeName.getSuffix());
    }

    @Override
    public void initialize(Map<String, String> options) {
        properties = new Properties();
        properties.setProperty("datapath", backendDirectory.toString());
        properties.setProperty("logpath", backendDirectory.toString());
        properties.setProperty("logfile", "${logpath}/dj_${timestamp}.log");
        properties.setProperty("buffer.count.16384", "64K");
        properties.setProperty("journalpath", "${datapath}/dj_journal");
        int i = 1;
        for (DN baseDN : config.getBaseDN()) {
            // TODO use VolumeSpecification  Configuration.setVolumeList()?
            properties.setProperty("volume." + i++,
                "${datapath}/" + toSuffixName(baseDN.toString())
                    + ",create,pageSize:16K"
                    + ",initialSize:50M"
                    + ",extensionSize:1M"
                    + ",maximumSize:10G");
        }

        if (options != null) {
            for (Entry<String, String> entry : options.entrySet()) {
                properties.setProperty(entry.getKey(), entry.getValue());
            }
        }
    }

    /**
     * Replace persistit reserved comma character with an underscore character.
     */
    public String toSuffixName(String prefix) {
        return prefix.replaceAll("[,=]", "_");
    }

    @Override
    public void open() {
        try {
            db = new Persistit(properties);
            db.initialize();
            for (DN baseDN : config.getBaseDN()) {
                final String volumeName = toSuffixName(baseDN.toString());
                final TreeName suffixName = TreeName.of(volumeName);
                volumes.put(suffixName, db.loadVolume(volumeName));
            }
        } catch (PersistitException e) {
            throw new StorageRuntimeException(e);
        }
    }

    @Override
    public void close() {
        if (db != null) {
            try {
                db.close();
                db = null;
            } catch (PersistitException e) {
                throw new IllegalStateException(e);
            }
        }
    }

    static void clearAndCreateDbDir(final File dbDir) {
        if (dbDir.exists()) {
            for (final File child : dbDir.listFiles()) {
                child.delete();
            }
        } else {
            dbDir.mkdirs();
        }
    }

    @Override
    public Importer startImport() {
        clearAndCreateDbDir(backendDirectory);
        open();
        return new ImporterImpl();
    }

    @Override
    public <T> T read(ReadOperation<T> operation) throws Exception {
        final Transaction txn = db.getTransaction();
        for (;;) {
            txn.begin();
            try {
                final StorageImpl storageImpl = new StorageImpl();
                try {
                    final T result = operation.run(storageImpl);
                    txn.commit();
                    return result;
                } catch (StorageRuntimeException e) {
                    throw (Exception) e.getCause();
                } finally {
                    storageImpl.release();
                }
            } catch (RollbackException e) {
                // retry
            } catch (Exception e) {
                txn.rollback();
                throw e;
            } finally {
                txn.end();
            }
        }
    }

    @Override
    public void write(WriteOperation operation) throws Exception {
        final Transaction txn = db.getTransaction();
        for (;;) {
            txn.begin();
            try {
                final StorageImpl storageImpl = new StorageImpl();
                try {
                    operation.run(storageImpl);
                    txn.commit();
                    return;
                } catch (StorageRuntimeException e) {
                    throw (Exception) e.getCause();
                } finally {
                    storageImpl.release();
                }
            } catch (RollbackException e) {
                // retry
            } catch (Exception e) {
                txn.rollback();
                throw e;
            } finally {
                txn.end();
            }
        }
    }

    @Override
    public Cursor openCursor(TreeName treeName) {
        try {
            return new CursorImpl(getExchange0(treeName, false));//FIXME JNR we must release the exchange
        } catch (PersistitException e) {
            throw new StorageRuntimeException(e);
        }
    }

    private Exchange getExchange0(TreeName treeName, boolean create) throws PersistitException {
        return db.getExchange(getVolume(treeName), treeName.toString(), create);
    }
}

 opendj3-server-dev/src/server/org/opends/server/backends/pluggable/AttributeIndex.java

@@ -49,7 +49,6 @@
import org.opends.server.backends.pluggable.BackendImpl.WriteOperation;
import org.opends.server.backends.pluggable.BackendImpl.WriteableStorage;
import org.opends.server.core.DirectoryServer;
import org.opends.server.monitors.DatabaseEnvironmentMonitor;
import org.opends.server.types.*;
import org.opends.server.util.StaticUtils;

@@ -716,11 +715,12 @@
        }
      }
    }
    removeIndexesForExtensibleMatchingRules(validRules, validIndexIds);
    removeIndexesForExtensibleMatchingRules(txn, validRules, validIndexIds);
  }

  /** Remove indexes which do not correspond to valid rules. */
  private void removeIndexesForExtensibleMatchingRules(Set<MatchingRule> validRules, Set<String> validIndexIds)
  private void removeIndexesForExtensibleMatchingRules(WriteableStorage txn, Set<MatchingRule> validRules,
      Set<String> validIndexIds)
  {
    final Set<MatchingRule> rulesToDelete = getCurrentExtensibleMatchingRules();
    rulesToDelete.removeAll(validRules);
@@ -746,7 +746,7 @@
            Index index = nameToIndexes.get(indexId);
            if (index != null)
            {
              entryContainer.deleteDatabase(index);
              entryContainer.deleteDatabase(txn, index);
              nameToIndexes.remove(index);
            }
          }
@@ -779,7 +779,7 @@
    Index index = nameToIndexes.get(indexId);
    if (!cfg.getIndexType().contains(indexType))
    {
      removeIndex(index, indexType);
      removeIndex(txn, index, indexType);
      return;
    }

@@ -811,7 +811,7 @@
    Index index = nameToIndexes.get(indexID);
    if (!cfg.getIndexType().contains(indexType))
    {
      removeIndex(index, indexType);
      removeIndex(txn, index, indexType);
      return;
    }

@@ -832,7 +832,7 @@
    }
  }

  private void removeIndex(Index index, IndexType indexType)
  private void removeIndex(WriteableStorage txn, Index index, IndexType indexType)
  {
    if (index != null)
    {
@@ -840,7 +840,7 @@
      try
      {
        nameToIndexes.remove(indexType.toString());
        entryContainer.deleteDatabase(index);
        entryContainer.deleteDatabase(txn, index);
      }
      finally
      {

 opendj3-server-dev/src/server/org/opends/server/backends/pluggable/BackendImpl.java

@@ -54,16 +54,12 @@
import org.opends.server.api.Backend;
import org.opends.server.api.DiskSpaceMonitorHandler;
import org.opends.server.api.MonitorProvider;
import org.opends.server.backends.pluggable.BackendImpl.WriteableStorage;
import org.opends.server.core.*;
import org.opends.server.extensions.DiskSpaceMonitor;
import org.opends.server.types.*;
import org.opends.server.util.RuntimeInformation;

import com.sleepycat.je.Durability;
import com.sleepycat.je.EnvironmentConfig;

import static com.sleepycat.je.EnvironmentConfig.*;

import static org.opends.messages.BackendMessages.*;
import static org.opends.messages.JebMessages.*;
import static org.opends.server.backends.jeb.ConfigurableEnvironment.*;
@@ -408,7 +404,7 @@

    if (mustOpenRootContainer())
    {
      rootContainer = initializeRootContainer(parseConfigEntry(cfg));
      rootContainer = initializeRootContainer();
    }

    // Preload the database cache.
@@ -417,7 +413,7 @@
    try
    {
      // Log an informational message about the number of entries.
      logger.info(NOTE_JEB_BACKEND_STARTED, cfg.getBackendId(), rootContainer.getEntryCount());
      logger.info(NOTE_JEB_BACKEND_STARTED, cfg.getBackendId(), getEntryCount());
    }
    catch (StorageRuntimeException e)
    {
@@ -640,7 +636,6 @@
        logger.traceException(e);
      }
    }

    return -1;
  }

@@ -988,7 +983,7 @@

      throw new NotImplementedException();
//      Importer importer = new Importer(importConfig, cfg, envConfig);
//      rootContainer = initializeRootContainer(envConfig);
//      rootContainer = initializeRootContainer();
//      return importer.processImport(rootContainer);
    }
    catch (ExecutionException execEx)
@@ -1005,10 +1000,10 @@
      logger.traceException(intEx);
      throw new DirectoryException(errorRC, ERR_INTERRUPTED_ERROR.get(intEx.getMessage()));
    }
    catch (JebException je)
    catch (StorageRuntimeException e)
    {
      logger.traceException(je);
      throw new DirectoryException(errorRC, je.getMessageObject());
      logger.traceException(e);
      throw new DirectoryException(errorRC, LocalizableMessage.raw(e.getMessage()));
    }
    catch (InitializationException ie)
    {
@@ -1093,12 +1088,6 @@
      logger.traceException(e);
      throw createDirectoryException(e);
    }
    catch (JebException e)
    {
      logger.traceException(e);
      throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
                                   e.getMessageObject());
    }
    finally
    {
      closeTemporaryRootContainer(openRootContainer);
@@ -1141,7 +1130,7 @@
      if (openRootContainer)
      {
        envConfig = getEnvConfigForImport();
        rootContainer = initializeRootContainer(envConfig);
        rootContainer = initializeRootContainer();
      }
      else
      {
@@ -1167,10 +1156,10 @@
      logger.traceException(ce);
      throw new DirectoryException(errorRC, ce.getMessageObject());
    }
    catch (JebException e)
    catch (StorageRuntimeException e)
    {
      logger.traceException(e);
      throw new DirectoryException(errorRC, e.getMessageObject());
      throw new DirectoryException(errorRC, LocalizableMessage.raw(e.getMessage()));
    }
    catch (InitializationException e)
    {
@@ -1270,50 +1259,52 @@

  /** {@inheritDoc} */
  @Override
  public ConfigChangeResult applyConfigurationChange(LocalDBBackendCfg newCfg)
  public ConfigChangeResult applyConfigurationChange(final LocalDBBackendCfg newCfg)
  {
    ResultCode resultCode = ResultCode.SUCCESS;
    ArrayList<LocalizableMessage> messages = new ArrayList<LocalizableMessage>();

    final ConfigChangeResult ccr = new ConfigChangeResult();
    try
    {
      if(rootContainer != null)
      {
        SortedSet<DN> newBaseDNs = newCfg.getBaseDN();
        DN[] newBaseDNsArray = newBaseDNs.toArray(new DN[newBaseDNs.size()]);

        // Check for changes to the base DNs.
        removeDeletedBaseDNs(newBaseDNs);
        ConfigChangeResult failure = createNewBaseDNs(newBaseDNsArray, messages);
        if (failure != null)
        rootContainer.getStorage().write(new WriteOperation()
        {
          return failure;
        }
          @Override
          public void run(WriteableStorage txn) throws Exception
          {
            SortedSet<DN> newBaseDNs = newCfg.getBaseDN();
            DN[] newBaseDNsArray = newBaseDNs.toArray(new DN[newBaseDNs.size()]);

        baseDNs = newBaseDNsArray;
            // Check for changes to the base DNs.
            removeDeletedBaseDNs(newBaseDNs, txn);
            if (!createNewBaseDNs(newBaseDNsArray, ccr, txn))
            {
              return;
            }

            baseDNs = newBaseDNsArray;

            if(cfg.getDiskFullThreshold() != newCfg.getDiskFullThreshold() ||
                cfg.getDiskLowThreshold() != newCfg.getDiskLowThreshold())
            {
              diskMonitor.setFullThreshold(newCfg.getDiskFullThreshold());
              diskMonitor.setLowThreshold(newCfg.getDiskLowThreshold());
            }
            
            // Put the new configuration in place.
            cfg = newCfg;
          }
        });
      }

      if(cfg.getDiskFullThreshold() != newCfg.getDiskFullThreshold() ||
          cfg.getDiskLowThreshold() != newCfg.getDiskLowThreshold())
      {
        diskMonitor.setFullThreshold(newCfg.getDiskFullThreshold());
        diskMonitor.setLowThreshold(newCfg.getDiskLowThreshold());
      }

      // Put the new configuration in place.
      this.cfg = newCfg;
    }
    catch (Exception e)
    {
      messages.add(LocalizableMessage.raw(stackTraceToSingleLineString(e)));
      return new ConfigChangeResult(
          DirectoryServer.getServerErrorResultCode(), false, messages);
      ccr.setResultCode(DirectoryServer.getServerErrorResultCode());
      ccr.addMessage(LocalizableMessage.raw(stackTraceToSingleLineString(e)));
    }

    return new ConfigChangeResult(resultCode, false, messages);
    return ccr;
  }

  private void removeDeletedBaseDNs(SortedSet<DN> newBaseDNs) throws DirectoryException
  private void removeDeletedBaseDNs(SortedSet<DN> newBaseDNs, WriteableStorage txn) throws DirectoryException
  {
    for (DN baseDN : cfg.getBaseDN())
    {
@@ -1323,12 +1314,12 @@
        DirectoryServer.deregisterBaseDN(baseDN);
        EntryContainer ec = rootContainer.unregisterEntryContainer(baseDN);
        ec.close();
        ec.delete();
        ec.delete(txn);
      }
    }
  }

  private ConfigChangeResult createNewBaseDNs(DN[] newBaseDNsArray, ArrayList<LocalizableMessage> messages)
  private boolean createNewBaseDNs(DN[] newBaseDNsArray, ConfigChangeResult ccr, WriteableStorage txn)
  {
    for (DN baseDN : newBaseDNsArray)
    {
@@ -1337,7 +1328,7 @@
        try
        {
          // The base DN was added.
          EntryContainer ec = rootContainer.openEntryContainer(baseDN, null);
          EntryContainer ec = rootContainer.openEntryContainer(baseDN, null, txn);
          rootContainer.registerEntryContainer(baseDN, ec);
          DirectoryServer.registerBaseDN(baseDN, this, false);
        }
@@ -1345,13 +1336,13 @@
        {
          logger.traceException(e);

          ResultCode resultCode = DirectoryServer.getServerErrorResultCode();
          messages.add(ERR_BACKEND_CANNOT_REGISTER_BASEDN.get(baseDN, e));
          return new ConfigChangeResult(resultCode, false, messages);
          ccr.setResultCode(DirectoryServer.getServerErrorResultCode());
          ccr.addMessage(ERR_BACKEND_CANNOT_REGISTER_BASEDN.get(baseDN, e));
          return false;
        }
      }
    }
    return null;
    return true;
  }

  /**
@@ -1380,15 +1371,7 @@
  public RootContainer getReadOnlyRootContainer()
      throws ConfigException, InitializationException
  {
    EnvironmentConfig envConfig = parseConfigEntry(cfg);

    envConfig.setReadOnly(true);
    envConfig.setAllowCreate(false);
    envConfig.setTransactional(false);
    envConfig.setConfigParam(ENV_IS_LOCKING, "true");
    envConfig.setConfigParam(ENV_RUN_CHECKPOINTER, "true");

    return initializeRootContainer(envConfig);
    return initializeRootContainer();
  }

  /**
@@ -1397,11 +1380,9 @@
   *
   * @throws  ConfigException  If an unrecoverable problem arises in the
   *                           process of performing the initialization.
   *
   * @throws  JebException     If an error occurs while removing the data.
   * @throws  StorageRuntimeException If an error occurs while removing the data.
   */
  public void clearBackend()
      throws ConfigException, JebException
  public void clearBackend() throws ConfigException, StorageRuntimeException
  {
    // Determine the backend database directory.
    File parentDirectory = getFileForPath(cfg.getDBDirectory());
@@ -1419,7 +1400,7 @@
   */
  DirectoryException createDirectoryException(StorageRuntimeException e)
  {
    if (true) {
    if (true) { // FIXME JNR
      throw new NotImplementedException();
    }
    if (/*e instanceof EnvironmentFailureException && */ !rootContainer.isValid()) {
@@ -1465,12 +1446,12 @@
    return cfg.dn();
  }

  private RootContainer initializeRootContainer(EnvironmentConfig envConfig)
  private RootContainer initializeRootContainer()
          throws ConfigException, InitializationException {
    // Open the database environment
    try {
      RootContainer rc = new RootContainer(this, cfg);
      rc.open(envConfig);
      rc.open();
      return rc;
    }
    catch (StorageRuntimeException e)

 opendj3-server-dev/src/server/org/opends/server/backends/pluggable/DN2URI.java

@@ -104,7 +104,6 @@
   * @throws StorageRuntimeException
   *           If an error occurs in the JE database.
   */
  @SuppressWarnings("unchecked")
  DN2URI(TreeName treeName, Storage storage, EntryContainer entryContainer)
      throws StorageRuntimeException
  {
@@ -502,7 +501,7 @@
  {
    if (containsReferrals == ConditionResult.UNDEFINED)
    {
      containsReferrals = containsReferrals(null);
      containsReferrals = containsReferrals(txn);
    }

    if (containsReferrals == ConditionResult.FALSE)

 opendj3-server-dev/src/server/org/opends/server/backends/pluggable/DatabaseContainer.java

@@ -211,7 +211,7 @@
   * @return The count of key/data pairs in the database.
   * @throws StorageRuntimeException If an error occurs in the JE operation.
   */
  public long getRecordCount() throws StorageRuntimeException
  public long getRecordCount(ReadableStorage txn) throws StorageRuntimeException
  {
    long count = treeName.count();
    if (logger.isTraceEnabled())

 opendj3-server-dev/src/server/org/opends/server/backends/pluggable/DatabaseEnvironmentMonitor.java

New file
@@ -0,0 +1,394 @@
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License, Version 1.0 only
 * (the "License").  You may not use this file except in compliance
 * with the License.
 *
 * You can obtain a copy of the license at legal-notices/CDDLv1_0.txt
 * or http://forgerock.org/license/CDDLv1.0.html.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at legal-notices/CDDLv1_0.txt.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information:
 *      Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 *
 *
 *      Copyright 2006-2010 Sun Microsystems, Inc.
 *      Portions Copyright 2014 ForgeRock AS
 */
package org.opends.server.backends.pluggable;

import java.lang.reflect.Method;
import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;

import org.forgerock.i18n.LocalizableMessage;
import org.forgerock.i18n.slf4j.LocalizedLogger;
import org.forgerock.opendj.config.server.ConfigException;
import org.opends.server.admin.std.server.MonitorProviderCfg;
import org.opends.server.api.AttributeSyntax;
import org.opends.server.api.MonitorProvider;
import org.opends.server.core.DirectoryServer;
import org.opends.server.types.*;
import org.opends.server.util.TimeThread;

/**
 * A monitor provider for a Berkeley DB JE environment.
 * It uses reflection on the environment statistics object
 * so that we don't need to keep a list of all the stats.
 */
public class DatabaseEnvironmentMonitor
       extends MonitorProvider<MonitorProviderCfg>
{
  private static final LocalizedLogger logger = LocalizedLogger.getLoggerForThisClass();

  /** Represents the statistical information kept for each search filter. */
  private static class FilterStats implements Comparable<FilterStats>
  {
    private volatile LocalizableMessage failureReason = LocalizableMessage.EMPTY;
    private long maxMatchingEntries = -1;
    private final AtomicInteger hits = new AtomicInteger();

    @Override
    public int compareTo(FilterStats that) {
      return this.hits.get() - that.hits.get();
    }

    private void update(int hitCount, LocalizableMessage failureReason)
    {
      this.hits.getAndAdd(hitCount);
      this.failureReason = failureReason;
    }

    private void update(int hitCount, long matchingEntries)
    {
      this.hits.getAndAdd(hitCount);
      this.failureReason = LocalizableMessage.EMPTY;
      synchronized(this)
      {
        if(matchingEntries > maxMatchingEntries)
        {
          maxMatchingEntries = matchingEntries;
        }
      }
    }
  }

  /** The name of this monitor instance. */
  private String name;

  /** The root container to be monitored. */
  private RootContainer rootContainer;

  private int maxEntries = 1024;
  private boolean filterUseEnabled = false;
  private String startTimeStamp;
  private final HashMap<SearchFilter, FilterStats> filterToStats =
      new HashMap<SearchFilter, FilterStats>();
  private final AtomicInteger indexedSearchCount = new AtomicInteger();
  private final AtomicInteger unindexedSearchCount = new AtomicInteger();

  /**
   * Creates a new database environment monitor.
   * @param name The monitor instance name.
   * @param rootContainer A root container handle for the database to be
   * monitored.
   */
  public DatabaseEnvironmentMonitor(String name, RootContainer rootContainer)
  {
    this.name = name;
    this.rootContainer = rootContainer;
  }

  /** {@inheritDoc} */
  @Override
  public void initializeMonitorProvider(MonitorProviderCfg configuration)
       throws ConfigException, InitializationException
  {
  }

  /**
   * Retrieves the name of this monitor provider.  It should be unique among all
   * monitor providers, including all instances of the same monitor provider.
   *
   * @return The name of this monitor provider.
   */
  @Override
  public String getMonitorInstanceName()
  {
    return name;
  }

  /**
   * Creates monitor attribute values for a given JE statistics object,
   * using reflection to call all the getter methods of the statistics object.
   * The attribute type names of the created attribute values are derived from
   * the names of the getter methods.
   * @param monitorAttrs The monitor attribute values are inserted into this
   * attribute list.
   * @param stats The JE statistics object.
   * @param attrPrefix A common prefix for the attribute type names of the
   * monitor attribute values, to distinguish the attributes of one
   * type of statistical object from another, and to avoid attribute name
   * collisions.
   */
  private void addAttributesForStatsObject(ArrayList<Attribute> monitorAttrs,
                                           Object stats, String attrPrefix)
  {
    Class<?> c = stats.getClass();
    Method[] methods = c.getMethods();

    // Iterate through all the statistic class methods.
    for (Method method : methods)
    {
      // Invoke all the getters returning integer values.
      if (method.getName().startsWith("get"))
      {
        Class<?> returnType = method.getReturnType();
        if (returnType.equals(int.class) || returnType.equals(long.class))
        {
          AttributeSyntax<?> integerSyntax =
               DirectoryServer.getDefaultIntegerSyntax();

          // Remove the 'get' from the method name and add the prefix.
          String attrName = attrPrefix + method.getName().substring(3);

          try
          {
            // Read the statistic.
            Object statValue = method.invoke(stats);

            // Create an attribute from the statistic.
            AttributeType attrType =
                 DirectoryServer.getDefaultAttributeType(attrName, integerSyntax);
            monitorAttrs.add(Attributes.create(attrType, String.valueOf(statValue)));

          } catch (Exception e)
          {
            logger.traceException(e);
          }
        }
      }
    }
  }

  /**
   * Retrieves a set of attributes containing monitor data that should be
   * returned to the client if the corresponding monitor entry is requested.
   *
   * @return A set of attributes containing monitor data that should be
   *         returned to the client if the corresponding monitor entry is
   *         requested.
   */
  @Override
  public ArrayList<Attribute> getMonitorData()
  {
    ArrayList<Attribute> monitorAttrs = new ArrayList<Attribute>();

    AttributeBuilder needReindex = new AttributeBuilder("need-reindex");
    for(EntryContainer ec : rootContainer.getEntryContainers())
    {
      List<DatabaseContainer> databases = new ArrayList<DatabaseContainer>();
      ec.listDatabases(databases);
      for(DatabaseContainer dc : databases)
      {
        if(dc instanceof Index && !((Index)dc).isTrusted())
        {
          needReindex.add(dc.getName().toString());
        }
      }
    }
    if(needReindex.size() > 0)
    {
      monitorAttrs.add(needReindex.toAttribute());
    }

    if(filterUseEnabled)
    {
      monitorAttrs.add(Attributes.create("filter-use-startTime",
          startTimeStamp));
      AttributeBuilder builder = new AttributeBuilder("filter-use");

      StringBuilder stringBuilder = new StringBuilder();
      synchronized(filterToStats)
      {
        for(Map.Entry<SearchFilter, FilterStats> entry :
            filterToStats.entrySet())
        {
          entry.getKey().toString(stringBuilder);
          stringBuilder.append(" hits:");
          stringBuilder.append(entry.getValue().hits.get());
          stringBuilder.append(" maxmatches:");
          stringBuilder.append(entry.getValue().maxMatchingEntries);
          stringBuilder.append(" message:");
          stringBuilder.append(entry.getValue().failureReason);
          builder.add(stringBuilder.toString());
          stringBuilder.setLength(0);
        }
      }
      monitorAttrs.add(builder.toAttribute());
      monitorAttrs.add(Attributes.create("filter-use-indexed",
          String.valueOf(indexedSearchCount.get())));
      monitorAttrs.add(Attributes.create("filter-use-unindexed",
          String.valueOf(unindexedSearchCount.get())));
    }

    return monitorAttrs;
  }


  /**
   * Updates the index filter statistics with this latest search filter
   * and the reason why an index was not used.
   *
   * @param searchFilter The search filter that was evaluated.
   * @param failureMessage The reason why an index was not used.
   */
  public void updateStats(SearchFilter searchFilter, LocalizableMessage failureMessage)
  {
    if(!filterUseEnabled)
    {
      return;
    }

    FilterStats stats;
    synchronized(filterToStats)
    {
      stats = filterToStats.get(searchFilter);


      if(stats != null)
      {
        stats.update(1, failureMessage);
      }
      else
      {
        stats = new FilterStats();
        stats.update(1, failureMessage);
        removeLowestHit();
        filterToStats.put(searchFilter, stats);
      }
    }
  }

  /**
   * Updates the index filter statistics with this latest search filter
   * and the number of entries matched by the index lookup.
   *
   * @param searchFilter The search filter that was evaluated.
   * @param matchingEntries The number of entries matched by the successful
   *                        index lookup.
   */
  public void updateStats(SearchFilter searchFilter, long matchingEntries)
  {
    if(!filterUseEnabled)
    {
      return;
    }

    FilterStats stats;
    synchronized(filterToStats)
    {
      stats = filterToStats.get(searchFilter);


      if(stats != null)
      {
        stats.update(1, matchingEntries);
      }
      else
      {
        stats = new FilterStats();
        stats.update(1, matchingEntries);
        removeLowestHit();
        filterToStats.put(searchFilter, stats);
      }
    }
  }

  /**
   * Enable or disable index filter statistics gathering.
   *
   * @param enabled <code>true></code> to enable index filter statics gathering.
   */
  public void enableFilterUseStats(boolean enabled)
  {
    if(enabled && !filterUseEnabled)
    {
      startTimeStamp = TimeThread.getGMTTime();
      indexedSearchCount.set(0);
      unindexedSearchCount.set(0);
    }
    else if(!enabled)
    {
      filterToStats.clear();
    }
    filterUseEnabled = enabled;
  }

  /**
   * Indicates if index filter statistics gathering is enabled.
   *
   * @return <code>true</code> If index filter statistics gathering is enabled.
   */
  public boolean isFilterUseEnabled()
  {
    return filterUseEnabled;
  }

  /**
   * Sets the maximum number of search filters statistics entries to keep
   * before ones with the least hits will be removed.
   *
   * @param maxEntries The maximum number of search filters statistics
   * entries to keep
   */
  public void setMaxEntries(int maxEntries) {
    this.maxEntries = maxEntries;
  }

  /**
   * Updates the statistics counter to include an indexed search.
   */
  public void updateIndexedSearchCount()
  {
    indexedSearchCount.getAndIncrement();
  }

  /**
   * Updates the statistics counter to include an unindexed search.
   */
  public void updateUnindexedSearchCount()
  {
    unindexedSearchCount.getAndIncrement();
  }

  private void removeLowestHit()
  {
    while(!filterToStats.isEmpty() && filterToStats.size() > maxEntries)
    {
      Iterator<Map.Entry<SearchFilter, FilterStats>> i =
          filterToStats.entrySet().iterator();
      Map.Entry<SearchFilter, FilterStats> lowest = i.next();
      Map.Entry<SearchFilter, FilterStats> entry;
      while(lowest.getValue().hits.get() > 1 && i.hasNext())
      {
        entry = i.next();
        if(entry.getValue().hits.get() < lowest.getValue().hits.get())
        {
          lowest = entry;
        }
      }

      filterToStats.remove(lowest.getKey());
    }
  }
}

 opendj3-server-dev/src/server/org/opends/server/backends/pluggable/EntryContainer.java

@@ -27,7 +27,13 @@
 */
package org.opends.server.backends.pluggable;

import java.util.*;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantReadWriteLock;

@@ -60,13 +66,38 @@
import org.opends.server.backends.pluggable.BackendImpl.TreeName;
import org.opends.server.backends.pluggable.BackendImpl.WriteOperation;
import org.opends.server.backends.pluggable.BackendImpl.WriteableStorage;
import org.opends.server.backends.pluggable.SuffixContainer;
import org.opends.server.controls.*;
import org.opends.server.core.*;
import org.opends.server.types.*;
import org.opends.server.controls.PagedResultsControl;
import org.opends.server.controls.ServerSideSortRequestControl;
import org.opends.server.controls.ServerSideSortResponseControl;
import org.opends.server.controls.SubtreeDeleteControl;
import org.opends.server.controls.VLVRequestControl;
import org.opends.server.core.AddOperation;
import org.opends.server.core.DeleteOperation;
import org.opends.server.core.DirectoryServer;
import org.opends.server.core.ModifyDNOperation;
import org.opends.server.core.ModifyOperation;
import org.opends.server.core.SearchOperation;
import org.opends.server.types.Attribute;
import org.opends.server.types.AttributeType;
import org.opends.server.types.Attributes;
import org.opends.server.types.CanceledOperationException;
import org.opends.server.types.ConfigChangeResult;
import org.opends.server.types.Control;
import org.opends.server.types.DN;
import org.opends.server.types.DirectoryException;
import org.opends.server.types.Entry;
import org.opends.server.types.Modification;
import org.opends.server.types.Operation;
import org.opends.server.types.Privilege;
import org.opends.server.types.RDN;
import org.opends.server.types.SearchFilter;
import org.opends.server.types.SortKey;
import org.opends.server.types.VirtualAttributeRule;
import org.opends.server.util.ServerConstants;
import org.opends.server.util.StaticUtils;

import com.sleepycat.je.TransactionConfig;

import static org.opends.messages.JebMessages.*;
import static org.opends.server.backends.pluggable.JebFormat.*;
import static org.opends.server.core.DirectoryServer.*;
@@ -156,6 +187,8 @@
    {
      try
      {
        // FIXME this should be a read operation, but I cannot change it
        // because of AttributeIndex ctor.
        storage.write(new WriteOperation()
        {
          @Override
@@ -216,30 +249,35 @@

    /** {@inheritDoc} */
    @Override
    public ConfigChangeResult applyConfigurationDelete(LocalDBIndexCfg cfg)
    public ConfigChangeResult applyConfigurationDelete(final LocalDBIndexCfg cfg)
    {
      boolean adminActionRequired = false;
      ArrayList<LocalizableMessage> messages = new ArrayList<LocalizableMessage>();
      final ConfigChangeResult ccr = new ConfigChangeResult();

      exclusiveLock.lock();
      try
      {
        AttributeIndex index = attrIndexMap.get(cfg.getAttribute());
        deleteAttributeIndex(index);
        attrIndexMap.remove(cfg.getAttribute());
        storage.write(new WriteOperation()
        {
          @Override
          public void run(WriteableStorage txn) throws Exception
          {
            AttributeIndex index = attrIndexMap.get(cfg.getAttribute());
            deleteAttributeIndex(txn, index, ccr);
            attrIndexMap.remove(cfg.getAttribute());
          }
        });
      }
      catch(StorageRuntimeException de)
      catch (Exception de)
      {
        messages.add(LocalizableMessage.raw(StaticUtils.stackTraceToSingleLineString(de)));
        return new ConfigChangeResult(
            DirectoryServer.getServerErrorResultCode(), adminActionRequired, messages);
        ccr.setResultCode(getServerErrorResultCode());
        ccr.addMessage(LocalizableMessage.raw(StaticUtils.stackTraceToSingleLineString(de)));
      }
      finally
      {
        exclusiveLock.unlock();
      }

      return new ConfigChangeResult(ResultCode.SUCCESS, adminActionRequired, messages);
      return ccr;
    }
  }

@@ -354,31 +392,33 @@

    /** {@inheritDoc} */
    @Override
    public ConfigChangeResult applyConfigurationDelete(LocalDBVLVIndexCfg cfg)
    public ConfigChangeResult applyConfigurationDelete(final LocalDBVLVIndexCfg cfg)
    {
      boolean adminActionRequired = false;
      List<LocalizableMessage> messages = new ArrayList<LocalizableMessage>();

      final ConfigChangeResult ccr = new ConfigChangeResult();
      exclusiveLock.lock();
      try
      {
        VLVIndex vlvIndex =
          vlvIndexMap.get(cfg.getName().toLowerCase());
        deleteDatabase(vlvIndex);
        vlvIndexMap.remove(cfg.getName());
        storage.write(new WriteOperation()
        {
          @Override
          public void run(WriteableStorage txn) throws Exception
          {
            VLVIndex vlvIndex = vlvIndexMap.get(cfg.getName().toLowerCase());
            deleteDatabase(txn, vlvIndex);
            vlvIndexMap.remove(cfg.getName());
          }
        });
      }
      catch(StorageRuntimeException de)
      catch (Exception e)
      {
        messages.add(LocalizableMessage.raw(StaticUtils.stackTraceToSingleLineString(de)));
        return new ConfigChangeResult(
            DirectoryServer.getServerErrorResultCode(), adminActionRequired, messages);
        ccr.setResultCode(getServerErrorResultCode());
        ccr.addMessage(LocalizableMessage.raw(StaticUtils.stackTraceToSingleLineString(e)));
      }
      finally
      {
        exclusiveLock.unlock();
      }

      return new ConfigChangeResult(ResultCode.SUCCESS, adminActionRequired, messages);
      return ccr;
    }

  }
@@ -403,7 +443,7 @@
   * @param rootContainer The root container this entry container is in.
   * @throws ConfigException if a configuration related error occurs.
   */
  public EntryContainer(DN baseDN, String databasePrefix, Backend<?> backend,
  public EntryContainer(DN baseDN, TreeName databasePrefix, Backend<?> backend,
      LocalDBBackendCfg config, Storage env, RootContainer rootContainer)
          throws ConfigException
  {
@@ -412,8 +452,7 @@
    this.config = config;
    this.storage = env;
    this.rootContainer = rootContainer;

    this.databasePrefix = preparePrefix(databasePrefix);
    this.databasePrefix = databasePrefix;

    config.addLocalDBChangeListener(this);

@@ -1316,14 +1355,12 @@
    boolean manageDsaIT = isManageDsaITOperation(searchOperation);
    boolean continueSearch = true;

    // Set the starting value.
    EntryID begin = null;
    if (pageRequest != null && pageRequest.getCookie().length() != 0)
    {
      // The cookie contains the ID of the next entry to be returned.
      try
      {
        begin = new EntryID(pageRequest.getCookie());
        new EntryID(pageRequest.getCookie());
      }
      catch (Exception e)
      {
@@ -1354,10 +1391,8 @@
    // Iterate through the index candidates.
    if (continueSearch)
    {
      for (Iterator<EntryID> it = entryIDList.iterator(begin); it.hasNext();)
      for (EntryID id : entryIDList)
      {
        final EntryID id = it.next();

        Entry entry;
        try
        {
@@ -1556,7 +1591,7 @@
                EntryID nodeID = dn2id.get(txn, dn, false);
                if (nodeID == null)
                {
                  throw new JebException(ERR_JEB_MISSING_DN2ID_RECORD.get(dn));
                  throw new StorageRuntimeException(ERR_JEB_MISSING_DN2ID_RECORD.get(dn).toString());
                }

                // Insert into id2subtree for this node.
@@ -1816,7 +1851,7 @@
      DN targetDN,
      ByteSequence leafDNKey,
      EntryID leafID)
  throws StorageRuntimeException, DirectoryException, JebException
  throws StorageRuntimeException, DirectoryException
  {
    if(leafID == null || leafDNKey == null)
    {
@@ -1884,7 +1919,7 @@
      EntryID parentID = dn2id.get(txn, parentDN, false);
      if (parentID == null)
      {
        throw new JebException(ERR_JEB_MISSING_DN2ID_RECORD.get(parentDN));
        throw new StorageRuntimeException(ERR_JEB_MISSING_DN2ID_RECORD.get(parentDN).toString());
      }

      ByteString parentIDBytes = parentID.toByteString();
@@ -2747,14 +2782,12 @@
   * @return The number of entries stored in this entry container.
   * @throws StorageRuntimeException If an error occurs in the JE database.
   */
  @Override
  public long getEntryCount() throws StorageRuntimeException
  public long getEntryCount(ReadableStorage txn) throws StorageRuntimeException
  {
    EntryID entryID = dn2id.get(null, baseDN, false);
    final EntryID entryID = dn2id.get(txn, baseDN, false);
    if (entryID != null)
    {
      EntryIDSet entryIDSet = id2subtree.readKey(entryID.toByteString(), null);

      final EntryIDSet entryIDSet = id2subtree.readKey(entryID.toByteString(), txn);
      long count = entryIDSet.size();
      if(count != Long.MAX_VALUE)
      {
@@ -2764,7 +2797,7 @@
      else
      {
        // The count is not maintained. Fall back to the slow method
        return id2entry.getRecordCount();
        return id2entry.getRecordCount(txn);
      }
    }
    else
@@ -2910,36 +2943,14 @@
   * @throws StorageRuntimeException If an error occurs while removing the entry
   *                           container.
   */
  public void delete() throws StorageRuntimeException
  public void delete(WriteableStorage txn) throws StorageRuntimeException
  {
    List<DatabaseContainer> databases = new ArrayList<DatabaseContainer>();
    listDatabases(databases);

    if(storage.getConfig().getTransactional())
    for (DatabaseContainer db : databases)
    {
      Transaction txn = beginTransaction();

      try
      {
        for(DatabaseContainer db : databases)
        {
          storage.removeDatabase(txn, db.getName());
        }

        transactionCommit(txn);
      }
      catch(StorageRuntimeException de)
      {
        transactionAbort(txn);
        throw de;
      }
    }
    else
    {
      for(DatabaseContainer db : databases)
      {
        storage.removeDatabase(null, db.getName());
      }
      storage.removeDatabase(txn, db.getName());
    }
  }

@@ -2950,8 +2961,7 @@
   * @throws StorageRuntimeException If an error occurs while attempting to delete the
   * database.
   */
  public void deleteDatabase(DatabaseContainer database)
  throws StorageRuntimeException
  public void deleteDatabase(WriteableStorage txn, DatabaseContainer database) throws StorageRuntimeException
  {
    if(database == state)
    {
@@ -2960,31 +2970,10 @@
    }

    database.close();
    if(storage.getConfig().getTransactional())
    storage.removeDatabase(txn, database.getName());
    if(database instanceof Index)
    {
      Transaction txn = beginTransaction();
      try
      {
        storage.removeDatabase(txn, database.getName());
        if(database instanceof Index)
        {
          state.removeIndexTrustState(txn, database);
        }
        transactionCommit(txn);
      }
      catch(StorageRuntimeException de)
      {
        transactionAbort(txn);
        throw de;
      }
    }
    else
    {
      storage.removeDatabase(null, database.getName());
      if(database instanceof Index)
      {
        state.removeIndexTrustState(null, database);
      }
      state.removeIndexTrustState(txn, database);
    }
  }

@@ -2995,31 +2984,14 @@
   * @throws StorageRuntimeException If an JE database error occurs while attempting
   * to delete the index.
   */
  private void deleteAttributeIndex(AttributeIndex attributeIndex)
  private void deleteAttributeIndex(WriteableStorage txn, AttributeIndex attributeIndex, ConfigChangeResult ccr)
      throws StorageRuntimeException
  {
    attributeIndex.close();
    Transaction txn = storage.getConfig().getTransactional()
      ? beginTransaction() : null;
    try
    for (Index index : attributeIndex.getAllIndexes())
    {
      for (Index index : attributeIndex.getAllIndexes())
      {
        storage.removeDatabase(txn, index.getName());
        state.removeIndexTrustState(txn, index);
      }
      if (txn != null)
      {
        transactionCommit(txn);
      }
    }
    catch(StorageRuntimeException de)
    {
      if (txn != null)
      {
        transactionAbort(txn);
      }
      throw de;
      storage.removeDatabase(txn, index.getName());
      state.removeIndexTrustState(txn, index);
    }
  }

@@ -3041,16 +3013,13 @@
   *
   * @param newDatabasePrefix The new database prefix to use.
   * @throws StorageRuntimeException If an error occurs in the JE database.
   * @throws JebException If an error occurs in the JE backend.
   */
  public void setDatabasePrefix(String newDatabasePrefix)
  throws StorageRuntimeException, JebException

  public void setDatabasePrefix(TreeName newDatabasePrefix) throws StorageRuntimeException, StorageRuntimeException
  {
    List<DatabaseContainer> databases = new ArrayList<DatabaseContainer>();
    final List<DatabaseContainer> databases = new ArrayList<DatabaseContainer>();
    listDatabases(databases);

    TreeName newDbPrefix = preparePrefix(newDatabasePrefix);
    final TreeName newDbPrefix = newDatabasePrefix;

    // close the containers.
    for(DatabaseContainer db : databases)
@@ -3060,11 +3029,10 @@

    try
    {
      if(storage.getConfig().getTransactional())
      storage.write(new WriteOperation()
      {
        //Rename under transaction
        Transaction txn = beginTransaction();
        try
        @Override
        public void run(WriteableStorage txn) throws Exception
        {
          for(DatabaseContainer db : databases)
          {
@@ -3072,52 +3040,60 @@
            String newName = oldName.replace(databasePrefix, newDbPrefix);
            storage.renameDatabase(txn, oldName, newName);
          }

          transactionCommit(txn);

          for(DatabaseContainer db : databases)
        }
      });
      storage.write(new WriteOperation()
      {
        @Override
        public void run(WriteableStorage txn) throws Exception
        {
          for (DatabaseContainer db : databases)
          {
            TreeName oldName = db.getName();
            String newName = oldName.replace(databasePrefix, newDbPrefix);
            db.setName(newName);
          }

          // Update the prefix.
          this.databasePrefix = newDbPrefix;
          databasePrefix = newDbPrefix;
        }
        catch(Exception e)
        {
          transactionAbort(txn);

          String msg = e.getMessage();
          if (msg == null)
          {
            msg = stackTraceToSingleLineString(e);
          }
          LocalizableMessage message = ERR_JEB_UNCHECKED_EXCEPTION.get(msg);
          throw new JebException(message, e);
        }
      }
      else
      });
    }
    catch (Exception e)
    {
      String msg = e.getMessage();
      if (msg == null)
      {
        for(DatabaseContainer db : databases)
        {
          TreeName oldName = db.getName();
          String newName = oldName.replace(databasePrefix, newDbPrefix);
          storage.renameDatabase(txn, oldName, newName);
          db.setName(newName);
        }

        // Update the prefix.
        this.databasePrefix = newDbPrefix;
        msg = stackTraceToSingleLineString(e);
      }
      LocalizableMessage message = ERR_JEB_UNCHECKED_EXCEPTION.get(msg);
      throw new StorageRuntimeException(message.toString(), e);
    }
    finally
    {
      // Open the containers backup.
      for(DatabaseContainer db : databases)
      try
      {
        db.open(txn);
        storage.write(new WriteOperation()
        {
          @Override
          public void run(WriteableStorage txn) throws Exception
          {
            // Open the containers backup.
            for(DatabaseContainer db : databases)
            {
              db.open(txn);
            }
          }
        });
      }
      catch (Exception e)
      {
        String msg = e.getMessage();
        if (msg == null)
        {
          msg = stackTraceToSingleLineString(e);
        }
        LocalizableMessage message = ERR_JEB_UNCHECKED_EXCEPTION.get(msg);
        throw new StorageRuntimeException(message.toString(), e);
      }
    }
  }
@@ -3232,19 +3208,6 @@
  }

  /**
   * Get the environment config of the JE environment used in this entry
   * container.
   *
   * @return The environment config of the JE environment.
   * @throws StorageRuntimeException If an error occurs while retrieving the
   *                           configuration object.
   */
  public EnvironmentConfig getEnvironmentConfig() throws StorageRuntimeException
  {
    return storage.getConfig();
  }

  /**
   * Clear the contents of this entry container.
   *
   * @return The number of records deleted.
@@ -3253,7 +3216,28 @@
   */
  public long clear() throws StorageRuntimeException
  {
    List<DatabaseContainer> databases = new ArrayList<DatabaseContainer>();
    final AtomicLong count = new AtomicLong();
    try
    {
      storage.write(new WriteOperation()
      {
        @Override
        public void run(WriteableStorage txn) throws Exception
        {
          count.set(clear0(txn));
        }
      });
      return count.get();
    }
    catch (Exception e)
    {
      throw new StorageRuntimeException(e);
    }
  }

  private long clear0(WriteableStorage txn) throws StorageRuntimeException
  {
    final List<DatabaseContainer> databases = new ArrayList<DatabaseContainer>();
    listDatabases(databases);
    long count = 0;

@@ -3263,31 +3247,9 @@
    }
    try
    {
      if(storage.getConfig().getTransactional())
      for (DatabaseContainer db : databases)
      {
        Transaction txn = beginTransaction();

        try
        {
          for(DatabaseContainer db : databases)
          {
            count += storage.truncateDatabase(txn, db.getName(), true);
          }

          transactionCommit(txn);
        }
        catch(StorageRuntimeException de)
        {
          transactionAbort(txn);
          throw de;
        }
      }
      else
      {
        for(DatabaseContainer db : databases)
        {
          count += storage.truncateDatabase(null, db.getName(), true);
        }
        count += storage.truncateDatabase(txn, db.getName(), true);
      }
    }
    finally
@@ -3297,39 +3259,11 @@
        db.open(txn);
      }

      Transaction txn = null;
      try
      for (DatabaseContainer db : databases)
      {
        if(storage.getConfig().getTransactional()) {
          txn = beginTransaction();
        }
        for(DatabaseContainer db : databases)
        if (db instanceof Index)
        {
          if (db instanceof Index)
          {
            Index index = (Index)db;
            index.setTrusted(txn, true);
          }
        }
        if(storage.getConfig().getTransactional()) {
          transactionCommit(txn);
        }
      }
      catch(Exception de)
      {
        logger.traceException(de);

        // This is mainly used during the unit tests, so it's not essential.
        try
        {
          if (txn != null)
          {
            transactionAbort(txn);
          }
        }
        catch (Exception e)
        {
          logger.traceException(de);
          ((Index) db).setTrusted(txn, true);
        }
      }
    }
@@ -3343,34 +3277,30 @@
   * @param database The database to clear.
   * @throws StorageRuntimeException if a JE database error occurs.
   */
  public void clearDatabase(DatabaseContainer database)
  throws StorageRuntimeException
  public void clearDatabase(final DatabaseContainer database) throws StorageRuntimeException
  {
    database.close();
    try
    {
      if(storage.getConfig().getTransactional())
      storage.write(new WriteOperation()
      {
        Transaction txn = beginTransaction();
        try
        @Override
        public void run(WriteableStorage txn) throws Exception
        {
          storage.removeDatabase(txn, database.getName());
          transactionCommit(txn);
          try
          {
            storage.removeDatabase(txn, database.getName());
          }
          finally
          {
            database.open(txn);
          }
        }
        catch(StorageRuntimeException de)
        {
          transactionAbort(txn);
          throw de;
        }
      }
      else
      {
        storage.removeDatabase(null, database.getName());
      }
      });
    }
    finally
    catch (Exception e)
    {
      database.open(txn);
      throw new StorageRuntimeException(e);
    }
    if(logger.isTraceEnabled())
    {
@@ -3507,31 +3437,6 @@
    return baseEntry;
  }


  /**
   * Transform a database prefix string to one usable by the DB.
   * @param databasePrefix the database prefix
   * @return a new string when non letter or digit characters
   *         have been replaced with underscore
   */
  private TreeName preparePrefix(String databasePrefix)
  {
    StringBuilder builder = new StringBuilder(databasePrefix.length());
    for (int i = 0; i < databasePrefix.length(); i++)
    {
      char ch = databasePrefix.charAt(i);
      if (Character.isLetterOrDigit(ch))
      {
        builder.append(ch);
      }
      else
      {
        builder.append('_');
      }
    }
    return TreeName.of(builder.toString());
  }

  /** Get the exclusive lock. */
  public void lock() {
    exclusiveLock.lock();

 opendj3-server-dev/src/server/org/opends/server/backends/pluggable/EntryIDSetSorter.java

@@ -35,7 +35,6 @@
import org.forgerock.opendj.ldap.ByteString;
import org.forgerock.opendj.ldap.ResultCode;
import org.forgerock.opendj.ldap.SearchScope;
import org.opends.server.backends.pluggable.SuffixContainer;
import org.opends.server.controls.VLVRequestControl;
import org.opends.server.controls.VLVResponseControl;
import org.opends.server.core.DirectoryServer;
@@ -68,7 +67,7 @@
   *
   * @throws  DirectoryException  If an error occurs while performing the sort.
   */
  public static EntryIDSet sort(SuffixContainer suffixContainer,
  public static EntryIDSet sort(EntryContainer suffixContainer,
                                EntryIDSet entryIDSet,
                                SearchOperation searchOperation,
                                SortOrder sortOrder,

 opendj3-server-dev/src/server/org/opends/server/backends/pluggable/EnvManager.java

@@ -25,14 +25,16 @@
 *      Portions Copyright 2014 ForgeRock AS
 */
package org.opends.server.backends.pluggable;
import org.forgerock.i18n.LocalizableMessage;

import org.forgerock.i18n.slf4j.LocalizedLogger;
import static org.opends.messages.JebMessages.*;

import java.io.File;
import java.io.FilenameFilter;

import org.forgerock.i18n.LocalizableMessage;
import org.forgerock.i18n.slf4j.LocalizedLogger;
import org.opends.server.backends.pluggable.BackendImpl.StorageRuntimeException;

import static org.opends.messages.JebMessages.*;

/**
 * A singleton class to manage the life-cycle of a JE database environment.
 */
@@ -40,10 +42,7 @@
{
  private static final LocalizedLogger logger = LocalizedLogger.getLoggerForThisClass();


  /**
   * A filename filter to match all kinds of JE files.
   */
  /** A filename filter to match all kinds of JE files. */
  private static final FilenameFilter jeAllFilesFilter;

  static
@@ -53,6 +52,7 @@
    // here but is not public.
    jeAllFilesFilter = new FilenameFilter()
    {
      @Override
      public boolean accept(File d, String name)
      {
        return name.endsWith(".jdb") ||
@@ -68,10 +68,9 @@
   * The environment must not be open.
   *
   * @param homeDir The backend home directory.
   * @throws JebException If an error occurs in the JE backend.
   * @throws StorageRuntimeException If an error occurs in the JE backend.
   */
  public static void createHomeDir(String homeDir)
       throws JebException
  public static void createHomeDir(String homeDir) throws StorageRuntimeException
  {
    File dir = new File(homeDir);

@@ -80,7 +79,7 @@
      if (!dir.isDirectory())
      {
        LocalizableMessage message = ERR_JEB_DIRECTORY_INVALID.get(homeDir);
        throw new JebException(message);
        throw new StorageRuntimeException(message.toString());
      }
      removeFiles(homeDir);
    }
@@ -94,7 +93,7 @@
      {
        logger.traceException(e);
        LocalizableMessage message = ERR_JEB_CREATE_FAIL.get(e.getMessage());
        throw new JebException(message, e);
        throw new StorageRuntimeException(message.toString(), e);
      }
    }
  }
@@ -104,22 +103,22 @@
   * The environment must not be open.
   *
   * @param homeDir The backend home directory
   * @throws JebException If an error occurs in the JE backend or if the
   * specified home directory does not exist.
   * @throws StorageRuntimeException
   *           If an error occurs in the JE backend or if the specified home
   *           directory does not exist.
   */
  public static void removeFiles(String homeDir)
       throws JebException
  public static void removeFiles(String homeDir) throws StorageRuntimeException
  {
    File dir = new File(homeDir);
    if (!dir.exists())
    {
      LocalizableMessage message = ERR_JEB_DIRECTORY_DOES_NOT_EXIST.get(homeDir);
      throw new JebException(message);
      throw new StorageRuntimeException(message.toString());
    }
    if (!dir.isDirectory())
    {
      LocalizableMessage message = ERR_JEB_DIRECTORY_INVALID.get(homeDir);
      throw new JebException(message);
      throw new StorageRuntimeException(message.toString());
    }

    try
@@ -134,7 +133,7 @@
    {
      logger.traceException(e);
      LocalizableMessage message = ERR_JEB_REMOVE_FAIL.get(e.getMessage());
      throw new JebException(message, e);
      throw new StorageRuntimeException(message.toString(), e);
    }
  }


 opendj3-server-dev/src/server/org/opends/server/backends/pluggable/ExportJob.java

@@ -88,12 +88,11 @@
   * @param rootContainer The root container to export.
   * @throws StorageRuntimeException If an error occurs in the JE database.
   * @throws IOException If an I/O error occurs while writing an entry.
   * @throws JebException If an error occurs in the JE backend.
   * @throws LDIFException If an error occurs while trying to determine whether
   * to write an entry.
   */
  public void exportLDIF(RootContainer rootContainer)
       throws IOException, LDIFException, StorageRuntimeException, JebException
       throws IOException, LDIFException, StorageRuntimeException
  {
    List<DN> includeBranches = exportConfig.getIncludeBranches();
    DN baseDN;

 opendj3-server-dev/src/server/org/opends/server/backends/pluggable/Index.java

@@ -136,7 +136,7 @@
    this.maintainCount = maintainCount;

    this.state = state;
    this.trusted = state.getIndexTrustState(null, this);
    this.trusted = state.getIndexTrustState(txn, this);
    if (!trusted && entryContainer.getHighestEntryID(txn).longValue() == 0)
    {
      // If there are no entries in the entry container then there

 opendj3-server-dev/src/server/org/opends/server/backends/pluggable/IndexFilter.java

@@ -33,7 +33,6 @@

import org.opends.server.backends.pluggable.AttributeIndex.IndexFilterType;
import org.opends.server.core.SearchOperation;
import org.opends.server.monitors.DatabaseEnvironmentMonitor;
import org.opends.server.types.AttributeType;
import org.opends.server.types.FilterType;
import org.opends.server.types.SearchFilter;

 opendj3-server-dev/src/server/org/opends/server/backends/pluggable/JebException.java

File was deleted

 opendj3-server-dev/src/server/org/opends/server/backends/pluggable/NullIndex.java

@@ -251,7 +251,7 @@

  /** {@inheritDoc} */
  @Override
  public long getRecordCount() throws StorageRuntimeException
  public long getRecordCount(ReadableStorage txn) throws StorageRuntimeException
  {
    return 0;
  }

 opendj3-server-dev/src/server/org/opends/server/backends/pluggable/RootContainer.java

@@ -34,16 +34,19 @@
import org.forgerock.i18n.LocalizableMessage;
import org.forgerock.i18n.slf4j.LocalizedLogger;
import org.forgerock.opendj.config.server.ConfigException;
import org.forgerock.opendj.ldap.ResultCode;
import org.opends.server.admin.server.ConfigurationChangeListener;
import org.opends.server.admin.std.server.LocalDBBackendCfg;
import org.opends.server.api.Backend;
import org.opends.server.backends.pluggable.BackendImpl.Storage;
import org.opends.server.api.CompressedSchema;
import org.opends.server.backends.persistit.PersistItStorage;
import org.opends.server.backends.pluggable.BackendImpl.ReadOperation;
import org.opends.server.backends.pluggable.BackendImpl.ReadableStorage;
import org.opends.server.backends.pluggable.BackendImpl.StorageRuntimeException;
import org.opends.server.backends.pluggable.BackendImpl.TreeName;
import org.opends.server.backends.pluggable.BackendImpl.WriteOperation;
import org.opends.server.backends.pluggable.BackendImpl.WriteableStorage;
import org.opends.server.core.DefaultCompressedSchema;
import org.opends.server.core.DirectoryServer;
import org.opends.server.monitors.DatabaseEnvironmentMonitor;
import org.opends.server.types.ConfigChangeResult;
import org.opends.server.types.DN;
import org.opends.server.types.FilePermission;
@@ -64,10 +67,7 @@
  private static final LocalizedLogger logger = LocalizedLogger.getLoggerForThisClass();

  /** The JE database environment. */
  private Storage storage;

  /** Used to force a checkpoint during import. */
  private final CheckpointConfig importForceCheckPoint = new CheckpointConfig();
  private PersistItStorage storage; // FIXME JNR do not hardcode here

  /** The backend configuration. */
  private LocalDBBackendCfg config;
@@ -84,8 +84,9 @@
  /** The cached value of the next entry identifier to be assigned. */
  private AtomicLong nextid = new AtomicLong(1);

  // FIXME JNR Switch back to a database persisted implementation of CompressedSchema
  /** The compressed schema manager for this backend. */
  private JECompressedSchema compressedSchema;
  private CompressedSchema compressedSchema;



@@ -106,13 +107,16 @@
    getMonitorProvider().setMaxEntries(config.getIndexFilterAnalyzerMaxFilters());

    config.addLocalDBChangeListener(this);
    importForceCheckPoint.setForce(true);
  }

  PersistItStorage getStorage()
  {
    return storage;
  }

  /**
   * Opens the root container using the JE configuration object provided.
   *
   * @param  envConfig               The JE environment configuration.
   * @throws StorageRuntimeException       If a database error occurs when creating
   *                                 the environment.
   * @throws InitializationException If an initialization error occurs while
@@ -120,7 +124,7 @@
   * @throws ConfigException         If an configuration error occurs while
   *                                 creating the environment.
   */
  public void open(EnvironmentConfig envConfig)
  public void open()
      throws StorageRuntimeException, InitializationException, ConfigException
  {
    // Determine the backend database directory.
@@ -183,39 +187,26 @@
    }

    // Open the database environment
    storage = new Storage(backendDirectory, envConfig);
    storage = new PersistItStorage(backendDirectory, this.config);

    if (logger.isTraceEnabled())
    compressedSchema = new DefaultCompressedSchema();
    try
    {
      logger.trace("JE (%s) environment opened with the following config: %n%s",
          JEVersion.CURRENT_VERSION, storage.getConfig());

      // Get current size of heap in bytes
      long heapSize = Runtime.getRuntime().totalMemory();

      // Get maximum size of heap in bytes. The heap cannot grow beyond this size.
      // Any attempt will result in an OutOfMemoryException.
      long heapMaxSize = Runtime.getRuntime().maxMemory();

      // Get amount of free memory within the heap in bytes. This size will increase
      // after garbage collection and decrease as new objects are created.
      long heapFreeSize = Runtime.getRuntime().freeMemory();

      logger.trace("Current size of heap: %d bytes", heapSize);
      logger.trace("Max size of heap: %d bytes", heapMaxSize);
      logger.trace("Free memory in heap: %d bytes", heapFreeSize);
    }

    compressedSchema = new JECompressedSchema(storage);

    storage.write(new WriteOperation()
    {
      @Override
      public void run(WriteableStorage txn) throws Exception
      storage.initialize(null);
      storage.open();
      storage.write(new WriteOperation()
      {
        openAndRegisterEntryContainers(txn, config.getBaseDN());
      }
    });
        @Override
        public void run(WriteableStorage txn) throws Exception
        {
          openAndRegisterEntryContainers(txn, config.getBaseDN());
        }
      });
    }
    catch (Exception e)
    {
      throw new StorageRuntimeException(e);
    }
  }

  /**
@@ -248,13 +239,26 @@
      databasePrefix = name;
    }

    EntryContainer ec = new EntryContainer(baseDN, databasePrefix,
    EntryContainer ec = new EntryContainer(baseDN, toSuffixName(databasePrefix),
                                           backend, config, storage, this);
    ec.open(txn);
    return ec;
  }

  /**
   * Transform a database prefix string to one usable by the DB.
   *
   * @param databasePrefix
   *          the database prefix
   * @return a new string when non letter or digit characters have been replaced
   *         with underscore
   */
  private TreeName toSuffixName(String databasePrefix)
  {
    return TreeName.of(storage.toSuffixName(databasePrefix));
  }

  /**
   * Registers the entry container for a base DN.
   *
   * @param baseDN The base DN of the entry container to close.
@@ -325,7 +329,7 @@
   *
   * @return  The compressed schema manager for this backend.
   */
  public JECompressedSchema getCompressedSchema()
  public CompressedSchema getCompressedSchema()
  {
    return compressedSchema;
  }
@@ -537,35 +541,6 @@
  }

  /**
   * Get the environment transaction stats of the JE environment used
   * in this root container.
   *
   * @param statsConfig The configuration to use for the EnvironmentStats
   *                    object.
   * @return The environment status of the JE environment.
   * @throws StorageRuntimeException If an error occurs while retrieving the stats
   *                           object.
   */
  public TransactionStats getEnvironmentTransactionStats(
      StatsConfig statsConfig) throws StorageRuntimeException
  {
    return storage.getTransactionStats(statsConfig);
  }

  /**
   * Get the environment config of the JE environment used in this root
   * container.
   *
   * @return The environment config of the JE environment.
   * @throws StorageRuntimeException If an error occurs while retrieving the
   *                           configuration object.
   */
  public EnvironmentConfig getEnvironmentConfig() throws StorageRuntimeException
  {
    return storage.getConfig();
  }

  /**
   * Get the backend configuration used by this root container.
   *
   * @return The JE backend configuration used by this root container.
@@ -584,21 +559,34 @@
   */
  public long getEntryCount() throws StorageRuntimeException
  {
    long entryCount = 0;
    for(EntryContainer ec : this.entryContainers.values())
    try
    {
      ec.sharedLock.lock();
      try
      return storage.read(new ReadOperation<Long>()
      {
        entryCount += ec.getEntryCount();
      }
      finally
      {
        ec.sharedLock.unlock();
      }
        @Override
        public Long run(ReadableStorage txn) throws Exception
        {
          long entryCount = 0;
          for (EntryContainer ec : entryContainers.values())
          {
            ec.sharedLock.lock();
            try
            {
              entryCount += ec.getEntryCount(txn);
            }
            finally
            {
              ec.sharedLock.unlock();
            }
          }
          return entryCount;
        }
      });
    }

    return entryCount;
    catch (Exception e)
    {
      throw new StorageRuntimeException(e);
    }
  }

  /**
@@ -705,81 +693,10 @@
  @Override
  public ConfigChangeResult applyConfigurationChange(LocalDBBackendCfg cfg)
  {
    boolean adminActionRequired = false;
    ArrayList<LocalizableMessage> messages = new ArrayList<LocalizableMessage>();
    final ConfigChangeResult ccr = new ConfigChangeResult();

    try
    {
      if(storage != null)
      {
        // Check if any JE non-mutable properties were changed.
        EnvironmentConfig oldEnvConfig = storage.getConfig();
        EnvironmentConfig newEnvConfig =
            ConfigurableEnvironment.parseConfigEntry(cfg);
        Map<?,?> paramsMap = EnvironmentParams.SUPPORTED_PARAMS;

        // Iterate through native JE properties.
        SortedSet<String> jeProperties = cfg.getJEProperty();
        for (String jeEntry : jeProperties) {
          // There is no need to validate properties yet again.
          StringTokenizer st = new StringTokenizer(jeEntry, "=");
          if (st.countTokens() == 2) {
            String jePropertyName = st.nextToken();
            String jePropertyValue = st.nextToken();
            ConfigParam param = (ConfigParam) paramsMap.get(jePropertyName);
            if (!param.isMutable()) {
              String oldValue = oldEnvConfig.getConfigParam(param.getName());
              if (!oldValue.equalsIgnoreCase(jePropertyValue)) {
                adminActionRequired = true;
                messages.add(INFO_CONFIG_JE_PROPERTY_REQUIRES_RESTART.get(jePropertyName));
                if(logger.isTraceEnabled()) {
                  logger.trace("The change to the following property " +
                    "will take effect when the component is restarted: " +
                    jePropertyName);
                }
              }
            }
          }
        }

        // Iterate through JE configuration attributes.
        for (Object o : paramsMap.values())
        {
          ConfigParam param = (ConfigParam) o;
          if (!param.isMutable())
          {
            String oldValue = oldEnvConfig.getConfigParam(param.getName());
            String newValue = newEnvConfig.getConfigParam(param.getName());
            if (!oldValue.equalsIgnoreCase(newValue))
            {
              adminActionRequired = true;
              String configAttr = ConfigurableEnvironment.
                  getAttributeForProperty(param.getName());
              if (configAttr != null)
              {
                messages.add(NOTE_JEB_CONFIG_ATTR_REQUIRES_RESTART.get(configAttr));
              }
              else
              {
                messages.add(NOTE_JEB_CONFIG_ATTR_REQUIRES_RESTART.get(param.getName()));
              }
              if(logger.isTraceEnabled())
              {
                logger.trace("The change to the following property will " +
                    "take effect when the backend is restarted: " +
                    param.getName());
              }
            }
          }
        }

        // This takes care of changes to the JE environment for those
        // properties that are mutable at runtime.
        storage.setMutableConfig(newEnvConfig);

        logger.trace("JE database configuration: %s", storage.getConfig());
      }

      // Create the directory if it doesn't exist.
      if(!cfg.getDBDirectory().equals(this.config.getDBDirectory()))
      {
@@ -791,31 +708,25 @@
        {
          if(!backendDirectory.mkdirs())
          {
            messages.add(ERR_JEB_CREATE_FAIL.get(backendDirectory.getPath()));
            return new ConfigChangeResult(
                DirectoryServer.getServerErrorResultCode(),
                adminActionRequired,
                messages);
            ccr.setResultCode(DirectoryServer.getServerErrorResultCode());
            ccr.addMessage(ERR_JEB_CREATE_FAIL.get(backendDirectory.getPath()));
            return ccr;
          }
        }
        //Make sure the directory is valid.
        else if (!backendDirectory.isDirectory())
        {
          messages.add(ERR_JEB_DIRECTORY_INVALID.get(backendDirectory.getPath()));
          return new ConfigChangeResult(
              DirectoryServer.getServerErrorResultCode(),
              adminActionRequired,
              messages);
          ccr.setResultCode(DirectoryServer.getServerErrorResultCode());
          ccr.addMessage(ERR_JEB_DIRECTORY_INVALID.get(backendDirectory.getPath()));
          return ccr;
        }

        adminActionRequired = true;
        messages.add(NOTE_JEB_CONFIG_DB_DIR_REQUIRES_RESTART.get(
                        this.config.getDBDirectory(), cfg.getDBDirectory()));
        ccr.setAdminActionRequired(true);
        ccr.addMessage(NOTE_JEB_CONFIG_DB_DIR_REQUIRES_RESTART.get(this.config.getDBDirectory(), cfg.getDBDirectory()));
      }

      if(!cfg.getDBDirectoryPermissions().equalsIgnoreCase(
          config.getDBDirectoryPermissions()) ||
          !cfg.getDBDirectory().equals(this.config.getDBDirectory()))
      if (!cfg.getDBDirectoryPermissions().equalsIgnoreCase(config.getDBDirectoryPermissions())
          || !cfg.getDBDirectory().equals(this.config.getDBDirectory()))
      {
        FilePermission backendPermission;
        try
@@ -825,25 +736,19 @@
        }
        catch(Exception e)
        {
          messages.add(ERR_CONFIG_BACKEND_MODE_INVALID.get(config.dn()));
          return new ConfigChangeResult(
              DirectoryServer.getServerErrorResultCode(),
              adminActionRequired,
              messages);
          ccr.setResultCode(DirectoryServer.getServerErrorResultCode());
          ccr.addMessage(ERR_CONFIG_BACKEND_MODE_INVALID.get(config.dn()));
          return ccr;
        }

        //Make sure the mode will allow the server itself access to
        //the database
        // Make sure the mode will allow the server itself access to the database
        if(!backendPermission.isOwnerWritable() ||
            !backendPermission.isOwnerReadable() ||
            !backendPermission.isOwnerExecutable())
        {
          messages.add(ERR_CONFIG_BACKEND_INSANE_MODE.get(
              cfg.getDBDirectoryPermissions()));
          return new ConfigChangeResult(
              DirectoryServer.getServerErrorResultCode(),
              adminActionRequired,
              messages);
          ccr.setResultCode(DirectoryServer.getServerErrorResultCode());
          ccr.addMessage(ERR_CONFIG_BACKEND_INSANE_MODE.get(cfg.getDBDirectoryPermissions()));
          return ccr;
        }

        // Get the backend database backendDirectory permissions and apply
@@ -866,22 +771,18 @@
        }
      }

      getMonitorProvider().enableFilterUseStats(
          cfg.isIndexFilterAnalyzerEnabled());
      getMonitorProvider()
          .setMaxEntries(cfg.getIndexFilterAnalyzerMaxFilters());
      getMonitorProvider().enableFilterUseStats(cfg.isIndexFilterAnalyzerEnabled());
      getMonitorProvider().setMaxEntries(cfg.getIndexFilterAnalyzerMaxFilters());

      this.config = cfg;
    }
    catch (Exception e)
    {
      messages.add(LocalizableMessage.raw(stackTraceToSingleLineString(e)));
      return new ConfigChangeResult(DirectoryServer.getServerErrorResultCode(),
                                   adminActionRequired,
                                   messages);
      ccr.setResultCode(DirectoryServer.getServerErrorResultCode());
      ccr.addMessage(LocalizableMessage.raw(stackTraceToSingleLineString(e)));
      return ccr;
    }

    return new ConfigChangeResult(ResultCode.SUCCESS, adminActionRequired, messages);
    return ccr;
  }

  /**

 opendj3-server-dev/src/server/org/opends/server/backends/pluggable/SortValuesSet.java

@@ -99,9 +99,9 @@
   * @param sv The sort values to add.
   * @param types The types of the values to add.
   * @throws DirectoryException If a Directory Server error occurs.
   * @throws DatabaseException If an error occurs in the JE database.
   * @throws StorageRuntimeException If an error occurs in the JE database.
   */
  void add(SortValues sv) throws DirectoryException
  void add(SortValues sv) throws StorageRuntimeException, DirectoryException
  {
    add(sv.getEntryID(), sv.getValues(), sv.getTypes());
  }
@@ -587,10 +587,8 @@
   * @return The sort values object at the specified index.
   * @throws DirectoryException If a Directory Server error occurs.
   * @throws StorageRuntimeException If an error occurs in the JE database.
   * @throws JebException If an error occurs in the JE database.
   **/
  public SortValues getSortValues(int index)
      throws JebException, StorageRuntimeException, DirectoryException
   */
  public SortValues getSortValues(int index) throws StorageRuntimeException, DirectoryException
  {
    if(entryIDs == null || entryIDs.length == 0)
    {

 opendj3-server-dev/src/server/org/opends/server/backends/pluggable/SuffixContainer.java

@@ -75,12 +75,4 @@
   * @return the baseDN that this suffix container is responsible for
   */
  DN getBaseDN();

  /**
   * Returns the number of entries stored in this suffix container.
   *
   * @return the number of entries stored in this suffix container, or -1 if the
   *         count not be determined
   */
  long getEntryCount();
}

 opendj3-server-dev/src/server/org/opends/server/backends/pluggable/VLVIndex.java

@@ -26,7 +26,6 @@
 */
package org.opends.server.backends.pluggable;

import java.util.ArrayList;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
@@ -197,7 +196,7 @@
    this.comparator = new VLVKeyComparator(orderingRules, ascending);

    this.state = state;
    this.trusted = state.getIndexTrustState(null, this);
    this.trusted = state.getIndexTrustState(txn, this);
    if (!trusted && entryContainer.getHighestEntryID(txn).longValue() == 0)
    {
      // If there are no entries in the entry container then there
@@ -264,12 +263,10 @@
   * @return True if the entry ID for the entry are added. False if
   *         the entry ID already exists.
   * @throws StorageRuntimeException If an error occurs in the JE database.
   * @throws org.opends.server.types.DirectoryException If a Directory Server
   * error occurs.
   * @throws JebException If an error occurs in the JE backend.
   * @throws DirectoryException If a Directory Server error occurs.
   */
  public boolean addEntry(WriteableStorage txn, EntryID entryID, Entry entry)
      throws StorageRuntimeException, DirectoryException, JebException
      throws StorageRuntimeException, DirectoryException
  {
    return shouldInclude(entry)
        && insertValues(txn, entryID.longValue(), entry);
@@ -470,13 +467,11 @@
   * found.
   * @throws StorageRuntimeException If an error occurs during an operation on a
   * JE database.
   * @throws JebException If an error occurs during an operation on a
   * JE database.
   * @throws DirectoryException If a Directory Server error occurs.
   */
  public boolean containsValues(ReadableStorage txn, long entryID,
      ByteString[] values, AttributeType[] types) throws JebException,
      StorageRuntimeException, DirectoryException
      ByteString[] values, AttributeType[] types)
          throws StorageRuntimeException, DirectoryException
  {
    SortValuesSet valuesSet = getSortValuesSet(txn, entryID, values, types);
    int pos = valuesSet.binarySearch(entryID, values);
@@ -484,7 +479,7 @@
  }

  private boolean insertValues(WriteableStorage txn, long entryID, Entry entry)
      throws JebException, StorageRuntimeException, DirectoryException
      throws StorageRuntimeException, DirectoryException
  {
    ByteString[] values = getSortValues(entry);
    AttributeType[] types = getSortTypes();
@@ -545,19 +540,6 @@
    return types;
  }

  private boolean getSearchKeyRange(ReadableStorage txn, ByteString key)
  {
    Cursor cursor = txn.openCursor(treeName);
    try
    {
      return cursor.positionToKeyOrNext(key);
    }
    finally
    {
      cursor.close();
    }
  }

  /**
   * Update the vlvIndex with the specified values to add and delete.
   *
@@ -1279,25 +1261,42 @@

  /** {@inheritDoc} */
  @Override
  public synchronized ConfigChangeResult applyConfigurationChange(
      LocalDBVLVIndexCfg cfg)
  public synchronized ConfigChangeResult applyConfigurationChange(final LocalDBVLVIndexCfg cfg)
  {
    ResultCode resultCode = ResultCode.SUCCESS;
    boolean adminActionRequired = false;
    ArrayList<LocalizableMessage> messages = new ArrayList<LocalizableMessage>();
    try
    {
      final ConfigChangeResult ccr = new ConfigChangeResult();
      storage.write(new WriteOperation()
      {
        @Override
        public void run(WriteableStorage txn) throws Exception
        {
          applyConfigurationChange0(txn, cfg, ccr);
        }
      });
      return ccr;
    }
    catch (Exception e)
    {
      throw new StorageRuntimeException(e);
    }
  }

  private synchronized void applyConfigurationChange0(WriteableStorage txn, LocalDBVLVIndexCfg cfg,
      ConfigChangeResult ccr)
  {
    // Update base DN only if changed..
    if(!config.getBaseDN().equals(cfg.getBaseDN()))
    {
      this.baseDN = cfg.getBaseDN();
      adminActionRequired = true;
      ccr.setAdminActionRequired(true);
    }

    // Update scope only if changed.
    if(!config.getScope().equals(cfg.getScope()))
    {
      this.scope = SearchScope.valueOf(cfg.getScope().name());
      adminActionRequired = true;
      ccr.setAdminActionRequired(true);
    }

    // Update sort set capacity only if changed.
@@ -1309,7 +1308,7 @@
      // Otherwise, we will lazily update the sorted sets.
      if (config.getMaxBlockSize() < cfg.getMaxBlockSize())
      {
        adminActionRequired = true;
        ccr.setAdminActionRequired(true);
      }
    }

@@ -1319,18 +1318,13 @@
      try
      {
        this.filter = SearchFilter.createFilterFromString(cfg.getFilter());
        adminActionRequired = true;
        ccr.setAdminActionRequired(true);
      }
      catch(Exception e)
      {
        LocalizableMessage msg = ERR_JEB_CONFIG_VLV_INDEX_BAD_FILTER.get(
                config.getFilter(), treeName,
                stackTraceToSingleLineString(e));
        messages.add(msg);
        if(resultCode == ResultCode.SUCCESS)
        {
          resultCode = ResultCode.INVALID_ATTRIBUTE_SYNTAX;
        }
        ccr.addMessage(ERR_JEB_CONFIG_VLV_INDEX_BAD_FILTER.get(
            config.getFilter(), treeName, stackTraceToSingleLineString(e)));
        ccr.setResultCode(ResultCode.INVALID_ATTRIBUTE_SYNTAX);
      }
    }

@@ -1361,22 +1355,16 @@
        }
        catch(Exception e)
        {
          messages.add(ERR_JEB_CONFIG_VLV_INDEX_UNDEFINED_ATTR.get(sortKeys[i], treeName));
          if(resultCode == ResultCode.SUCCESS)
          {
            resultCode = ResultCode.INVALID_ATTRIBUTE_SYNTAX;
          }
          ccr.addMessage(ERR_JEB_CONFIG_VLV_INDEX_UNDEFINED_ATTR.get(sortKeys[i], treeName));
          ccr.setResultCode(ResultCode.INVALID_ATTRIBUTE_SYNTAX);
        }

        AttributeType attrType =
            DirectoryServer.getAttributeType(sortAttrs[i].toLowerCase());
        if(attrType == null)
        {
          messages.add(ERR_JEB_CONFIG_VLV_INDEX_UNDEFINED_ATTR.get(sortKeys[i], treeName));
          if(resultCode == ResultCode.SUCCESS)
          {
            resultCode = ResultCode.INVALID_ATTRIBUTE_SYNTAX;
          }
          ccr.addMessage(ERR_JEB_CONFIG_VLV_INDEX_UNDEFINED_ATTR.get(sortKeys[i], treeName));
          ccr.setResultCode(ResultCode.INVALID_ATTRIBUTE_SYNTAX);
        }
        else
        {
@@ -1392,22 +1380,15 @@
      entryContainer.exclusiveLock.lock();
      try
      {
        storage.write(new WriteOperation()
        {
          @Override
          public void run(WriteableStorage txn) throws Exception
          {
            close();
            open(txn);
          }
        });
        close();
        open(txn);
      }
      catch (Exception e)
      catch (StorageRuntimeException de)
      {
        messages.add(LocalizableMessage.raw(StaticUtils.stackTraceToSingleLineString(e)));
        if(resultCode == ResultCode.SUCCESS)
        ccr.addMessage(LocalizableMessage.raw(StaticUtils.stackTraceToSingleLineString(de)));
        if (ccr.getResultCode() == ResultCode.SUCCESS)
        {
          resultCode = DirectoryServer.getServerErrorResultCode();
          ccr.setResultCode(DirectoryServer.getServerErrorResultCode());
        }
      }
      finally
@@ -1415,29 +1396,28 @@
        entryContainer.exclusiveLock.unlock();
      }

      adminActionRequired = true;
      ccr.setAdminActionRequired(true);
    }


    if(adminActionRequired)
    if (ccr.adminActionRequired())
    {
      trusted = false;
      messages.add(NOTE_JEB_INDEX_ADD_REQUIRES_REBUILD.get(treeName));
      ccr.addMessage(NOTE_JEB_INDEX_ADD_REQUIRES_REBUILD.get(treeName));
      try
      {
        state.putIndexTrustState(null, this, false);
      }
      catch(StorageRuntimeException de)
      {
        messages.add(LocalizableMessage.raw(StaticUtils.stackTraceToSingleLineString(de)));
        if(resultCode == ResultCode.SUCCESS)
        ccr.addMessage(LocalizableMessage.raw(StaticUtils.stackTraceToSingleLineString(de)));
        if (ccr.getResultCode() == ResultCode.SUCCESS)
        {
          resultCode = DirectoryServer.getServerErrorResultCode();
          ccr.setResultCode(DirectoryServer.getServerErrorResultCode());
        }
      }
    }

    this.config = cfg;
    return new ConfigChangeResult(resultCode, adminActionRequired, messages);
  }
}

 opendj3-server-dev/src/server/org/opends/server/backends/pluggable/VerifyJob.java

@@ -51,7 +51,6 @@
import org.opends.server.backends.pluggable.BackendImpl.Cursor;
import org.opends.server.backends.pluggable.BackendImpl.ReadOperation;
import org.opends.server.backends.pluggable.BackendImpl.ReadableStorage;
import org.opends.server.backends.pluggable.BackendImpl.Storage;
import org.opends.server.backends.pluggable.BackendImpl.StorageRuntimeException;
import org.opends.server.core.DirectoryServer;
import org.opends.server.types.Attribute;
@@ -146,16 +145,14 @@
   * @param statEntry Optional statistics entry.
   * @return The error count.
   * @throws StorageRuntimeException If an error occurs in the JE database.
   * @throws JebException If an error occurs in the JE backend.
   * @throws DirectoryException If an error occurs while verifying the backend.
   */
  public long verifyBackend(final RootContainer rootContainer, final Entry statEntry) throws StorageRuntimeException,
      JebException, DirectoryException
      DirectoryException
  {
    Storage s;
    try
    {
      return s.read(new ReadOperation<Long>()
      return rootContainer.getStorage().read(new ReadOperation<Long>()
      {
        @Override
        public Long run(ReadableStorage txn) throws Exception
@@ -171,7 +168,7 @@
  }

  private long verifyBackend0(ReadableStorage txn, RootContainer rootContainer, Entry statEntry)
      throws StorageRuntimeException, JebException, DirectoryException
      throws StorageRuntimeException, DirectoryException
  {
    this.rootContainer = rootContainer;
    EntryContainer entryContainer =
@@ -224,7 +221,7 @@
            {
              LocalizableMessage msg = NOTE_JEB_SUBORDINATE_INDEXES_DISABLED
                  .get(rootContainer.getConfiguration().getBackendId());
              throw new JebException(msg);
              throw new StorageRuntimeException(msg.toString());
            }
          }
          else if ("id2subtree".equals(lowerName))
@@ -237,7 +234,7 @@
            {
              LocalizableMessage msg = NOTE_JEB_SUBORDINATE_INDEXES_DISABLED
                  .get(rootContainer.getConfiguration().getBackendId());
              throw new JebException(msg);
              throw new StorageRuntimeException(msg.toString());
            }
          }
          else if(lowerName.startsWith("vlv."))
@@ -245,7 +242,7 @@
            if(lowerName.length() < 5)
            {
              LocalizableMessage msg = ERR_JEB_VLV_INDEX_NOT_CONFIGURED.get(lowerName);
              throw new JebException(msg);
              throw new StorageRuntimeException(msg.toString());
            }

            VLVIndex vlvIndex =
@@ -254,35 +251,31 @@
            {
              LocalizableMessage msg =
                  ERR_JEB_VLV_INDEX_NOT_CONFIGURED.get(lowerName.substring(4));
              throw new JebException(msg);
              throw new StorageRuntimeException(msg.toString());
            }

            vlvIndexList.add(vlvIndex);
          }
          else
          {
            AttributeType attrType =
                DirectoryServer.getAttributeType(lowerName);
            AttributeType attrType = DirectoryServer.getAttributeType(lowerName);
            if (attrType == null)
            {
              LocalizableMessage msg = ERR_JEB_ATTRIBUTE_INDEX_NOT_CONFIGURED.get(index);
              throw new JebException(msg);
              throw new StorageRuntimeException(msg.toString());
            }
            AttributeIndex attrIndex =
                entryContainer.getAttributeIndex(attrType);
            AttributeIndex attrIndex = entryContainer.getAttributeIndex(attrType);
            if (attrIndex == null)
            {
              LocalizableMessage msg = ERR_JEB_ATTRIBUTE_INDEX_NOT_CONFIGURED.get(index);
              throw new JebException(msg);
              throw new StorageRuntimeException(msg.toString());
            }
            attrIndexList.add(attrIndex);
          }
        }
      }

      entryLimitMap =
          new IdentityHashMap<Index,HashMap<ByteString,Long>>(
              attrIndexList.size());
      entryLimitMap = new IdentityHashMap<Index, HashMap<ByteString, Long>>(attrIndexList.size());

      // We will be updating these files independently of the indexes
      // so we need direct access to them rather than going through
@@ -421,7 +414,7 @@
    Cursor cursor = id2entry.openCursor(txn);
    try
    {
      long storedEntryCount = id2entry.getRecordCount();
      long storedEntryCount = id2entry.getRecordCount(txn);
      while (cursor.next())
      {
        ByteString key = cursor.getKey();
@@ -487,12 +480,10 @@
   * index cleanliness. For each ID in the index we check that the
   * entry it refers to does indeed contain the expected value.
   *
   * @throws JebException If an error occurs in the JE backend.
   * @throws StorageRuntimeException If an error occurs in the JE database.
   * @throws DirectoryException If an error occurs reading values in the index.
   */
  private void iterateIndex(ReadableStorage txn)
      throws JebException, StorageRuntimeException, DirectoryException
  private void iterateIndex(ReadableStorage txn) throws StorageRuntimeException, DirectoryException
  {
    if (verifyDN2ID)
    {
@@ -598,10 +589,9 @@
   * Iterate through the entries in ID2Children to perform a check for
   * index cleanliness.
   *
   * @throws JebException If an error occurs in the JE backend.
   * @throws StorageRuntimeException If an error occurs in the JE database.
   */
  private void iterateID2Children(ReadableStorage txn) throws JebException, StorageRuntimeException
  private void iterateID2Children(ReadableStorage txn) throws StorageRuntimeException
  {
    Cursor cursor = id2c.openCursor(txn);
    try
@@ -727,10 +717,9 @@
   * Iterate through the entries in ID2Subtree to perform a check for
   * index cleanliness.
   *
   * @throws JebException If an error occurs in the JE backend.
   * @throws StorageRuntimeException If an error occurs in the JE database.
   */
  private void iterateID2Subtree(ReadableStorage txn) throws JebException, StorageRuntimeException
  private void iterateID2Subtree(ReadableStorage txn) throws StorageRuntimeException
  {
    Cursor cursor = id2s.openCursor(txn);
    try
@@ -906,12 +895,11 @@
   *
   * @param vlvIndex The VLV index to perform the check against.
   * @param verifyID True to verify the IDs against id2entry.
   * @throws JebException If an error occurs in the JE backend.
   * @throws StorageRuntimeException If an error occurs in the JE database.
   * @throws DirectoryException If an error occurs reading values in the index.
   */
  private void iterateVLVIndex(ReadableStorage txn, VLVIndex vlvIndex, boolean verifyID)
      throws JebException, StorageRuntimeException, DirectoryException
      throws StorageRuntimeException, DirectoryException
  {
    if(vlvIndex == null)
    {
@@ -1012,11 +1000,10 @@
   * Iterate through the entries in an attribute index to perform a check for
   * index cleanliness.
   * @param index The index database to be checked.
   * @throws JebException If an error occurs in the JE backend.
   * @throws StorageRuntimeException If an error occurs in the JE database.
   */
  private void iterateAttrIndex(ReadableStorage txn, Index index, IndexingOptions options)
      throws JebException, StorageRuntimeException
      throws StorageRuntimeException
  {
    if (index == null)
    {
@@ -1487,16 +1474,6 @@
        }
        errorCount++;
      }
      catch (JebException e)
      {
        if (logger.isTraceEnabled())
        {
          logger.traceException(e);
          logger.trace("Error reading VLV index %s for entry %s: %s",
              vlvIndex.getName(), entry.getName(), StaticUtils.getBacktrace(e));
        }
        errorCount++;
      }
    }
  }