OPENDJ-1602 (CR-5566) New pluggable storage based backend

Jean-Noel Rouvignac

17.47.2014 efde45a134da54ccac6bac24d7de48c5633139d1

OPENDJ-1602 (CR-5566) New pluggable storage based backend

Reduced the number of compilation errors in package "org.opends.server.backends.pluggable".


*.java:
Passed in (Readable|Writeable)Storage objects rather than null in several method calls.
Removed more references to Berkeley JE classes.

TreeName.java:
Added replaceSuffix() and getIndex().

WriteableStorage.java:
Added truncateTree() and deleteTree().

EntryContainer.java:
Removed beginTransaction(), transactionCommit(), transactionAbort(),

RootContainer.java:
In preload(), removed the implementation as it is unsupported for now.

VerifyJob.java
VLVIndex.java

 opendj-sdk/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/AttributeIndex.java

@@ -672,7 +672,7 @@
    {
      final Set<MatchingRule> validRules = Collections.emptySet();
      final Set<String> validIndexIds = Collections.emptySet();
      removeIndexesForExtensibleMatchingRules(validRules, validIndexIds);
      removeIndexesForExtensibleMatchingRules(txn, validRules, validIndexIds);
      return;
    }


 opendj-sdk/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/DatabaseContainer.java

@@ -83,7 +83,7 @@
    txn.openTree(treeName);
    if (logger.isTraceEnabled())
    {
      logger.trace("JE database %s opened. txnid=%d", treeName, txn.getId());
      logger.trace("JE database %s opened. txn=%s", treeName, txn);
    }
  }

@@ -105,11 +105,7 @@
  @Override
  public synchronized void close() throws StorageRuntimeException
  {
    if(dbConfig.getDeferredWrite())
    {
      treeName.sync();
    }
    treeName.close();
    storage.closeTree(treeName);
    treeName = null;

    if(logger.isTraceEnabled())
@@ -213,7 +209,7 @@
   */
  public long getRecordCount(ReadableStorage txn) throws StorageRuntimeException
  {
    long count = treeName.count();
    long count = count(txn);
    if (logger.isTraceEnabled())
    {
      logger.trace(messageToLog(true, treeName, null, null, null));
@@ -221,6 +217,24 @@
    return count;
  }

  private long count(ReadableStorage txn)
  {
    final Cursor cursor = txn.openCursor(treeName);
    try
    {
      long count = 0;
      while (cursor.next())
      {
        count++;
      }
      return count;
    }
    finally
    {
      cursor.close();
    }
  }

  /**
   * Get a string representation of this object.
   * @return return A string representation of this object.
@@ -242,20 +256,6 @@
  }

  /**
   * Preload the database into cache.
   *
   * @param config The preload configuration.
   * @return Statistics about the preload process.
   * @throws StorageRuntimeException If an JE database error occurs
   * during the preload.
   */
  public PreloadStats preload(PreloadConfig config)
      throws StorageRuntimeException
  {
    return treeName.preload(config);
  }

  /**
   * Set the JE database name to use for this container.
   *
   * @param name The database name to use for this container.
@@ -277,15 +277,8 @@
    builder.append(treeName);
    if (txn != null)
    {
      builder.append(" txnid=");
      try
      {
        builder.append(txn.getId());
      }
      catch (StorageRuntimeException de)
      {
        builder.append(de);
      }
      builder.append(" txn=");
      builder.append(txn);
    }
    else
    {

 opendj-sdk/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/DbPreloadComparator.java

@@ -47,11 +47,12 @@
  static private int priority(DatabaseContainer database)
  {
    TreeName name = database.getName();
    if (name.endsWith(EntryContainer.ID2ENTRY_DATABASE_NAME))
    String indexName = name.getIndex().toString();
    if (indexName.endsWith(SuffixContainer.ID2ENTRY_INDEX_NAME))
    {
      return 1;
    }
    else if (name.endsWith(EntryContainer.DN2ID_DATABASE_NAME))
    else if (indexName.endsWith(SuffixContainer.DN2ID_INDEX_NAME))
    {
      return 2;
    }

 opendj-sdk/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/EntryContainer.java

@@ -33,7 +33,6 @@
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantReadWriteLock;

@@ -96,8 +95,6 @@
import org.opends.server.util.ServerConstants;
import org.opends.server.util.StaticUtils;

import com.sleepycat.je.TransactionConfig;

import static org.opends.messages.JebMessages.*;
import static org.opends.server.backends.pluggable.JebFormat.*;
import static org.opends.server.core.DirectoryServer.*;
@@ -499,19 +496,13 @@
        // Use a null index and ensure that future attempts to use the real
        // subordinate indexes will fail.
        id2children = new NullIndex(databasePrefix.child(ID2CHILDREN_DATABASE_NAME),
            new ID2CIndexer(), state, storage, this);
        if (!storage.getConfig().getReadOnly())
        {
          state.putIndexTrustState(null, id2children, false);
        }
            new ID2CIndexer(), state, storage, txn, this);
        state.putIndexTrustState(txn, id2children, false);
        id2children.open(txn); // No-op

        id2subtree = new NullIndex(databasePrefix.child(ID2SUBTREE_DATABASE_NAME),
            new ID2SIndexer(), state, storage, this);
        if (!storage.getConfig().getReadOnly())
        {
          state.putIndexTrustState(null, id2subtree, false);
        }
            new ID2SIndexer(), state, storage, txn, this);
        state.putIndexTrustState(txn, id2subtree, false);
        id2subtree.open(txn); // No-op

        logger.info(NOTE_JEB_SUBORDINATE_INDEXES_DISABLED, backend.getBackendID());
@@ -1606,9 +1597,6 @@
              addOperation.checkIfCanceled(true);
            }

            // Commit the transaction.
            EntryContainer.transactionCommit(txn);

            // Update the entry cache.
            EntryCache<?> entryCache = DirectoryServer.getEntryCache();
            if (entryCache != null)
@@ -1618,23 +1606,18 @@
          }
          catch (StorageRuntimeException StorageRuntimeException)
          {
            EntryContainer.transactionAbort(txn);
            throw StorageRuntimeException;
          }
          catch (DirectoryException directoryException)
          {
            EntryContainer.transactionAbort(txn);
            throw directoryException;
          }
          catch (CanceledOperationException coe)
          {
            EntryContainer.transactionAbort(txn);
            throw coe;
          }
          catch (Exception e)
          {
            EntryContainer.transactionAbort(txn);

            String msg = e.getMessage();
            if (msg == null)
            {
@@ -1793,9 +1776,6 @@
              deleteOperation.checkIfCanceled(true);
            }

            // Commit the transaction.
            EntryContainer.transactionCommit(txn);

            if (isSubtreeDelete)
            {
              deleteOperation.addAdditionalLogItem(unquotedKeyValue(getClass(), "deletedEntries",
@@ -1804,23 +1784,18 @@
          }
          catch (StorageRuntimeException StorageRuntimeException)
          {
            EntryContainer.transactionAbort(txn);
            throw StorageRuntimeException;
          }
          catch (DirectoryException directoryException)
          {
            EntryContainer.transactionAbort(txn);
            throw directoryException;
          }
          catch (CanceledOperationException coe)
          {
            EntryContainer.transactionAbort(txn);
            throw coe;
          }
          catch (Exception e)
          {
            EntryContainer.transactionAbort(txn);

            String msg = e.getMessage();
            if (msg == null)
            {
@@ -2133,9 +2108,6 @@
              modifyOperation.checkIfCanceled(true);
            }

            // Commit the transaction.
            EntryContainer.transactionCommit(txn);

            // Update the entry cache.
            EntryCache<?> entryCache = DirectoryServer.getEntryCache();
            if (entryCache != null)
@@ -2145,23 +2117,18 @@
          }
          catch (StorageRuntimeException StorageRuntimeException)
          {
            EntryContainer.transactionAbort(txn);
            throw StorageRuntimeException;
          }
          catch (DirectoryException directoryException)
          {
            EntryContainer.transactionAbort(txn);
            throw directoryException;
          }
          catch (CanceledOperationException coe)
          {
            EntryContainer.transactionAbort(txn);
            throw coe;
          }
          catch (Exception e)
          {
            EntryContainer.transactionAbort(txn);

            String msg = e.getMessage();
            if (msg == null)
            {
@@ -2400,29 +2367,21 @@
              // One last check before committing
              modifyDNOperation.checkIfCanceled(true);
            }

            // Commit the transaction.
            EntryContainer.transactionCommit(txn);
          }
          catch (StorageRuntimeException StorageRuntimeException)
          {
            EntryContainer.transactionAbort(txn);
            throw StorageRuntimeException;
          }
          catch (DirectoryException directoryException)
          {
            EntryContainer.transactionAbort(txn);
            throw directoryException;
          }
          catch (CanceledOperationException coe)
          {
            EntryContainer.transactionAbort(txn);
            throw coe;
          }
          catch (Exception e)
          {
            EntryContainer.transactionAbort(txn);

            String msg = e.getMessage();
            if (msg == null)
            {
@@ -2877,66 +2836,6 @@
  }

  /**
   * Begin a leaf transaction using the default configuration.
   * Provides assertion debug logging.
   * @return A JE transaction handle.
   * @throws StorageRuntimeException If an error occurs while attempting to begin
   * a new transaction.
   */
  public Transaction beginTransaction()
  throws StorageRuntimeException
  {
    Transaction parentTxn = null;
    TransactionConfig txnConfig = null;
    Transaction txn = storage.beginTransaction(parentTxn, txnConfig);
    if (logger.isTraceEnabled())
    {
      logger.trace("beginTransaction", "begin txnid=" + txn.getId());
    }
    return txn;
  }

  /**
   * Commit a transaction.
   * Provides assertion debug logging.
   * @param txn The JE transaction handle.
   * @throws StorageRuntimeException If an error occurs while attempting to commit
   * the transaction.
   */
  public static void transactionCommit(WriteableStorage txn)
  throws StorageRuntimeException
  {
    if (txn != null)
    {
      txn.commit();
      if (logger.isTraceEnabled())
      {
        logger.trace("commit txnid=%d", txn.getId());
      }
    }
  }

  /**
   * Abort a transaction.
   * Provides assertion debug logging.
   * @param txn The JE transaction handle.
   * @throws StorageRuntimeException If an error occurs while attempting to abort the
   * transaction.
   */
  public static void transactionAbort(WriteableStorage txn)
  throws StorageRuntimeException
  {
    if (txn != null)
    {
      txn.abort();
      if (logger.isTraceEnabled())
      {
        logger.trace("abort txnid=%d", txn.getId());
      }
    }
  }

  /**
   * Delete this entry container from disk. The entry container should be
   * closed before calling this method.
   *
@@ -2950,7 +2849,7 @@

    for (DatabaseContainer db : databases)
    {
      storage.removeDatabase(txn, db.getName());
      txn.deleteTree(db.getName());
    }
  }

@@ -2970,7 +2869,7 @@
    }

    database.close();
    storage.removeDatabase(txn, database.getName());
    txn.deleteTree(database.getName());
    if(database instanceof Index)
    {
      state.removeIndexTrustState(txn, database);
@@ -2990,7 +2889,7 @@
    attributeIndex.close();
    for (Index index : attributeIndex.getAllIndexes())
    {
      storage.removeDatabase(txn, index.getName());
      txn.deleteTree(index.getName());
      state.removeIndexTrustState(txn, index);
    }
  }
@@ -3014,13 +2913,11 @@
   * @param newDatabasePrefix The new database prefix to use.
   * @throws StorageRuntimeException If an error occurs in the JE database.
   */
  public void setDatabasePrefix(TreeName newDatabasePrefix) throws StorageRuntimeException, StorageRuntimeException
  public void setDatabasePrefix(final TreeName newDatabasePrefix) throws StorageRuntimeException
  {
    final List<DatabaseContainer> databases = new ArrayList<DatabaseContainer>();
    listDatabases(databases);

    final TreeName newDbPrefix = newDatabasePrefix;

    // close the containers.
    for(DatabaseContainer db : databases)
    {
@@ -3037,8 +2934,8 @@
          for(DatabaseContainer db : databases)
          {
            TreeName oldName = db.getName();
            String newName = oldName.replace(databasePrefix, newDbPrefix);
            storage.renameDatabase(txn, oldName, newName);
            TreeName newName = oldName.replaceSuffix(newDatabasePrefix);
            txn.renameTree(oldName, newName);
          }
        }
      });
@@ -3050,11 +2947,9 @@
          for (DatabaseContainer db : databases)
          {
            TreeName oldName = db.getName();
            String newName = oldName.replace(databasePrefix, newDbPrefix);
            TreeName newName = oldName.replaceSuffix(newDatabasePrefix);
            db.setName(newName);
          }

          databasePrefix = newDbPrefix;
        }
      });
    }
@@ -3157,14 +3052,14 @@
              // future attempts to use the real indexes will fail.
              id2children.close();
              id2children = new NullIndex(databasePrefix.child(ID2CHILDREN_DATABASE_NAME),
                  new ID2CIndexer(), state, storage, EntryContainer.this);
              state.putIndexTrustState(null, id2children, false);
                  new ID2CIndexer(), state, storage, txn, EntryContainer.this);
              state.putIndexTrustState(txn, id2children, false);
              id2children.open(txn); // No-op

              id2subtree.close();
              id2subtree = new NullIndex(databasePrefix.child(ID2SUBTREE_DATABASE_NAME),
                  new ID2SIndexer(), state, storage, EntryContainer.this);
              state.putIndexTrustState(null, id2subtree, false);
                  new ID2SIndexer(), state, storage, txn, EntryContainer.this);
              state.putIndexTrustState(txn, id2subtree, false);
              id2subtree.open(txn); // No-op

              logger.info(NOTE_JEB_SUBORDINATE_INDEXES_DISABLED, cfg.getBackendId());
@@ -3210,13 +3105,11 @@
  /**
   * Clear the contents of this entry container.
   *
   * @return The number of records deleted.
   * @throws StorageRuntimeException If an error occurs while removing the entry
   *                           container.
   */
  public long clear() throws StorageRuntimeException
  public void clear() throws StorageRuntimeException
  {
    final AtomicLong count = new AtomicLong();
    try
    {
      storage.write(new WriteOperation()
@@ -3224,10 +3117,9 @@
        @Override
        public void run(WriteableStorage txn) throws Exception
        {
          count.set(clear0(txn));
          clear0(txn);
        }
      });
      return count.get();
    }
    catch (Exception e)
    {
@@ -3235,11 +3127,10 @@
    }
  }

  private long clear0(WriteableStorage txn) throws StorageRuntimeException
  private void clear0(WriteableStorage txn) throws StorageRuntimeException
  {
    final List<DatabaseContainer> databases = new ArrayList<DatabaseContainer>();
    listDatabases(databases);
    long count = 0;

    for(DatabaseContainer db : databases)
    {
@@ -3249,7 +3140,7 @@
    {
      for (DatabaseContainer db : databases)
      {
        count += storage.truncateDatabase(txn, db.getName(), true);
        txn.truncateTree(db.getName());
      }
    }
    finally
@@ -3267,8 +3158,6 @@
        }
      }
    }

    return count;
  }

  /**
@@ -3289,7 +3178,7 @@
        {
          try
          {
            storage.removeDatabase(txn, database.getName());
            txn.deleteTree(database.getName());
          }
          finally
          {

 opendj-sdk/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/NullIndex.java

@@ -39,9 +39,6 @@
import org.opends.server.types.Entry;
import org.opends.server.types.Modification;

import com.sleepycat.je.PreloadConfig;
import com.sleepycat.je.PreloadStats;

/**
 * A null index which replaces id2children and id2subtree when they have been
 * disabled.
@@ -66,10 +63,10 @@
   * @throws StorageRuntimeException
   *           If an error occurs in the JE database.
   */
  public NullIndex(TreeName name, Indexer indexer, State state, Storage storage,
  public NullIndex(TreeName name, Indexer indexer, State state, Storage storage, WriteableStorage txn,
      EntryContainer entryContainer) throws StorageRuntimeException
  {
    super(name, indexer, state, 0, 0, false, storage, entryContainer);
    super(name, indexer, state, 0, 0, false, storage, txn, entryContainer);
  }

  /** {@inheritDoc} */
@@ -255,12 +252,4 @@
  {
    return 0;
  }

  /** {@inheritDoc} */
  @Override
  public PreloadStats preload(PreloadConfig config) throws StorageRuntimeException
  {
    return new PreloadStats();
  }

}

 opendj-sdk/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/RootContainer.java

@@ -383,61 +383,7 @@
      // is filled.
      try
      {
        // Configure preload of Leaf Nodes (LNs) containing the data values.
        PreloadConfig preloadConfig = new PreloadConfig();
        preloadConfig.setLoadLNs(true);

        logger.info(NOTE_JEB_CACHE_PRELOAD_STARTED, backend.getBackendID());

        boolean isInterrupted = false;

        long timeEnd = System.currentTimeMillis() + timeLimit;

        for (DatabaseContainer db : dbList)
        {
          // Calculate the remaining time.
          long timeRemaining = timeEnd - System.currentTimeMillis();
          if (timeRemaining <= 0)
          {
            break;
          }

          preloadConfig.setMaxMillisecs(timeRemaining);
          PreloadStats preloadStats = db.preload(preloadConfig);

          if(logger.isTraceEnabled())
          {
            logger.trace("file=" + db.getName() + " LNs=" + preloadStats.getNLNsLoaded());
          }

          // Stop if the cache is full or the time limit has been exceeded.
          PreloadStatus preloadStatus = preloadStats.getStatus();
          if (preloadStatus != PreloadStatus.SUCCESS)
          {
            if (preloadStatus == PreloadStatus.EXCEEDED_TIME) {
              logger.info(NOTE_JEB_CACHE_PRELOAD_INTERRUPTED_BY_TIME, backend.getBackendID(), db.getName());
            } else if (preloadStatus == PreloadStatus.FILLED_CACHE) {
              logger.info(NOTE_JEB_CACHE_PRELOAD_INTERRUPTED_BY_SIZE, backend.getBackendID(), db.getName());
            } else {
              logger.info(NOTE_JEB_CACHE_PRELOAD_INTERRUPTED_UNKNOWN, backend.getBackendID(), db.getName());
            }

            isInterrupted = true;
            break;
          }

          logger.info(NOTE_JEB_CACHE_DB_PRELOADED, db.getName());
        }

        if (!isInterrupted) {
          logger.info(NOTE_JEB_CACHE_PRELOAD_DONE, backend.getBackendID());
        }

        // Log an informational message about the size of the cache.
        EnvironmentStats stats = storage.getStats(new StatsConfig());
        long total = stats.getCacheTotalBytes();

        logger.info(NOTE_JEB_CACHE_SIZE_AFTER_PRELOAD, total / (1024 * 1024));
        throw new NotImplementedException();
      }
      catch (StorageRuntimeException e)
      {
@@ -471,7 +417,8 @@
      }
    }

    compressedSchema.close();
    // FIXME JNR call close() for a DB stored compressed schema
    // compressedSchema.close();
    config.removeLocalDBChangeListener(this);

    if (storage != null)
@@ -525,22 +472,6 @@
  }

  /**
   * Get the environment stats of the JE environment used in this root
   * container.
   *
   * @param statsConfig The configuration to use for the EnvironmentStats
   *                    object.
   * @return The environment status of the JE environment.
   * @throws StorageRuntimeException If an error occurs while retrieving the stats
   *                           object.
   */
  public EnvironmentStats getEnvironmentStats(StatsConfig statsConfig)
      throws StorageRuntimeException
  {
    return storage.getStats(statsConfig);
  }

  /**
   * Get the backend configuration used by this root container.
   *
   * @return The JE backend configuration used by this root container.
@@ -676,7 +607,7 @@

    try
    {
      ConfigurableEnvironment.parseConfigEntry(cfg);
      // FIXME JNR validate database specific configuration
    }
    catch (Exception e)
    {

 opendj-sdk/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/VLVIndex.java

@@ -70,8 +70,6 @@
import org.opends.server.types.SortOrder;
import org.opends.server.util.StaticUtils;

import com.sleepycat.je.LockMode;

import static org.opends.messages.JebMessages.*;
import static org.opends.server.util.StaticUtils.*;

@@ -721,7 +719,7 @@
    {
      debugBuilder.append("vlv=");
      debugBuilder.append("[INDEX:");
      debugBuilder.append(treeName.replace(entryContainer.getDatabasePrefix() + "_", ""));
      debugBuilder.append(treeName.getIndex());
      debugBuilder.append("]");
    }

@@ -837,7 +835,6 @@
        Cursor cursor = openCursor(txn);
        try
        {
          LockMode lockMode = LockMode.DEFAULT;
          ByteSequence vBytes = vlvRequest.getGreaterThanOrEqualAssertion();
          ByteStringBuilder keyBytes = new ByteStringBuilder(vBytes.length() + 4);
          keyBytes.appendBERLength(vBytes.length());
@@ -850,8 +847,7 @@
            {
              logSearchKeyResult(cursor.getKey());
            }
            SortValuesSet sortValuesSet =
 new SortValuesSet(cursor.getKey(), cursor.getValue(), this);
            SortValuesSet sortValuesSet = new SortValuesSet(cursor.getKey(), cursor.getValue(), this);

            int adjustedTargetOffset = sortValuesSet.binarySearch(
                -1, vlvRequest.getGreaterThanOrEqualAssertion());
@@ -1406,7 +1402,7 @@
      ccr.addMessage(NOTE_JEB_INDEX_ADD_REQUIRES_REBUILD.get(treeName));
      try
      {
        state.putIndexTrustState(null, this, false);
        state.putIndexTrustState(txn, this, false);
      }
      catch(StorageRuntimeException de)
      {

 opendj-sdk/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/VerifyJob.java

@@ -62,9 +62,6 @@
import org.opends.server.util.ServerConstants;
import org.opends.server.util.StaticUtils;

import com.sleepycat.je.EnvironmentStats;
import com.sleepycat.je.StatsConfig;

import static org.opends.messages.JebMessages.*;
import static org.opends.server.backends.pluggable.JebFormat.*;

@@ -290,14 +287,8 @@

      // Start a timer for the progress report.
      Timer timer = new Timer();
      TimerTask progressTask = new ProgressTask();
      if (cleanMode)
      {
        // Create a new progressTask based on the index count.
        progressTask = new ProgressTask(true);
      }
      timer.scheduleAtFixedRate(progressTask, progressInterval,
                                progressInterval);
      TimerTask progressTask = new ProgressTask(cleanMode, txn);
      timer.scheduleAtFixedRate(progressTask, progressInterval, progressInterval);

      // Iterate through the index keys.
      try
@@ -685,8 +676,7 @@
              errorCount++;
              if (logger.isTraceEnabled())
              {
                logger.trace("File id2children has ID %d referencing " +
 "unknown ID %d%n", entryID, id);
                logger.trace("File id2children has ID %d referencing unknown ID %d%n", entryID, id);
              }
              continue;
            }
@@ -1419,11 +1409,10 @@
    {
      try
      {
        List<Attribute> attrList =
             entry.getAttribute(attrIndex.getAttributeType());
        List<Attribute> attrList = entry.getAttribute(attrIndex.getAttributeType());
        if (attrList != null)
        {
          verifyAttribute(attrIndex, entryID, attrList);
          verifyAttribute(txn, attrIndex, entryID, attrList);
        }
      }
      catch (DirectoryException e)
@@ -1432,8 +1421,7 @@
        {
          logger.traceException(e);

          logger.trace("Error normalizing values of attribute %s in " +
              "entry <%s>: %s.%n",
          logger.trace("Error normalizing values of attribute %s in entry <%s>: %s.%n",
                     attrIndex.getAttributeType(), entry.getName(), e.getMessageObject());
        }
      }
@@ -1485,8 +1473,7 @@
   * @param attrList The attribute to be checked.
   * @throws DirectoryException If a Directory Server error occurs.
   */
  private void verifyAttribute(AttributeIndex attrIndex, EntryID entryID,
                              List<Attribute> attrList)
  private void verifyAttribute(ReadableStorage txn, AttributeIndex attrIndex, EntryID entryID, List<Attribute> attrList)
       throws DirectoryException
  {
    if (attrList == null || attrList.isEmpty())
@@ -1494,7 +1481,6 @@
      return;
    }

    ReadableStorage txn = null; // FIXME JNR
    Index equalityIndex = attrIndex.getEqualityIndex();
    Index presenceIndex = attrIndex.getPresenceIndex();
    Index substringIndex = attrIndex.getSubstringIndex();
@@ -1607,9 +1593,7 @@
   */
  private class ProgressTask extends TimerTask
  {
    /**
     * The total number of records to process.
     */
    /** The total number of records to process. */
    private long totalCount;

    /**
@@ -1618,17 +1602,10 @@
     */
    private long previousCount;

    /**
     * The time in milliseconds of the previous progress report.
     */
    /** The time in milliseconds of the previous progress report. */
    private long previousTime;

    /**
     * The environment statistics at the time of the previous report.
     */
    private EnvironmentStats prevEnvStats;

    /**
     * The number of bytes in a megabyte.
     * Note that 1024*1024 bytes may eventually become known as a mebibyte(MiB).
     */
@@ -1636,85 +1613,61 @@

    /**
     * Create a new verify progress task.
     * @throws StorageRuntimeException An error occurred while accessing the JE
     * database.
     */
    public ProgressTask() throws StorageRuntimeException
    {
      previousTime = System.currentTimeMillis();
      prevEnvStats =
          rootContainer.getEnvironmentStats(new StatsConfig());
      totalCount = rootContainer.getEntryContainer(
        verifyConfig.getBaseDN()).getEntryCount();
    }

    /**
     * Create a new verify progress task.
     * @param indexIterator boolean, indicates if the task is iterating
     * through indexes or the entries.
     * @throws StorageRuntimeException An error occurred while accessing the JE
     * database.
     */
    private ProgressTask(boolean indexIterator) throws StorageRuntimeException
    private ProgressTask(boolean indexIterator, ReadableStorage txn) throws StorageRuntimeException
    {
      previousTime = System.currentTimeMillis();
      prevEnvStats = rootContainer.getEnvironmentStats(new StatsConfig());

      if (indexIterator)
      {
        if (verifyDN2ID)
        {
          totalCount = dn2id.getRecordCount();
          totalCount = dn2id.getRecordCount(txn);
        }
        else if (verifyID2Children)
        {
          totalCount = id2c.getRecordCount();
          totalCount = id2c.getRecordCount(txn);
        }
        else if (verifyID2Subtree)
        {
          totalCount = id2s.getRecordCount();
          totalCount = id2s.getRecordCount(txn);
        }
        else if(attrIndexList.size() > 0)
        {
          AttributeIndex attrIndex = attrIndexList.get(0);
          totalCount = 0;
          if (attrIndex.getEqualityIndex() != null)
          {
            totalCount += attrIndex.getEqualityIndex().getRecordCount();
          }
          if (attrIndex.getPresenceIndex() != null)
          {
            totalCount += attrIndex.getPresenceIndex().getRecordCount();
          }
          if (attrIndex.getSubstringIndex() != null)
          {
            totalCount += attrIndex.getSubstringIndex().getRecordCount();
          }
          if (attrIndex.getOrderingIndex() != null)
          {
            totalCount += attrIndex.getOrderingIndex().getRecordCount();
          }
          if (attrIndex.getApproximateIndex() != null)
          {
            totalCount += attrIndex.getApproximateIndex().getRecordCount();
          }
          totalCount += getRecordCount(txn, attrIndex.getEqualityIndex());
          totalCount += getRecordCount(txn, attrIndex.getPresenceIndex());
          totalCount += getRecordCount(txn, attrIndex.getSubstringIndex());
          totalCount += getRecordCount(txn, attrIndex.getOrderingIndex());
          totalCount += getRecordCount(txn, attrIndex.getApproximateIndex());
          // TODO: Add support for Extended Matching Rules indexes.
        }
        else if (vlvIndexList.size() > 0)
        {
          totalCount = vlvIndexList.get(0).getRecordCount();
          totalCount = vlvIndexList.get(0).getRecordCount(txn);
        }
      }
      else
      {
        totalCount = rootContainer.getEntryContainer(
          verifyConfig.getBaseDN()).getEntryCount();
        totalCount = rootContainer.getEntryContainer(verifyConfig.getBaseDN()).getEntryCount(txn);
      }
    }

    /**
     * The action to be performed by this timer task.
     */
    private long getRecordCount(ReadableStorage txn, Index index)
    {
      if (index != null)
      {
        return index.getRecordCount(txn);
      }
      return 0;
    }

    /** The action to be performed by this timer task. */
    @Override
    public void run()
    {
@@ -1737,20 +1690,10 @@
        Runtime runtime = Runtime.getRuntime();
        long freeMemory = runtime.freeMemory() / bytesPerMegabyte;

        EnvironmentStats envStats =
            rootContainer.getEnvironmentStats(new StatsConfig());
        long nCacheMiss =
             envStats.getNCacheMiss() - prevEnvStats.getNCacheMiss();

        // FIXME JNR compute the cache miss rate
        float cacheMissRate = 0;
        if (deltaCount > 0)
        {
          cacheMissRate = nCacheMiss/(float)deltaCount;
        }

        logger.debug(INFO_JEB_VERIFY_CACHE_AND_MEMORY_REPORT, freeMemory, cacheMissRate);

        prevEnvStats = envStats;
      }
      catch (StorageRuntimeException e)
      {

 opendj-sdk/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/spi/TreeName.java

@@ -23,7 +23,6 @@
 *
 *      Copyright 2014 ForgeRock AS
 */

package org.opends.server.backends.pluggable.spi;

import java.util.ArrayList;
@@ -76,15 +75,26 @@
    return new TreeName(Collections.singletonList(names.get(0)));
  }

  public boolean isSuffixOf(TreeName tree)
  public TreeName replaceSuffix(TreeName newSuffix)
  {
    if (names.size() > tree.names.size())
    if (names.size() == 0)
    {
      throw new IllegalStateException();
    }
    final ArrayList<String> newNames = new ArrayList<String>(names);
    newNames.set(0, newSuffix.names.get(0));
    return new TreeName(newNames);
  }

  public boolean isSuffixOf(TreeName treeName)
  {
    if (names.size() > treeName.names.size())
    {
      return false;
    }
    for (int i = 0; i < names.size(); i++)
    {
      if (!tree.names.get(i).equals(names.get(i)))
      if (!treeName.names.get(i).equals(names.get(i)))
      {
        return false;
      }
@@ -92,6 +102,15 @@
    return true;
  }

  public TreeName getIndex()
  {
    if (names.size() == 1)
    {
      return null;
    }
    return new TreeName(names.subList(1, names.size()));
  }

  @Override
  public boolean equals(final Object obj)
  {

 opendj-sdk/opendj3-server-dev/src/server/org/opends/server/backends/pluggable/spi/WriteableStorage.java

@@ -23,7 +23,6 @@
 *
 *      Copyright 2014 ForgeRock AS
 */

package org.opends.server.backends.pluggable.spi;

import org.forgerock.opendj.ldap.ByteSequence;
@@ -32,6 +31,10 @@
{
  void openTree(TreeName name);

  void truncateTree(TreeName name);

  void deleteTree(TreeName name);

  void create(TreeName name, ByteSequence key, ByteSequence value);

  boolean putIfAbsent(TreeName treeName, ByteSequence key, ByteSequence value);