Rebuild index using new import engine.

dugan

26.33.2009 3e7410b2ec4ceeef110ed4864d86cd56993512fe

Rebuild index using new import engine.

 opends/resource/schema/02-config.ldif

@@ -1423,8 +1423,8 @@
  SYNTAX 1.3.6.1.4.1.1466.115.121.1.15
  X-ORIGIN 'OpenDS Directory Server' )
attributeTypes: ( 1.3.6.1.4.1.26027.1.1.291
  NAME 'ds-task-rebuild-max-threads'
  SYNTAX 1.3.6.1.4.1.1466.115.121.1.27
  NAME 'ds-task-rebuild-tmp-directory'
  SYNTAX 1.3.6.1.4.1.1466.115.121.1.15
  SINGLE-VALUE
  X-ORIGIN 'OpenDS Directory Server' )
attributeTypes: ( 1.3.6.1.4.1.26027.1.1.292
@@ -3398,7 +3398,7 @@
  STRUCTURAL
  MUST ( ds-task-rebuild-base-dn $
         ds-task-rebuild-index )
  MAY ds-task-rebuild-max-threads
  MAY ds-task-rebuild-tmp-directory
  X-ORIGIN 'OpenDS Directory Server' )
objectClasses: ( 1.3.6.1.4.1.26027.1.2.98
  NAME 'ds-virtual-static-group'

 opends/src/server/org/opends/server/backends/jeb/BackendImpl.java

@@ -1136,8 +1136,7 @@
      envConfig.setConfigParam(EnvironmentConfig.ENV_RUN_CLEANER, "false");
      envConfig.setConfigParam(EnvironmentConfig.EVICTOR_LRU_ONLY, "false");
      envConfig.setConfigParam(EnvironmentConfig.EVICTOR_NODES_PER_SCAN, "128");
      Importer importer = new Importer(importConfig, cfg);
      importer.initialize(envConfig);
      Importer importer = Importer.getInstance(importConfig, cfg, envConfig);
      rootContainer = initializeRootContainer(envConfig);
      return importer.processImport(rootContainer);
    }
@@ -1337,35 +1336,73 @@
   * @throws DirectoryException If a Directory Server error occurs.
   */
  public void rebuildBackend(RebuildConfig rebuildConfig)
      throws InitializationException, ConfigException, DirectoryException
          throws InitializationException, ConfigException, DirectoryException
  {
    // If the backend already has the root container open, we must use the same
    // underlying root container
    boolean openRootContainer = rootContainer == null;

    // If the rootContainer is open, the backend is initialized by something
    // else.
    // We can't do any rebuild of system indexes while others are using this
    // backend. Throw error. TODO: Need to make baseDNs disablable.
    /*
      If the rootContainer is open, the backend is initialized by something
      else. We can't do any rebuild of system indexes while others are using
      this backend.
   */
    if(!openRootContainer && rebuildConfig.includesSystemIndex())
    {
      Message message = ERR_JEB_REBUILD_BACKEND_ONLINE.get();
      throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
                                   message);
              message);
    }

    Importer importer;
    try
    {
      EnvironmentConfig envConfig =
              ConfigurableEnvironment.parseConfigEntry(cfg);
      importer = Importer.getInstance(rebuildConfig, cfg, envConfig);
      if (openRootContainer)
      {
        EnvironmentConfig envConfig =
            ConfigurableEnvironment.parseConfigEntry(cfg);

        rootContainer = initializeRootContainer(envConfig);
      }

      RebuildJob rebuildJob = new RebuildJob(rebuildConfig);
      rebuildJob.rebuildBackend(rootContainer);
      importer.rebuildIndexes(rootContainer);
    }
    catch (ExecutionException execEx)
    {
      if (debugEnabled())
      {
        TRACER.debugCaught(DebugLogLevel.ERROR, execEx);
      }
      Message message = ERR_EXECUTION_ERROR.get(execEx.getMessage());
      throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
              message);
    }
    catch (InterruptedException intEx)
    {
      if (debugEnabled())
      {
        TRACER.debugCaught(DebugLogLevel.ERROR, intEx);
      }
      Message message = ERR_INTERRUPTED_ERROR.get(intEx.getMessage());
      throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
              message);
    }
    catch (IOException ioe)
    {
      if (debugEnabled())
      {
        TRACER.debugCaught(DebugLogLevel.ERROR, ioe);
      }
      Message message = ERR_JEB_IO_ERROR.get(ioe.getMessage());
      throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
              message);
    }
    catch (ConfigException ce)
    {
      if (debugEnabled())
      {
        TRACER.debugCaught(DebugLogLevel.ERROR, ce);
      }
      throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
              ce.getMessageObject());
    }
    catch (DatabaseException e)
    {
@@ -1382,7 +1419,7 @@
        TRACER.debugCaught(DebugLogLevel.ERROR, e);
      }
      throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
                                   e.getMessageObject());
              e.getMessageObject());
    }
    finally
    {
@@ -1407,7 +1444,6 @@
  }



  /**
   * {@inheritDoc}
   */
@@ -1652,7 +1688,7 @@
   */
  DirectoryException createDirectoryException(DatabaseException e) {
    ResultCode resultCode = DirectoryServer.getServerErrorResultCode();
    Message message = null;
    Message message;
    if (e instanceof RunRecoveryException) {
      message = NOTE_BACKEND_ENVIRONMENT_UNUSABLE.get(getBackendID());
      logError(message);

 opends/src/server/org/opends/server/backends/jeb/RebuildConfig.java

@@ -22,7 +22,7 @@
 * CDDL HEADER END
 *
 *
 *      Copyright 2006-2008 Sun Microsystems, Inc.
 *      Copyright 2006-2009 Sun Microsystems, Inc.
 */
package org.opends.server.backends.jeb;

@@ -45,11 +45,9 @@
   */
  private ArrayList<String> rebuildList;

  /**
   * The maximum number of rebuild threads to use at one time. An negative
   * number indicates unlimited max number of threads.
   */
  private int maxRebuildThreads = -1;
  private boolean rebuildAll = false;

  private String tmpDirectory;

  /**
   * Create a new rebuild configuraiton.
@@ -173,28 +171,6 @@
  }

  /**
   * Get the maximum number of rebuild threads to use for the rebuild job
   * at one time.
   *
   * @return The maximum number of rebuild threads.
   */
  public int getMaxRebuildThreads()
  {
    return maxRebuildThreads;
  }

  /**
   * Set the maximum number of rebuild threads to use for the rebuild
   * job at one time.
   *
   * @param maxRebuildThreads The maximum number of rebuild threads.
   */
  public void setMaxRebuildThreads(int maxRebuildThreads)
  {
    this.maxRebuildThreads = maxRebuildThreads;
  }

  /**
   * Test if this rebuild config includes any system indexes to rebuild.
   *
   * @return True if rebuilding of system indexes are included. False otherwise.
@@ -215,17 +191,52 @@
      {
        return true;
      }
      if(index.equalsIgnoreCase("id2children"))
      {
        return true;
      }
      if(index.equalsIgnoreCase("id2subtree"))
      {
        return true;
      }
    }

    return false;
  }


  /**
   * Set the temporary directory to the specified path.
   *
   * @param path The path to set the temporary directory to.
   */
  public void setTmpDirectory(String path)
  {
    tmpDirectory = path;
  }

  /**
   * Return the temporary directory path.
   *
   * @return  The temporary directory string.
   */
  public String getTmpDirectory()
  {
    return tmpDirectory;
  }


  /**
   * Set the rebuild all boolean to the specified value.
   *
   * @param v The value to set the rebuild all boolean to.
   */
  public void setRebuildAll(boolean v)
  {
    rebuildAll = v;
  }

  /**
   * Return the rebuild all boolean value.
   *
   * @return Return the rebuild all boolean value.
   */
  public boolean isRebuildAll()
  {
    return rebuildAll;
  }


}

 opends/src/server/org/opends/server/backends/jeb/importLDIF/Importer.java

@@ -45,6 +45,8 @@
import org.opends.messages.Category;
import org.opends.messages.Severity;
import org.opends.server.admin.std.server.LocalDBBackendCfg;
import org.opends.server.admin.std.server.LocalDBIndexCfg;
import org.opends.server.admin.std.meta.LocalDBIndexCfgDefn;
import org.opends.server.backends.jeb.*;
import org.opends.server.config.ConfigException;
import org.opends.server.core.DirectoryServer;
@@ -54,7 +56,7 @@


/**
 * Performs a LDIF import.
 * Performs LDIF import and rebuild of indexes.
 */
public class Importer
{
@@ -134,8 +136,9 @@
          new ConcurrentHashMap<Integer, EntryContainer>();

  private final Object synObj = new Object();
  private final RebuildManager rebuildManager;

    static
  static
  {
    if ((dnType = DirectoryServer.getAttributeType("dn")) == null)
    {
@@ -143,18 +146,79 @@
    }
  }

  private void initialize()
  {

  }

  private
  Importer(RebuildConfig rebuildConfig, LocalDBBackendCfg cfg,
            EnvironmentConfig envConfig) throws IOException,
          InitializationException, JebException, ConfigException
  {
    this.importConfiguration = null;
    this.threadCount = 1;
    this.rebuildManager = new RebuildManager(rebuildConfig, cfg);
    indexCount = rebuildManager.getIndexCount();
    indexWriterList = new ArrayList<IndexFileWriterTask>(indexCount);
    indexWriterFutures = new CopyOnWriteArrayList<Future<?>>();
    File parentDir;
    if(rebuildConfig.getTmpDirectory() == null)
    {
      parentDir = getFileForPath("import-tmp");
    }
    else
    {
       parentDir = getFileForPath(rebuildConfig.getTmpDirectory());
    }
    tempDir = new File(parentDir, cfg.getBackendId());
    if(!tempDir.exists() && !tempDir.mkdirs())
    {
      Message message =
                ERR_JEB_IMPORT_CREATE_TMPDIR_ERROR.get(String.valueOf(tempDir));
      throw new IOException(message.toString());
    }
    if (tempDir.listFiles() != null)
    {
      for (File f : tempDir.listFiles())
      {
        f.delete();
      }
    }
    skipDNValidation = true;
    String propString = System.getProperty(DIRECT_PROPERTY);
    if(propString != null)
    {
      int directSize = Integer.valueOf(propString);
      directBuffer = ByteBuffer.allocateDirect(directSize);
    }
    else
    {
     directBuffer = null;
    }
    if(envConfig != null)
    {
      initializeDBEnv(envConfig);
    }
  }


  /**
   * Create a new import job with the specified ldif import config.
   *
   * @param importConfiguration The LDIF import configuration.
   * @param dbCfg The local DB back-end configuration.
   * @param localDBBackendCfg The local DB back-end configuration.
   * @param envConfig The JEB environment config.
   * @throws IOException  If a problem occurs while opening the LDIF file for
   *                      reading.
   * @throws  InitializationException If a problem occurs during initialization.
   */
  public Importer(LDIFImportConfig importConfiguration, LocalDBBackendCfg dbCfg)
          throws IOException, InitializationException
  private Importer(LDIFImportConfig importConfiguration,
                   LocalDBBackendCfg localDBBackendCfg,
                   EnvironmentConfig envConfig) throws IOException,
          InitializationException
  {
    this.rebuildManager = null;
    this.importConfiguration = importConfiguration;
    if(importConfiguration.getThreadCount() == 0)
    {
@@ -164,7 +228,9 @@
    {
      threadCount = importConfiguration.getThreadCount();
    }
    indexCount = dbCfg.listLocalDBIndexes().length + 2;
    indexCount = localDBBackendCfg.listLocalDBIndexes().length + 2;


    indexWriterList = new ArrayList<IndexFileWriterTask>(indexCount);
    indexWriterFutures = new CopyOnWriteArrayList<Future<?>>();
    File parentDir;
@@ -177,7 +243,7 @@
       parentDir = getFileForPath(importConfiguration.getTmpDirectory());
    }

    tempDir = new File(parentDir, dbCfg.getBackendId());
    tempDir = new File(parentDir, localDBBackendCfg.getBackendId());
    if(!tempDir.exists() && !tempDir.mkdirs())
    {
      Message message =
@@ -202,9 +268,53 @@
    {
     directBuffer = null;
    }
    initializeDBEnv(envConfig);
  }

    private void getBufferSizes(long availMem, int buffers)
  /**
   * Return and import LDIF instance using the specified arguments.
   *
   * @param importCfg The import config to use.
   * @param localDBBackendCfg The local DB backend config to use.
   * @param envCfg The JEB environment config to use.
   * @return A import LDIF instance.
   *
   * @throws IOException If an I/O error occurs.
   * @throws InitializationException If the instance cannot be initialized.
   */
  public static
  Importer getInstance(LDIFImportConfig importCfg,
                       LocalDBBackendCfg localDBBackendCfg,
                       EnvironmentConfig envCfg)
          throws IOException, InitializationException
  {
     return  new Importer(importCfg, localDBBackendCfg, envCfg);
  }


  /**
   * Return an import rebuild index instance using the specified arguments.
   *
   * @param rebuildCfg The rebuild config to use.
   * @param localDBBackendCfg The local DB backend config to use.
   * @param envCfg The JEB environment config to use.
   * @return An import rebuild index instance.
   *
   * @throws IOException If an I/O error occurs.
   * @throws InitializationException If the instance cannot be initialized.
   * @throws JebException If a JEB exception occurs.
   * @throws ConfigException If the instance cannot be configured.
   */
  public static synchronized
  Importer getInstance(RebuildConfig rebuildCfg,
                       LocalDBBackendCfg localDBBackendCfg,
                       EnvironmentConfig envCfg)
  throws IOException, InitializationException, JebException, ConfigException
  {
      return new Importer(rebuildCfg, localDBBackendCfg, envCfg);
  }

  private void getBufferSizes(long availMem, int buffers)
  {
    long memory = availMem - (MAX_DB_CACHE_SIZE + MAX_DB_LOG_BUFFER_BYTES);
    bufferSize = (int) (memory/buffers);
@@ -240,7 +350,6 @@
    }
  }


  /**
   * Return the suffix instance in the specified map that matches the specified
   * DN.
@@ -272,7 +381,7 @@
   *
   * @throws InitializationException If a problem occurs during calculation.
   */
  public void initialize(EnvironmentConfig envConfig)
  private void initializeDBEnv(EnvironmentConfig envConfig)
          throws InitializationException
  {
      Message message;
@@ -447,6 +556,33 @@
 }


  /**
   * Rebuild the indexes using the specified rootcontainer.
   *
   * @param rootContainer The rootcontainer to rebuild indexes in.
   *
   * @throws ConfigException If a configuration error occurred.
   * @throws InitializationException If an initialization error occurred.
   * @throws IOException If an IO error occurred.
   * @throws JebException If the JEB database had an error.
   * @throws DatabaseException If a database error occurred.
   * @throws InterruptedException If an interrupted error occurred.
   * @throws ExecutionException If an execution error occurred.
   */
  public void
  rebuildIndexes(RootContainer rootContainer) throws ConfigException,
          InitializationException, IOException, JebException, DatabaseException,
          InterruptedException, ExecutionException
  {
    this.rootContainer = rootContainer;
    long startTime = System.currentTimeMillis();
    rebuildManager.initialize();
    rebuildManager.printStartMessage();
    rebuildManager.rebuldIndexes();
    tempDir.delete();
    rebuildManager.printStopMessage(startTime);
  }


  /**
   * Import a LDIF using the specified root container.
@@ -493,9 +629,10 @@
      float rate = 0;
      if (importTime > 0)
        rate = 1000f * reader.getEntriesRead() / importTime;
      message = NOTE_JEB_IMPORT_FINAL_STATUS.get(reader.getEntriesRead(),
              importCount.get(), reader.getEntriesIgnored(), reader
                 .getEntriesRejected(), migratedCount, importTime / 1000, rate);
        message = NOTE_JEB_IMPORT_FINAL_STATUS.get(reader.getEntriesRead(),
                  importCount.get(), reader.getEntriesIgnored(),
                  reader.getEntriesRejected(), migratedCount,
                  importTime / 1000, rate);
      logError(message);
    }
    finally
@@ -566,8 +703,11 @@

    tasks.add(new MigrateExistingTask());
    List<Future<Void>> results = execService.invokeAll(tasks);
    for (Future<Void> result : results)
      assert result.isDone();
    for (Future<Void> result : results) {
      if(!result.isDone()) {
        result.get();
      }
    }
    tasks.clear();
    results.clear();

@@ -588,21 +728,23 @@
    }
    results = execService.invokeAll(tasks);
    for (Future<Void> result : results)
      assert result.isDone();


      if(!result.isDone()) {
        result.get();
      }
    tasks.clear();
    results.clear();
    tasks.add(new MigrateExcludedTask());
    results = execService.invokeAll(tasks);
    for (Future<Void> result : results)
      assert result.isDone();


      if(!result.isDone()) {
        result.get();
      }
    stopIndexWriterTasks();
    for (Future<?> result : indexWriterFutures)
    {
      result.get();
     if(!result.isDone()) {
        result.get();
      }
    }
    indexWriterList.clear();
    indexWriterFutures.clear();
@@ -618,7 +760,7 @@
  private void processPhaseTwo() throws InterruptedException
  {
    SecondPhaseProgressTask progress2Task =
            new SecondPhaseProgressTask(indexMgrList);
            new SecondPhaseProgressTask(indexMgrList, reader.getEntriesRead());
    Timer timer2 = new Timer();
    timer2.scheduleAtFixedRate(progress2Task, TIMER_INTERVAL, TIMER_INTERVAL);
    processIndexFiles();
@@ -975,24 +1117,23 @@
          Index index;
          if((index=attributeIndex.getEqualityIndex()) != null) {
            processAttribute(index, entry, entryID,
                      new IndexKey(attributeType,IndexType.EQUALITY));
                      new IndexKey(attributeType, ImportIndexType.EQUALITY));
          }
          if((index=attributeIndex.getPresenceIndex()) != null) {
            processAttribute(index, entry, entryID,
                      new IndexKey(attributeType, IndexType.PRESENCE));
                      new IndexKey(attributeType, ImportIndexType.PRESENCE));
          }
          if((index=attributeIndex.getSubstringIndex()) != null) {
            int subLen = ((SubstringIndexer)index.indexer).getSubStringLen();
            processAttribute(index, entry, entryID,
                      new IndexKey(attributeType, IndexType.SUBSTRING, subLen));
                new IndexKey(attributeType, ImportIndexType.SUBSTRING));
          }
          if((index=attributeIndex.getOrderingIndex()) != null) {
            processAttribute(index, entry, entryID,
                      new IndexKey(attributeType, IndexType.ORDERING));
                      new IndexKey(attributeType, ImportIndexType.ORDERING));
          }
          if((index=attributeIndex.getApproximateIndex()) != null) {
            processAttribute(index, entry, entryID,
                       new IndexKey(attributeType,IndexType.APPROXIMATE));
                      new IndexKey(attributeType, ImportIndexType.APPROXIMATE));
          }
          for(VLVIndex vlvIdx : suffix.getEntryContainer().getVLVIndexes()) {
            Transaction transaction = null;
@@ -1007,7 +1148,7 @@
            if(subIndexes != null) {
              for(Index subIndex: subIndexes) {
                processAttribute(subIndex, entry, entryID,
                          new IndexKey(attributeType, IndexType.EX_SUBSTRING));
                     new IndexKey(attributeType, ImportIndexType.EX_SUBSTRING));
              }
            }
            Collection<Index> sharedIndexes =
@@ -1016,7 +1157,7 @@
            if(sharedIndexes !=null) {
              for(Index sharedIndex:sharedIndexes) {
                processAttribute(sharedIndex, entry, entryID,
                          new IndexKey(attributeType, IndexType.EX_SHARED));
                       new IndexKey(attributeType, ImportIndexType.EX_SHARED));
              }
            }
          }
@@ -1274,24 +1415,23 @@
          Index index;
          if((index=attributeIndex.getEqualityIndex()) != null) {
            processAttribute(index, entry, entryID,
                      new IndexKey(attributeType,IndexType.EQUALITY));
                      new IndexKey(attributeType, ImportIndexType.EQUALITY));
          }
          if((index=attributeIndex.getPresenceIndex()) != null) {
            processAttribute(index, entry, entryID,
                      new IndexKey(attributeType, IndexType.PRESENCE));
                      new IndexKey(attributeType, ImportIndexType.PRESENCE));
          }
          if((index=attributeIndex.getSubstringIndex()) != null) {
            int subLen = ((SubstringIndexer)index.indexer).getSubStringLen();
            processAttribute(index, entry, entryID,
                      new IndexKey(attributeType, IndexType.SUBSTRING, subLen));
                new IndexKey(attributeType, ImportIndexType.SUBSTRING));
          }
          if((index=attributeIndex.getOrderingIndex()) != null) {
            processAttribute(index, entry, entryID,
                      new IndexKey(attributeType, IndexType.ORDERING));
                      new IndexKey(attributeType, ImportIndexType.ORDERING));
          }
          if((index=attributeIndex.getApproximateIndex()) != null) {
            processAttribute(index, entry, entryID,
                       new IndexKey(attributeType,IndexType.APPROXIMATE));
                      new IndexKey(attributeType, ImportIndexType.APPROXIMATE));
          }
          for(VLVIndex vlvIdx : suffix.getEntryContainer().getVLVIndexes()) {
            Transaction transaction = null;
@@ -1306,7 +1446,7 @@
            if(subIndexes != null) {
              for(Index subIndex: subIndexes) {
                processAttribute(subIndex, entry, entryID,
                          new IndexKey(attributeType, IndexType.EX_SUBSTRING));
                     new IndexKey(attributeType, ImportIndexType.EX_SUBSTRING));
              }
            }
            Collection<Index> sharedIndexes =
@@ -1315,7 +1455,7 @@
            if(sharedIndexes !=null) {
              for(Index sharedIndex:sharedIndexes) {
                processAttribute(sharedIndex, entry, entryID,
                          new IndexKey(attributeType, IndexType.EX_SHARED));
                        new IndexKey(attributeType, ImportIndexType.EX_SHARED));
              }
            }
          }
@@ -1346,8 +1486,8 @@
        {
          IndexKey indexKey = e.getKey();
          IndexBuffer indexBuffer = e.getValue();
          IndexType indexType = indexKey.getIndexType();
          if(indexType.equals(IndexType.DN))
          ImportIndexType indexType = indexKey.getIndexType();
          if(indexType.equals(ImportIndexType.DN))
          {
            indexBuffer.setComparator(dnComparator);
          }
@@ -1413,7 +1553,7 @@
      DatabaseContainer dn2id = suffix.getDN2ID();
      byte[] dnBytes = StaticUtils.getBytes(dn.toNormalizedString());
      int id = processKey(dn2id, dnBytes, entryID, dnComparator,
                 new IndexKey(dnType, IndexType.DN), true);
                 new IndexKey(dnType, ImportIndexType.DN), true);
      idECMap.putIfAbsent(id, suffix.getEntryContainer());
    }

@@ -2070,7 +2210,8 @@
     */
    public Void call() throws Exception
    {
      if (importConfiguration.isCancelled())
      if (importConfiguration != null &&
          importConfiguration.isCancelled())
      {
        return null;
      }
@@ -2100,7 +2241,7 @@
        {
          return;
        }
        if(indexKey.getIndexType().equals(IndexType.DN))
        if(indexKey.getIndexType().equals(ImportIndexType.DN))
        {
          isDN = true;
        }
@@ -2522,6 +2663,837 @@
    }
  }


  /**
   * The rebuild manager handles all rebuild index related tasks.
   */
  class RebuildManager extends ImportTask {

   private final RebuildConfig rebuildConfig;
   private final LocalDBBackendCfg cfg;
   private final Map<IndexKey, Index> indexMap =
                          new LinkedHashMap<IndexKey, Index>();
   private final Map<IndexKey, Collection<Index>> extensibleIndexMap =
                               new LinkedHashMap<IndexKey, Collection<Index>>();
   private DN2ID dn2id = null;
   private DN2URI dn2uri = null;
   private long totalEntries =0;
   private final AtomicLong entriesProcessed = new AtomicLong(0);
   private Suffix suffix = null;
   private final boolean rebuildAll;
   private EntryContainer ec;


    /**
     * Create an instance of the rebuild index manager using the specified
     * parameters.
     *
     * @param rebuildConfig  The rebuild configuration to use.
     * @param cfg The local DB configuration to use.
     */
    public RebuildManager(RebuildConfig rebuildConfig, LocalDBBackendCfg cfg)
    {
      this.rebuildConfig = rebuildConfig;
      this.cfg = cfg;
      this.rebuildAll = rebuildConfig.isRebuildAll();
    }

    /**
     * Initialize a rebuild manager to start rebuilding indexes.
     *
     * @throws ConfigException If an configuration error occurred.
     * @throws InitializationException If an initialization error occurred.
     */
    public void initialize() throws ConfigException, InitializationException
    {
      ec = rootContainer.getEntryContainer(rebuildConfig.getBaseDN());
      suffix = Suffix.createSuffixContext(ec, null, null, null);
      if(suffix == null)
      {
        Message msg = ERR_JEB_REBUILD_SUFFIX_ERROR.get(rebuildConfig.
                getBaseDN().toString());
        throw new InitializationException(msg);
      }
    }

    /**
     * Print start message.
     *
     * @throws DatabaseException If an database error occurred.
     */
    public void printStartMessage() throws DatabaseException
    {
      StringBuilder sb = new StringBuilder();
      List<String> rebuildList = rebuildConfig.getRebuildList();
      for(String index : rebuildList)
      {
        if(sb.length() > 0)
        {
          sb.append(", ");
        }
        sb.append(index);
      }
      totalEntries = suffix.getID2Entry().getRecordCount();
      Message message = NOTE_JEB_REBUILD_START.get(sb.toString(), totalEntries);
      if(rebuildAll) {
        message = NOTE_JEB_REBUILD_ALL_START.get(totalEntries);
      }
      logError(message);
    }

    /**
     * Print stop message.
     *
     * @param startTime The time the rebuild started.
     */
    public void printStopMessage(long startTime)
   {
      long finishTime = System.currentTimeMillis();
      long totalTime = (finishTime - startTime);
      float rate = 0;
      if (totalTime > 0)
      {
        rate = 1000f* entriesProcessed.get() / totalTime;
      }
      Message message =
                     NOTE_JEB_REBUILD_FINAL_STATUS.get(entriesProcessed.get(),
                                                       totalTime/1000, rate);
     logError(message);
   }


    /**
     * {@inheritDoc}
     */
    public Void call() throws Exception
    {
      ID2Entry id2entry = ec.getID2Entry();
      Cursor cursor = id2entry.openCursor(null, CursorConfig.READ_COMMITTED);
      DatabaseEntry key = new DatabaseEntry();
      DatabaseEntry data = new DatabaseEntry();
      LockMode lockMode = LockMode.DEFAULT;
      OperationStatus status;
      try {
      for (status = cursor.getFirst(key, data, lockMode);
           status == OperationStatus.SUCCESS;
           status = cursor.getNext(key, data, lockMode))
      {
        EntryID entryID = new EntryID(key);
        Entry entry = ID2Entry.entryFromDatabase(
                ByteString.wrap(data.getData()),
                ec.getRootContainer().getCompressedSchema());
        processEntry(entry, entryID);
        entriesProcessed.getAndIncrement();
      }
      flushIndexBuffers();
      cursor.close();
      } catch (Exception e) {
        System.out.println("here");
        e.printStackTrace();
      }
      return null;
  }

    /**
     * Perform the index rebuild.
     *
     * @throws DatabaseException If an database error occurred.
     * @throws InterruptedException If an interrupted error occurred.
     * @throws ExecutionException If an Excecution error occurred.
     * @throws JebException If an JEB error occurred.
     */
    public void rebuldIndexes() throws DatabaseException, InterruptedException,
                                      ExecutionException, JebException
   {
     processPhaseOne();
     processPhaseTwo();
     setIndexesTrusted();
   }

   private void setIndexesTrusted() throws JebException
   {
     try {
       suffix.setIndexesTrusted();
     }
     catch (DatabaseException ex)
     {
       Message message =
               NOTE_JEB_IMPORT_LDIF_TRUSTED_FAILED.get(ex.getMessage());
       throw new JebException(message);
     }
   }

   private void processPhaseOne() throws DatabaseException,
           InterruptedException, ExecutionException {
     if(rebuildAll)
     {
       clearAllIndexes();
     }
     else
     {
       clearRebuildListIndexes();
     }
     initializeIndexBuffers(threadCount);
     RBFirstPhaseProgressTask progressTask = new RBFirstPhaseProgressTask();
     Timer timer = new Timer();
     timer.scheduleAtFixedRate(progressTask, TIMER_INTERVAL, TIMER_INTERVAL);
     indexProcessService = Executors.newFixedThreadPool(2 * indexCount);
     sortService = Executors.newFixedThreadPool(threadCount);
     ExecutorService execService = Executors.newFixedThreadPool(threadCount);
     List<Callable<Void>> tasks = new ArrayList<Callable<Void>>(threadCount);
     for (int i = 0; i < threadCount; i++)
     {
       tasks.add(this);
     }
     List<Future<Void>> results = execService.invokeAll(tasks);
     for (Future<Void> result : results) {
       if(!result.isDone()) {
         result.get();
       }
     }
     stopIndexWriterTasks();
     for (Future<?> result : indexWriterFutures)
     {
       if(!result.isDone()) {
         result.get();
       }
     }
     tasks.clear();
     results.clear();
     execService.shutdown();
     freeBufferQueue.clear();
     sortService.shutdown();
     timer.cancel();
   }


   private void processPhaseTwo() throws InterruptedException
   {
     SecondPhaseProgressTask progress2Task =
            new SecondPhaseProgressTask(indexMgrList, entriesProcessed.get());
     Timer timer2 = new Timer();
     timer2.scheduleAtFixedRate(progress2Task, TIMER_INTERVAL, TIMER_INTERVAL);
     processIndexFiles();
     timer2.cancel();
   }

   private int getIndexCount() throws ConfigException, JebException
   {
    int indexCount;
    if(!rebuildAll)
    {
      indexCount = getRebuildListIndexCount(cfg);
    }
    else
    {
      indexCount = getAllIndexesCount(cfg);
    }
     return indexCount;
   }

   private int getAllIndexesCount(LocalDBBackendCfg cfg)
   {
     int indexCount = cfg.listLocalDBIndexes().length;
     indexCount += cfg.listLocalDBVLVIndexes().length;
     indexCount += 4;
     return indexCount;
   }

   private int getRebuildListIndexCount(LocalDBBackendCfg cfg)
           throws JebException, ConfigException
   {
     int indexCount = 0;
     List<String> rebuildList = rebuildConfig.getRebuildList();
     if(!rebuildList.isEmpty())
     {
       for (String index : rebuildList)
       {
         String lowerName = index.toLowerCase();
         if (lowerName.equals("dn2id"))
         {
           indexCount += 3;
         }
         else if (lowerName.equals("dn2uri"))
         {
           indexCount++;
         }
         else if (lowerName.startsWith("vlv."))
         {
           if(lowerName.length() < 5)
           {
             Message msg = ERR_JEB_VLV_INDEX_NOT_CONFIGURED.get(lowerName);
             throw new JebException(msg);
           }
           indexCount++;
         } else if(lowerName.equals("id2subtree") ||
                   lowerName.equals("id2children"))
         {
             Message msg = ERR_JEB_ATTRIBUTE_INDEX_NOT_CONFIGURED.get(index);
             throw new JebException(msg);
         }
         else
         {
           String[] attrIndexParts = lowerName.split("\\.");
           if((attrIndexParts.length <= 0) || (attrIndexParts.length > 3))
           {
             Message msg = ERR_JEB_ATTRIBUTE_INDEX_NOT_CONFIGURED.get(index);
             throw new JebException(msg);
           }
           AttributeType attrType =
                   DirectoryServer.getAttributeType(attrIndexParts[0]);
           if (attrType == null)
           {
             Message msg = ERR_JEB_ATTRIBUTE_INDEX_NOT_CONFIGURED.get(index);
             throw new JebException(msg);
           }
           if(attrIndexParts.length != 1)
           {
             if(attrIndexParts.length == 2)
             {
               if(attrIndexParts[1].equals("presence"))
               {
                 indexCount++;
               }
               else if(attrIndexParts[1].equals("equality"))
               {
                 indexCount++;
               }
               else if(attrIndexParts[1].equals("substring"))
               {
                 indexCount++;
               }
               else if(attrIndexParts[1].equals("ordering"))
               {
                 indexCount++;
               }
               else if(attrIndexParts[1].equals("approximate"))
               {
                 indexCount++;
               } else {
                 Message msg =
                              ERR_JEB_ATTRIBUTE_INDEX_NOT_CONFIGURED.get(index);
                 throw new JebException(msg);
               }
             }
             else
             {
               boolean found = false;
               String s = attrIndexParts[1] + "." + attrIndexParts[2];
               for (String idx : cfg.listLocalDBIndexes())
               {
                 LocalDBIndexCfg indexCfg = cfg.getLocalDBIndex(idx);
                 if (indexCfg.getIndexType().
                     contains(LocalDBIndexCfgDefn.IndexType.EXTENSIBLE))
                 {
                   Set<String> extensibleRules =
                           indexCfg.getIndexExtensibleMatchingRule();
                   for(String exRule : extensibleRules)
                   {
                     if(exRule.equalsIgnoreCase(s))
                     {
                       found = true;
                       break;
                     }
                   }
                 }
                 if(found)
                 {
                   break;
                 }
               }
               if(!found) {
                 Message msg =
                             ERR_JEB_ATTRIBUTE_INDEX_NOT_CONFIGURED.get(index);
                 throw new JebException(msg);
               }
               indexCount++;
             }
           }
           else
           {
             for (String idx : cfg.listLocalDBIndexes())
             {
               if(!idx.equalsIgnoreCase(index))
               {
                 continue;
               }
               LocalDBIndexCfg indexCfg = cfg.getLocalDBIndex(idx);
               if(indexCfg.getIndexType().
                  contains(LocalDBIndexCfgDefn.IndexType.EQUALITY))
               {
                 indexCount++;
               }
               if(indexCfg.getIndexType().
                  contains(LocalDBIndexCfgDefn.IndexType.ORDERING))
               {
                 indexCount++;
               }
               if(indexCfg.getIndexType().
                  contains(LocalDBIndexCfgDefn.IndexType.PRESENCE))
               {
                 indexCount++;
               }
               if(indexCfg.getIndexType().
                 contains(LocalDBIndexCfgDefn.IndexType.SUBSTRING))
               {
                 indexCount++;
               }
               if(indexCfg.getIndexType().
                 contains(LocalDBIndexCfgDefn.IndexType.APPROXIMATE))
               {
                 indexCount++;
               }
               if (indexCfg.getIndexType().
                   contains(LocalDBIndexCfgDefn.IndexType.EXTENSIBLE))
               {
                 Set<String> extensibleRules =
                         indexCfg.getIndexExtensibleMatchingRule();
                 boolean shared = false;
                 for(String exRule : extensibleRules)
                 {
                   if(exRule.endsWith(".sub"))
                   {
                     indexCount++;
                   }
                   else
                   {
                     if(!shared)
                     {
                       shared=true;
                       indexCount++;
                     }
                   }
                 }
               }
             }
           }
         }
       }
     }
     return indexCount;
   }

   private void clearRebuildListIndexes() throws DatabaseException
   {
     List<String> rebuildList = rebuildConfig.getRebuildList();
     if(!rebuildList.isEmpty())
     {
       for (String index : rebuildList)
       {
         String lowerName = index.toLowerCase();
         if (lowerName.equals("dn2id"))
         {
            clearDN2IDIndexes(ec);
         }
         else if (lowerName.equals("dn2uri"))
         {
           clearDN2URI(ec);
         }
         else if (lowerName.startsWith("vlv."))
         {
           clearVLVIndex(lowerName.substring(4), ec);
         }
         else
         {
           String[] attrIndexParts = lowerName.split("\\.");
           AttributeType attrType =
                   DirectoryServer.getAttributeType(attrIndexParts[0]);
           AttributeIndex attrIndex = ec.getAttributeIndex(attrType);

           if(attrIndexParts.length != 1)
           {
             Index partialAttrIndex;
             if(attrIndexParts[1].equals("presence"))
             {
               partialAttrIndex = attrIndex.getPresenceIndex();
               ec.clearDatabase(partialAttrIndex);
               IndexKey indexKey =
                       new IndexKey(attrType, ImportIndexType.PRESENCE);
               indexMap.put(indexKey, partialAttrIndex);
             }
             else if(attrIndexParts[1].equals("equality"))
             {
               partialAttrIndex = attrIndex.getEqualityIndex();
               ec.clearDatabase(partialAttrIndex);
               IndexKey indexKey =
                       new IndexKey(attrType, ImportIndexType.EQUALITY);
               indexMap.put(indexKey, partialAttrIndex);
             }
             else if(attrIndexParts[1].equals("substring"))
             {
               partialAttrIndex = attrIndex.getSubstringIndex();
               ec.clearDatabase(partialAttrIndex);
               IndexKey indexKey =
                       new IndexKey(attrType, ImportIndexType.SUBSTRING);
               indexMap.put(indexKey, partialAttrIndex);
             }
             else if(attrIndexParts[1].equals("ordering"))
             {
               partialAttrIndex = attrIndex.getOrderingIndex();
               ec.clearDatabase(partialAttrIndex);
               IndexKey indexKey =
                       new IndexKey(attrType, ImportIndexType.ORDERING);
               indexMap.put(indexKey, partialAttrIndex);
             }
             else if(attrIndexParts[1].equals("approximate"))
             {
               partialAttrIndex = attrIndex.getApproximateIndex();
               ec.clearDatabase(partialAttrIndex);
               IndexKey indexKey =
                       new IndexKey(attrType, ImportIndexType.APPROXIMATE);
               indexMap.put(indexKey, partialAttrIndex);
             }
             else
             {
               String dbPart = "shared";
               if(attrIndexParts[2].startsWith("sub"))
               {
                 dbPart = "substring";
               }
               StringBuilder nameBldr = new StringBuilder();
               nameBldr.append(ec.getDatabasePrefix());
               nameBldr.append("_");
               nameBldr.append(attrIndexParts[0]);
               nameBldr.append(".");
               nameBldr.append(attrIndexParts[1]);
               nameBldr.append(".");
               nameBldr.append(dbPart);
               String indexName = nameBldr.toString();
               Map<String,Collection<Index>> extensibleMap =
                       attrIndex.getExtensibleIndexes();
               if(!extensibleMap.isEmpty()) {
                 Collection<Index> subIndexes =
                         attrIndex.getExtensibleIndexes().get(
                                 EXTENSIBLE_INDEXER_ID_SUBSTRING);
                 if(subIndexes != null) {
                   for(Index subIndex : subIndexes) {
                     String name = subIndex.getName();
                     if(name.equalsIgnoreCase(indexName))
                     {
                       ec.clearDatabase(subIndex);
                       Collection<Index> substring = new ArrayList<Index>();
                       substring.add(subIndex);
                       extensibleIndexMap.put(new IndexKey(attrType,
                               ImportIndexType.EX_SUBSTRING),substring);
                       break;
                     }
                   }
                   Collection<Index> sharedIndexes = attrIndex.
                       getExtensibleIndexes().get(EXTENSIBLE_INDEXER_ID_SHARED);
                   if(sharedIndexes !=null) {
                     for(Index sharedIndex : sharedIndexes) {
                       String name = sharedIndex.getName();
                       if(name.equalsIgnoreCase(indexName))
                       {
                         ec.clearDatabase(sharedIndex);
                         Collection<Index> shared = new ArrayList<Index>();
                         shared.add(sharedIndex);
                         extensibleIndexMap.put(new IndexKey(attrType,
                                            ImportIndexType.EX_SHARED), shared);
                         break;
                       }
                     }
                   }
                 }
               }
             }
           }
           else
           {
             clearAttributeIndexes(attrIndex, attrType, ec);
           }
         }
       }
     }
   }


   private void clearAllIndexes() throws DatabaseException
   {
     for(Map.Entry<AttributeType, AttributeIndex> mapEntry :
             suffix.getAttrIndexMap().entrySet()) {
       AttributeType attributeType = mapEntry.getKey();
       AttributeIndex attributeIndex = mapEntry.getValue();
       clearAttributeIndexes(attributeIndex, attributeType, ec);
     }
     for(VLVIndex vlvIndex : suffix.getEntryContainer().getVLVIndexes()) {
       ec.clearDatabase(vlvIndex);
     }
     clearDN2IDIndexes(ec);
     if(ec.getDN2URI() != null)
     {
       clearDN2URI(ec);
     }
   }

   private void clearVLVIndex(String name, EntryContainer ec)
           throws DatabaseException
   {
     VLVIndex vlvIndex = ec.getVLVIndex(name);
     ec.clearDatabase(vlvIndex);
   }

   private void clearDN2URI(EntryContainer ec) throws DatabaseException
   {
     ec.clearDatabase(ec.getDN2URI());
     dn2uri = ec.getDN2URI();
   }

   private void clearDN2IDIndexes(EntryContainer ec) throws DatabaseException
   {
     ec.clearDatabase(ec.getDN2ID());
     ec.clearDatabase(ec.getID2Children());
     ec.clearDatabase(ec.getID2Subtree());
     dn2id = ec.getDN2ID();
   }

   private void clearAttributeIndexes(AttributeIndex attrIndex,
                               AttributeType attrType, EntryContainer ec)
   throws DatabaseException
   {
     Index partialAttrIndex;
     if(attrIndex.getSubstringIndex() != null)
     {
       partialAttrIndex = attrIndex.getSubstringIndex();
       ec.clearDatabase(partialAttrIndex);
       IndexKey indexKey =
               new IndexKey(attrType, ImportIndexType.SUBSTRING);
       indexMap.put(indexKey, partialAttrIndex);
     }
     if(attrIndex.getOrderingIndex() != null)
     {
       partialAttrIndex = attrIndex.getOrderingIndex();
       ec.clearDatabase(partialAttrIndex);
       IndexKey indexKey =
               new IndexKey(attrType, ImportIndexType.ORDERING);
       indexMap.put(indexKey, partialAttrIndex);
     }
     if(attrIndex.getEqualityIndex() != null)
     {
       partialAttrIndex = attrIndex.getEqualityIndex();
       ec.clearDatabase(partialAttrIndex);
       IndexKey indexKey =
               new IndexKey(attrType, ImportIndexType.EQUALITY);
       indexMap.put(indexKey, partialAttrIndex);
     }
     if(attrIndex.getPresenceIndex() != null)
     {
       partialAttrIndex = attrIndex.getPresenceIndex();
       ec.clearDatabase(partialAttrIndex);
       IndexKey indexKey =
               new IndexKey(attrType, ImportIndexType.PRESENCE);
       indexMap.put(indexKey, partialAttrIndex);

     }
     if(attrIndex.getApproximateIndex() != null)
     {
       partialAttrIndex = attrIndex.getApproximateIndex();
       ec.clearDatabase(partialAttrIndex);
       IndexKey indexKey =
               new IndexKey(attrType, ImportIndexType.APPROXIMATE);
       indexMap.put(indexKey, partialAttrIndex);
     }
     Map<String,Collection<Index>> extensibleMap =
             attrIndex.getExtensibleIndexes();
     if(!extensibleMap.isEmpty()) {
       Collection<Index> subIndexes =
               attrIndex.getExtensibleIndexes().get(
                       EXTENSIBLE_INDEXER_ID_SUBSTRING);
       if(subIndexes != null) {
         for(Index subIndex : subIndexes) {
           ec.clearDatabase(subIndex);
         }
         extensibleIndexMap.put(new IndexKey(attrType,
                               ImportIndexType.EX_SUBSTRING), subIndexes);
       }
       Collection<Index> sharedIndexes =
             attrIndex.getExtensibleIndexes().get(EXTENSIBLE_INDEXER_ID_SHARED);
       if(sharedIndexes !=null) {
         for(Index sharedIndex : sharedIndexes) {
           ec.clearDatabase(sharedIndex);
         }
         extensibleIndexMap.put(new IndexKey(attrType,
                                     ImportIndexType.EX_SHARED), sharedIndexes);
       }
     }
   }


   private
   void processEntry(Entry entry, EntryID entryID) throws DatabaseException,
           ConfigException, DirectoryException, JebException
   {
     if(dn2id != null)
     {
        processDN2ID(suffix, entry.getDN(), entryID);
     }
     if(dn2uri != null)
     {
        processDN2URI(suffix, null, entry);
     }
     processIndexes(entry, entryID);
     processExtensibleIndexes(entry, entryID);
     processVLVIndexes(entry, entryID);
   }

   private void processVLVIndexes(Entry entry, EntryID entryID)
           throws DatabaseException, JebException, DirectoryException
   {
     for(VLVIndex vlvIdx : suffix.getEntryContainer().getVLVIndexes()) {
       Transaction transaction = null;
       vlvIdx.addEntry(transaction, entryID, entry);
     }
   }


   private void processExtensibleIndexes(Entry entry, EntryID entryID) throws
          DatabaseException, DirectoryException, JebException, ConfigException
   {
     for(Map.Entry<IndexKey, Collection<Index>> mapEntry :
             this.extensibleIndexMap.entrySet()) {
       IndexKey key = mapEntry.getKey();
       AttributeType attrType = key.getType();
       if(entry.hasAttribute(attrType)) {
         Collection<Index> indexes = mapEntry.getValue();
         for(Index index : indexes) {
            processAttribute(index, entry, entryID, key);
         }
       }
     }
   }

   private void
   processIndexes(Entry entry, EntryID entryID) throws
           DatabaseException, DirectoryException, JebException, ConfigException
   {

     for(Map.Entry<IndexKey, Index> mapEntry :
             indexMap.entrySet()) {
       IndexKey key = mapEntry.getKey();
       AttributeType attrType = key.getType();
       if(entry.hasAttribute(attrType)) {
         ImportIndexType indexType = key.getIndexType();
         Index index = mapEntry.getValue();
         if(indexType == ImportIndexType.SUBSTRING)
         {
           processAttribute(index, entry, entryID,
                   new IndexKey(attrType, ImportIndexType.SUBSTRING));
         }
         else
         {
            processAttribute(index, entry, entryID,
                             new IndexKey(attrType, indexType));
         }
       }
     }
   }

   /**
    * Return the number of entries processed by the rebuild manager.
    *
    * @return The number of entries processed.
    */
   public long getEntriesProcess()
   {
     return this.entriesProcessed.get();
   }

   /**
    * Return the total number of entries to process by the rebuild manager.
    *
    * @return The total number for entries to process.
    */
   public long getTotEntries()
   {
     return this.totalEntries;
   }
  }

  /**
    * This class reports progress of the rebuild job at fixed intervals.
    */
   class RBFirstPhaseProgressTask extends TimerTask
   {
     /**
      * The number of records that had been processed at the time of the
      * previous progress report.
      */
     private long previousProcessed = 0;

     /**
      * The time in milliseconds of the previous progress report.
      */
     private long previousTime;

     /**
      * The environment statistics at the time of the previous report.
      */
     private EnvironmentStats prevEnvStats;

    /**
      * Create a new verify progress task.
      * @throws DatabaseException An error occurred while accessing the JE
      * database.
      */
     public RBFirstPhaseProgressTask() throws DatabaseException
     {
       previousTime = System.currentTimeMillis();
       prevEnvStats =
           rootContainer.getEnvironmentStats(new StatsConfig());
     }

     /**
      * The action to be performed by this timer task.
      */
     public void run()
     {
       long latestTime = System.currentTimeMillis();
       long deltaTime = latestTime - previousTime;

       if (deltaTime == 0)
       {
         return;
       }
       long currentRBProcessed = rebuildManager.getEntriesProcess();
       long deltaCount = (currentRBProcessed - previousProcessed);
       float rate = 1000f*deltaCount / deltaTime;
       float completed = 0;
       if(rebuildManager.getTotEntries() > 0)
       {
         completed = 100f*currentRBProcessed / rebuildManager.getTotEntries();
       }
       Message message = NOTE_JEB_REBUILD_PROGRESS_REPORT.get(
           completed, currentRBProcessed, rebuildManager.getTotEntries(), rate);
       logError(message);
       try
       {
         Runtime runtime = Runtime.getRuntime();
         long freeMemory = runtime.freeMemory() / MB;
         EnvironmentStats envStats =
             rootContainer.getEnvironmentStats(new StatsConfig());
         long nCacheMiss =
              envStats.getNCacheMiss() - prevEnvStats.getNCacheMiss();

         float cacheMissRate = 0;
         if (deltaCount > 0)
         {
           cacheMissRate = nCacheMiss/(float)deltaCount;
         }
         message = NOTE_JEB_REBUILD_CACHE_AND_MEMORY_REPORT.get(
             freeMemory, cacheMissRate);
         logError(message);
         prevEnvStats = envStats;
       }
       catch (DatabaseException e)
       {

       }
       previousProcessed = currentRBProcessed;
       previousTime = latestTime;
     }
   }


  /**
   * This class reports progress of the import job at fixed intervals.
   */
@@ -2665,7 +3637,7 @@
  /**
   * This class reports progress of the import job at fixed intervals.
   */
  private final class SecondPhaseProgressTask extends TimerTask
  class SecondPhaseProgressTask extends TimerTask
  {
    /**
     * The number of entries that had been read at the time of the
@@ -2687,16 +3659,19 @@
    private boolean evicting = false;

    private final List<IndexManager> indexMgrList;

    private long latestCount;

      /**
     * Create a new import progress task.
     * @param indexMgrList List of index managers.
     * @param  latestCount The latest count of entries processed in phase one.
     */
    public SecondPhaseProgressTask (List<IndexManager> indexMgrList)
    public SecondPhaseProgressTask (List<IndexManager> indexMgrList,
                                    long latestCount)
    {
      previousTime = System.currentTimeMillis();
      this.indexMgrList = indexMgrList;
      this.latestCount = latestCount;
      try
      {
        previousStats =
@@ -2715,7 +3690,6 @@
    @Override
    public void run()
    {
      long latestCount = reader.getEntriesRead() + 0;
      long deltaCount = (latestCount - previousCount);
      long latestTime = System.currentTimeMillis();
      long deltaTime = latestTime - previousTime;
@@ -2847,7 +3821,7 @@
   * This class defines the individual index type available.
   *
   */
  public enum IndexType {
  public enum ImportIndexType {
    /**
     * The DN index type.
     **/
@@ -2886,9 +3860,13 @@
    /**
     * The extensible shared index type.
     **/
    EX_SHARED
  }
    EX_SHARED,

    /**
     * The vlv index type.
     */
    VLV
  }

  /**
   * This class is used as an index key for hash maps that need to
@@ -2901,22 +3879,8 @@
  public class IndexKey {

    private final AttributeType type;
    private final IndexType indexType;
    private byte[] keyBytes = null;
    private final ImportIndexType indexType;

    /**
     * Create index key instance using the specified attribute type, index type
     * and sub-string length. Used only for sub-string indexes.
     *
     * @param type The attribute type.
     * @param indexType The index type.
     * @param subLen The sub-string length.
     */
    IndexKey(AttributeType type, IndexType indexType, int subLen)
    {
      this(type, indexType);
      keyBytes = new byte[subLen];
    }

   /**
     * Create index key instance using the specified attribute type, index type.
@@ -2924,31 +3888,31 @@
     * @param type The attribute type.
     * @param indexType The index type.
     */
    IndexKey(AttributeType type, IndexType indexType)
    IndexKey(AttributeType type, ImportIndexType indexType)
    {
      this.type = type;
      this.indexType = indexType;
    }

      /**
       * An equals method that uses both the attribute type and the index type.
       *
       * @param obj the object to compare.
       * @return <CODE>true</CODE> if the objects are equal.
       */
      public boolean equals(Object obj)
      {
          boolean returnCode = false;
          if (obj instanceof IndexKey) {
              IndexKey oKey = (IndexKey) obj;
              if(type.equals(oKey.getType()) &&
                 indexType.equals(oKey.getIndexType()))
              {
                  returnCode = true;
              }
          }
          return returnCode;
    /**
     * An equals method that uses both the attribute type and the index type.
     *
     * @param obj the object to compare.
     * @return <CODE>true</CODE> if the objects are equal.
     */
    public boolean equals(Object obj)
    {
      boolean returnCode = false;
      if (obj instanceof IndexKey) {
        IndexKey oKey = (IndexKey) obj;
        if(type.equals(oKey.getType()) &&
                indexType.equals(oKey.getIndexType()))
        {
          returnCode = true;
        }
      }
      return returnCode;
    }

    /**
     * A hash code method that adds the hash codes of the attribute type and
@@ -2975,7 +3939,7 @@
     * Return the index type.
     * @return The index type.
     */
    public IndexType getIndexType()
    public ImportIndexType getIndexType()
    {
      return indexType;
    }
@@ -2990,7 +3954,7 @@
    public String getName()
    {
      return type.getPrimaryName() + "." +
             StaticUtils.toLowerCase(indexType.name());
              StaticUtils.toLowerCase(indexType.name());
    }
  }
}

 opends/src/server/org/opends/server/backends/jeb/importLDIF/Suffix.java

@@ -27,16 +27,13 @@

package org.opends.server.backends.jeb.importLDIF;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import org.opends.server.backends.jeb.*;
import org.opends.server.config.ConfigException;
import org.opends.server.types.*;
import static org.opends.server.loggers.ErrorLogger.logError;
import static org.opends.server.util.ServerConstants.*;
import org.opends.messages.Message;
import org.opends.messages.Category;
import org.opends.messages.Severity;
@@ -277,7 +274,7 @@
    entryContainer.getID2Children().setTrusted(null,true);
    entryContainer.getID2Subtree().setTrusted(null, true);
    for(AttributeIndex attributeIndex :
        entryContainer.getAttributeIndexes()) {
            entryContainer.getAttributeIndexes()) {
      Index index;
      if((index = attributeIndex.getEqualityIndex()) != null) {
        index.setTrusted(null, true);
@@ -294,9 +291,28 @@
      if((index=attributeIndex.getApproximateIndex()) != null) {
        index.setTrusted(null, true);
      }
      Map<String,Collection<Index>> exIndexes =
              attributeIndex.getExtensibleIndexes();
      if(!exIndexes.isEmpty())
      {
        Collection<Index> subIndexes = attributeIndex.getExtensibleIndexes().
                get(EXTENSIBLE_INDEXER_ID_SUBSTRING);
        if(subIndexes != null) {
          for(Index subIndex : subIndexes) {
            subIndex.setTrusted(null, true);
          }
        }
        Collection<Index> sharedIndexes = attributeIndex.
                getExtensibleIndexes().get(EXTENSIBLE_INDEXER_ID_SHARED);
        if(sharedIndexes !=null) {
          for(Index sharedIndex : sharedIndexes) {
            sharedIndex.setTrusted(null, true);
          }
        }
      }
    }
    for(VLVIndex vlvIdx : entryContainer.getVLVIndexes()) {
        vlvIdx.setTrusted(null, true);
      vlvIdx.setTrusted(null, true);
    }
  }


 opends/src/server/org/opends/server/config/ConfigConstants.java

@@ -4332,10 +4332,15 @@

  /**
   * The name of the attribute in an rebuild task definition that specifies the
   * maximum number of threads.
   * temporary scratch file.
   */
  public static final String ATTR_REBUILD_MAX_THREADS =
       NAME_PREFIX_TASK + "rebuild-max-threads";
  public static final String ATTR_REBUILD_TMP_DIRECTORY =
       NAME_PREFIX_TASK + "rebuild-tmp-directory";

  /**
   * Used to specify that the rebuild all boolean should be set.
   */
  public static final String REBUILD_ALL = "rebuildall";

  /**
   * The name of the objectclass that will be used for a Directory Server

 opends/src/server/org/opends/server/tasks/RebuildTask.java

@@ -52,26 +52,10 @@
import static org.opends.server.util.StaticUtils.*;
import static org.opends.server.loggers.debug.DebugLogger.*;
import org.opends.server.loggers.debug.DebugTracer;
import static org.opends.messages.TaskMessages.
    ERR_TASK_INDEXREBUILD_INSUFFICIENT_PRIVILEGES;
import static org.opends.messages.ToolMessages.
    ERR_REBUILDINDEX_ERROR_DURING_REBUILD;
import static org.opends.messages.ToolMessages.
    ERR_REBUILDINDEX_WRONG_BACKEND_TYPE;
import static org.opends.messages.ToolMessages.
    ERR_NO_BACKENDS_FOR_BASE;
import static org.opends.messages.ToolMessages.
    ERR_CANNOT_DECODE_BASE_DN;
import static org.opends.messages.ToolMessages.
    ERR_REBUILDINDEX_CANNOT_EXCLUSIVE_LOCK_BACKEND;
import static org.opends.messages.ToolMessages.
    WARN_REBUILDINDEX_CANNOT_UNLOCK_BACKEND;
import static org.opends.server.config.ConfigConstants.
    ATTR_REBUILD_BASE_DN;
import static org.opends.server.config.ConfigConstants.
    ATTR_REBUILD_INDEX;
import static org.opends.server.config.ConfigConstants.
    ATTR_REBUILD_MAX_THREADS;
import static org.opends.messages.TaskMessages.*;
import static org.opends.messages.ToolMessages.*;
import static org.opends.server.config.ConfigConstants.*;


import java.util.List;
import java.util.ArrayList;
@@ -89,7 +73,8 @@

  String baseDN = null;
  ArrayList<String> indexes = null;
  int maxThreads = -1;
  private String tmpDirectory = null;
  private boolean rebuildAll = false;

  /**
   * {@inheritDoc}
@@ -123,14 +108,14 @@

    AttributeType typeBaseDN;
    AttributeType typeIndex;
    AttributeType typeMaxThreads;
    AttributeType typeTmpDirectory;

    typeBaseDN =
         getAttributeType(ATTR_REBUILD_BASE_DN, true);
    typeIndex =
         getAttributeType(ATTR_REBUILD_INDEX, true);
    typeMaxThreads =
         getAttributeType(ATTR_REBUILD_MAX_THREADS, true);
    typeTmpDirectory =
         getAttributeType(ATTR_REBUILD_TMP_DIRECTORY, true);

    List<Attribute> attrList;

@@ -140,9 +125,32 @@
    attrList = taskEntry.getAttribute(typeIndex);
    indexes = TaskUtils.getMultiValueString(attrList);

    if(isRebuildAll(indexes))
    {
      if(indexes.size() != 1)
      {
        Message msg = ERR_TASK_INDEXREBUILD_ALL_ERROR.get();
        throw new DirectoryException(ResultCode.UNWILLING_TO_PERFORM, msg);
      }
      rebuildAll = true;
      indexes.clear();
    }

    attrList = taskEntry.getAttribute(typeMaxThreads);
    maxThreads = TaskUtils.getSingleValueInteger(attrList, -1);
    attrList = taskEntry.getAttribute(typeTmpDirectory);
    tmpDirectory = TaskUtils.getSingleValueString(attrList);

  }

  private boolean isRebuildAll(List<String> indexList)
  {
    for(String s : indexList)
    {
      if(s.equalsIgnoreCase(REBUILD_ALL))
      {
        return true;
      }
    }
    return false;
  }

  /**
@@ -169,7 +177,6 @@
      rebuildConfig.addRebuildIndex(index);
    }

    rebuildConfig.setMaxRebuildThreads(maxThreads);

    Backend backend =
        DirectoryServer.getBackendWithBaseDN(rebuildConfig.getBaseDN());
@@ -228,7 +235,12 @@
      return TaskState.STOPPED_BY_ERROR;
    }


     if(tmpDirectory == null)
    {
      tmpDirectory = "import-tmp";
    }
    rebuildConfig.setTmpDirectory(tmpDirectory);
    rebuildConfig.setRebuildAll(rebuildAll);
    TaskState returnCode = TaskState.COMPLETED_SUCCESSFULLY;
    // Launch the rebuild process.
    try

 opends/src/server/org/opends/server/tools/RebuildIndex.java

@@ -80,6 +80,8 @@
  private StringArgument  configFile              = null;
  private StringArgument  baseDNString            = null;
  private StringArgument  indexList               = null;
  private StringArgument  tmpDirectory            = null;
  private BooleanArgument rebuildAll               = null;

  /**
   * Processes the command-line arguments and invokes the rebuild process.
@@ -140,7 +142,7 @@
    }

    // Define the command-line arguments that may be used with this program.
    BooleanArgument displayUsage            = null;
    BooleanArgument displayUsage ;


    // Create the command-line argument parser for use with this program.
@@ -187,6 +189,19 @@
      argParser.addArgument(indexList);


      rebuildAll =
           new BooleanArgument("rebuildAll", null, "rebuildAll",
                    INFO_REBUILDINDEX_DESCRIPTION_REBUILD_ALL.get());
      argParser.addArgument(rebuildAll);


      tmpDirectory =
           new StringArgument("tmpdirectory", null, "tmpdirectory", false,
                   false, true, INFO_REBUILDINDEX_TEMP_DIR_PLACEHOLDER.get(),
                   "import-tmp",
                    null, INFO_REBUILDINDEX_DESCRIPTION_TEMP_DIRECTORY.get());
      argParser.addArgument(tmpDirectory);

      displayUsage =
           new BooleanArgument("help", 'H', "help",
                               INFO_DESCRIPTION_USAGE.get());
@@ -236,7 +251,7 @@
    }


    if (indexList.getValues().size() <= 0)
    if (indexList.getValues().size() <= 0 && !rebuildAll.isPresent())
    {
      Message message = ERR_REBUILDINDEX_REQUIRES_AT_LEAST_ONE_INDEX.get();

@@ -245,6 +260,13 @@
      return 1;
    }

    if(rebuildAll.isPresent() && indexList.isPresent())
    {
      Message msg = ERR_REBUILDINDEX_REBUILD_ALL_ERROR.get();
      err.println(wrapText(msg, MAX_LINE_WIDTH));
      out.println(argParser.getUsage());
      return 1;
    }
    return process(argParser, initializeServer, out, err);
  }

@@ -424,7 +446,7 @@
    ArrayList<Backend>     backendList = new ArrayList<Backend>();
    ArrayList<BackendCfg>  entryList   = new ArrayList<BackendCfg>();
    ArrayList<List<DN>> dnList = new ArrayList<List<DN>>();
    int code = BackendToolUtils.getBackends(backendList, entryList, dnList);
    BackendToolUtils.getBackends(backendList, entryList, dnList);

    int numBackends = backendList.size();
    for (int i=0; i < numBackends; i++)
@@ -498,6 +520,9 @@
      return 1;
    }

   rebuildConfig.setRebuildAll(rebuildAll.isPresent());
   rebuildConfig.setTmpDirectory(tmpDirectory.getValue());

    // Launch the rebuild process.
    int returnCode = 0;
    try
@@ -567,6 +592,25 @@
      values.add(ByteString.valueOf(s));
    }
    attributes.add(new LDAPAttribute(ATTR_REBUILD_INDEX, values));


    if (tmpDirectory.getValue() != null &&
            !tmpDirectory.getValue().equals(
                    tmpDirectory.getDefaultValue())) {
      values = new ArrayList<ByteString>(1);
      values.add(ByteString.valueOf(tmpDirectory.getValue()));
      attributes.add(new LDAPAttribute(ATTR_REBUILD_TMP_DIRECTORY, values));
    }


    if (rebuildAll.getValue() != null &&
            !rebuildAll.getValue().equals(
                    rebuildAll.getDefaultValue())) {
      values = new ArrayList<ByteString>(1);
      values.add(ByteString.valueOf(REBUILD_ALL));
      attributes.add(
              new LDAPAttribute(ATTR_REBUILD_INDEX, values));
    }
  }

  /**

 opends/src/server/org/opends/server/util/LDIFReader.java

@@ -236,7 +236,7 @@



  private final Entry readEntry(boolean checkSchema, Map<DN, Suffix> map,
  private  Entry readEntry(boolean checkSchema, Map<DN, Suffix> map,
                                Importer.EntryInformation entryInfo)
          throws IOException, LDIFException
  {
@@ -244,8 +244,8 @@
    {
      LinkedList<StringBuilder> lines;
      DN entryDN;
      EntryID entryID = null;
      Suffix suffix = null;
      EntryID entryID;
      Suffix suffix;
      synchronized (this)
      {
        // Read the set of lines that make up the next entry.
@@ -260,7 +260,14 @@

        // Read the DN of the entry and see if it is one that should be included
        // in the import.
        entryDN = readDN(lines);

        try
        {
          entryDN = readDN(lines);
        } catch (LDIFException le) {
          entriesIgnored.incrementAndGet();
          continue;
        }
        if (entryDN == null)
        {
          // This should only happen if the LDIF starts with the "version:" line
@@ -904,7 +911,8 @@
        // The value did not have a valid base64-encoding.
        if (debugEnabled())
        {
          TRACER.debugCaught(DebugLogLevel.ERROR, e);
          TRACER.debugInfo("Base64 decode failed for dn: ",
                            line.substring(pos));
        }

        Message message =
@@ -925,7 +933,7 @@
      {
        if (debugEnabled())
        {
          TRACER.debugCaught(DebugLogLevel.ERROR, de);
          TRACER.debugInfo("DN decode failed for: ", dnStr);
        }

        Message message = ERR_LDIF_INVALID_DN.get(
@@ -940,9 +948,8 @@
      {
        if (debugEnabled())
        {
          TRACER.debugCaught(DebugLogLevel.ERROR, e);
          TRACER.debugInfo("DN decode failed for: ", dnStr);
        }

        Message message = ERR_LDIF_INVALID_DN.get(
                lastEntryLineNumber, line.toString(),
                String.valueOf(e));
@@ -972,9 +979,8 @@
      {
        if (debugEnabled())
        {
          TRACER.debugCaught(DebugLogLevel.ERROR, de);
          TRACER.debugInfo("DN decode failed for: ", line.substring(pos));
        }

        Message message = ERR_LDIF_INVALID_DN.get(
                lastEntryLineNumber, line.toString(), de.getMessageObject());

@@ -986,7 +992,7 @@
      {
        if (debugEnabled())
        {
          TRACER.debugCaught(DebugLogLevel.ERROR, e);
          TRACER.debugInfo("DN decode failed for: ", line.substring(pos));
        }

        Message message = ERR_LDIF_INVALID_DN.get(
@@ -1103,9 +1109,7 @@
        pos++;
      }

      String changeTypeString = line.substring(pos);

      return changeTypeString;
      return line.substring(pos);
    }
  }

@@ -1269,51 +1273,41 @@

      // Check to see if any of the attributes in the list have the same set of
      // options.  If so, then try to add a value to that attribute.
      for (int i = 0; i < attrList.size(); i++) {
        AttributeBuilder a = attrList.get(i);

        if (a.optionsEqual(attribute.getOptions()))
        {
          if (a.contains(attributeValue))
          {
            if (! checkSchema)
            {
      for (AttributeBuilder a : attrList) {
        if (a.optionsEqual(attribute.getOptions())) {
          if (a.contains(attributeValue)) {
            if (!checkSchema) {
              // If we're not doing schema checking, then it is possible that
              // the attribute type should use case-sensitive matching and the
              // values differ in capitalization.  Only reject the proposed
              // value if we find another value that is exactly the same as the
              // one that was provided.
              for (AttributeValue v : a)
              {
                if (v.getValue().equals(attributeValue.getValue()))
                {
              for (AttributeValue v : a) {
                if (v.getValue().equals(attributeValue.getValue())) {
                  Message message = WARN_LDIF_DUPLICATE_ATTR.get(
                          String.valueOf(entryDN),
                          lastEntryLineNumber, attrName,
                          value.toString());
                  logToRejectWriter(lines, message);
                  throw new LDIFException(message, lastEntryLineNumber,
                                          true);
                          true);
                }
              }
            }
            else
            {
            } else {
              Message message = WARN_LDIF_DUPLICATE_ATTR.get(
                      String.valueOf(entryDN),
                      lastEntryLineNumber, attrName,
                      value.toString());
              logToRejectWriter(lines, message);
              throw new LDIFException(message, lastEntryLineNumber,
                                      true);
                      true);
            }
          }

          if (attrType.isSingleValue() && !a.isEmpty() && checkSchema)
          {
          if (attrType.isSingleValue() && !a.isEmpty() && checkSchema) {
            Message message = ERR_LDIF_MULTIPLE_VALUES_FOR_SINGLE_VALUED_ATTR
                    .get(String.valueOf(entryDN),
                         lastEntryLineNumber, attrName);
                            lastEntryLineNumber, attrName);
            logToRejectWriter(lines, message);
            throw new LDIFException(message, lastEntryLineNumber, true);
          }
@@ -1328,7 +1322,6 @@
      AttributeBuilder builder = new AttributeBuilder(attribute, true);
      builder.add(attributeValue);
      attrList.add(builder);
      return;
    }
  }

@@ -2356,7 +2349,6 @@
      AttributeBuilder builder = new AttributeBuilder(attribute, true);
      builder.add(attributeValue);
      attrList.add(builder.toAttribute());
      return;
    }
  }
}

 opends/tests/unit-tests-testng/src/server/org/opends/server/backends/jeb/TestRebuildJob.java

@@ -22,7 +22,7 @@
 * CDDL HEADER END
 *
 *
 *      Copyright 2006-2008 Sun Microsystems, Inc.
 *      Copyright 2006-2009 Sun Microsystems, Inc.
 */
package org.opends.server.backends.jeb;

@@ -32,6 +32,7 @@
import org.testng.annotations.Test;
import static org.testng.Assert.*;
import org.opends.server.TestCaseUtils;
import org.opends.server.config.ConfigConstants;
import org.opends.server.tasks.TaskUtils;
import static org.opends.server.util.ServerConstants.OC_TOP;
import static org.opends.server.util.ServerConstants.OC_EXTENSIBLE_OBJECT;
@@ -58,8 +59,6 @@
  @DataProvider(name = "systemIndexes")
  public Object[][] systemIndexes() {
    return new Object[][] {
        { "id2subtree" },
        { "id2children" },
        { "dn2id" },
        { "dn2uri" }
    };
@@ -83,6 +82,8 @@
    return new Object[][] {
        { "id2entry" },
        { "nonindex" },
        { "id2subtree" },
        { "id2children" },
        { "mail.nonindex" }
    };
  }
@@ -221,13 +222,32 @@
  }

  @Test
  public void testRebuildDependentIndexes() throws Exception
  public void testRebuildAll() throws Exception
  {
    cleanAndLoad(10);
    RebuildConfig rebuildConfig = new RebuildConfig();
    rebuildConfig.setBaseDN(baseDNs[0]);
    rebuildConfig.setRebuildAll(true);

    be=(BackendImpl) DirectoryServer.getBackend(beID);

    TaskUtils.disableBackend(beID);

    be.rebuildBackend(rebuildConfig);

    assertEquals(verifyBackend(null), 0);

    TaskUtils.enableBackend(beID);
  }


  @Test
  public void testRebuildDN2ID() throws Exception
  {
    cleanAndLoad(10);
    RebuildConfig rebuildConfig = new RebuildConfig();
    rebuildConfig.setBaseDN(baseDNs[0]);
    rebuildConfig.addRebuildIndex("dn2id");
    rebuildConfig.addRebuildIndex("id2children");

    be=(BackendImpl) DirectoryServer.getBackend(beID);


 opends/tests/unit-tests-testng/src/server/org/opends/server/tasks/TestRebuildTask.java

@@ -22,7 +22,7 @@
 * CDDL HEADER END
 *
 *
 *      Copyright 2006-2008 Sun Microsystems, Inc.
 *      Copyright 2006-2009 Sun Microsystems, Inc.
 */
package org.opends.server.tasks;

@@ -122,10 +122,7 @@
                   "ds-task-rebuild-base-dn: " + suffix,
                   "ds-task-rebuild-index: dn2id",
                   "ds-task-rebuild-index: dn2uri",
                   "ds-task-rebuild-index: id2children",
                   "ds-task-rebuild-index: id2subtree",
                   "ds-task-rebuild-index: mail",
                   "ds-task-rebuild-max-threads: 3"
                   "ds-task-rebuild-index: mail"
              ),
              TaskState.COMPLETED_SUCCESSFULLY
         },
@@ -150,6 +147,6 @@
  public void testRebuildTask(Entry taskEntry, TaskState expectedState)
       throws Exception
  {
    testTask(taskEntry, expectedState, 60);    
    testTask(taskEntry, expectedState, 60);
 }
}