OPENDJ-799 - CR-1391 rebuild-index disables backend after trying to rebuild...

Violette Roche-Montane

11.39.2013 7a895d5ebc47738b22920b26b7f9794100018f14

OPENDJ-799 - CR-1391 rebuild-index disables backend after trying to rebuild a nonexistent index.

 opends/src/server/org/opends/server/backends/jeb/BackendImpl.java

@@ -23,6 +23,7 @@
 *
 *
 *      Copyright 2007-2010 Sun Microsystems, Inc.
 *      Portions Copyright 2013 ForgeRock AS
 */
package org.opends.server.backends.jeb;
import org.opends.messages.Message;
@@ -1414,6 +1415,14 @@
      throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
              e.getMessageObject());
    }
    catch (InitializationException e)
    {
      if (debugEnabled())
      {
        TRACER.debugCaught(DebugLogLevel.ERROR, e);
      }
      throw new InitializationException(e.getMessageObject());
    }
    finally
    {
      //If a root container was opened in this method as read only, close it

 opends/src/server/org/opends/server/backends/jeb/RebuildConfig.java

@@ -203,15 +203,18 @@
   * Test if this rebuild config includes any system indexes to rebuild.
   *
   * @return True if rebuilding of system indexes are included. False otherwise.
   * @throws InitializationException
   */
  public boolean includesSystemIndex()
  {
    for (String index : rebuildList)
    {
      if (index.equalsIgnoreCase("id2entry"))
      // Removed because the id2entry is not A system indexes is THE
      // primary system index. It cannot be rebuilt.
      /*if (index.equalsIgnoreCase("id2entry"))
      {
        return true;
      }
      }*/
      if (index.equalsIgnoreCase("dn2id"))
      {
        return true;

 opends/src/server/org/opends/server/backends/jeb/importLDIF/Importer.java

@@ -473,57 +473,60 @@
    final long phaseOneBufferMemory =
        usableMemory - dbCacheSize - tmpEnvCacheSize;
    final int oldThreadCount = threadCount;
    while (true)
    if (indexCount != 0) // Avoid / by zero
    {
      phaseOneBufferCount = 2 * indexCount * threadCount;

      // Scratch writers allocate 4 buffers per index as well.
      final int totalPhaseOneBufferCount =
          phaseOneBufferCount + (4 * indexCount);
      bufferSize = (int) (phaseOneBufferMemory / totalPhaseOneBufferCount);

      if (bufferSize > MAX_BUFFER_SIZE)
      while (true)
      {
        if (!skipDNValidation)
        phaseOneBufferCount = 2 * indexCount * threadCount;

        // Scratch writers allocate 4 buffers per index as well.
        final int totalPhaseOneBufferCount =
            phaseOneBufferCount + (4 * indexCount);
        bufferSize = (int) (phaseOneBufferMemory / totalPhaseOneBufferCount);

        if (bufferSize > MAX_BUFFER_SIZE)
        {
          // The buffers are big enough: the memory is best used for the DN2ID
          // temp DB.
          bufferSize = MAX_BUFFER_SIZE;
          if (!skipDNValidation)
          {
            // The buffers are big enough: the memory is best used for the DN2ID
            // temp DB.
            bufferSize = MAX_BUFFER_SIZE;

          final long extraMemory =
              phaseOneBufferMemory - (totalPhaseOneBufferCount * bufferSize);
          if (!clearedBackend)
          {
            dbCacheSize += extraMemory / 2;
            tmpEnvCacheSize += extraMemory / 2;
            final long extraMemory =
                phaseOneBufferMemory - (totalPhaseOneBufferCount * bufferSize);
            if (!clearedBackend)
            {
              dbCacheSize += extraMemory / 2;
              tmpEnvCacheSize += extraMemory / 2;
            }
            else
            {
              tmpEnvCacheSize += extraMemory;
            }
          }
          else
          {
            tmpEnvCacheSize += extraMemory;
          }

          break;
        }

        break;
      }
      else if (bufferSize > MIN_BUFFER_SIZE)
      {
        // This is acceptable.
        break;
      }
      else if (threadCount > 1)
      {
        // Retry using less threads.
        threadCount--;
      }
      else
      {
        // Not enough memory.
        final long minimumPhaseOneBufferMemory =
            totalPhaseOneBufferCount * MIN_BUFFER_SIZE;
        Message message =
            ERR_IMPORT_LDIF_LACK_MEM.get(usableMemory,
                minimumPhaseOneBufferMemory + dbCacheSize + tmpEnvCacheSize);
        throw new InitializationException(message);
        else if (bufferSize > MIN_BUFFER_SIZE)
        {
          // This is acceptable.
          break;
        }
        else if (threadCount > 1)
        {
          // Retry using less threads.
          threadCount--;
        }
        else
        {
          // Not enough memory.
          final long minimumPhaseOneBufferMemory =
              totalPhaseOneBufferCount * MIN_BUFFER_SIZE;
          Message message =
              ERR_IMPORT_LDIF_LACK_MEM.get(usableMemory,
                  minimumPhaseOneBufferMemory + dbCacheSize + tmpEnvCacheSize);
          throw new InitializationException(message);
        }
      }
    }

@@ -3210,7 +3213,8 @@
        }
        break;
      }
      if ( message != null ) {
      if ( message != null )
      {
        logError(message);
      }
    }
@@ -3318,8 +3322,8 @@
      else
      {
        Message message =
            NOTE_JEB_REBUILD_CLEARDEGRADEDSTATE_FINAL_STATUS.get(
                rebuildConfig.getRebuildList().toString());
            NOTE_JEB_REBUILD_CLEARDEGRADEDSTATE_FINAL_STATUS.get(rebuildConfig
                .getRebuildList().toString());
        logError(message);
      }

@@ -3683,7 +3687,8 @@
      timer2.cancel();
    }

    private int getIndexCount() throws ConfigException, JebException
    private int getIndexCount() throws ConfigException, JebException,
        InitializationException
    {
      switch (rebuildConfig.getRebuildMode())
      {
@@ -3700,7 +3705,7 @@
    }

    private int getRebuildListIndexCount(LocalDBBackendCfg cfg)
        throws JebException, ConfigException
        throws JebException, ConfigException, InitializationException
    {
      int indexCount = 0;
      List<String> rebuildList = rebuildConfig.getRebuildList();
@@ -3730,7 +3735,7 @@
              || lowerName.equals("id2children"))
          {
            Message msg = ERR_JEB_ATTRIBUTE_INDEX_NOT_CONFIGURED.get(index);
            throw new JebException(msg);
            throw new InitializationException(msg);
          }
          else
          {
@@ -3738,14 +3743,14 @@
            if ((attrIndexParts.length <= 0) || (attrIndexParts.length > 3))
            {
              Message msg = ERR_JEB_ATTRIBUTE_INDEX_NOT_CONFIGURED.get(index);
              throw new JebException(msg);
              throw new InitializationException(msg);
            }
            AttributeType attrType =
                DirectoryServer.getAttributeType(attrIndexParts[0]);
            if (attrType == null)
            {
              Message msg = ERR_JEB_ATTRIBUTE_INDEX_NOT_CONFIGURED.get(index);
              throw new JebException(msg);
              throw new InitializationException(msg);
            }
            if (attrIndexParts.length != 1)
            {
@@ -3775,7 +3780,7 @@
                {
                  Message msg =
                      ERR_JEB_ATTRIBUTE_INDEX_NOT_CONFIGURED.get(index);
                  throw new JebException(msg);
                  throw new InitializationException(msg);
                }
              }
              else
@@ -3808,19 +3813,21 @@
                {
                  Message msg =
                      ERR_JEB_ATTRIBUTE_INDEX_NOT_CONFIGURED.get(index);
                  throw new JebException(msg);
                  throw new InitializationException(msg);
                }
                indexCount++;
              }
            }
            else
            {
              for (String idx : cfg.listLocalDBIndexes())
              boolean found = false;
              for (final String idx : cfg.listLocalDBIndexes())
              {
                if (!idx.equalsIgnoreCase(index))
                {
                  continue;
                }
                found = true;
                LocalDBIndexCfg indexCfg = cfg.getLocalDBIndex(idx);
                if (indexCfg.getIndexType().contains(
                    LocalDBIndexCfgDefn.IndexType.EQUALITY))
@@ -3853,7 +3860,7 @@
                  Set<String> extensibleRules =
                      indexCfg.getIndexExtensibleMatchingRule();
                  boolean shared = false;
                  for (String exRule : extensibleRules)
                  for (final String exRule : extensibleRules)
                  {
                    if (exRule.endsWith(".sub"))
                    {
@@ -3870,6 +3877,12 @@
                  }
                }
              }
              if (!found)
              {
                Message msg =
                    ERR_JEB_ATTRIBUTE_INDEX_NOT_CONFIGURED.get(index);
                throw new InitializationException(msg);
              }
            }
          }
        }
@@ -3908,7 +3921,7 @@
        throws InterruptedException
    {
      for (Map.Entry<IndexKey, Collection<Index>> mapEntry :
          this.extensibleIndexMap.entrySet())
        this.extensibleIndexMap.entrySet())
      {
        IndexKey key = mapEntry.getKey();
        AttributeType attrType = key.getAttributeType();

 opends/src/server/org/opends/server/tasks/RebuildTask.java

@@ -41,6 +41,7 @@
import org.opends.server.types.DebugLogLevel;
import org.opends.server.types.DirectoryException;
import org.opends.server.types.Entry;
import org.opends.server.types.InitializationException;

import org.opends.server.types.Operation;
import org.opends.server.types.Privilege;
@@ -192,6 +193,7 @@
    // The degraded state is set(if present in args)
    // during the initialization.
    rebuildConfig.isClearDegradedState(isClearDegradedState);
    boolean isBackendNeedToBeEnabled = false;

    if (tmpDirectory == null)
    {
@@ -222,7 +224,7 @@
    String lockFile = LockFileManager.getBackendLockFileName(backend);
    StringBuilder failureReason = new StringBuilder();

    // Disable the backend.
    // Disable the backend
    // Except in 'cleardegradedstate' mode we don't need to disable it.
    if (!isClearDegradedState)
    {
@@ -293,6 +295,20 @@
      BackendImpl jebBackend = (BackendImpl) backend;
      jebBackend.rebuildBackend(rebuildConfig);
    }
    catch (InitializationException e)
    {
      // This exception catches all 'index not found'
      // The backend needs to be re-enabled at the end of the process.
      Message message =
          ERR_REBUILDINDEX_ERROR_DURING_REBUILD.get(e.getMessage());
      if (debugEnabled())
      {
        TRACER.debugCaught(DebugLogLevel.ERROR, e);
      }
      logError(message);
      isBackendNeedToBeEnabled = true;
      returnCode = TaskState.STOPPED_BY_ERROR;
    }
    catch (Exception e)
    {
      if (debugEnabled())
@@ -305,10 +321,9 @@
      logError(message);
      returnCode = TaskState.STOPPED_BY_ERROR;
    }

    // Release the lock on the backend.
    finally
    {
      // Release the lock on the backend.
      try
      {
        lockFile = LockFileManager.getBackendLockFileName(backend);
@@ -334,7 +349,8 @@

    // The backend must be enabled only if the task is successful
    // for prevent potential risks of database corruption.
    if (returnCode == TaskState.COMPLETED_SUCCESSFULLY && !isClearDegradedState)
    if ((returnCode == TaskState.COMPLETED_SUCCESSFULLY
        || (isBackendNeedToBeEnabled)) && !isClearDegradedState)
    {
      // Enable the backend.
      try

 opends/src/server/org/opends/server/tools/RebuildIndex.java

@@ -598,6 +598,13 @@
      BackendImpl jebBackend = (BackendImpl)backend;
      jebBackend.rebuildBackend(rebuildConfig);
    }
    catch (InitializationException e)
    {
      Message message =
          ERR_REBUILDINDEX_ERROR_DURING_REBUILD.get(e.getMessage());
      logError(message);
      returnCode = 1;
    }
    catch (Exception e)
    {
      Message message =

 opends/tests/unit-tests-testng/src/server/org/opends/server/backends/jeb/TestRebuildJob.java

@@ -23,7 +23,7 @@
 *
 *
 *      Copyright 2006-2009 Sun Microsystems, Inc.
 *      Portions Copyright 2011-2012 ForgeRock AS
 *      Portions Copyright 2011-2013 ForgeRock AS
 */
package org.opends.server.backends.jeb;

@@ -61,6 +61,7 @@
    return new Object[][] {
        { "dn2id" },
        { "dn2uri" }
        // { "id2entry" } internal index
    };
  }

@@ -80,7 +81,6 @@
  @DataProvider(name = "badIndexes")
  public Object[][] badIndexes() {
    return new Object[][] {
        { "id2entry" },
        { "nonindex" },
        { "id2subtree" },
        { "id2children" },
@@ -178,14 +178,60 @@
    }
  }

  /**
   * Try to rebuild the main system index id2entry.
   * (primary index from which all other indexes are derived).
   * It cannot ever be rebuilt. Online mode.
   *
   * @throws InitializationException There is no index configured
   *  for attribute type 'id2entry'.
   */
  @Test(expectedExceptions = InitializationException.class)
  public void testRebuildForbiddenSystemIndexId2EntryOnline() throws Exception
  {
    RebuildConfig rebuildConfig = new RebuildConfig();
    rebuildConfig.setBaseDN(baseDNs[0]);
    rebuildConfig.addRebuildIndex("id2entry");
    be=(BackendImpl) DirectoryServer.getBackend(beID);

    be.rebuildBackend(rebuildConfig);

  }

  /**
   * Try to rebuild the main system index id2entry.
   * (primary index from which all other indexes are derived).
   * It cannot ever be rebuilt. Offline mode.
   *
   * @throws InitializationException There is no index configured
   *  for attribute type 'id2entry'.
   */
  @Test(expectedExceptions = InitializationException.class)
  public void testRebuildForbiddenSystemIndexId2EntryOffline() throws Exception
  {
    RebuildConfig rebuildConfig = new RebuildConfig();
    rebuildConfig.setBaseDN(baseDNs[0]);
    rebuildConfig.addRebuildIndex("id2entry");
    be=(BackendImpl) DirectoryServer.getBackend(beID);

    TaskUtils.disableBackend(beID);

    try {
      be.rebuildBackend(rebuildConfig);
    } finally {
      TaskUtils.enableBackend(beID);
    }
  }

  @Test(dataProvider = "badIndexes",
        expectedExceptions = DirectoryException.class)
  public void testRebuildBadIndexes(String index) throws Exception
        expectedExceptions = InitializationException.class)
  public void testRebuildBadIndexes(final String index) throws Exception
  {
    RebuildConfig rebuildConfig = new RebuildConfig();
    rebuildConfig.setBaseDN(baseDNs[0]);
    rebuildConfig.addRebuildIndex(index);
    be=(BackendImpl) DirectoryServer.getBackend(beID);

    be.rebuildBackend(rebuildConfig);
  }