OPENDJ-1719: implement upgrade tasks for migrating JE backends

Matthew Swift

07.10.2015 642ee854211fe638a4c47b44eae0d1405f20caf7

OPENDJ-1719: implement upgrade tasks for migrating JE backends

The task comprises of several steps:

* aborts the upgrade if there are enabled local-db backends but no JE
  in the class-path
* aborts the upgrade if the enabled local-db backends cannot be opened
  or if the upgrade task does not support automated migration, in which
  case the user must manually export their data, disable the backend(s),
  upgrade, and re-import
* migrates any local-db backend configuration to the new je-backend
  config model
* prepares existing local-db backend databases for a rebuild-all by
  making the database compatible with the pluggable storge architecture.
  In particular, the task renames the compressed schema indexes and
  id2entry to the naming scheme used in the new JE backend, and removes
  all other indexes. The updates are performed as a transaction to
  prevent partial migration
* registers the rebuild-all post-upgrade task.

 opendj-server-legacy/src/main/java/org/opends/server/tools/upgrade/Upgrade.java

@@ -383,10 +383,6 @@
        deleteConfigEntry(INFO_UPGRADE_TASK_11339_SUMMARY.get(),
            "dn: cn=Extensions,cn=config"));

    /** See OPENDJ-1637 */
    register("2.8.0",
        rebuildAllIndexes(INFO_UPGRADE_TASK_11260_SUMMARY.get()));

    /** See OPENDJ-1701 */
    register("2.8.0",
        deleteConfigEntry(INFO_UPGRADE_TASK_11476_SUMMARY.get(),
@@ -407,6 +403,49 @@
    register("2.7.0",
        rerunJavaPropertiesTool(INFO_UPGRADE_TASK_9206_SUMMARY.get()));

    register("3.0.0",
        migrateLocalDBBackendsToJEBackends(),
        modifyConfigEntry(INFO_UPGRADE_TASK_MIGRATE_JE_SUMMARY_2.get(),
            "(objectClass=ds-cfg-local-db-backend)",
            "replace: objectClass",
            "objectClass: top",
            "objectClass: ds-cfg-backend",
            "objectClass: ds-cfg-pluggable-backend",
            "objectClass: ds-cfg-je-backend",
            "-",
            "replace: ds-cfg-java-class",
            "ds-cfg-java-class: org.opends.server.backends.jeb.JEBackend",
            "-",
            "delete: ds-cfg-import-thread-count",
            "-",
            "delete: ds-cfg-import-queue-size",
            "-",
            "delete: ds-cfg-subordinate-indexes-enabled",
            "-"),
        modifyConfigEntry(INFO_UPGRADE_TASK_MIGRATE_JE_SUMMARY_3.get(),
            "(objectClass=ds-cfg-local-db-index)",
            "replace: objectClass",
            "objectClass: top",
            "objectClass: ds-cfg-backend-index",
            "-"),
        modifyConfigEntry(INFO_UPGRADE_TASK_MIGRATE_JE_SUMMARY_4.get(),
            "(objectClass=ds-cfg-local-db-vlv-index)",
            "replace: objectClass",
            "objectClass: top",
            "objectClass: ds-cfg-backend-vlv-index",
            "-",
            "delete: ds-cfg-max-block-size",
            "-"));

    /**
     * Rebuild all indexes when upgrading to 3.0.0.
     *
     * 1) matching rules have changed in 2.8.0 and again in 3.0.0- see OPENDJ-1637
     * 2) JE backend has been migrated to pluggable architecture.
     */
    register("3.0.0",
        rebuildAllIndexes(INFO_UPGRADE_TASK_11260_SUMMARY.get()));

    /*
     * All upgrades will refresh the server configuration schema and generate
     * a new upgrade folder.
@@ -485,27 +524,27 @@
      return;
    }

    /*
     * Let tasks interact with the user in order to obtain user's selection.
     */
    context.notify(INFO_UPGRADE_REQUIREMENTS.get(), TITLE_CALLBACK);
    for (final UpgradeTask task : tasks)
    {
      task.prepare(context);
    }

    // Starts upgrade
    final int userResponse = context.confirmYN(INFO_UPGRADE_DISPLAY_CONFIRM_START.get(), ConfirmationCallback.YES);
    if (userResponse == ConfirmationCallback.NO)
    {
      final LocalizableMessage message = INFO_UPGRADE_ABORTED_BY_USER.get();
      context.notify(message, WARNING);
      throw new ClientException(ReturnCode.ERROR_UNEXPECTED, message);
    }

    try
    {
      /*
       * Let tasks interact with the user in order to obtain user's selection.
       */
      context.notify(INFO_UPGRADE_REQUIREMENTS.get(), TITLE_CALLBACK);
      for (final UpgradeTask task : tasks)
      {
        task.prepare(context);
      }

      // Starts upgrade
      final int userResponse = context.confirmYN(INFO_UPGRADE_DISPLAY_CONFIRM_START.get(), ConfirmationCallback.YES);
      if (userResponse == ConfirmationCallback.NO)
      {
        final LocalizableMessage message = INFO_UPGRADE_ABORTED_BY_USER.get();
        context.notify(message, WARNING);
        throw new ClientException(ReturnCode.ERROR_UNEXPECTED, message);
      }

      /*
       * Perform the upgrade tasks.
       */
      context.notify(INFO_UPGRADE_PERFORMING_TASKS.get(), TITLE_CALLBACK);

 opendj-server-legacy/src/main/java/org/opends/server/tools/upgrade/UpgradeTasks.java

@@ -25,32 +25,49 @@
 */
package org.opends.server.tools.upgrade;

import static javax.security.auth.callback.ConfirmationCallback.NO;
import static javax.security.auth.callback.ConfirmationCallback.YES;
import static org.opends.messages.ToolMessages.*;
import static org.opends.server.tools.upgrade.FileManager.copy;
import static org.opends.server.tools.upgrade.Installation.CURRENT_CONFIG_FILE_NAME;
import static org.opends.server.tools.upgrade.UpgradeUtils.*;

import java.io.File;
import java.io.IOException;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;

import javax.security.auth.callback.TextOutputCallback;

import org.forgerock.i18n.LocalizableMessage;
import org.forgerock.i18n.slf4j.LocalizedLogger;
import org.forgerock.opendj.ldap.DN;
import org.forgerock.opendj.ldap.Entry;
import org.forgerock.opendj.ldap.Filter;
import org.forgerock.opendj.ldap.SearchScope;
import org.forgerock.opendj.ldap.requests.Requests;
import org.forgerock.opendj.ldap.requests.SearchRequest;
import org.forgerock.opendj.ldif.EntryReader;
import org.opends.server.backends.pluggable.spi.TreeName;
import org.opends.server.tools.JavaPropertiesTool;
import org.opends.server.tools.RebuildIndex;
import org.opends.server.util.BuildVersion;
import org.opends.server.util.ChangeOperationType;
import org.opends.server.util.StaticUtils;

import com.forgerock.opendj.cli.ClientException;
import com.forgerock.opendj.cli.ReturnCode;

import static javax.security.auth.callback.ConfirmationCallback.*;
import static org.opends.messages.ToolMessages.*;
import static org.opends.server.tools.upgrade.FileManager.*;
import static org.opends.server.tools.upgrade.Installation.*;
import static org.opends.server.tools.upgrade.UpgradeUtils.*;
import com.sleepycat.je.DatabaseException;
import com.sleepycat.je.Environment;
import com.sleepycat.je.EnvironmentConfig;
import com.sleepycat.je.Transaction;
import com.sleepycat.je.TransactionConfig;

/**
 * Factory methods for create new upgrade tasks.
@@ -638,7 +655,7 @@
         * the post upgrade tasks succeed and a message is printed in the
         * upgrade log file.
         */
        final List<String> backends = UpgradeUtils.getLocalBackendsFromConfig();
        final List<String> backends = UpgradeUtils.getIndexedBackendsFromConfig();
        if (!backends.isEmpty())
        {
          for (final String be : backends)
@@ -799,6 +816,201 @@
    };
  }

  /**
   * Creates an upgrade task which is responsible for preparing local-db backend JE databases for a full rebuild once
   * they have been converted to pluggable JE backends.
   *
   * @return An upgrade task which is responsible for preparing local-db backend JE databases.
   */
  public static UpgradeTask migrateLocalDBBackendsToJEBackends() {
    return new AbstractUpgradeTask() {
      /** Properties of JE backends to be migrated. */
      class Backend {
        final String id;
        final boolean isEnabled;
        final Set<DN> baseDNs;
        final File envDir;
        final Map<String, String> renamedDbs = new HashMap<>();

        private Backend(Entry config) {
          id = config.parseAttribute("ds-cfg-backend-id").asString();
          isEnabled = config.parseAttribute("ds-cfg-enabled").asBoolean(false);
          baseDNs = config.parseAttribute("ds-cfg-base-dn").asSetOfDN();
          String dbDirectory = config.parseAttribute("ds-cfg-db-directory").asString();
          File backendParentDirectory = new File(dbDirectory);
          if (!backendParentDirectory.isAbsolute()) {
            backendParentDirectory = new File(getInstancePath(), dbDirectory);
          }
          envDir = new File(backendParentDirectory, id);
          for (String db : Arrays.asList("compressed_attributes", "compressed_object_classes")) {
            renamedDbs.put(db, new TreeName("compressed_schema", db).toString());
          }
          for (DN baseDN : baseDNs) {
            renamedDbs.put(oldName(baseDN), newName(baseDN));
          }
        }
      }

      private final List<Backend> backends = new LinkedList<>();

      /**
       * Finds all the existing JE backends and determines if they can be migrated or not. It will not be possible to
       * migrate a JE backend if the id2entry database name cannot easily be determined, which may happen because
       * matching rules have changed significantly in 3.0.0.
       */
      @Override
      public void prepare(final UpgradeContext context) throws ClientException {
        // Requires answer from the user.
        if (context.confirmYN(INFO_UPGRADE_TASK_MIGRATE_JE_DESCRIPTION.get(), NO) != YES) {
          throw new ClientException(ReturnCode.ERROR_USER_CANCELLED,
                                    INFO_UPGRADE_TASK_MIGRATE_JE_CANCELLED.get());
        }

        final SearchRequest sr = Requests.newSearchRequest("", SearchScope.WHOLE_SUBTREE,
                                                           "(objectclass=ds-cfg-local-db-backend)");
        try (final EntryReader entryReader = searchConfigFile(sr)) {
          // Abort the upgrade if there are JE backends but no JE library.
          if (entryReader.hasNext() && !isJeLibraryAvailable()) {
            throw new ClientException(ReturnCode.CONSTRAINT_VIOLATION, INFO_UPGRADE_TASK_MIGRATE_JE_NO_JE_LIB.get());
          }
          while (entryReader.hasNext()) {
            Backend backend = new Backend(entryReader.readEntry());
            if (backend.isEnabled) {
              abortIfBackendCannotBeMigrated(backend);
            }
            backends.add(backend);
          }
        } catch (IOException e) {
          throw new ClientException(ReturnCode.APPLICATION_ERROR,
                                    INFO_UPGRADE_TASK_MIGRATE_JE_CONFIG_READ_FAIL.get(), e);
        }
      }

      private void abortIfBackendCannotBeMigrated(final Backend backend) throws ClientException {
        Set<String> existingDatabases = JEHelper.listDatabases(backend.envDir);
        for (DN baseDN : backend.baseDNs) {
          final String oldName = oldName(baseDN);
          if (!existingDatabases.contains(oldName)) {
            throw new ClientException(ReturnCode.CONSTRAINT_VIOLATION,
                                      INFO_UPGRADE_TASK_MIGRATE_JE_UGLY_DN.get(backend.id, baseDN));
          }
        }
      }

      /**
       * Renames the compressed schema indexes and id2entry in a 2.x environment to
       * the naming scheme used in 3.0.0. Before 3.0.0 JE databases were named as follows:
       *
       * 1) normalize the base DN
       * 2) replace all non-alphanumeric characters with '_'
       * 3) append '_'
       * 4) append the index name.
       *
       * For example, id2entry in the base DN dc=white space,dc=com would be named
       * dc_white_space_dc_com_id2entry. In 3.0.0 JE databases are named as follows:
       *
       * 1) normalize the base DN and URL encode it (' '  are converted to %20)
       * 2) format as '/' + URL encoded base DN + '/' + index name.
       *
       * The matching rules in 3.0.0 are not compatible with previous versions, so we need
       * to do a best effort attempt to figure out the old database name from a given base DN.
       */
      @Override
      public void perform(final UpgradeContext context) throws ClientException {
        if (!isJeLibraryAvailable()) {
          return;
        }

        for (Backend backend : backends) {
          if (backend.isEnabled) {
            ProgressNotificationCallback pnc = new ProgressNotificationCallback(
                    0, INFO_UPGRADE_TASK_MIGRATE_JE_SUMMARY_1.get(backend.id), 0);
            context.notifyProgress(pnc);
            try {
              JEHelper.migrateDatabases(backend.envDir, backend.renamedDbs);
              context.notifyProgress(pnc.setProgress(100));
            } catch (ClientException e) {
              manageTaskException(context, e.getMessageObject(), pnc);
            }
          } else {
            // Skip backends which have been disabled.
            final ProgressNotificationCallback pnc = new ProgressNotificationCallback(
                    0, INFO_UPGRADE_TASK_MIGRATE_JE_SUMMARY_5.get(backend.id), 0);
            context.notifyProgress(pnc);
            context.notifyProgress(pnc.setProgress(100));
          }
        }
      }

      private boolean isJeLibraryAvailable() {
        try {
          Class.forName("com.sleepycat.je.Environment");
          return true;
        } catch (Exception e) {
          return false;
        }
      }

      private String newName(final DN baseDN) {
        return new TreeName(baseDN.toNormalizedUrlSafeString(), "id2entry").toString();
      }

      private String oldName(final DN baseDN) {
        String s = baseDN.toString();
        StringBuilder builder = new StringBuilder();
        for (int i = 0; i < s.length(); i++) {
          char c = s.charAt(i);
          builder.append(Character.isLetterOrDigit(c) ? c : '_');
        }
        builder.append("_id2entry");
        return builder.toString();
      }
    };
  }

  /** This inner classes causes JE to be lazily linked and prevents runtime errors if JE is not in the classpath. */
  static final class JEHelper {
    private static ClientException clientException(final File backendDirectory, final DatabaseException e) {
      logger.error(LocalizableMessage.raw(StaticUtils.stackTraceToString(e)));
      return new ClientException(ReturnCode.CONSTRAINT_VIOLATION,
                                 INFO_UPGRADE_TASK_MIGRATE_JE_ENV_UNREADABLE.get(backendDirectory), e);
    }

    static Set<String> listDatabases(final File backendDirectory) throws ClientException {
      try (Environment je = new Environment(backendDirectory, null)) {
        Set<String> databases = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
        databases.addAll(je.getDatabaseNames());
        return databases;
      } catch (DatabaseException e) {
        throw clientException(backendDirectory, e);
      }
    }

    static void migrateDatabases(final File envDir, final Map<String, String> renamedDbs) throws ClientException {
      EnvironmentConfig config = new EnvironmentConfig().setTransactional(true);
      try (Environment je = new Environment(envDir, config)) {
        final Transaction txn = je.beginTransaction(null, new TransactionConfig());
        try {
          for (String dbName : je.getDatabaseNames()) {
            String newDbName = renamedDbs.get(dbName);
            if (newDbName != null) {
              // id2entry or compressed schema should be kept
              je.renameDatabase(txn, dbName, newDbName);
            } else {
              // This index will need rebuilding
              je.removeDatabase(txn, dbName);
            }
          }
          txn.commit();
        } finally {
          txn.abort();
        }
      } catch (DatabaseException e) {
        throw JEHelper.clientException(envDir, e);
      }
    }
  }

  private static void displayChangeCount(final String fileName,
      final int changeCount)
  {

 opendj-server-legacy/src/main/java/org/opends/server/tools/upgrade/UpgradeUtils.java

@@ -371,44 +371,35 @@
   *
   * @return A backend list.
   */
  static List<String> getLocalBackendsFromConfig()
  static List<String> getIndexedBackendsFromConfig()
  {
    final Schema schema = getUpgradeSchema();

    final SearchRequest sr = Requests.newSearchRequest("", SearchScope.WHOLE_SUBTREE,
            "(&(objectclass=ds-cfg-pluggable-backend)(ds-cfg-enabled=true))",
            "ds-cfg-base-dn");
    final List<String> listBackends = new LinkedList<>();
    LDIFEntryReader entryReader = null;
    try
    try (final EntryReader entryReader = searchConfigFile(sr))
    {
      entryReader =
          new LDIFEntryReader(new FileInputStream(new File(configDirectory,
              CURRENT_CONFIG_FILE_NAME))).setSchema(schema);

      final SearchRequest sr =
          Requests.newSearchRequest("", SearchScope.WHOLE_SUBTREE,
              "(&(objectclass=ds-cfg-local-db-backend)(ds-cfg-enabled=true))",
              "ds-cfg-base-dn");

      final EntryReader resultReader = LDIF.search(entryReader, sr, schema);

      while (resultReader.hasNext())
      while (entryReader.hasNext())
      {
        final Entry entry = resultReader.readEntry();
        listBackends.add(entry.getAttribute("ds-cfg-base-dn")
            .firstValueAsString());
        final Entry entry = entryReader.readEntry();
        listBackends.addAll(entry.parseAttribute("ds-cfg-base-dn").asSetOfString());
      }
    }
    catch (Exception ex)
    {
      logger.error(LocalizableMessage.raw(ex.getMessage()));
    }
    finally
    {
      StaticUtils.close(entryReader);
    }

    return listBackends;
  }

  static EntryReader searchConfigFile(final SearchRequest sr) throws FileNotFoundException
  {
    final Schema schema = getUpgradeSchema();
    final File configFile = new File(configDirectory, CURRENT_CONFIG_FILE_NAME);
    final LDIFEntryReader entryReader = new LDIFEntryReader(new FileInputStream(configFile)).setSchema(schema);
    return LDIF.search(entryReader, sr, schema);
  }

  /**
   * Updates the config file during the upgrade process.
   *

 opendj-server-legacy/src/messages/org/opends/messages/tool.properties

@@ -2658,6 +2658,28 @@
INFO_UPGRADE_TASK_11476_SUMMARY_10037=Removing config for 'File System Entry Cache'
INFO_UPGRADE_TASK_12226_SUMMARY_10038=Removing config for 'Entry Cache Preload'
INFO_UPGRADE_TASK_9206_SUMMARY_10039=Rerunning dsjavaproperties
INFO_UPGRADE_TASK_MIGRATE_JE_NO_JE_LIB_10040=The upgrade will not be performed because the current instance contains \
  one or more JE based backends which cannot be migrated because the JE library is not present in the class-path. \
  The upgrade should be restarted once the JE backends have been exported and disabled
INFO_UPGRADE_TASK_MIGRATE_JE_UGLY_DN_10041=The upgrade will not be performed because the JE backend '%s' contains a \
  base DN '%s' that cannot be migrated. The upgrade should be restarted once the JE backend has been exported and \
  disabled
INFO_UPGRADE_TASK_MIGRATE_JE_CONFIG_READ_FAIL_10042=The upgrade will not be performed because the configuration file \
  config.ldif could not be parsed
INFO_UPGRADE_TASK_MIGRATE_JE_ENV_UNREADABLE_10043=The upgrade cannot continue because the JE backend with \
  environment directory '%s' could not be accessed for migration
INFO_UPGRADE_TASK_MIGRATE_JE_SUMMARY_1_10044=Migrating JE backend '%s'
INFO_UPGRADE_TASK_MIGRATE_JE_SUMMARY_2_10045=Convert local DB backends to JE backends
INFO_UPGRADE_TASK_MIGRATE_JE_SUMMARY_3_10046=Convert local DB indexes to backend indexes
INFO_UPGRADE_TASK_MIGRATE_JE_SUMMARY_4_10047=Convert local DB VLV indexes to backend VLV indexes
INFO_UPGRADE_TASK_MIGRATE_JE_SUMMARY_5_10048=Skipping migration of disabled JE backend '%s'
INFO_UPGRADE_TASK_MIGRATE_JE_DESCRIPTION_10049=OpenDJ 3.0.0 introduced changes to the JE backend configuration and \
  database format. The upgrade will update all JE backend configurations, but will only migrate JE backend databases \
  which are associated with *enabled* JE backends. It is very strongly recommended that any existing data has been \
  backed up and that you have read the upgrade documentation before proceeding. Do you want to proceed with the \
  upgrade?
INFO_UPGRADE_TASK_MIGRATE_JE_CANCELLED_10050=The upgrade will not be performed because some JE backends need to be \
  migrated

# Strings for generated reference documentation.
REF_SHORT_DESC_BACKUP_15000=back up OpenDJ directory data