OPENDJ-2364 PR-148 Change backend type during upgrade

Gaetan Boismal

13.42.2015 a1c05a12054b831b0aead0bd5eac9c6bdac6a655

OPENDJ-2364 PR-148 Change backend type during upgrade

Perform the migration from local-db backend to JE only is the OpenDJ edition is standard.
If the OpenDJ edition is OEM, runs the added upgrade task to migrate
local-db backends to pdb.

 opendj-server-legacy/src/main/java/org/opends/server/tools/upgrade/Installation.java

@@ -169,4 +169,7 @@
   */
  static final String BACKUP = "backup";

  /** The relative path to the lib directory. */
  static final String LIB_RELATIVE_PATH = "lib";

}

 opendj-server-legacy/src/main/java/org/opends/server/tools/upgrade/Upgrade.java

@@ -51,7 +51,9 @@
import static org.opends.server.tools.upgrade.FormattedNotificationCallback.*;
import static org.opends.server.tools.upgrade.LicenseFile.*;
import static org.opends.server.tools.upgrade.UpgradeTasks.*;
import static org.opends.server.tools.upgrade.UpgradeUtils.*;
import static org.opends.server.tools.upgrade.UpgradeUtils.batDirectory;
import static org.opends.server.tools.upgrade.UpgradeUtils.binDirectory;
import static org.opends.server.tools.upgrade.UpgradeUtils.libDirectory;
import static org.opends.server.util.StaticUtils.*;

/**
@@ -179,7 +181,7 @@
        "ds-cfg-strict-format: false"));

    register("2.5.0",
        requireConfirmation(INFO_UPGRADE_TASK_8214_DESCRIPTION.get(),
        requireConfirmation(INFO_UPGRADE_TASK_8214_DESCRIPTION.get(), YES,
            modifyConfigEntry(INFO_UPGRADE_TASK_8214_SUMMARY.get(),
                "(ds-cfg-java-class=org.opends.server.extensions.IsMemberOfVirtualAttributeProvider)",
                "add: ds-cfg-filter",
@@ -403,39 +405,151 @@
    register("2.7.0",
        rerunJavaPropertiesTool(INFO_UPGRADE_TASK_9206_SUMMARY.get()));

    /** If the upgraded version is a non OEM one, migrates local-db backends to JE Backend, see OPENDJ-2364 **/
    register("3.0.0",
        migrateLocalDBBackendsToJEBackends(),
        modifyConfigEntry(INFO_UPGRADE_TASK_MIGRATE_JE_SUMMARY_2.get(),
            "(objectClass=ds-cfg-local-db-backend)",
            "replace: objectClass",
            "objectClass: top",
            "objectClass: ds-cfg-backend",
            "objectClass: ds-cfg-pluggable-backend",
            "objectClass: ds-cfg-je-backend",
            "-",
            "replace: ds-cfg-java-class",
            "ds-cfg-java-class: org.opends.server.backends.jeb.JEBackend",
            "-",
            "delete: ds-cfg-import-thread-count",
            "-",
            "delete: ds-cfg-import-queue-size",
            "-",
            "delete: ds-cfg-subordinate-indexes-enabled",
            "-"),
        modifyConfigEntry(INFO_UPGRADE_TASK_MIGRATE_JE_SUMMARY_3.get(),
            "(objectClass=ds-cfg-local-db-index)",
            "replace: objectClass",
            "objectClass: top",
            "objectClass: ds-cfg-backend-index",
            "-"),
        modifyConfigEntry(INFO_UPGRADE_TASK_MIGRATE_JE_SUMMARY_4.get(),
            "(objectClass=ds-cfg-local-db-vlv-index)",
            "replace: objectClass",
            "objectClass: top",
            "objectClass: ds-cfg-backend-vlv-index",
            "-",
            "delete: ds-cfg-max-block-size",
            "-"));
        conditionalUpgradeTasks(
          new UpgradeCondition() {
              @Override
              public boolean shouldPerformUpgradeTasks(UpgradeContext context) throws ClientException {
                return !isOEMVersion();
              }
          },
          migrateLocalDBBackendsToJEBackends(),
          modifyConfigEntry(INFO_UPGRADE_TASK_MIGRATE_JE_SUMMARY_2.get(),
              "(objectClass=ds-cfg-local-db-backend)",
              "replace: objectClass",
              "objectClass: top",
              "objectClass: ds-cfg-backend",
              "objectClass: ds-cfg-pluggable-backend",
              "objectClass: ds-cfg-je-backend",
              "-",
              "replace: ds-cfg-java-class",
              "ds-cfg-java-class: org.opends.server.backends.jeb.JEBackend",
              "-",
              "delete: ds-cfg-import-thread-count",
              "-",
              "delete: ds-cfg-import-queue-size",
              "-",
              "delete: ds-cfg-subordinate-indexes-enabled",
              "-"
          ),
          modifyConfigEntry(INFO_UPGRADE_TASK_MIGRATE_JE_SUMMARY_3.get(),
              "(objectClass=ds-cfg-local-db-index)",
              "replace: objectClass",
              "objectClass: top",
              "objectClass: ds-cfg-backend-index",
              "-"
          ),
          modifyConfigEntry(INFO_UPGRADE_TASK_MIGRATE_JE_SUMMARY_4.get(),
              "(objectClass=ds-cfg-local-db-vlv-index)",
              "replace: objectClass",
              "objectClass: top",
              "objectClass: ds-cfg-backend-vlv-index",
              "-",
              "delete: ds-cfg-max-block-size",
              "-"
          )
        )
    );

    /** If the upgraded version is OEM, migrates local-db backends to PDB, see OPENDJ-2364 **/
    register("3.0.0",
      conditionalUpgradeTasks(
        new UpgradeCondition() {
          @Override
          public boolean shouldPerformUpgradeTasks(UpgradeContext context) throws ClientException {
            return isOEMVersion();
          }
        },
        deleteFile(new File(libDirectory, "je.jar")),
        requireConfirmation(INFO_UPGRADE_TASK_LOCAL_DB_TO_PDB_1_SUMMARY.get(), NO,
                renameLocalDBBackendDirectories(),
                // Convert JE backends to PDB backends.
                modifyConfigEntry(INFO_UPGRADE_TASK_LOCAL_DB_TO_PDB_2_SUMMARY.get(),
                        "(objectclass=ds-cfg-local-db-backend)",
                        "delete: objectclass",
                        "objectclass: ds-cfg-local-db-backend",
                        "-",
                        "add: objectclass",
                        "objectclass: ds-cfg-pluggable-backend",
                        "objectclass: ds-cfg-pdb-backend",
                        "-",
                        "replace: ds-cfg-java-class",
                        "ds-cfg-java-class: org.opends.server.backends.pdb.PDBBackend",
                        "-",
                        "delete: ds-cfg-preload-time-limit",
                        "-",
                        "delete: ds-cfg-import-thread-count",
                        "-",
                        "delete: ds-cfg-import-queue-size",
                        "-",
                        "delete: ds-cfg-db-txn-write-no-sync",
                        "-",
                        "delete: ds-cfg-db-run-cleaner",
                        "-",
                        "delete: ds-cfg-db-cleaner-min-utilization",
                        "-",
                        "delete: ds-cfg-db-evictor-lru-only",
                        "-",
                        "delete: ds-cfg-db-evictor-core-threads",
                        "-",
                        "delete: ds-cfg-db-evictor-max-threads",
                        "-",
                        "delete: ds-cfg-db-evictor-keep-alive",
                        "-",
                        "delete: ds-cfg-db-evictor-nodes-per-scan",
                        "-",
                        "delete: ds-cfg-db-log-file-max",
                        "-",
                        "delete: ds-cfg-db-log-filecache-size",
                        "-",
                        "delete: ds-cfg-db-logging-file-handler-on",
                        "-",
                        "delete: ds-cfg-db-logging-level",
                        "-",
                        "delete: ds-cfg-db-checkpointer-bytes-interval",
                        "-",
                        "delete: ds-cfg-db-checkpointer-wakeup-interval",
                        "-",
                        "delete: ds-cfg-db-num-lock-tables",
                        "-",
                        "delete: ds-cfg-db-num-cleaner-threads",
                        "-",
                        "delete: ds-cfg-je-property",
                        "-",
                        "delete: ds-cfg-subordinate-indexes-enabled",
                        "-"
                ),
                // Convert JE backend indexes to PDB backend indexes.
                modifyConfigEntry(INFO_UPGRADE_TASK_LOCAL_DB_TO_PDB_3_SUMMARY.get(),
                        "(objectclass=ds-cfg-local-db-index)",
                        "delete: objectclass",
                        "objectclass: ds-cfg-local-db-index",
                        "-",
                        "add: objectclass",
                        "objectclass: ds-cfg-backend-index",
                        "-"
                ),
                // Convert JE backend VLV indexes to PDB backend VLV indexes.
                modifyConfigEntry(INFO_UPGRADE_TASK_LOCAL_DB_TO_PDB_4_SUMMARY.get(),
                        "(objectclass=ds-cfg-local-db-vlv-index)",
                        "delete: objectclass",
                        "objectclass: ds-cfg-local-db-vlv-index",
                        "-",
                        "add: objectclass",
                        "objectclass: ds-cfg-backend-vlv-index",
                        "-",
                        "delete: ds-cfg-max-block-size",
                        "-"
                )
        )
      )
    );

    /** Remove dbtest tool (replaced by backendstat in 3.0.0) - see OPENDJ-1791 **/
    register("3.0.0",
            deleteFile(new File(binDirectory, "dbtest")),
            deleteFile(new File(batDirectory, "dbtest.bat")));

    /**
     * Rebuild all indexes when upgrading to 3.0.0.
@@ -444,7 +558,7 @@
     * 2) JE backend has been migrated to pluggable architecture.
     */
    register("3.0.0",
        rebuildAllIndexes(INFO_UPGRADE_TASK_11260_SUMMARY.get()));
            rebuildAllIndexes(INFO_UPGRADE_TASK_11260_SUMMARY.get()));

    /** See OPENDJ-1742 */
    register("3.0.0",

 opendj-server-legacy/src/main/java/org/opends/server/tools/upgrade/UpgradeTasks.java

@@ -420,27 +420,26 @@
   *          The group of tasks to invoke if the user agrees.
   * @return An upgrade task which will only be invoked if the user confirms agreement.
   */
  public static UpgradeTask requireConfirmation(final LocalizableMessage message, final UpgradeTask... tasks)
  static UpgradeTask requireConfirmation(
          final LocalizableMessage message, final int defaultResponse, final UpgradeTask... tasks)
  {
    return conditionalUpgradeTasks(new UpgradeCondition()
    {
      @Override
      public boolean shouldPerformUpgradeTasks(final UpgradeContext context) throws ClientException
      {
        return context.confirmYN(INFO_UPGRADE_TASK_NEEDS_USER_CONFIRM.get(message), YES) == YES;
        return context.confirmYN(INFO_UPGRADE_TASK_NEEDS_USER_CONFIRM.get(message), defaultResponse) == YES;
      }
    }, tasks);
  }

  /**
   * Determines whether conditional tasks should be performed.
   */
  private static interface UpgradeCondition
  /** Determines whether conditional tasks should be performed. */
  interface UpgradeCondition
  {
    boolean shouldPerformUpgradeTasks(final UpgradeContext context) throws ClientException;
  }

  private static UpgradeTask conditionalUpgradeTasks(final UpgradeCondition condition, final UpgradeTask... tasks)
  static UpgradeTask conditionalUpgradeTasks(final UpgradeCondition condition, final UpgradeTask... tasks)
  {
    return new AbstractUpgradeTask()
    {
@@ -937,12 +936,7 @@
      }

      private boolean isJeLibraryAvailable() {
        try {
          Class.forName("com.sleepycat.je.Environment");
          return true;
        } catch (Exception e) {
          return false;
        }
        return isClassAvailable("com.sleepycat.je.Environment");
      }

      private String newName(final DN baseDN) {
@@ -962,6 +956,89 @@
    };
  }

  /**
   * Creates backups of the local DB backends directories by renaming adding them a ".bak" suffix.
   *  e.g "userRoot" would become "userRoot.bak"
   */
  static UpgradeTask renameLocalDBBackendDirectories()
  {
    return new AbstractUpgradeTask()
    {
      private boolean reimportRequired = false;

      @Override
      public void perform(UpgradeContext context) throws ClientException
      {
        try
        {
          Filter filter = Filter.equality("objectclass", "ds-cfg-local-db-backend");
          SearchRequest findLocalDBBackends = Requests.newSearchRequest(DN.rootDN(), SearchScope.WHOLE_SUBTREE, filter);
          try (final EntryReader jeBackends = searchConfigFile(findLocalDBBackends))
          {
            while (jeBackends.hasNext())
            {
              Upgrade.setHasPostUpgradeTask(true);
              reimportRequired = true;

              Entry jeBackend = jeBackends.readEntry();
              File dbParent = UpgradeUtils.getFileForPath(jeBackend.parseAttribute("ds-cfg-db-directory").asString());
              String id = jeBackend.parseAttribute("ds-cfg-backend-id").asString();

              // Use canonical paths so that the progress message is more readable.
              File dbDirectory = new File(dbParent, id).getCanonicalFile();
              File dbDirectoryBackup = new File(dbParent, id + ".bak").getCanonicalFile();
              if (dbDirectory.exists() && !dbDirectoryBackup.exists())
              {
                LocalizableMessage msg = INFO_UPGRADE_TASK_RENAME_JE_DB_DIR.get(dbDirectory, dbDirectoryBackup);
                ProgressNotificationCallback pnc = new ProgressNotificationCallback(0, msg, 0);
                context.notifyProgress(pnc);
                boolean renameSucceeded = dbDirectory.renameTo(dbDirectoryBackup);
                context.notifyProgress(pnc.setProgress(renameSucceeded ? 100 : -1));
              }
            }
          }
        }
        catch (Exception e)
        {
          logger.error(LocalizableMessage.raw(e.getMessage()));
        }
      }

      @Override
      public void postUpgrade(UpgradeContext context) throws ClientException
      {
        postponePostUpgrade(context);
      }

      @Override
      public void postponePostUpgrade(UpgradeContext context) throws ClientException
      {
        if (reimportRequired)
        {
          context.notify(INFO_UPGRADE_TASK_RENAME_JE_DB_DIR_WARNING.get(), TextOutputCallback.WARNING);
        }
      }
    };
  }

  static boolean isOEMVersion()
  {
    return !isClassAvailable("org.opends.server.backends.jeb.JEBackend");
  }

  private static boolean isClassAvailable(final String className)
  {
    try
    {
      Class.forName(className);
      return true;
    }
    catch (Exception e)
    {
      return false;
    }
  }

  /** This inner classes causes JE to be lazily linked and prevents runtime errors if JE is not in the classpath. */
  static final class JEHelper {
    private static ClientException clientException(final File backendDirectory, final DatabaseException e) {

 opendj-server-legacy/src/main/java/org/opends/server/tools/upgrade/UpgradeUtils.java

@@ -91,6 +91,9 @@
      configDirectory + File.separator + Installation.SNMP_PATH_RELATIVE
          + File.separator + Installation.SECURITY_PATH_RELATIVE);

  /** The lib folder of the current installation. */
  static final File libDirectory = new File(getInstallationPath(), Installation.LIB_RELATIVE_PATH);

  /** The bin folder of the current installation. */
  static final File binDirectory = new File(getInstallationPath(), Installation.UNIX_BINARIES_PATH_RELATIVE);

@@ -265,6 +268,12 @@
    return getPath(new File(new File(parentPath), relativePath));
  }

  static File getFileForPath(String path)
  {
    final File f = new File(path);
    return f.isAbsolute() ? f : new File(getInstancePath() + File.separator + path);
  }

  /**
   * Determines whether one file is the parent of another.
   *

 opendj-server-legacy/src/messages/org/opends/messages/tool.properties

@@ -2752,3 +2752,15 @@
SUPPLEMENT_DESCRIPTION_BACKEND_TOOL_SUBCMD_LIST_INDEX_STATUS_20016=\
  <xinclude:include href="variablelist-backendstat-index-status.xml" />
INFO_INSTALLDS_BACKEND_TYPE_USED_20017=Backend Type: %s
INFO_UPGRADE_TASK_LOCAL_DB_TO_PDB_1_SUMMARY_20018=WARNING: OpenDJ 3.0.0 OEM Edition removes support for the \
 Berkeley JE backend.\n\nThe upgrade tool will reconfigure all JE \
 backends as PDB backends.\n\nAfter the upgrade the new PDB backend(s) \
 will be empty. It is therefore very strongly recommended that any data that was \
 in the JE backends be exported to LDIF so that it can be re-imported once the \
 upgrade completes.\n\n
INFO_UPGRADE_TASK_LOCAL_DB_TO_PDB_2_SUMMARY_20019=Reconfiguring local-db backends to PDB backends
INFO_UPGRADE_TASK_LOCAL_DB_TO_PDB_3_SUMMARY_20020=Reconfiguring local-db backend indexes to PDB backend indexes
INFO_UPGRADE_TASK_LOCAL_DB_TO_PDB_4_SUMMARY_20021=Reconfiguring local-db backend VLV indexes to PDB backend VLV indexes
INFO_UPGRADE_TASK_RENAME_JE_DB_DIR_20022=Renaming local-db backend directory '%s' to '%s'
INFO_UPGRADE_TASK_RENAME_JE_DB_DIR_WARNING_20023=You must reimport all your data into the PDB \
 backends in order to have a fully functional server