OPENDJ-3263: import with DN validation on JE is using high disk space

Yannick Lecaillez

05.59.2016 fc6ff5cd4f579042078b4103af7931804895f344

OPENDJ-3263: import with DN validation on JE is using high disk space

Remove --skipDNValidation option.

 opendj-server-legacy/src/main/java/org/opends/guitools/controlpanel/ui/ImportLDIFPanel.java

@@ -69,7 +69,6 @@
  private JTextField file;
  private JCheckBox dataCompressed;
  private JCheckBox rejectNotSchemaCompliant;
  private JCheckBox doDNValidationAfter;
  private JCheckBox writeRejects;
  private JCheckBox writeSkips;
  private JTextField threads;
@@ -233,19 +232,6 @@
    gbc.gridx = 0;
    gbc.gridy ++;
    gbc.insets.left = 0;
    lDNValidation = Utilities.createPrimaryLabel(INFO_CTRL_PANEL_DN_VALIDATION_LABEL.get());
    add(lDNValidation, gbc);

    gbc.gridx = 1;
    doDNValidationAfter = Utilities.createCheckBox(INFO_CTRL_PANEL_DO_DN_VALIDATION_LATER_LABEL.get());
    doDNValidationAfter.setSelected(false);
    gbc.insets.left = 10;
    add(doDNValidationAfter, gbc);
    lDNValidation.setLabelFor(doDNValidationAfter);

    gbc.gridx = 0;
    gbc.gridy ++;
    gbc.insets.left = 0;
    lThreads = Utilities.createPrimaryLabel(INFO_CTRL_PANEL_IMPORT_THREADS_LABEL.get());
    add(lThreads, gbc);

@@ -682,10 +668,6 @@
      {
        args.add("--skipSchemaValidation");
      }
      if (doDNValidationAfter.isSelected())
      {
        args.add("--skipDNValidation");
      }

      String sThread = threads.getText().trim();
      if (sThread.length() > 0)

 opendj-server-legacy/src/main/java/org/opends/server/config/ConfigConstants.java

@@ -3974,14 +3974,6 @@
       NAME_PREFIX_TASK + "import-tmp-directory";

    /**
   * The name of the attribute in an import task definition that specifies
   * that minimal DN validation should be done during phase 2.
   */

   public static final String ATTR_IMPORT_SKIP_DN_VALIDATION =
       NAME_PREFIX_TASK + "import-skip-dn-validation";

  /**
   * The name of the objectclass that will be used for a Directory Server
   * initialize task definition.
   */

 opendj-server-legacy/src/main/java/org/opends/server/replication/plugin/LDAPReplicationDomain.java

@@ -3552,7 +3552,6 @@

      importConfig = new LDIFImportConfig(input);
      importConfig.setIncludeBranches(newLinkedHashSet(getBaseDN()));
      importConfig.setSkipDNValidation(true);
      // We should not validate schema for replication
      importConfig.setValidateSchema(false);
      // Allow fractional replication ldif import plugin to be called

 opendj-server-legacy/src/main/java/org/opends/server/tasks/ImportTask.java

@@ -90,7 +90,6 @@
  private boolean overwrite;
  private boolean skipSchemaValidation;
  private boolean clearBackend;
  private boolean skipDNValidation;
  private String tmpDirectory;
  private int threadCount;
  private String backendID;
@@ -153,7 +152,6 @@
    AttributeType typeRandomSeed = getSchema().getAttributeType(ATTR_IMPORT_RANDOM_SEED);
    AttributeType typeThreadCount = getSchema().getAttributeType(ATTR_IMPORT_THREAD_COUNT);
    AttributeType typeTmpDirectory = getSchema().getAttributeType(ATTR_IMPORT_TMP_DIRECTORY);
    AttributeType typeDNCheckPhase2 = getSchema().getAttributeType(ATTR_IMPORT_SKIP_DN_VALIDATION);

    ArrayList<String> ldifFilestmp = asListOfStrings(taskEntry, typeLdifFile);
    ldifFiles = new ArrayList<>(ldifFilestmp.size());
@@ -189,7 +187,6 @@
      }
    }

    skipDNValidation = asBoolean(taskEntry, typeDNCheckPhase2);
    tmpDirectory = asString(taskEntry, typeTmpDirectory);
    backendID = asString(taskEntry, typeBackendID);
    includeBranchStrings = asListOfStrings(taskEntry, typeIncludeBranch);
@@ -581,7 +578,6 @@
    importConfig.setIncludeBranches(includeBranches);
    importConfig.setIncludeFilters(includeFilters);
    importConfig.setValidateSchema(!skipSchemaValidation);
    importConfig.setSkipDNValidation(skipDNValidation);
    importConfig.setTmpDirectory(tmpDirectory);
    importConfig.setThreadCount(threadCount);

@@ -669,16 +665,7 @@
        logger.traceException(de);

        DirectoryServer.notifyImportEnded(backend, importConfig, false);
        LocalizableMessage msg;
        if (de.getResultCode() == ResultCode.CONSTRAINT_VIOLATION)
        {
          msg = ERR_LDIFIMPORT_ERROR_CONSTRAINT_VIOLATION.get();
        }
        else
        {
          msg = de.getMessageObject();
        }
        logger.error(ERR_LDIFIMPORT_ERROR_DURING_IMPORT.get(msg));
        logger.error(ERR_LDIFIMPORT_ERROR_DURING_IMPORT.get(de.getMessageObject()));
        return TaskState.STOPPED_BY_ERROR;
      }
      catch (Exception e)

 opendj-server-legacy/src/main/java/org/opends/server/tools/ImportLDIF.java

@@ -35,7 +35,6 @@
import org.forgerock.i18n.LocalizableMessage;
import org.forgerock.i18n.slf4j.LocalizedLogger;
import org.forgerock.opendj.ldap.DN;
import org.forgerock.opendj.ldap.ResultCode;
import org.forgerock.opendj.ldap.schema.AttributeType;
import org.forgerock.opendj.server.config.server.BackendCfg;
import org.opends.server.api.Backend;
@@ -134,7 +133,6 @@
  private StringArgument  rejectFile;
  private StringArgument  skipFile;
  private StringArgument  templateFile;
  private BooleanArgument skipDNValidation;
  private IntegerArgument threadCount;
  private StringArgument  tmpDirectory;

@@ -348,10 +346,6 @@
                      .shortIdentifier('S')
                      .description(INFO_LDIFIMPORT_DESCRIPTION_SKIP_SCHEMA_VALIDATION.get())
                      .buildAndAddToParser(argParser);
      skipDNValidation =
              BooleanArgument.builder("skipDNValidation")
                      .description(INFO_LDIFIMPORT_DESCRIPTION_DN_VALIDATION.get())
                      .buildAndAddToParser(argParser);
      threadCount =
              IntegerArgument.builder("threadCount")
                      .description(INFO_LDIFIMPORT_DESCRIPTION_THREAD_COUNT.get())
@@ -413,7 +407,6 @@
    addAttribute2(attributes, ATTR_IMPORT_OVERWRITE, overwrite);
    addAttribute2(attributes, ATTR_IMPORT_SKIP_SCHEMA_VALIDATION, skipSchemaValidation);
    addAttribute2(attributes, ATTR_IMPORT_TMP_DIRECTORY, tmpDirectory);
    addAttribute2(attributes, ATTR_IMPORT_SKIP_DN_VALIDATION, skipDNValidation);
    addAttribute2(attributes, ATTR_IMPORT_IS_COMPRESSED, isCompressed);
    addAttribute2(attributes, ATTR_IMPORT_IS_ENCRYPTED, isEncrypted);
    addAttribute2(attributes, ATTR_IMPORT_CLEAR_BACKEND, clearBackend);
@@ -771,7 +764,6 @@
    importConfig.setIncludeBranches(includeBranches);
    importConfig.setIncludeFilters(includeFilters);
    importConfig.setValidateSchema(!skipSchemaValidation.isPresent());
    importConfig.setSkipDNValidation(skipDNValidation.isPresent());
    importConfig.setTmpDirectory(tmpDirectory.getValue());

    try
@@ -872,14 +864,7 @@
    catch (DirectoryException de)
    {
      LocalizableMessage msg;
      if (de.getResultCode() == ResultCode.CONSTRAINT_VIOLATION)
      {
        msg = ERR_LDIFIMPORT_ERROR_CONSTRAINT_VIOLATION.get();
      }
      else
      {
        msg = de.getMessageObject();
      }
      logger.error(ERR_LDIFIMPORT_ERROR_DURING_IMPORT.get(msg));
      retCode = 1;
    }

 opendj-server-legacy/src/main/java/org/opends/server/types/LDIFImportConfig.java

@@ -125,7 +125,6 @@
  private boolean excludeAllOpAttrs;

  private String tmpDirectory;
  private boolean skipDNValidation;
  private int threadCount;

  /**
@@ -1038,26 +1037,6 @@
  }

  /**
   * Set the dn check in phase two boolean to the specified value.
   *
   * @param v The value to set the dn check in phase two boolean to.
   */
  public void setSkipDNValidation(boolean v)
  {
    skipDNValidation = v;
  }

  /**
   * Return the dn check in phase two boolean.
   *
   * @return Return the dn check in phase two boolean value.
   */
  public boolean getSkipDNValidation()
  {
    return skipDNValidation;
  }

  /**
   * Set the thread count.
   *
   * @param c The thread count value.

 opendj-server-legacy/src/messages/org/opends/messages/admin_tool.properties

@@ -1689,9 +1689,6 @@
INFO_CTRL_PANEL_SCHEMA_VALIDATION_LABEL=Schema Validation:
INFO_CTRL_PANEL_REJECT_NOT_SCHEMA_COMPLIANT_LABEL=Reject Entries That are Not \
 Schema-Compliant
INFO_CTRL_PANEL_DN_VALIDATION_LABEL=DN Validation:
INFO_CTRL_PANEL_DO_DN_VALIDATION_LATER_LABEL=Perform DN Validation during \
 Later Part of Import
INFO_CTRL_PANEL_IMPORT_THREADS_LABEL=Thread Number:
INFO_CTRL_PANEL_IMPORT_THREADS_TOOLTIP=Number of threads used to read LDIF \
 file during import.

 opendj-server-legacy/src/messages/org/opends/messages/admin_tool_de.properties

@@ -1034,8 +1034,6 @@
INFO_CTRL_PANEL_FILE_TO_IMPORT_LABEL=Datei f\u00fcr Import:
INFO_CTRL_PANEL_SCHEMA_VALIDATION_LABEL=Schemavalidierung
INFO_CTRL_PANEL_REJECT_NOT_SCHEMA_COMPLIANT_LABEL=Eintr\u00e4ge ablehnen, die nicht mit dem Schema kompatibel sind
INFO_CTRL_PANEL_DN_VALIDATION_LABEL=DN-Validierung:
INFO_CTRL_PANEL_DO_DN_VALIDATION_LATER_LABEL=DN-Validierung sp\u00e4ter im Verlauf des Imports durchf\u00fchren
INFO_CTRL_PANEL_IMPORT_THREADS_LABEL=Anzahl der Threads:
INFO_CTRL_PANEL_IMPORT_THREADS_TOOLTIP=Anzahl der Threads, die zum Lesen der LDIF-Datei w\u00e4hrend des Imports verwendet werden.
INFO_CTRL_PANEL_IMPORT_THREADS_HELP=Wenn keine Angabe gemacht wird, ist die Anzahl der Threads das Doppelte von der Anzahl der CPUs auf dem Serversystem.

 opendj-server-legacy/src/messages/org/opends/messages/admin_tool_es.properties

@@ -1034,8 +1034,6 @@
INFO_CTRL_PANEL_FILE_TO_IMPORT_LABEL=Archivo para importar:
INFO_CTRL_PANEL_SCHEMA_VALIDATION_LABEL=Validaci\u00f3n de esquema:
INFO_CTRL_PANEL_REJECT_NOT_SCHEMA_COMPLIANT_LABEL=Rechazar las entradas no compatibles con el esquema
INFO_CTRL_PANEL_DN_VALIDATION_LABEL=Validaci\u00f3n de ND:
INFO_CTRL_PANEL_DO_DN_VALIDATION_LATER_LABEL=Realizar la validaci\u00f3n de ND durante la parte final de la importaci\u00f3n
INFO_CTRL_PANEL_IMPORT_THREADS_LABEL=N\u00famero de subprocesos:
INFO_CTRL_PANEL_IMPORT_THREADS_TOOLTIP=N\u00famero de subprocesos utilizados para leer el archivo LDIF durante la importaci\u00f3n.
INFO_CTRL_PANEL_IMPORT_THREADS_HELP=Si se deja vac\u00edo, el n\u00famero de subprocesos equivaldr\u00e1 al doble del n\u00famero de unidades CPU en la m\u00e1quina servidor.

 opendj-server-legacy/src/messages/org/opends/messages/admin_tool_fr.properties

@@ -1034,8 +1034,6 @@
INFO_CTRL_PANEL_FILE_TO_IMPORT_LABEL=Fichier \u00e0 importer\u00a0:
INFO_CTRL_PANEL_SCHEMA_VALIDATION_LABEL=Validation de sch\u00e9ma\u00a0:
INFO_CTRL_PANEL_REJECT_NOT_SCHEMA_COMPLIANT_LABEL=Refuser les entr\u00e9es qui ne sont pas conformes au sch\u00e9ma
INFO_CTRL_PANEL_DN_VALIDATION_LABEL=Validation de DN\u00a0:
INFO_CTRL_PANEL_DO_DN_VALIDATION_LATER_LABEL=Ex\u00e9cuter la validation de DN \u00e0 une \u00e9tape ult\u00e9rieure de l'importation
INFO_CTRL_PANEL_IMPORT_THREADS_LABEL=Num\u00e9ro de thread\u00a0:
INFO_CTRL_PANEL_IMPORT_THREADS_TOOLTIP=Nombre de threads utilis\u00e9s pour lire le fichier LDIF durant l'importation.
INFO_CTRL_PANEL_IMPORT_THREADS_HELP=Si vous laissez ce champ vide, le nombre de threads utilis\u00e9 est \u00e9gal \u00e0 2 x le nombre de CPU de la machine serveur.

 opendj-server-legacy/src/messages/org/opends/messages/admin_tool_zh_CN.properties

@@ -1034,8 +1034,6 @@
INFO_CTRL_PANEL_FILE_TO_IMPORT_LABEL=\u8981\u5bfc\u5165\u7684\u6587\u4ef6:
INFO_CTRL_PANEL_SCHEMA_VALIDATION_LABEL=\u6a21\u5f0f\u9a8c\u8bc1:
INFO_CTRL_PANEL_REJECT_NOT_SCHEMA_COMPLIANT_LABEL=\u62d2\u7edd\u4e0d\u7b26\u5408\u6a21\u5f0f\u7684\u6761\u76ee
INFO_CTRL_PANEL_DN_VALIDATION_LABEL=DN \u9a8c\u8bc1:
INFO_CTRL_PANEL_DO_DN_VALIDATION_LATER_LABEL=\u5728\u5bfc\u5165\u671f\u95f4\u7684\u540e\u671f\u6267\u884c DN \u9a8c\u8bc1
INFO_CTRL_PANEL_IMPORT_THREADS_LABEL=\u7ebf\u7a0b\u6570:
INFO_CTRL_PANEL_IMPORT_THREADS_TOOLTIP=\u5728\u5bfc\u5165\u671f\u95f4\u7528\u4e8e\u8bfb\u53d6 LDIF \u6587\u4ef6\u7684\u7ebf\u7a0b\u6570\u3002
INFO_CTRL_PANEL_IMPORT_THREADS_HELP=\u5982\u679c\u7559\u7a7a\uff0c\u5219\u7ebf\u7a0b\u6570\u7b49\u4e8e 2 x \u670d\u52a1\u5668\u673a\u5668\u4e0a\u7684 CPU \u6570\u91cf\u3002

 opendj-server-legacy/src/messages/org/opends/messages/tool.properties

@@ -179,10 +179,6 @@
 trying to open the rejects file %s for writing:  %s
ERR_LDIFIMPORT_ERROR_DURING_IMPORT_96=An error occurred while \
 attempting to process the LDIF import:  %s
ERR_LDIFIMPORT_ERROR_CONSTRAINT_VIOLATION_97=One or more DN \
indexes could not be built due to invalid DNs or missing parent entries. \
Please re-import the data without the --skipDNValidation option in order \
to determine the exact cause
INFO_PROCESSING_OPERATION_104=Processing %s request for %s
INFO_OPERATION_FAILED_105=%s operation failed
INFO_OPERATION_SUCCESSFUL_106=%s operation successful for DN %s
@@ -2231,8 +2227,6 @@
INFO_LDIFIMPORT_DESCRIPTION_TEMP_DIRECTORY_1683=Path to temporary directory \
for index scratch files during LDIF import
INFO_LDIFIMPORT_TEMP_DIR_PLACEHOLDER_1684={directory}
INFO_LDIFIMPORT_DESCRIPTION_DN_VALIDATION_1685=Perform DN validation \
 during later part of LDIF import
INFO_LDIFIMPORT_DESCRIPTION_THREAD_COUNT_1686=Number of threads used to \
 read LDIF file during import. Default value (0) equals: 2 x (number of CPUs)
INFO_LDIFIMPORT_THREAD_COUNT_PLACEHOLDER_1687={count}

 opendj-server-legacy/src/messages/org/opends/messages/tool_de.properties

@@ -1145,7 +1145,6 @@
INFO_INSTALLDS_DO_NOT_ENABLE_WINDOWS_SERVICE_1682=Den Server nicht als Windows-Dienst aktivieren
INFO_LDIFIMPORT_DESCRIPTION_TEMP_DIRECTORY_1683=Pfad zu einem tempor\u00e4ren Verzeichnis f\u00fcr tempor\u00e4re Index-Dateien w\u00e4hrend des LDIF-Imports
INFO_LDIFIMPORT_TEMP_DIR_PLACEHOLDER_1684={directory}
INFO_LDIFIMPORT_DESCRIPTION_DN_VALIDATION_1685=DN-Validierung sp\u00e4ter im Verlauf des LDIF-Imports durchf\u00fchren
INFO_LDIFIMPORT_DESCRIPTION_THREAD_COUNT_1686=Anzahl der Threads, die zum Lesen der LDIF-Datei w\u00e4hrend des Imports verwendet werden. Der Standardwert (0) entspricht: 2 x (Zahl der CPUs)
INFO_LDIFIMPORT_THREAD_COUNT_PLACEHOLDER_1687={count}
ERR_LDIFIMPORT_CANNOT_PARSE_THREAD_COUNT_1688=Der Wert %s f\u00fcr threadCount kann nicht analysiert werden: %s

 opendj-server-legacy/src/messages/org/opends/messages/tool_es.properties

@@ -1145,7 +1145,6 @@
INFO_INSTALLDS_DO_NOT_ENABLE_WINDOWS_SERVICE_1682=No habilite el servidor para ejecutarlo como un servicio de Windows
INFO_LDIFIMPORT_DESCRIPTION_TEMP_DIRECTORY_1683=Ruta a un directorio temporal para archivos temporales de \u00edndice durante la importaci\u00f3n de LDIF
INFO_LDIFIMPORT_TEMP_DIR_PLACEHOLDER_1684={directory}
INFO_LDIFIMPORT_DESCRIPTION_DN_VALIDATION_1685=Realizar la validaci\u00f3n de ND durante la parte final de la importaci\u00f3n de LDIF
INFO_LDIFIMPORT_DESCRIPTION_THREAD_COUNT_1686=N\u00famero de subprocesos utilizados para leer el archivo LDIF durante la importaci\u00f3n. El valor predeterminado (0) es igual a: 2 x (n\u00famero de CPU)
INFO_LDIFIMPORT_THREAD_COUNT_PLACEHOLDER_1687={count}
ERR_LDIFIMPORT_CANNOT_PARSE_THREAD_COUNT_1688=No puede analizarse el valor %s para threadCount: %s

 opendj-server-legacy/src/messages/org/opends/messages/tool_fr.properties

@@ -1145,7 +1145,6 @@
INFO_INSTALLDS_DO_NOT_ENABLE_WINDOWS_SERVICE_1682=Ne pas activer le serveur pour qu'il s'ex\u00e9cute en tant que service Windows
INFO_LDIFIMPORT_DESCRIPTION_TEMP_DIRECTORY_1683=Chemin au r\u00e9pertoire temporaire des fichiers d'index durant l'importation LDIF
INFO_LDIFIMPORT_TEMP_DIR_PLACEHOLDER_1684={r\u00e9pertoire}
INFO_LDIFIMPORT_DESCRIPTION_DN_VALIDATION_1685=Ex\u00e9cuter la validation DN \u00e0 une \u00e9tape ult\u00e9rieure de l'importation LDIF
INFO_LDIFIMPORT_DESCRIPTION_THREAD_COUNT_1686=Nombre de threads utilis\u00e9s pour lire le fichier LDIF durant l'importation. La valeur par d\u00e9faut (0) est \u00e9gale \u00e0 2 x (nombre de CPU)
INFO_LDIFIMPORT_THREAD_COUNT_PLACEHOLDER_1687={Nombre}
ERR_LDIFIMPORT_CANNOT_PARSE_THREAD_COUNT_1688=Impossible d'analyser la valeur %s pour threadCount\u00a0: %s

 opendj-server-legacy/src/messages/org/opends/messages/tool_zh_CN.properties

@@ -1145,7 +1145,6 @@
INFO_INSTALLDS_DO_NOT_ENABLE_WINDOWS_SERVICE_1682=\u4e0d\u542f\u7528\u670d\u52a1\u5668\u4ee5\u4f5c\u4e3a Windows \u670d\u52a1\u8fd0\u884c
INFO_LDIFIMPORT_DESCRIPTION_TEMP_DIRECTORY_1683=\u5728 LDIF \u5bfc\u5165\u671f\u95f4\u6307\u5411\u7d22\u5f15\u4e34\u65f6\u6587\u4ef6\u7684\u4e34\u65f6\u76ee\u5f55\u7684\u8def\u5f84
INFO_LDIFIMPORT_TEMP_DIR_PLACEHOLDER_1684={directory}
INFO_LDIFIMPORT_DESCRIPTION_DN_VALIDATION_1685=\u5728 LDIF \u5bfc\u5165\u671f\u95f4\u7684\u540e\u671f\u6267\u884c DN \u9a8c\u8bc1
INFO_LDIFIMPORT_DESCRIPTION_THREAD_COUNT_1686=\u5728\u5bfc\u5165\u671f\u95f4\u7528\u4e8e\u8bfb\u53d6 LDIF \u6587\u4ef6\u7684\u7ebf\u7a0b\u6570\u3002\u9ed8\u8ba4\u503c (0) \u7b49\u4e8e\uff1a2 x\uff08CPU \u6570\uff09
INFO_LDIFIMPORT_THREAD_COUNT_PLACEHOLDER_1687={count}
ERR_LDIFIMPORT_CANNOT_PARSE_THREAD_COUNT_1688=\u65e0\u6cd5\u89e3\u6790 threadCount \u7684\u503c %s: %s

 opendj-server-legacy/src/test/java/org/opends/server/backends/pluggable/PluggableBackendImplTestCase.java

@@ -953,7 +953,6 @@
      importConf.setClearBackend(true);
      importConf.writeRejectedEntries(rejectedEntries);
      importConf.setIncludeBranches(Collections.singleton(testBaseDN));
      importConf.setSkipDNValidation(true);
      importConf.setThreadCount(0);
      backend.importLDIF(importConf, DirectoryServer.getInstance().getServerContext());
    }