Fix for issue 4148 (setup fails : The import LDIF tool returned error code ...

jvergara

28.34.2009 8f59b2f80eab5a18bc35028b29b9160195409e9c

Fix for issue 4148 (setup fails : The import LDIF tool returned error code 1)
Since the import-ldif utility does not support any more (temporarily) the append option the setup (and the control-panel) were broken.

Temporarily the following modifications have been made in the control panel:
If the user decides to create data in the new base DN and there was already data in the backend, a confirmation dialog will be displayed informing that all the data will be overwritten.
The appending options in the Import LDIF panel have been hidden.

Concerning the setup, the modifications made are permanent.  What basically has been done is:
Use a single template LDIF file when the user decides to import automatically generated data.  Before this, one file per base DN was generated, which required the option 'append' to be used.  With a single file, no append is required.  This is a clearer solution, since I think that appending data affects to performance.
I have found some memory issues with the default ergonomics of the new import (see issue 4151).  Since some ergonomics issues may occur, the import is now launched in a separate process.  If we manage to improve the code that determines the default java arguments to be used for the import, this will help avoiding problems with the import.

 opends/src/guitools/org/opends/guitools/controlpanel/ui/ImportLDIFPanel.java

@@ -288,6 +288,12 @@
    gbc.gridy ++;
    add(replaceEntries, gbc);

//  TODO: change this once the append is supported by the import again.
    lImportType.setVisible(false);
    overwrite.setVisible(false);
    append.setVisible(false);
    replaceEntries.setVisible(false);

    gbc.gridx = 0;
    gbc.gridy ++;
    gbc.insets.left = 0;
@@ -709,7 +715,8 @@
      }
      if (append.isSelected())
      {
        args.add("--append");
        // TODO: change this once the append is supported again by the import.
        //args.add("--append");
        if (replaceEntries.isSelected())
        {
          args.add("--replaceExisting");

 opends/src/guitools/org/opends/guitools/controlpanel/ui/NewBaseDNPanel.java

@@ -625,6 +625,11 @@
      }
      if (errors.isEmpty())
      {
        // TODO: delete this check
        if (!continueToOverwriteBackend())
        {
          return;
        }
        launchOperation(newTask,
            INFO_CTRL_PANEL_CREATING_BASE_DN_SUMMARY.get(dn),
            INFO_CTRL_PANEL_CREATING_BASE_DN_COMPLETE.get(),
@@ -645,6 +650,38 @@
    }
  }

  /**
   * TODO: once the append is supported in the import-ldif, delete this method.
   * @return <CODE>true</CODE> if the user accepted to overwrite the backend
   * contents and <CODE>false</CODE> otherwise.
   */
  private boolean continueToOverwriteBackend()
  {
    boolean userConfirmed = true;
    if (!isNewBackend() && isServerRunning())
    {
      String backendName = getBackendName();
      for (BackendDescriptor bck :
        getInfo().getServerDescriptor().getBackends())
      {
        if (bck.getBackendID().equalsIgnoreCase(backendName))
        {
          if (bck.getEntries() > 0)
          {
            // Ask confirmation: append is not supported and we are going to
            // overwrite the contents of the backend.
            userConfirmed = displayConfirmationDialog(
                INFO_CTRL_PANEL_CONFIRMATION_REQUIRED_SUMMARY.get(),
                INFO_CTRL_PANEL_CONFIRMATION_IMPORT_LDIF_DETAILS.get(
                    backendName));
          }
          break;
        }
      }
    }
    return userConfirmed;
  }

  private String getBackendName()
  {
    Object backendName = backends.getSelectedItem();
@@ -829,10 +866,12 @@
          }
          args.add("--backendID");
          args.add(getBackendName());
          args.add("--append");
          // TODO: uncomment this line once import supports append again
          // args.add("--append");
        }
        else
        {
          // If we are not local, we use ldapmodify to update the contents.
          args.add("-a");
          args.add("-f");
          args.add(ldifFile);

 opends/src/quicksetup/org/opends/quicksetup/Installation.java

@@ -289,6 +289,16 @@
  public static final String WINDOWS_STATUSCLI_FILE_NAME = "status.bat";

  /**
   * The UNIX import LDIF script file name.
   */
  public static final String UNIX_IMPORT_LDIF = "import-ldif";

  /**
   * The Windows import LDIF batch file name.
   */
  public static final String WINDOWS_IMPORT_LDIF = "import-ldif.bat";

  /**
   * Name of the file kept in the history directory containing logs
   * of upgrade and reversions.
   */

 opends/src/quicksetup/org/opends/quicksetup/installer/Installer.java

@@ -789,16 +789,14 @@
   * @return a list of file objects pointing to the create template files.
   * @throws ApplicationException if an error occurs.
   */
  private LinkedList<File> createTemplateFiles() throws ApplicationException {
    LinkedList<File> files = new LinkedList<File>();
  private File createTemplateFile() throws ApplicationException {
    File file;
    try
    {
      Set<String> baseDNs = new LinkedHashSet<String>(
          getUserData().getNewSuffixOptions().getBaseDns());
      int nEntries = getUserData().getNewSuffixOptions().getNumberEntries();

      for (String baseDn : getUserData().getNewSuffixOptions().getBaseDns())
      {
        files.add(SetupUtils.createTemplateFile(baseDn, nEntries));
      }
      file = SetupUtils.createTemplateFile(baseDNs, nEntries);
    }
    catch (IOException ioe)
    {
@@ -808,7 +806,7 @@
          ReturnCode.FILE_SYSTEM_ACCESS_ERROR,
          failedMsg, ioe);
    }
    return files;
    return file;
  }

  /**
@@ -1155,11 +1153,6 @@
    checkAbort();

    ArrayList<String> argList = new ArrayList<String>();
    argList.add("-C");
    argList.add(getConfigurationClassName());

    argList.add("-f");
    argList.add(getConfigurationFile());

    argList.add("-n");
    argList.add(getBackendName());
@@ -1187,18 +1180,18 @@
      {
        try
        {
          int result = helper.invokeImportLDIF(args);
          int result = helper.invokeImportLDIF(Installer.this, args);

          if (result != 0)
          {
            ae = new ApplicationException(
                ReturnCode.CONFIGURATION_ERROR,
                ReturnCode.IMPORT_ERROR,
                INFO_ERROR_CREATING_BASE_ENTRY.get(), null);
          }
        } catch (Throwable t)
        {
          ae = new ApplicationException(
              ReturnCode.CONFIGURATION_ERROR,
              ReturnCode.IMPORT_ERROR,
              getThrowableMsg(INFO_ERROR_CREATING_BASE_ENTRY.get(), t), t);
        }
        finally
@@ -1266,11 +1259,6 @@
    }

    ArrayList<String> argList = new ArrayList<String>();
    argList.add("-C");
    argList.add(getConfigurationClassName());

    argList.add("-f");
    argList.add(getConfigurationFile());
    argList.add("-n");
    argList.add(getBackendName());
    for (String ldifPath : ldifPaths)
@@ -1301,26 +1289,26 @@
        try
        {
          InstallerHelper helper = new InstallerHelper();
          int result = helper.invokeImportLDIF(args);
          int result = helper.invokeImportLDIF(Installer.this, args);

          if (result != 0)
          {
            ae = new ApplicationException(
                ReturnCode.CONFIGURATION_ERROR,
                ReturnCode.IMPORT_ERROR,
                INFO_ERROR_IMPORTING_LDIF.get(), null);
          }
        } catch (Throwable t)
        {
          ae = new ApplicationException(
              ReturnCode.CONFIGURATION_ERROR,
              ReturnCode.IMPORT_ERROR,
              getThrowableMsg(INFO_ERROR_IMPORTING_LDIF.get(), t), t);
        }
        finally
        {
          if (!isVerbose())
          {
            pointAdder.stop();
            setNotifyListeners(true);
            pointAdder.stop();
          }
        }
        isOver = true;
@@ -1367,7 +1355,7 @@
   * @throws ApplicationException if something goes wrong.
   */
  private void importAutomaticallyGenerated() throws ApplicationException {
    LinkedList<File> templatePaths = createTemplateFiles();
    File templatePath = createTemplateFile();
    int nEntries = getUserData().getNewSuffixOptions().getNumberEntries();
    MessageBuilder mb = new MessageBuilder();
    if (isVerbose())
@@ -1390,18 +1378,12 @@
    {
      pointAdder.start();
    }
    for (File templatePath : templatePaths)
    {

      if (!isVerbose())
      {
        setNotifyListeners(false);
      }
      final ArrayList<String> argList = new ArrayList<String>();
      argList.add("-C");
      argList.add(getConfigurationClassName());

      argList.add("-f");
      argList.add(getConfigurationFile());
      argList.add("-n");
      argList.add(getBackendName());
      argList.add("-A");
@@ -1409,8 +1391,7 @@
      argList.add("-s"); // seed
      argList.add("0");

      // append: each file contains data for each base DN.
      argList.add("-a");
    argList.add("-F");

      final String[] args = new String[argList.size()];
      argList.toArray(args);
@@ -1422,19 +1403,19 @@
          try
          {
            InstallerHelper helper = new InstallerHelper();
            int result = helper.invokeImportLDIF(args);
          int result = helper.invokeImportLDIF(Installer.this, args);

            if (result != 0)
            {
              ae = new ApplicationException(
                  ReturnCode.CONFIGURATION_ERROR,
                ReturnCode.IMPORT_ERROR,
                  INFO_ERROR_IMPORT_LDIF_TOOL_RETURN_CODE.get(
                      Integer.toString(result)), null);
            }
          } catch (Throwable t)
          {
            ae = new ApplicationException(
                ReturnCode.CONFIGURATION_ERROR,
              ReturnCode.IMPORT_ERROR,
                getThrowableMsg(INFO_ERROR_IMPORT_AUTOMATICALLY_GENERATED.get(
                    listToString(argList, " "), t.getLocalizedMessage()), t),
                    t);
@@ -1464,7 +1445,6 @@
        notifyListeners(getFormattedDoneWithLineBreak());
      }
    }
  }

  /**
   * This method undoes the modifications made in other servers in terms of

 opends/src/quicksetup/org/opends/quicksetup/installer/InstallerHelper.java

@@ -27,11 +27,13 @@

package org.opends.quicksetup.installer;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Map;
@@ -44,12 +46,15 @@

import javax.naming.ldap.InitialLdapContext;

import org.opends.quicksetup.Application;
import org.opends.quicksetup.ApplicationException;
import org.opends.quicksetup.Installation;
import org.opends.quicksetup.ReturnCode;
import org.opends.quicksetup.util.OutputReader;
import org.opends.quicksetup.util.Utils;

import static org.opends.quicksetup.util.Utils.*;

import org.opends.server.admin.DefaultBehaviorException;
import org.opends.server.admin.ManagedObjectNotFoundException;
import org.opends.server.admin.client.ManagementContext;
@@ -63,9 +68,9 @@
import org.opends.messages.ReplicationMessages;
import org.opends.messages.Message;
import static org.opends.messages.QuickSetupMessages.*;

import org.opends.server.tools.ConfigureDS;
import org.opends.server.tools.ConfigureWindowsService;
import org.opends.server.tools.ImportLDIF;
import org.opends.server.tools.JavaPropertiesTool;
import org.opends.server.types.DN;
import org.opends.server.types.DirectoryException;
@@ -109,14 +114,99 @@
  }

  /**
   * Invokes the method ImportLDIF.mainImportLDIF with the provided parameters.
   * @param args the arguments to be passed to ImportLDIF.mainImportLDIF.
   * @return the return code of the ImportLDIF.mainImportLDIF method.
   * @throws org.opends.quicksetup.ApplicationException if something goes wrong.
   * @see org.opends.server.tools.ImportLDIF#mainImportLDIF(String[]).
   * Invokes the import-ldif command-line with the provided parameters.
   * @param application the application that is launching this.
   * @param args the arguments to be passed to import-ldif.
   * @return the return code of the import-ldif call.
   * @throws IOException if the process could not be launched.
   * @throws InterruptedException if the process was interrupted.
   */
  public int invokeImportLDIF(String[] args) throws ApplicationException {
    return ImportLDIF.mainImportLDIF(args);
  public int invokeImportLDIF(final Application application, String[] args)
  throws IOException, InterruptedException
  {
    File installPath = new File(application.getInstallationPath());
    ArrayList<String> argList = new ArrayList<String>();
    File binPath;
    if (Utils.isWindows())
    {
      binPath =
        new File(installPath, Installation.WINDOWS_BINARIES_PATH_RELATIVE);
    } else
    {
      binPath =
        new File(installPath, Installation.UNIX_BINARIES_PATH_RELATIVE);
    }
    File importPath;
    if (Utils.isWindows())
    {
      importPath = new File(binPath, Installation.WINDOWS_IMPORT_LDIF);
    } else
    {
      importPath = new File(binPath, Installation.UNIX_IMPORT_LDIF);
    }
    argList.add(Utils.getScriptPath(importPath.getAbsolutePath()));
    for (String arg : args)
    {
      argList.add(arg);
    }
    String[] allArgs = new String[argList.size()];
    argList.toArray(allArgs);
    LOG.log(Level.INFO, "import-ldif arg list: "+argList);
    ProcessBuilder pb = new ProcessBuilder(allArgs);
    Map<String, String> env = pb.environment();
    env.remove(SetupUtils.OPENDS_JAVA_HOME);
    env.remove(SetupUtils.OPENDS_JAVA_ARGS);
    pb.directory(installPath);
    Process process = null;
    try
    {
      process = pb.start();
      final BufferedReader err =
        new BufferedReader(new InputStreamReader(process.getErrorStream()));
      new OutputReader(err)
      {
        public void processLine(String line)
        {
          LOG.log(Level.WARNING, "import-ldif error log: "+line);
          application.notifyListeners(Message.raw(line));
          application.notifyListeners(application.getLineBreak());
        }
      };
      BufferedReader out =
        new BufferedReader(new InputStreamReader(process.getInputStream()));
      new OutputReader(out)
      {
        public void processLine(String line)
        {
          LOG.log(Level.INFO, "import-ldif out log: "+line);
          application.notifyListeners(Message.raw(line));
          application.notifyListeners(application.getLineBreak());
        }
      };
      return process.waitFor();
    }
    finally
    {
      if (process != null)
      {
        try
        {
          process.getErrorStream().close();
        }
        catch (Throwable t)
        {
          LOG.log(Level.WARNING, "Error closing error stream: "+t, t);
        }
        try
        {
          process.getOutputStream().close();
        }
        catch (Throwable t)
        {
          LOG.log(Level.WARNING, "Error closing output stream: "+t, t);
        }
      }
    }
  }

  /**

 opends/src/server/org/opends/server/util/SetupUtils.java

@@ -22,7 +22,7 @@
 * CDDL HEADER END
 *
 *
 *      Copyright 2006-2008 Sun Microsystems, Inc.
 *      Copyright 2006-2009 Sun Microsystems, Inc.
 */
package org.opends.server.util;

@@ -39,7 +39,9 @@
import java.security.KeyStoreException;
import java.security.cert.Certificate;
import java.security.cert.CertificateEncodingException;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.Set;

import java.util.Random;
import org.opends.server.types.OperatingSystem;
@@ -146,21 +148,48 @@
  public static File createTemplateFile(String baseDN, int numEntries)
         throws IOException
  {
    Set<String> baseDNs = new HashSet<String>(1);
    baseDNs.add(baseDN);
    return createTemplateFile(baseDNs, numEntries);
  }

  /**
   * Creates a MakeLDIF template file using the provided information.
   *
   * @param  baseDNs     The base DNs for the data in the template file.
   * @param  numEntries  The number of user entries the template file should
   *                     create.
   *
   * @return  The {@code File} object that references the created template file.
   *
   * @throws  IOException  If a problem occurs while writing the template file.
   */
  public static File createTemplateFile(Set<String> baseDNs,
      int numEntries)
         throws IOException
  {
    File templateFile = File.createTempFile("opends-install", ".template");
    templateFile.deleteOnExit();

    LinkedList<String> lines = new LinkedList<String>();
    lines.add("define suffix=" + baseDN);

    int i = 0;
    for (String baseDN : baseDNs)
    {
      i++;
      lines.add("define suffix"+i+"=" + baseDN);
    }
    if (numEntries > 0)
    {
      lines.add("define numusers=" + numEntries);
    }

    for (i=0; i<baseDNs.size(); i++)
    {
      i++;
    lines.add("");
    lines.add("branch: [suffix]");
      lines.add("branch: [suffix"+i+"]");
    lines.add("");
    lines.add("branch: ou=People,[suffix]");
      lines.add("branch: ou=People,[suffix"+i+"]");

    if (numEntries > 0)
    {
@@ -192,6 +221,7 @@
      lines.add("postalAddress: {cn}${street}${l}, {st}  {postalCode}");
      lines.add("description: This is the description for {cn}.");
    }
    }

    BufferedWriter writer = new BufferedWriter(new FileWriter(templateFile));
    for (String line : lines)