OPENDJ-1075 Port server make-ldif tool to the SDK CR-2603

Nicolas Capponi

19.50.2013 13da562e3e11baa35131967d3b1a1dd12a6996a5

OPENDJ-1075 Port server make-ldif tool to the SDK
CR-2603

CLI for MakeLDIF 

* Class MakeLDIF.java
  - uses the new EntryGenerator class from opendj-core to generate entries
  - uses LDIFEntryWriter class from opendj-core to write entries to LDIF
* Differences from opendj server version :
 - remove configfile and configclass options
 - add "--constant" ("-c") option to provide constants from the command line
   those constants override pre-defined constants in template file
 - if no "--ldiffile" option is provided, output LDIF to the standard output
* Add MakeLDIFTestCase to test basic behavior of the tool

 opendj3/opendj-ldap-toolkit/src/main/java/com/forgerock/opendj/ldap/tools/ConsoleApplication.java

@@ -49,13 +49,13 @@
 * a console-based application.
 */
abstract class ConsoleApplication {
    private final PrintStream err = new PrintStream(System.err);
    private final PrintStream err;

    private final BufferedReader reader = new BufferedReader(new InputStreamReader(System.in));

    private final InputStream in = System.in;

    private final PrintStream out = new PrintStream(System.out);
    private final PrintStream out;

    private final Console console = System.console();

@@ -63,7 +63,16 @@
     * Creates a new console application instance.
     */
    ConsoleApplication() {
        // Nothing to do.
        this(System.out, System.err);
    }

    /**
     * Creates a new console application instance with
     * provided standard and error out streams.
     */
    ConsoleApplication(PrintStream out, PrintStream err) {
        this.out = out;
        this.err = err;
    }

    /**

 opendj3/opendj-ldap-toolkit/src/main/java/com/forgerock/opendj/ldap/tools/MakeLDIF.java

New file
@@ -0,0 +1,256 @@
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License, Version 1.0 only
 * (the "License").  You may not use this file except in compliance
 * with the License.
 *
 * You can obtain a copy of the license at legal-notices/CDDLv1_0.txt
 * or http://forgerock.org/license/CDDLv1.0.html.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at legal-notices/CDDLv1_0.txt.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information:
 *      Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 *
 *
 *      Copyright 2006-2010 Sun Microsystems, Inc.
 *      Portions Copyright 2013 ForgeRock AS
 */
package com.forgerock.opendj.ldap.tools;

import static com.forgerock.opendj.ldap.tools.ToolConstants.*;
import static com.forgerock.opendj.ldap.tools.ToolsMessages.*;
import static com.forgerock.opendj.ldap.tools.Utils.*;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintStream;
import org.forgerock.i18n.LocalizableMessage;
import org.forgerock.opendj.ldap.Entry;
import org.forgerock.opendj.ldif.EntryGenerator;
import org.forgerock.opendj.ldif.LDIFEntryWriter;

/**
 * Program that generate LDIF content based on a template.
 */
public final class MakeLDIF extends ConsoleApplication {

    private static final int EXIT_CODE_SUCCESS = 0;
    private static final int EXIT_CODE_FAILURE = 1;

    /** The total number of entries that have been written. */
    private long numberOfEntriesWritten;

    /**
     * Main method for MakeLDIF tool.
     *
     * @param args
     *            The command-line arguments provided to this program.
     */
    public static void main(final String[] args) {
        final int retCode = new MakeLDIF().run(args);
        System.exit(filterExitCode(retCode));
    }

    /** Run Make LDIF with provided command-line arguments. */
    int run(final String[] args) {
        final LocalizableMessage toolDescription = INFO_MAKELDIF_TOOL_DESCRIPTION.get();
        final ArgumentParser argParser = new ArgumentParser(MakeLDIF.class.getName(), toolDescription,
                false, true, 1, 1, "template-file-path");

        BooleanArgument showUsage;
        IntegerArgument randomSeed;
        StringArgument ldifFile;
        StringArgument resourcePath;
        StringArgument constants;
        try {
            resourcePath = new StringArgument("resourcepath", 'r', "resourcePath", false, false, true,
                    INFO_PATH_PLACEHOLDER.get(), null, null, INFO_MAKELDIF_DESCRIPTION_RESOURCE_PATH.get());
            resourcePath.setHidden(true);
            argParser.addArgument(resourcePath);

            ldifFile = new StringArgument("ldiffile", OPTION_SHORT_OUTPUT_LDIF_FILENAME,
                    OPTION_LONG_OUTPUT_LDIF_FILENAME, false, false, true, INFO_FILE_PLACEHOLDER.get(),
                    null, null, INFO_MAKELDIF_DESCRIPTION_LDIF.get());
            argParser.addArgument(ldifFile);

            randomSeed = new IntegerArgument("randomseed", OPTION_SHORT_RANDOM_SEED, OPTION_LONG_RANDOM_SEED, false,
                    false, true, INFO_SEED_PLACEHOLDER.get(), 0, null, INFO_MAKELDIF_DESCRIPTION_SEED.get());
            argParser.addArgument(randomSeed);

            constants = new StringArgument("constant", 'c', "constant", false, true, true, INFO_FILE_PLACEHOLDER.get(),
                    null, null, INFO_MAKELDIF_DESCRIPTION_LDIF.get());
            argParser.addArgument(constants);

            showUsage = new BooleanArgument("help", OPTION_SHORT_HELP, OPTION_LONG_HELP,
                    INFO_MAKELDIF_DESCRIPTION_HELP.get());
            argParser.addArgument(showUsage);
            argParser.setUsageArgument(showUsage, getOutputStream());
        } catch (ArgumentException ae) {
            println(ERR_CANNOT_INITIALIZE_ARGS.get(ae.getMessage()));
            return EXIT_CODE_FAILURE;
        }

        // Parse the command-line arguments provided to the program.
        try {
            argParser.parseArguments(args);
        } catch (ArgumentException ae) {
            println(ERR_ERROR_PARSING_ARGS.get(ae.getMessage()));
            println(argParser.getUsageMessage());
            return EXIT_CODE_FAILURE;
        }

        if (argParser.usageOrVersionDisplayed()) {
            return 0;
        }
        final String templatePath = argParser.getTrailingArguments().get(0);
        return run(templatePath, resourcePath, ldifFile, randomSeed, constants);
    }

    /** Run Make LDIF with provided arguments. */
    private int run(final String templatePath, final StringArgument resourcePath,
            final StringArgument ldifFile, final IntegerArgument randomSeedArg, final StringArgument constants) {
        EntryGenerator generator = null;
        LDIFEntryWriter writer = null;
        try {
            generator = createGenerator(templatePath, resourcePath, randomSeedArg, constants);
            if (generator == null) {
                return EXIT_CODE_FAILURE;
            }

            if (generator.hasWarning()) {
                for (LocalizableMessage warn : generator.getWarnings()) {
                    println(warn);
                }
            }

            if (ldifFile.isPresent()) {
                try {
                    writer = new LDIFEntryWriter(new BufferedWriter(new FileWriter(new File(ldifFile.getValue()))));
                } catch (IOException e) {
                    println(ERR_MAKELDIF_UNABLE_TO_CREATE_LDIF.get(ldifFile.getValue(), e.getMessage()));
                    return EXIT_CODE_FAILURE;
                }
            } else {
                writer = new LDIFEntryWriter(getOutputStream());
            }

            if (!generateEntries(generator, writer, ldifFile)) {
                return EXIT_CODE_FAILURE;
            }

            println(INFO_MAKELDIF_PROCESSING_COMPLETE.get(numberOfEntriesWritten));

        } finally {
            closeIfNotNull(generator, writer);
        }

        return EXIT_CODE_SUCCESS;
    }

    /**
     * Returns the initialized generator, or null if a problem occurs.
     */
    private EntryGenerator createGenerator(final String templatePath, final StringArgument resourcePath,
            final IntegerArgument randomSeedArg, final StringArgument constants) {
        final EntryGenerator generator = new EntryGenerator(templatePath);

        if (resourcePath.isPresent()) {
            final File resourceDir = new File(resourcePath.getValue());
            if (!resourceDir.exists()) {
                println(ERR_MAKELDIF_NO_SUCH_RESOURCE_DIRECTORY.get(resourcePath.getValue()));
                generator.close();
                return null;
            }
            generator.setResourcePath(resourcePath.getValue());
        }

        if (randomSeedArg.isPresent()) {
            try {
                generator.setRandomSeed(randomSeedArg.getIntValue());
            } catch (ArgumentException ae) {
                println(ERR_ERROR_PARSING_ARGS.get(ae.getMessage()));
                generator.close();
                return null;
            }
        }

        if (constants.isPresent()) {
            if (!addConstantsToGenerator(constants, generator)) {
                generator.close();
                return null;
            }
        }

        // Force initialization of generator
        try {
            generator.hasNext();
        } catch (IOException e) {
            println(ERR_MAKELDIF_EXCEPTION_DURING_PARSE.get(e.getMessage()));
            generator.close();
            return null;
        }

        return generator;
    }

    /**
     * Returns true if all constants are added to generator, false otherwise.
     */
    private boolean addConstantsToGenerator(StringArgument constants, EntryGenerator generator) {
        for (final String constant : constants.getValues()) {
            final String[] chunks = constant.split("=");
            if (chunks.length != 2) {
                println(ERR_CONSTANT_ARG_CANNOT_DECODE.get(constant));
                return false;
            }
            generator.setConstant(chunks[0], chunks[1]);
        }
        return true;
    }

    /**
     * Returns true if generation is successfull, false otherwise.
     */
    private boolean generateEntries(final EntryGenerator generator, final LDIFEntryWriter writer,
            final StringArgument ldifFile) {
        try {
            while (generator.hasNext()) {
                final Entry entry = generator.readEntry();
                try {
                    writer.writeEntry(entry);
                } catch (IOException e) {
                    println(ERR_MAKELDIF_ERROR_WRITING_LDIF.get(ldifFile.getValue(), e.getMessage()));
                    return false;
                }
                if ((++numberOfEntriesWritten % 1000) == 0) {
                    println(INFO_MAKELDIF_PROCESSED_N_ENTRIES.get(numberOfEntriesWritten));
                }
            }
        } catch (Exception e) {
            println(ERR_MAKELDIF_EXCEPTION_DURING_PROCESSING.get(e.getMessage()));
            return false;
        }
        return true;
    }

    private MakeLDIF() {
        // nothing to do
    }

    // To allow tests
    MakeLDIF(PrintStream out, PrintStream err) {
        super(out, err);
    }

}

 opendj3/opendj-ldap-toolkit/src/main/resources/com/forgerock/opendj/ldap/tools/tools.properties

@@ -117,7 +117,6 @@
INFO_TIME_IN_HOURS_MINUTES_SECONDS=%d hours, %d minutes, %d seconds
INFO_TIME_IN_DAYS_HOURS_MINUTES_SECONDS=%d days, %d hours, %d minutes, %d \
 seconds
INFO_ARGPARSER_USAGE=Usage:
INFO_SUBCMDPARSER_WHERE_OPTIONS_INCLUDE=Command options:
ERR_MENU_BAD_CHOICE_SINGLE=Invalid response. Please enter a valid \
menu option
@@ -472,3 +471,31 @@
 files and report the differences in LDIF format
INFO_LDIFSEARCH_TOOL_DESCRIPTION=This utility can be used to perform search \
 operations against entries contained in an LDIF file
 #
 # MakeLDIF tool
 #
INFO_MAKELDIF_TOOL_DESCRIPTION=This utility can be used to generate LDIF \
 data based on a definition in a template file
INFO_SEED_PLACEHOLDER={seed}
INFO_PATH_PLACEHOLDER={path}
INFO_MAKELDIF_DESCRIPTION_LDIF=The path to the LDIF file to be written
INFO_MAKELDIF_DESCRIPTION_SEED=The seed to use to initialize the random \
 number generator
INFO_MAKELDIF_DESCRIPTION_HELP=Show this usage information
INFO_MAKELDIF_DESCRIPTION_RESOURCE_PATH=Path to look for \
 MakeLDIF resources (e.g., data files)
ERR_MAKELDIF_NO_SUCH_RESOURCE_DIRECTORY=The specified resource \
 directory %s could not be found
INFO_MAKELDIF_PROCESSED_N_ENTRIES=Processed %d entries
INFO_MAKELDIF_PROCESSING_COMPLETE=LDIF processing complete. %d entries \
 written
ERR_MAKELDIF_EXCEPTION_DURING_PARSE=An error occurred while \
 parsing template file :  %s
ERR_MAKELDIF_UNABLE_TO_CREATE_LDIF=An error occurred while \
 attempting to open LDIF file %s for writing:  %s
ERR_MAKELDIF_ERROR_WRITING_LDIF=An error occurred while writing data \
 to LDIF file %s:  %s
ERR_MAKELDIF_EXCEPTION_DURING_PROCESSING=An error occurred while \
 processing :  %s
ERR_CONSTANT_ARG_CANNOT_DECODE=Unable to parse a constant argument, \
 expecting name=value but got %s

 opendj3/opendj-ldap-toolkit/src/test/java/com/forgerock/opendj/ldap/tools/MakeLDIFTestCase.java

New file
@@ -0,0 +1,130 @@
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License, Version 1.0 only
 * (the "License").  You may not use this file except in compliance
 * with the License.
 *
 * You can obtain a copy of the license at legal-notices/CDDLv1_0.txt
 * or http://forgerock.org/license/CDDLv1.0.html.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at legal-notices/CDDLv1_0.txt.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information:
 *      Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 *
 *
 *      Copyright 2013 ForgeRock AS.
 */
package com.forgerock.opendj.ldap.tools;

import static org.fest.assertions.Assertions.*;

import static com.forgerock.opendj.ldap.tools.ToolsMessages.*;

import java.io.ByteArrayOutputStream;
import java.io.PrintStream;
import java.io.UnsupportedEncodingException;

import org.forgerock.i18n.LocalizableMessage;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;

import com.forgerock.opendj.util.StaticUtils;

@SuppressWarnings("javadoc")
public class MakeLDIFTestCase extends ToolsTestCase {

    @DataProvider(name = "validArguments")
    Object[][] createValidArguments() throws Exception {
        Object[][] data = new Object[][] {
            { args("-c", "numusers=1", "example.template"),
              // 2 base entries + users
              expectedErrOutput(INFO_MAKELDIF_PROCESSING_COMPLETE.get(3)) },

            { args("-c", "numusers=5", "example.template"),
              // 2 base entries + users
              expectedErrOutput(INFO_MAKELDIF_PROCESSING_COMPLETE.get(7)) },
        };
        return data;
    }

    @DataProvider(name = "invalidArguments")
    Object[][] createInValidArguments() throws Exception {
        Object[][] data = new Object[][] {
            { // check that usage is written to output when arguments are invalid
              args(),
              expectedErrOutput(INFO_MAKELDIF_TOOL_DESCRIPTION.get()) },

            { // check that there is an argument error when no arg provided
              args(),
              expectedErrOutput(ERR_ERROR_PARSING_ARGS.get("")) },

            { args("-r", "unknown/path" , "example.template"),
              expectedErrOutput(ERR_MAKELDIF_NO_SUCH_RESOURCE_DIRECTORY.get("unknown/path")) },

            { args("-o", "unknown/path" , "example.template"),
              expectedErrOutput(ERR_MAKELDIF_UNABLE_TO_CREATE_LDIF.get("unknown/path", "")) },

            { args("-s", "non-numeric" , "example.template"),
              expectedErrOutput(ERR_ERROR_PARSING_ARGS.get("")) },
        };
        return data;
    }

    @Test(dataProvider = "validArguments")
    public void testRunValidArguments(String[] arguments, LocalizableMessage expectedErrOutput)
            throws Exception {
        run(arguments, true, expectedErrOutput);
    }

    @Test(dataProvider = "invalidArguments")
    public void testRunInvalidArguments(String[] arguments, LocalizableMessage expectedErrOutput)
            throws Exception {
        run(arguments, false, expectedErrOutput);
    }

    private void run(String[] arguments, boolean expectsResults, LocalizableMessage expectedErrOutput)
            throws UnsupportedEncodingException {
        PrintStream outStream = null;
        PrintStream errStream = null;
        try {
            ByteArrayOutputStream out = new ByteArrayOutputStream();
            outStream = new PrintStream(out);
            ByteArrayOutputStream err = new ByteArrayOutputStream();
            errStream = new PrintStream(err);

            MakeLDIF makeLDIF = new MakeLDIF(outStream, errStream);
            makeLDIF.run(arguments);

            if (expectsResults) {
                assertThat(out.size()).isGreaterThan(0);
            } else {
                assertThat(out.size()).isEqualTo(0);
            }

            assertThat(err.toString("UTF-8")).contains(Utils.wrapText(expectedErrOutput, Utils.MAX_LINE_WIDTH));
        } finally {
            StaticUtils.closeSilently(outStream, errStream);
        }
    }

    /** Arguments passed to the command */
    private String[] args(String...arguments) {
        return arguments;
    }

    /** A message the error output is expected to contain. */
    private LocalizableMessage expectedErrOutput(LocalizableMessage val) {
        return val;
    }


}