Various fixes to the unit testing framework.

davidely

22.56.2007 350bf10d4dbe64399058247d9bce39ed16d7a1b0

Various fixes to the unit testing framework.

1.  Wrapped TestNG suite runner with our own (SuiteRunner.java).  When there is an uncaught exception in the tests, this gives us the stack trace and forces the tests to exit.  This should make it easier to track down OutOfMemoryErrors in the future.

2.  Added our own listener for the tests.  This does the following
  * As soon as a test fails, it prints it out all of the info to the screen.  Test method, error message, stacktrace, parameters
  * At the end of the tests, it summarizes all of the tests that failed.
  * It generates a new test report in text format.  The html report generator was a real memory hog.

3.  Changes to build.xml
   * Test output is suppressed by default.
   * The coverage report is generated even if the tests failed.
   * Exposed the ability to rerun only the failed tests with -Dtest.failed=true.
   * The 'runtests' target prints out information about additional test options.
   * Two small fixes so that ant doesn't recompile files that haven't changed.
   * Removed tabs.  This was mostly in the status panel code.
   * Reverted the -Xmx256M temporary fix for the unit tests.

 opends/build.xml

@@ -31,7 +31,6 @@
    file in this directory for build instructions.
  </description>


  <!-- General server-wide properties                               -->
  <property name="src.dir"        location="src/server"              />
  <property name="build.dir"      location="build"                   />
@@ -52,7 +51,7 @@
  <!-- Properties for use in unit testing.                           -->
  <property name="unittest.testng.dir" location="tests/unit-tests-testng"/>
  <property name="unittest.testng.src.dir"
    location="${unittest.testng.dir}/src"/>
    location="${unittest.testng.dir}/src/server"/>

  <property name="unittest.classes.dir"
       location="${build.dir}/unit-tests/classes" />
@@ -226,7 +225,6 @@
        <expandproperties/>
      </filterchain>
    </copy>

  </target>


@@ -294,6 +292,7 @@
    <mkdir dir="${classes.dir}" />

    <javac srcdir="${src.dir}" destdir="${classes.dir}" optimize="true"
         excludes="**/package-info.java"
         debug="on" debuglevel="lines,source" source="1.5" target="1.5"
         deprecation="true" fork="true" memoryInitialSize="${MEM}"
         memoryMaximumSize="${MEM}">
@@ -779,8 +778,33 @@
                classList="${test.classes}" 
                methodList="${test.methods}" />

    <antcall target="testinit.checkFailedTestsOnly"/>
  </target>

 <!-- If we were asked to run only the tests that failed,
      then we overwrite the testng.xml that we just generated
      with testng-failed.xml, which TestNG generated. -->
  <target name="testinit.checkFailedTestsOnly" if="test.failures">
    <!-- Ensure that some of the tests failed last time. -->
    <available property="testng-failed.xml.exists" 
               file="${unittest.report.dir}/testng-failed.xml"/>
    <fail message="No unit tests failed in the previous run." 
          unless="testng-failed.xml.exists"/>

    <!-- We replace the 'Failed suite [OpenDS]' with 'OpenDS' so we
         don't end up with 'Failed suite [Failed suite [OpenDS]]]' etc. -->
    <replace file="${unittest.report.dir}/testng-failed.xml"
             token="Failed suite [OpenDS]"
             value="OpenDS"/>
    <replace file="${unittest.report.dir}/testng-failed.xml"
             token="default(failed)"
             value="default"/>

    <copy file="${unittest.report.dir}/testng-failed.xml"
          tofile="${unittest.resource.dir}/testng.xml"
          overwrite="true"/>
  
    <echo message="Will run the failed unit tests only"/>
  </target>

  <!-- Generate coverage diff report -->
@@ -827,7 +851,7 @@

    <coveragediff emmadatapath="${coverage.data.dir}"
                  outputpath="${cvgdiff.report.dir}"
                  diffpath="${basedir}/${test.diff.srcpath}"
                  diffpath="${test.diff.srcpath}"
                  svnpath="${test.diff.svnpath}" 
                  enabled="${test.diff.enabled}"
                  verbose="${test.diff.verbose}" />
@@ -847,14 +871,14 @@
  <!-- Execute all of the Directory Server TestNG unit tests in text mode. -->
  <target name="testall"
          depends="enableTestNGAssertions,prepdefaultalltest,testinit,runtests"
          description="Run all of the TestNG tests with assertions enabled.  See 'testwithcoverage' for properties you can set.">
          description="Run all of the TestNG tests (including 'slow' ones) with assertions enabled.  See 'testwithcoverage' for properties you can set.">
  </target>


  <!-- Execute the Directory Server TestNG unit tests in text mode. -->
  <target name="test"
          depends="testinit,runtests"
          description="Execute the Directory Server TestNG unit tests in text mode.  Set '-Dorg.opends.test.suppressOutput=true' to suppress output from the unit tests.">
          description="Execute the Directory Server TestNG unit tests in text mode.  Set '-Dorg.opends.test.suppressOutput=false' to see the output from the unit tests.  Set '-Dtest.failures=true' to run only the tests that failed previously.">
  </target>

  <!-- Execute the Directory Server TestNG unit tests in text mode with a coverage report. -->
@@ -874,8 +898,43 @@
    <echo message="This target is deprecated. Please use the testwithcoverage target as it now supports the test.* properties." />
  </target>

  <target name="testhelp" unless="disable.test.help">
    <echo message="About to run the unit tests.  Ant options to control the tests:"/>
    <echo message=""/>
    <echo message="  -Dorg.opends.test.suppressOutput=false"/>
    <echo message="      writes the unit test output to the screen"/>
    <echo message=""/>
    <echo message="  -Dtest.failed=true"/>
    <echo message="      runs only the tests that failed last time"/>
    <echo message=""/>
    <echo message="  -Dtest.packages=org.opends.server.api"/>
    <echo message="      for example runs only the tests in the api package"/>
    <echo message="      For multiple packages, separate them with a ',' and "/>
    <echo message="      quote the entire value"/>
    <echo message=""/>
    <echo message="  -Dtest.classes=org.opends.server.types.TestDN"/>
    <echo message="      for example only runs the TestDN class"/>
    <echo message="      For multiple classes, separate them with a ',' and "/>
    <echo message="      quote the entire value"/>
    <echo message=""/>
    <echo message="  -Dtest.methods=org.opends.server.types.TestDN.testGetRDN"/>
    <echo message="      for example only runs the testGetRDN method"/>
    <echo message="      For multiple methods, separate them with a ',' and "/>
    <echo message="      quote the entire value"/>
    <echo message=""/>
    <echo message="  -Dtest.diff.srcpath=src/server/org/opends/server/core"/>
    <echo message="      for example includes only the classes in"/>
    <echo message="      src/server/org/opends/server/core in the coveragediff report."/>
    <echo message="      To list multiple directories or files, separate them with"/>
    <echo message="      a space as you would an argument list to 'svn diff' and quote"/>
    <echo message="      the whole value."/>
    <echo message=""/>
  </target>

  <!-- Internal target to execute the Directory Server TestNG unit tests in text mode after everything has been initialized. -->
  <target name="runtests">
    <antcall target="testhelp"/>

    <mkdir dir="${unittest.report.dir}" />

    <taskdef resource="testngtasks">
@@ -895,14 +954,29 @@

    <!-- This sets org.opends.test.suppressOutput if and only if it's not
         already set. -->
    <condition property="org.opends.test.suppressOutput" value="false">
    <condition property="org.opends.test.suppressOutput" value="true">
      <not>
        <isset property="org.opends.test.suppressOutput" />
      </not>
    </condition>
    
    <testng outputdir="${unittest.report.dir}" haltonfailure="true"
            enableAssert="${TESTASSERT}">
    <!-- Cleanout the old reports.  Otherwise, the old testng-failed.xml 
         will hang around even if all of the tests pass. -->
    <delete>
      <fileset dir="${unittest.report.dir}" includes="*"/>
    </delete>

    <!-- Our testng listener will remove this file if all of the 
         tests passed.  This allows us to generate the coverage
         report even if the tests failed and still fail the build. -->
    <touch file="${unittest.report.dir}/.tests-failed-marker"/>

    <testng outputdir="${unittest.report.dir}"
            haltonfailure="false"
            enableAssert="${TESTASSERT}"
            listeners="org.opends.server.TestListener org.testng.reporters.FailedReporter"
            useDefaultListeners="false"
            suiteRunnerClass="org.opends.server.SuiteRunner">
      <classpath>
        <pathelement location="${coverage.instr.dir}" />
        <pathelement location="${classes.dir}" />
@@ -924,14 +998,22 @@
      <jvmarg value="-Dorg.opends.server.BuildRoot=${basedir}" />
      <jvmarg value="-Dorg.opends.test.suppressOutput=${org.opends.test.suppressOutput}" />
      <jvmarg value="-Xms${MEM}" />
      <jvmarg value="-Xmx256M" />
      <jvmarg value="-Xmx${MEM}" />
      <xmlfileset dir="${unittest.resource.dir}" includes="testng.xml" />
    </testng>

    <echo message="Generating unit tests coverage report..."/>
    <!-- Our testng listener will create this file if any of the
         tests failed.  This allows us to generate the coverage
         report even if the tests failed. -->
    <available property="testng.tests.failed" 
               file="${unittest.report.dir}/.tests-failed-marker"/>

    <!-- Delete all of the report suite sub-directories since we only
         have a single suite. -->
    <delete dir="${unittest.report.dir}/OpenDS"/>

    <emma enabled="${coverage.enabled}" >
      <report sourcepath="${src.dir}" >
        <!-- <property name="verbosity.level" value="verbose" /> -->
        <fileset dir="${coverage.data.dir}" >
           <include name="unit.emma"     />
           <include name="metadata.emma" />
@@ -940,8 +1022,12 @@
        <txt outfile="${coverage.report.dir}/coverage.txt" />
        <html outfile="${coverage.report.dir}/index.html" />
      </report>

    </emma>

    <!-- We delay failing until after the coverage report is generated. -->
    <fail message="The unit tests failed." if="testng.tests.failed"/>

  </target>

  <target name="prepdefaultalltest">