From 3556e2e0dcbf77010f7b230861b6fc32d95760c9 Mon Sep 17 00:00:00 2001
From: al_xipe <al_xipe@localhost>
Date: Wed, 29 Aug 2007 02:56:33 +0000
Subject: [PATCH] 1. added the ability to save the server error and access log.\n2. the testcases are now timed down to the second. subsecond timing would have been too platform(C library actually) dependent in python\n3. adapted the XSL to report the number of known issues\n4. Still need to fix coverage % XSL template\n5. other minor XSL fixes\n6. added windows new line support in environment.xml
---
opendj-sdk/opends/tests/functional-tests/shared/functions/utils.xml | 136 +++++++++++++++++++++------
opendj-sdk/opends/tests/functional-tests/shared/xsl/my-report.xsl | 106 ++++++++++++++++-----
opendj-sdk/opends/tests/functional-tests/shared/functions/environment.xml | 2
opendj-sdk/opends/tests/functional-tests/testcases/sample/sample.xml | 43 ++++++++
4 files changed, 230 insertions(+), 57 deletions(-)
diff --git a/opendj-sdk/opends/tests/functional-tests/shared/functions/environment.xml b/opendj-sdk/opends/tests/functional-tests/shared/functions/environment.xml
index 892e57e..cb43d88 100755
--- a/opendj-sdk/opends/tests/functional-tests/shared/functions/environment.xml
+++ b/opendj-sdk/opends/tests/functional-tests/shared/functions/environment.xml
@@ -134,10 +134,12 @@
fileExt='.bat'
fileFolder='bat'
pathSeparator=';'
+ newLine='\r\n'
else:
fileExt=''
fileFolder='bin'
pathSeparator=':'
+ newLine='\n'
</script>
<!-- Set the signal handlers -->
diff --git a/opendj-sdk/opends/tests/functional-tests/shared/functions/utils.xml b/opendj-sdk/opends/tests/functional-tests/shared/functions/utils.xml
index a6ad166..6708ac8 100755
--- a/opendj-sdk/opends/tests/functional-tests/shared/functions/utils.xml
+++ b/opendj-sdk/opends/tests/functional-tests/shared/functions/utils.xml
@@ -553,8 +553,18 @@
<function-no-args />
<sequence>
<script>
+ import time
+ testcaseStart=int(time.time())
testcaseStartTime=strftime("%Y%m%d@%H:%M:%S",localtime())
issuesList=[]
+ errorlog=open('%s/%s/logs/errors' % (TMPDIR,OPENDSNAME),'a')
+ errorlog.seek(0,2)
+ errorlog.write('Begin testcase %s\n' % STAXCurrentTestcase)
+ errorlog.close()
+ accesslog=open('%s/%s/logs/access' % (TMPDIR,OPENDSNAME),'a')
+ accesslog.seek(0,2)
+ accesslog.write('Begin testcase %s\n' % STAXCurrentTestcase)
+ accesslog.close()
</script>
<call function="'testCase_StartBanner'" />
</sequence>
@@ -582,7 +592,7 @@
SignalRaised=''
</script>
- <!-- Query the test case results -->
+ <!-- Query the test case results -->
<call function="'queryTestcase'" />
<script>
@@ -616,9 +626,21 @@
'endat' : testcaseEndTime }
</call>
<script>
+ import re
+
+ class Matcher:
+ def search(self, r,s):
+ _tmp = re.compile(r)
+ self.value = _tmp.match(s)
+ return self.value
+ def __getitem__(self, i):
+ return self.value.group(i)
+
class Test:
- def __init__(self, group, suite, fullname, start, stop, failures, successes, issues):
+ def __init__(self, group, suite, fullname, start, stop, failures, successes, issues, duration):
self.log=''
+ self.error=''
+ self.access=''
self.group=group
self.suite=suite
self.fullname=fullname
@@ -630,7 +652,7 @@
self.name=''.join(tmp)
else:
self.name=fullname
- self.duration = 0
+ self.duration = duration
if len(issues) == 0:
if failures == 0:
if successes == 0:
@@ -645,42 +667,55 @@
def toXML(self):
- xml = ' <test>\n'
- xml += ' <name>%s</name>\n' % self.name
- xml += ' <group>%s</group>\n' % self.group
- xml += ' <suite>%s</suite>\n' % self.suite
- xml += ' <start>%s</start>\n' % self.start
- xml += ' <stop>%s</stop>\n' % self.stop
- xml += ' <result>%s</result>\n' % self.result
- xml += ' <duration>%d</duration>\n' % self.duration
- xml += ' <issues>\n'
+ xml = ' <test>%s' % newLine
+ xml += ' <name>%s</name>%s' % (self.name,newLine)
+ xml += ' <group>%s</group>%s' % (self.group,newLine)
+ xml += ' <suite>%s</suite>%s' % (self.suite,newLine)
+ xml += ' <start>%s</start>%s' % (self.start,newLine)
+ xml += ' <stop>%s</stop>%s' % (self.stop,newLine)
+ xml += ' <result>%s</result>%s' % (self.result,newLine)
+ xml += ' <duration>%d</duration>%s' % (self.duration,newLine)
+ xml += ' <issues>%s' % newLine
for issue in self.issues:
- xml += ' <issue>%s</issue>\n' % issue
- xml += ' </issues>\n'
- xml += ' <log>\n<![CDATA['
- xml += '%s' % self.log
- xml += ' ]]></log>\n'
- xml += ' </test>\n'
+ xml += ' <issue>%s</issue>%s' % (issue,newLine)
+ xml += ' </issues>%s' % newLine
+ xml += ' <log>\n<![CDATA[%s' % newLine
+ xml += self.log
+ xml += ' ]]></log>%s' % newLine
+ xml += ' <error>\n<![CDATA[%s' % newLine
+ xml += self.error
+ xml += ' ]]></error>%s' % newLine
+ xml += ' <access>\n<![CDATA[%s' % newLine
+ xml += self.access
+ xml += ' ]]></access>%s' % newLine
+ xml += ' </test>%s' % newLine
return xml
-
- def appendLog(self, log):
+
+ def appendLog(self, category, log):
_leftPadding=''
+ wrappedLog=''
while len(log)>100:
_logChunk=log[:100]
log=log[100:]
- self.log+='%s%s\n' % (_leftPadding,_logChunk)
+ wrappedLog+='%s%s%s' % (_leftPadding,_logChunk,newLine)
_leftPadding='... '
- self.log += '%s%s\n' % (_leftPadding,log)
+ wrappedLog += '%s%s%s' % (_leftPadding,log,newLine)
+ if category == 'access':
+ self.access+=wrappedLog
+ elif category == 'error':
+ self.error+=wrappedLog
+ else:
+ self.log += wrappedLog
def getName(self):
return self.name
-
- thisTest = Test(CurrentTestPath['group'],CurrentTestPath['suite'],STAXCurrentTestcase, testcaseStartTime,testcaseEndTime,numFail,numPass,issuesList)
-
+
+ testcaseStop=int(time.time())
+ testcaseDuration=testcaseStop-testcaseStart
+ thisTest = Test(CurrentTestPath['group'],CurrentTestPath['suite'],STAXCurrentTestcase, testcaseStartTime,testcaseEndTime,numFail,numPass,issuesList,testcaseDuration)
+
doLog = False
- testlog=open('%s/tests-log.xml' % TMPDIR,'a')
- testlog.seek(0,2)
-
+
# loop through the log lines
for element in STAFResult:
level=element['level']
@@ -692,7 +727,7 @@
# if the current element is actually this test's output then log it
if doLog == True:
- thisTest.appendLog(element['message'])
+ thisTest.appendLog('test',element['message'])
# this test is AFTER the append log so we don't get the
# "starting testcase ..." header in the XML
@@ -704,7 +739,48 @@
# log facility)
if tmp[3] == thisTest.getName():
doLog=True
-
+
+ # parse the server's error log
+ # TODO: figure out how to do this for multiple instance for the replication
+ # tests for example
+ errorlog=open('%s/%s/logs/errors' % (TMPDIR,OPENDSNAME),'a')
+ errorlog.seek(0,2)
+ errorlog.write('End testcase %s\n' % STAXCurrentTestcase)
+ errorlog.close()
+ accesslog=open('%s/%s/logs/access' % (TMPDIR,OPENDSNAME),'a')
+ accesslog.seek(0,2)
+ accesslog.write('End testcase %s\n' % STAXCurrentTestcase)
+ accesslog.close()
+
+ errorlog=open('%s/%s/logs/errors' % (TMPDIR,OPENDSNAME), 'r')
+ _log=''
+ _doLog=False
+ for line in errorlog.readlines():
+ if line.startswith('End testcase %s' % STAXCurrentTestcase):
+ _doLog=False
+ if _doLog:
+ _log+=line
+ if line.startswith('Begin testcase %s' % STAXCurrentTestcase):
+ _doLog=True
+ errorlog.close()
+ thisTest.appendLog('error', _log)
+
+ accesslog=open('%s/%s/logs/access' % (TMPDIR,OPENDSNAME), 'r')
+ _log=''
+ _doLog=False
+ for line in accesslog.readlines():
+ if line.startswith('End testcase %s' % STAXCurrentTestcase):
+ _doLog=False
+ if _doLog:
+ _log+=line
+ if line.startswith('Begin testcase %s' % STAXCurrentTestcase):
+ _doLog=True
+ accesslog.close()
+ thisTest.appendLog('access', _log)
+
+ # save to test log
+ testlog=open('%s/tests-log.xml' % TMPDIR,'a')
+ testlog.seek(0,2)
testlog.write(thisTest.toXML())
testlog.close()
</script>
diff --git a/opendj-sdk/opends/tests/functional-tests/shared/xsl/my-report.xsl b/opendj-sdk/opends/tests/functional-tests/shared/xsl/my-report.xsl
index 99198b0..250ba02 100644
--- a/opendj-sdk/opends/tests/functional-tests/shared/xsl/my-report.xsl
+++ b/opendj-sdk/opends/tests/functional-tests/shared/xsl/my-report.xsl
@@ -110,6 +110,7 @@
<xsl:variable name="total" select="count($tests)"/>
<xsl:variable name="pass" select="count($tests[result='pass'])"/>
<xsl:variable name="fail" select="count($tests[result='fail'])"/>
+ <xsl:variable name="known" select="count($tests[result='known'])"/>
<xsl:variable name="unknown" select="count($tests[result='inconclusive'])"/>
<xsl:variable name="rate" select="round((($pass div $total) * 100) - 0.5)"/>
<!--- Test Report Header Variables -->
@@ -164,11 +165,12 @@
<th align="center">Platform</th>
<th align="center">JVM Version</th>
<th align="center">JVM Vendor</th>
- <th align="center">Total</th>
- <th align="center">Pass</th>
- <th align="center">Fail</th>
- <th align="center">Inconclusive</th>
- <th align="center">Coverage</th>
+ <th align="center" width="5%">Total</th>
+ <th align="center" width="5%">Pass</th>
+ <th align="center" width="5%">Fail</th>
+ <th align="center" width="5%">Known Issues</th>
+ <th align="center" width="5%">Inconclusive</th>
+ <th align="center" width="5%">Coverage</th>
</tr>
<tr>
<th align="center"><xsl:value-of select="$identification/buildid"/></th>
@@ -179,6 +181,7 @@
<th align="center"><xsl:value-of select="$total"/></th>
<th align="center"><xsl:value-of select="$pass"/></th>
<th align="center"><xsl:value-of select="$fail"/></th>
+ <th align="center"><xsl:value-of select="$known"/></th>
<th align="center"><xsl:value-of select="$unknown"/></th>
<th align="center">
<a>
@@ -201,6 +204,7 @@
<td width="5%">Success Rate</td>
<td width="5%">Pass</td>
<td width="5%">Fail</td>
+ <td width="5%">Known</td>
<td width="5%">Inc.</td>
<td width="5%">Cov.</td>
</tr>
@@ -213,6 +217,7 @@
<xsl:variable name="group-total" select="count($tests[group=$group])"/>
<xsl:variable name="group-pass" select="count($tests[group=$group][result='pass'])"/>
<xsl:variable name="group-fail" select="count($tests[group=$group][result='fail'])"/>
+ <xsl:variable name="group-known" select="count($tests[group=$group][result='known'])"/>
<xsl:variable name="group-unknown" select="count($tests[group=$group][result='inconclusive'])"/>
<xsl:variable name="group-rate" select="round((($group-pass div $group-total) * 100) - 0.5)"/>
<li>
@@ -255,6 +260,16 @@
<td width="5%">
<xsl:attribute name="class">
<xsl:choose>
+ <xsl:when test="$group-known > 0">
+ <xsl:value-of select="'warning'" />
+ </xsl:when>
+ </xsl:choose>
+ </xsl:attribute>
+ <xsl:value-of select="$group-known"/>
+ </td>
+ <td width="5%">
+ <xsl:attribute name="class">
+ <xsl:choose>
<xsl:when test="$group-unknown > 0">
<xsl:value-of select="'warning'" />
</xsl:when>
@@ -262,6 +277,14 @@
</xsl:attribute>
<xsl:value-of select="$group-unknown"/>
</td>
+ <td width="5%">
+ <a>
+ <xsl:attribute name="href">
+ <xsl:value-of select="concat(concat('coverage/',$group),'/coverage.html')"/>
+ </xsl:attribute>
+ <xsl:value-of select="/qa/functional-tests/results/group[name='$group']/coverage"/>%
+ </a>
+ </td>
</tr>
</table>
<ul>
@@ -271,6 +294,7 @@
<xsl:variable name="suite-total" select="count($tests[group=$group][suite=$suite])"/>
<xsl:variable name="suite-pass" select="count($tests[group=$group][suite=$suite][result='pass'])"/>
<xsl:variable name="suite-fail" select="count($tests[group=$group][suite=$suite][result='fail'])"/>
+ <xsl:variable name="suite-known" select="count($tests[group=$group][suite=$suite][result='known'])"/>
<xsl:variable name="suite-unknown" select="count($tests[group=$group][suite=$suite][result='inconclusive'])"/>
<xsl:variable name="suite-rate" select="round((($suite-pass div $suite-total) * 100) - 0.5)"/>
<li>
@@ -316,6 +340,16 @@
<td width="5%">
<xsl:attribute name="class">
<xsl:choose>
+ <xsl:when test="$suite-known > 0">
+ <xsl:value-of select="'warning'" />
+ </xsl:when>
+ </xsl:choose>
+ </xsl:attribute>
+ <xsl:value-of select="$suite-known"/>
+ </td>
+ <td width="5%">
+ <xsl:attribute name="class">
+ <xsl:choose>
<xsl:when test="$suite-unknown > 0">
<xsl:value-of select="'warning'" />
</xsl:when>
@@ -366,8 +400,26 @@
</tr>
</table>
<ul>
- <li>
- <pre><xsl:value-of select="log" /></pre>
+ <li>Test Log<br />
+ <ul>
+ <li>
+ <pre><xsl:value-of select="log" /></pre>
+ </li>
+ </ul>
+ </li>
+ <li>Server access Log<br />
+ <ul>
+ <li>
+ <pre><xsl:value-of select="access" /></pre>
+ </li>
+ </ul>
+ </li>
+ <li>Server error Log<br />
+ <ul>
+ <li>
+ <pre><xsl:value-of select="error" /></pre>
+ </li>
+ </ul>
</li>
</ul>
</li>
@@ -388,10 +440,12 @@
<xsl:variable name="group" select="group"/>
<xsl:if test="generate-id(.)=generate-id($tests[group=$group])">
<xsl:variable name="group-total" select="count($tests[group=$group])"/>
- <xsl:variable name="group-pass" select="count($tests[group=$group][result='pass'])"/>
- <xsl:variable name="group-fail" select="count($tests[group=$group][result='fail'])"/>
- <xsl:variable name="group-unknown" select="count($tests[group=$group][result='inconclusive'])"/>
- <xsl:variable name="group-rate" select="round((($group-pass div $group-total) * 100) - 0.5)"/>
+ <xsl:variable name="group-pass" select="count($tests[group=$group][result='pass'])"/>
+ <xsl:variable name="group-fail" select="count($tests[group=$group][result='fail'])"/>
+ <xsl:variable name="group-known" select="count($tests[group=$group][result='known'])"/>
+ <xsl:variable name="group-unknown" select="count($tests[group=$group][result='inconclusive'])"/>
+ <xsl:variable name="group-rate" select="round((($group-pass div $group-total) * 100) - 0.5)"/>
+ <xsl:variable name="group-coverage" select="coverage"/>
<p>
<table>
<tr>
@@ -399,6 +453,7 @@
<th width="5%">Success Rate</th>
<th width="5%">Pass</th>
<th width="5%">Fail</th>
+ <th width="5%">Known</th>
<th width="5%">Inc.</th>
<th width="5%">Cov.</th>
</tr>
@@ -427,28 +482,25 @@
<td align="center"><xsl:value-of select="$group-rate"/>%</td>
<td align="center"><xsl:value-of select="$group-pass"/></td>
<td align="center"><xsl:value-of select="$group-fail"/></td>
+ <td align="center"><xsl:value-of select="$group-known"/></td>
<td align="center"><xsl:value-of select="$group-unknown"/></td>
- <xsl:for-each select="$groups">
- <xsl:variable name="thisgroup" select="name"/>
- <xsl:if test="generate-id(.)=generate-id($groups[name=$thisgroup])">
- <td>
- <a>
- <xsl:attribute name="href">
- <xsl:value-of select="concat(concat('coverage/',$group),'/coverage.html')"/>
- </xsl:attribute>
- <xsl:value-of select="group"/>
- <xsl:value-of select="coverage"/>%
- </a>
- </td>
- </xsl:if>
- </xsl:for-each>
+ <td>
+ <a>
+ <xsl:attribute name="href">
+ <xsl:value-of select="concat(concat('coverage/',$group),'/coverage.html')"/>
+ </xsl:attribute>
+ <xsl:variable name="group-cov" select="/qa/functional-tests/group[name=$group]" />
+ <xsl:value-of select="$group-cov/coverage"/>%
+ </a>
+ </td>
</tr>
<xsl:for-each select="$tests[group=$group]">
<xsl:variable name="suite" select="suite"/>
<xsl:if test="generate-id(.)=generate-id($tests[group=$group][suite=$suite])">
- <xsl:variable name="suite-total" select="count($tests[group=$group][suite=$suite])"/>
+ <xsl:variable name="suite-total" select="count($tests[group=$group][suite=$suite])"/>
<xsl:variable name="suite-pass" select="count($tests[group=$group][suite=$suite][result='pass'])"/>
<xsl:variable name="suite-fail" select="count($tests[group=$group][suite=$suite][result='fail'])"/>
+ <xsl:variable name="suite-known" select="count($tests[group=$group][suite=$suite][result='known'])"/>
<xsl:variable name="suite-unknown" select="count($tests[group=$group][suite=$suite][result='inconclusive'])"/>
<xsl:variable name="suite-rate" select="round((($suite-pass div $suite-total) * 100) - 0.5)"/>
<tr>
@@ -460,6 +512,7 @@
<th width="5%">Success rate</th>
<th width="5%">Pass</th>
<th width="5%">Fail</th>
+ <th width="5%">Known</th>
<th width="5%">Inc.</th>
</tr>
<tr>
@@ -488,6 +541,7 @@
<td><xsl:value-of select="$suite-rate"/>%</td>
<td><xsl:value-of select="$suite-pass"/></td>
<td><xsl:value-of select="$suite-fail"/></td>
+ <td><xsl:value-of select="$suite-known"/></td>
<td><xsl:value-of select="$suite-unknown"/></td>
</tr>
</table>
diff --git a/opendj-sdk/opends/tests/functional-tests/testcases/sample/sample.xml b/opendj-sdk/opends/tests/functional-tests/testcases/sample/sample.xml
index 7a718b5..8f01496 100644
--- a/opendj-sdk/opends/tests/functional-tests/testcases/sample/sample.xml
+++ b/opendj-sdk/opends/tests/functional-tests/testcases/sample/sample.xml
@@ -55,7 +55,7 @@
-->
<call function="setupStep" />
</iterate>
- <iterate in="['passing_test']" var="testStep">
+ <iterate in="['sample_test','knownIssue_test','passing_test','failing_test']" var="testStep">
<!-- the sequence tag is superfluous here since there is only a
single instruction to execute within the iterate block
-->
@@ -126,6 +126,47 @@
</sequence>
</block>
</function>
+
+ <!-- the test -->
+ <function name="knownIssue_test">
+ <!---
+ Place test-specific test information here.
+ The tag, TestMarker, must be the same as the tag, TestSuiteName.
+ #@TestMarker Sample
+ #@TestName sample_test
+ #@TestIssue xyz
+ #@TestPurpose Illustrate how to issue a query on OpenDS
+ #@TestPreamble none
+ #@TestStep Fetch all the entries in the server
+ #@TestPostamble none
+ #@TestResult Success if OpenDS returns 0
+ -->
+ <block name="'knownIssue_test'">
+ <sequence>
+ <script>
+ CurrentTestPath['suite']= 'search'
+ </script>
+ <testcase name="getTestCaseName(STAXCurrentBlock)">
+ <sequence>
+ <call function="'testCase_Preamble'" />
+ <!-- search for an entry that does not exist -->
+ <call function="'SearchObject'">
+ { 'dsInstanceHost' : DIRECTORY_INSTANCE_HOST ,
+ 'dsInstancePort' : DIRECTORY_INSTANCE_PORT ,
+ 'dsInstanceDn' : DIRECTORY_INSTANCE_DN ,
+ 'dsInstancePswd' : DIRECTORY_INSTANCE_PSWD ,
+ 'dsBaseDN' : DIRECTORY_INSTANCE_SFX ,
+ 'dsFilter' : 'uid=wronguid' }
+ </call>
+ <script>
+ knownIssue(828)
+ </script>
+ <call function="'testCase_Postamble'" />
+ </sequence>
+ </testcase>
+ </sequence>
+ </block>
+ </function>
<!-- the dummy failing test -->
<function name="failing_test">
--
Gitblit v1.10.0