1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package net.sf.jameleon.plugin.junit;
20
21
22 import net.sf.jameleon.ExecuteTestCase;
23 import net.sf.jameleon.TestCaseTag;
24 import net.sf.jameleon.event.TestCaseEvent;
25 import net.sf.jameleon.event.TestCaseEventHandler;
26 import net.sf.jameleon.event.TestCaseListener;
27 import net.sf.jameleon.reporting.ResultsReporter;
28 import net.sf.jameleon.reporting.TestCaseCounter;
29 import net.sf.jameleon.result.*;
30 import net.sf.jameleon.util.JameleonUtility;
31 import org.apache.log4j.LogManager;
32 import org.apache.log4j.spi.LoggerRepository;
33 import org.apache.log4j.varia.DenyAllFilter;
34 import org.dom4j.Document;
35 import org.dom4j.DocumentException;
36 import org.dom4j.Node;
37 import org.dom4j.io.SAXReader;
38
39 import java.io.File;
40 import java.io.StringReader;
41 import java.io.StringWriter;
42 import java.io.Writer;
43 import java.util.ArrayList;
44 import java.util.Iterator;
45 import java.util.List;
46
47 /***
48 * Used to acceptance test Jameleon tags. Executes a Jameleon script and checks various outputs.
49 * @jameleon.function name="execute-jameleon-script" type="action"
50 * @jameleon.step Execute the given script
51 * @jameleon.step Validate that the script ran correctly
52 */
53 public class ExecuteJameleonScriptTag extends JUnitFunctionTag {
54
55 protected static final boolean DEBUG = false;
56 /***
57 * The script to execute
58 * @jameleon.attribute required="true"
59 */
60 protected File script;
61 /***
62 * Pass only if the script passed
63 * @jameleon.attribute
64 */
65 protected boolean checkOutcomePassed;
66 /***
67 * Pass only if the script failed
68 * @jameleon.attribute
69 */
70 protected boolean checkOutcomeFailed;
71 /***
72 * TODO
73 * @jameleon.attribute
74 */
75 protected boolean noTestCaseResults;
76 /***
77 * The number of functional points that should be run from the script being executed
78 * @jameleon.attribute default="-1"
79 */
80 protected int numOfFunctionsRun;
81 /***
82 * The number of failures that should fail from the script being executed
83 * @jameleon.attribute default="-1"
84 */
85 protected int numOfFailures;
86 /***
87 * A functionId where no failure should occur
88 * @jameleon.attribute
89 */
90 protected String noFailOnFunctionId;
91 /***
92 * The number of test cases that should run from the script being executed (for when useCSV is set to true in the testcase tag)
93 * @jameleon.attribute default="-1"
94 */
95 protected int numOfTestCasesRun;
96 /***
97 * The number of test cases that should fail from the script being executed (for when useCSV is set to true in the testcase tag)
98 * @jameleon.attribute default="-1"
99 */
100 protected int numOfTestCasesFailed;
101 /***
102 * The name of the generated test case doc file.
103 * @jameleon.attribute
104 */
105 protected String testCaseDocsFile;
106 /***
107 * The execution time should be greater than this number if set
108 * @jameleon.attribute default="-1"
109 */
110 protected int executionTimeGreaterThan;
111 /***
112 * The execution time should be greater than this number if set
113 * @jameleon.attribute default="-1"
114 */
115 protected int executionTimeLessThan;
116 /***
117 * The line # of the script that is stated as failing in the HTML . This requires the testCaseName attribute be set
118 * @jameleon.attribute default="-1"
119 */
120 protected int lineNumFailed;
121 /***
122 * The given error message for failure in the HTML results
123 * @jameleon.attribute
124 */
125 protected String lineFailedReason;
126 /***
127 * The functionId of the row that failed
128 * @jameleon.attribute
129 */
130 protected String lineFailedFunctionId;
131 /***
132 * A snippet of the error message or reason for that is expected in stdout.
133 * @jameleon.attribute
134 */
135 protected String errorMsgContains;
136 /***
137 * A snippet of the error message or reason for that is expected in stdout.
138 * @jameleon.attribute
139 */
140 protected String testCaseName;
141
142 protected StringWriter htmlResults;
143 protected Writer timestampedResults;
144 protected ExecuteTestCase jmln;
145 protected DenyAllFilter denyAll;
146 protected static SAXReader parser = new SAXReader();
147 protected TestCaseResult result;
148 protected TestCaseCounter originalCounter;
149 protected Writer stdOutWriter;
150
151 protected TestCaseListener listener = new TestCaseListener(){
152 public void beginTestCase(TestCaseEvent event) {
153 result = null;
154 }
155 public void endTestCase(TestCaseEvent event) {
156 TestCaseTag tct = (TestCaseTag)event.getSource();
157 result = tct.getResults();
158 }
159 };
160
161 public void testBlock(){
162 String errMsg = executeScript();
163 if (errorMsgContains != null) {
164 assertTrue("An expected error message was not provided! <"+errMsg+">", errMsg.indexOf(errorMsgContains) >= 0);
165 }
166 if (checkOutcomePassed) {
167 assertTrue("Errors NOT expected, but found!", errMsg.length() == 0);
168 assertOutcomePassed();
169 }
170 if (checkOutcomeFailed) {
171 assertTrue("Errors expected, but none found!", errMsg.length() > 0);
172 assertOutcomeFailed();
173 }
174 if (noTestCaseResults) {
175 assertNull("No information about this test case should be given.", result.getTestCaseDocsFile());
176 }
177 if (numOfFailures != -1) {
178 assertNumOfFailures(numOfFailures);
179 }
180 if (numOfFunctionsRun != -1) {
181 assertNumOfFunctionsRun(numOfFunctionsRun);
182 }
183 if (numOfTestCasesRun != -1) {
184 assertNumOfTestCasesRun(numOfTestCasesRun);
185 }
186 if (numOfTestCasesFailed != -1) {
187 assertNumOfTestCasesFailed(numOfTestCasesFailed);
188 }
189 if (testCaseDocsFile != null) {
190 assertTestCaseDocFileEndsWith(testCaseDocsFile);
191 }
192 if (testCaseName != null) {
193 assertHtmlTestCaseName(testCaseName);
194 }
195 if (lineNumFailed != -1) {
196 try{
197 JameleonTestResult res = null;
198
199 for (Iterator it = result.getFailedResults().iterator(); it.hasNext(); ){
200 JameleonTestResult tmpRes = (JameleonTestResult)it.next();
201 if (tmpRes.getLineNumber() == lineNumFailed){
202 res = tmpRes;
203 }
204 }
205
206 assertNotNull("No matching line # found", res);
207 assertEquals("Failed Line #:", lineNumFailed, res.getLineNumber());
208 if (lineFailedReason != null) {
209 assertEquals("Error Message: ", lineFailedReason, res.getErrorMsg());
210 }
211 if (lineFailedFunctionId != null) {
212 assertEquals("Function ID: ", lineFailedFunctionId, res.getIdentifier());
213 }
214 }catch(Exception e){
215 System.err.println("####################################");
216 e.printStackTrace();
217 System.err.println("####################################");
218 }
219 }
220 if (executionTimeGreaterThan != -1) {
221 assertExecutionTimeGreaterThan(executionTimeGreaterThan);
222 }
223 if (executionTimeLessThan != -1) {
224 assertExecutionTimeLessThan(executionTimeLessThan);
225 }
226 if (noFailOnFunctionId != null) {
227 assertFunctionIdNotFailed(noFailOnFunctionId);
228 }
229 }
230
231 protected void setupEnvironment(){
232 super.setupEnvironment();
233 postcondition = true;
234 }
235
236 public void setup(){
237 postcondition = true;
238 denyAll = new DenyAllFilter();
239 jmln = new ExecuteTestCase(DEBUG);
240 timestampedResults = ResultsReporter.getInstance().getHtmlTestRunReporter().getWriter();
241 htmlResults = new StringWriter();
242 ResultsReporter reporter = ResultsReporter.getInstance();
243 reporter.getHtmlTestRunReporter().setWriter(htmlResults);
244 originalCounter = reporter.getTestCaseCounter();
245 stdOutWriter = reporter.getSimpleTestRunReporter().getWriter();
246 reporter.getSimpleTestRunReporter().setWriter(new StringWriter());
247 reporter.setTestCaseCounter(new TestCaseCounter());
248 LoggerRepository repos = LogManager.getLoggerRepository();
249
250 TestCaseEventHandler.getInstance().addTestCaseListener(listener);
251 }
252
253 protected void setUpFunctionResults(){
254 fResults = new CountableFunctionResult(fp);
255 fResults.copyLocationAwareProperties(this);
256 Object obj = findAncestorWithClass(FunctionResultRecordable.class);
257 if (obj != null) {
258 ((FunctionResultRecordable)obj).recordFunctionResult(fResults);
259 }
260 }
261
262 public void tearDown(){
263 ResultsReporter.getInstance().getHtmlTestRunReporter().setWriter(timestampedResults);
264 jmln = null;
265 TestCaseEventHandler.getInstance().removeTestCaseListener(listener);
266 reverseTestCaseCount();
267 ResultsReporter.getInstance().getSimpleTestRunReporter().setWriter(stdOutWriter);
268 }
269
270 /***
271 * Rolls back the test case execution count. We do this so that the test case
272 * inside the script being run does not increment in the total # runs in the case
273 * it is data-driven. Basically, one script equals one test case run.
274 */
275 private void reverseTestCaseCount(){
276 ResultsReporter.getInstance().setTestCaseCounter(originalCounter);
277 }
278
279 protected String executeScript(){
280 return jmln.executeJellyScript(script);
281 }
282
283 protected void assertFunctionIdNotFailed(String noFailOnFunctionId){
284 boolean found = false;
285 JameleonTestResult jtr;
286 for (Iterator it = result.getFailedResults().iterator(); it.hasNext() && !found;) {
287 jtr = (JameleonTestResult)it.next();
288 found = noFailOnFunctionId.equals(jtr.getIdentifier());
289 }
290 assertFalse(noFailOnFunctionId +" was found in the list of failed results", found);
291 }
292
293 protected void assertOutcomeEquals(String expected){
294 assertEquals("Outcome of testcase: ", expected, getOutcome());
295 }
296
297 protected void assertOutcomePassed(){
298 assertOutcomeEquals("PASSED");
299 }
300
301 protected void assertOutcomeFailed(){
302 assertOutcomeEquals("FAILED");
303 }
304
305 protected void assertNumOfFunctionsRun(int expected){
306 assertEquals("Num of functional points run: ", expected, getFunctionsRun().size());
307 }
308
309 protected void assertNumOfFailures(int expected){
310 assertEquals("Num of tags failed: ", expected, result.getFailedResults().size());
311 }
312
313 protected void assertNumOfTestCasesRun(int expected){
314 assertEquals("Num of test cases run: ", expected, getTestCasesRun());
315 }
316
317 protected void assertNumOfTestCasesFailed(int expected){
318 assertEquals("Num of test cases failed: ", expected, getTestCasesFailed());
319 }
320
321 protected void assertTestCaseDocFileEndsWith(String expected){
322 String pathFixedDocFile = JameleonUtility.fixFileSeparators(expected);
323 assertTrue("Test Case doc file should end with <"+expected+"> was <"+getTestCaseDocFile()+">", getTestCaseDocFile().endsWith(pathFixedDocFile));
324 }
325
326 protected void assertExecutionTimeLessThan(long lessThanTime){
327 long execTime = getExecutionTime();
328 assertTrue("Execution Time <"+execTime+"> should be less than <"+lessThanTime+">", execTime < lessThanTime);
329 }
330
331 protected void assertExecutionTimeGreaterThan(long greaterThanTime){
332 long execTime = getExecutionTime();
333 assertTrue("Execution Time <"+execTime+"> should be greater than <"+greaterThanTime+">", execTime > greaterThanTime);
334 }
335
336
337 protected String getOutcome(){
338 return result.getOutcome();
339 }
340
341 protected List getFunctionsRun(){
342 List functionsRun = new ArrayList();
343 JameleonTestResult jtr;
344 for (Iterator it = result.getAllChildrenResults().iterator(); it.hasNext();) {
345 jtr = (JameleonTestResult)it.next();
346 if (jtr instanceof FunctionResult) {
347 functionsRun.add(jtr);
348 }
349 }
350 return functionsRun;
351 }
352
353 protected int getTestCasesRun(){
354 return result.getCountableResults().size();
355 }
356
357 protected int getTestCasesFailed(){
358 return result.getFailedCountableResults().size();
359 }
360
361 protected long getExecutionTime(){
362 return result.getExecutionTime();
363 }
364
365 protected String getTestCaseDocFile(){
366 return result.getTestCaseDocsFile();
367 }
368
369 protected void assertHtmlTestCaseName(String expected){
370 assertEquals("The displayed test case name: ", expected, getHtmlDisplayedTestCaseName());
371 }
372
373 protected String getHtmlDisplayedOutcome(){
374 String row = stripHtmlRow();
375 row = row.substring(row.indexOf(">")+1, row.indexOf("</td>"));
376 return row;
377 }
378
379 protected String getHtmlDisplayedTestCaseName(){
380 Document pHtml = getHtmlDocument();
381 Node node = pHtml.selectSingleNode( "//span[@title='Test Case Name']/a" );
382 return node.getText().trim();
383 }
384
385 protected String stripHtmlRow(){
386 String row = getRowFromHtml();
387 String subString = null;
388 try{
389 subString = row.substring(row.indexOf("</td>")+6);
390 }catch(Exception e){
391 System.err.println(row);
392 e.printStackTrace();
393 }
394 return subString;
395 }
396
397 protected String getRowFromHtml(){
398 return htmlResults.toString();
399 }
400
401 protected String getValueFromElement(String elementName, String message){
402 String startElement = "<"+elementName+">";
403 String endElement = "</"+elementName+">";
404 int indexS = message.indexOf(startElement);
405 int indexE = message.indexOf(endElement);
406 String value;
407 if (indexS >= 0 && indexE >= 0) {
408 value = message.substring((indexS+startElement.length()),indexE);
409 }else{
410 value = "N/A";
411 }
412 return value;
413 }
414
415 protected Document getHtmlDocument(){
416 return getDocument(htmlResults);
417 }
418
419 protected Document getDocument(StringWriter results){
420
421 String res = "<root>" + results.toString()+"</root>";
422 StringReader sr = new StringReader(res);
423 Document doc = null;
424 try{
425 doc = parser.read(sr);
426 }catch(DocumentException de){
427 de.printStackTrace();
428 fail("Error happened executing: "+script.getPath());
429 }finally{
430 sr.close();
431 }
432 return doc;
433
434 }
435
436 protected class FailedRow{
437 public int lineNum = 0;
438 public String functionId = null;
439 public String errMsg = null;
440 }
441
442 }