qm.test.classes.dejagnu

33 """A 'DejaGNUStream' formats its output like DejaGNU.""" 34 35 arguments = [ 36 qm.fields.BooleanField( 37 name = "show_expected_outcomes", 38 title = "Show Expected Outcomes", 39 description = """True if expected outcomes should be displayed. 40 41 By default, only information about unexpected outcomes is 42 displayed.""", 43 default_value = "false" 44 ) 45 ] 46 47 __summary_outcomes = [ 48 DejaGNUTest.PASS, 49 DejaGNUTest.FAIL, 50 DejaGNUTest.KFAIL, 51 DejaGNUTest.KPASS, 52 DejaGNUTest.XPASS, 53 DejaGNUTest.XFAIL, 54 DejaGNUTest.UNRESOLVED, 55 DejaGNUTest.UNTESTED, 56 DejaGNUTest.UNSUPPORTED 57 ] 58 """The outcomes for which summary output should be produced.""" 59 60 __outcome_descs = { 61 DejaGNUTest.PASS: "expected passes", 62 DejaGNUTest.FAIL: "unexpected failures", 63 DejaGNUTest.KPASS: "unexpected successes", 64 DejaGNUTest.KFAIL: "expected failures", 65 DejaGNUTest.XPASS: "unexpected successes", 66 DejaGNUTest.XFAIL: "expected failures", 67 DejaGNUTest.UNRESOLVED: "unresolved testcases", 68 DejaGNUTest.UNTESTED: "untested testcases", 69 DejaGNUTest.UNSUPPORTED: "unsupported tests", 70 } 71 """A map from DejaGNU outcomes to descriptions. 72 73 See 'init_testcounts' in the DejaGNU distribution for the code 74 emulated by this table.""" 75 76 __expected_outcomes = ( 77 DejaGNUTest.PASS, 78 DejaGNUTest.KFAIL, 79 DejaGNUTest.XFAIL, 80 DejaGNUTest.UNRESOLVED, 81 DejaGNUTest.UNSUPPORTED, 82 DejaGNUTest.UNTESTED 83 ) 84 """The DejaGNU outcomes that are considered "expected" results. 85 86 DejaGNU results with these outcomes are not displayed unless 87 'show_expected_outcomes' is true.""" 88

89 - def __init__(self, arguments = None, **args):

90 91 super(DejaGNUStream, self).__init__(arguments, **args) 92 self.__outcomes = {} 93 for o in DejaGNUTest.dejagnu_outcomes: 94 self.__outcomes[o] = 0

95 96

97 - def WriteResult(self, result):

98 99 # Get the DejaGNU annotations in sorted order. 100 keys = filter(lambda k: k.startswith(DejaGNUTest.RESULT_PREFIX), 101 result.keys()) 102 keys.sort(lambda k1, k2: cmp(int(k1[len(DejaGNUTest.RESULT_PREFIX):]), 103 int(k2[len(DejaGNUTest.RESULT_PREFIX):]))) 104 has_error = 0 105 for k in keys: 106 r = result[k] 107 outcome = r[:r.find(":")] 108 if (self.show_expected_outcomes == "true" 109 or outcome not in self.__expected_outcomes): 110 self.file.write(r + "\n") 111 # Keep track of the outcomes. 112 self.__outcomes[outcome] += 1 113 if outcome == DejaGNUTest.ERROR: 114 has_error = 1 115 116 # If something went wrong running the test, emit an ERROR 117 # message even if DejaGNU would not have. These cases usually 118 # indicate problems with the DejaGNU test itself. 119 if not has_error and result.GetOutcome () == result.ERROR: 120 self.file.write("ERROR: %s (%s)\n" 121 % (result.GetId(), result.GetCause())) 122 self.__outcomes[DejaGNUTest.ERROR] += 1

123 124

125 - def Summarize(self):

126 127 self.file.write("\n\t\t=== Summary ===\n\n") 128 # This function emulates log_summary from the DejaGNU 129 # distribution. 130 for o in self.__summary_outcomes: 131 if self.__outcomes[o]: 132 desc = "# of %s" % self.__outcome_descs[o] 133 self.file.write(desc) 134 if len(desc) < 24: 135 self.file.write("\t") 136 self.file.write("\t%d\n" % self.__outcomes[o])

141 """A 'DejaGNUReader' reads a DejaGNU log file. 142 143 The DejaGNU log file may then be processed by QMTest. For 144 example, QMTest may generate results in an alternative format, or 145 display them in the QMTest GUI. Therefore, this reader may be 146 used to obtain the benefits of QMTest's reporting characteristics, 147 when using a legacy DejaGNU testsuite. 148 149 Unfortunately, DejaGNU log files are relativley unstructured. 150 Therefore, this result reader uses heuristics that may not always 151 be 100% robust. Therefore, for optimal behavior, DejaGNU 152 testsuites should be converted to QMTest testsuites.""" 153 154 arguments = [ 155 qm.fields.BooleanField( 156 name = "is_combined", 157 title = "Combined Format?", 158 description=\ 159 """True if multiple results for the same test should be combined. 160 161 DejaGNU will sometimes print multiple results for the same 162 test. For example, when testing a compiler, DejaGNU may 163 issue one result indicating whether or not a file was 164 successfully compiled and another result indicating 165 whether or not the file was successfully executed. When 166 using the combined format, these two results will be treated as 167 subparts of a single test. When not using the combined 168 format, these results will be treated as separate 169 tests. 170 171 The combined format is the default. However, if you want 172 to see statistics that precisely match DejaGNU, you should 173 not use the combined format.""", 174 default_value="true", 175 ), 176 qm.fields.BooleanField( 177 name = "expectations", 178 title = "GenerateExpectations?", 179 description=\ 180 """True if expected (not actual) results should be generated. 181 182 In this mode, the actual results will be ignored. 183 Instead, a results file indicated expected failures as 184 actual failures will be generated.""", 185 default_value="false", 186 ), 187 ] 188 189 __id_regexp = re.compile("^[^:]*:[\\s]*(?P<id>[^\\s]*)") 190 """A regular expression for determining test names. 191 192 When applied to an outcome line from DejaGNU, this regular 193 expression's 'id' field gives the name of the test, in the 194 combined mode.""" 195 196 __cause_regexp = re.compile("\$(?P<cause>.*)\$\\s*$") 197 """A regular expression for determining failure causes. 198 199 When applied to an outcome line from DejaGNU, this regular 200 expression's 'cause' field gives the cause of the failure.""" 201

202 - def __init__(self, arguments = None, **args):

203 204 # Initialize the base class. 205 super(DejaGNUReader, self).__init__(arguments, **args) 206 # DejaGNU files start with "Test Run". 207 if self.file.read(len("Test Run")) != "Test Run": 208 raise FileResultReader.InvalidFile, \ 209 "file is not a DejaGNU result stream" 210 self.file.seek(0) 211 if self.__UseCombinedMode(): 212 test_id, dejagnu_outcome, cause = self.__NextOutcome() 213 if test_id: 214 self.__next_result = Result(Result.TEST, test_id) 215 self.__UpdateResult(self.__next_result, 216 dejagnu_outcome, 217 cause)

218 219

220 - def GetResult(self):

221 222 if self.__UseCombinedMode(): 223 result = self.__next_result 224 if not result: 225 return None 226 self.__next_result = None 227 else: 228 result = None 229 while True: 230 test_id, dejagnu_outcome, cause = self.__NextOutcome() 231 # If there are no more results, stop. 232 if not test_id: 233 break 234 if self.__UseCombinedMode() and test_id != result.GetId(): 235 self.__next_result = Result(Result.TEST, test_id) 236 self.__UpdateResult(self.__next_result, 237 dejagnu_outcome, 238 cause) 239 break 240 elif not self.__UseCombinedMode(): 241 result = Result(Result.TEST, test_id) 242 self.__UpdateResult(result, dejagnu_outcome, cause) 243 if not self.__UseCombinedMode(): 244 break 245 return result

246 247

248 - def __NextOutcome(self):

249 """The next DejaGNU outcome in the file. 250 251 returns -- A triplet ('test_id', 'outcome', 'cause'). The 252 'test_id' is the name of the test. The 'outcome' is the 253 DejaGNU outcome (one of the 'DejaGNUTest.dejagnu_outcomes'). 254 The 'cause' is a string giving the cause (if known) of 255 failure, if the test did not pass.""" 256 257 # Assume that there are no more results in the file. 258 dejagnu_outcome = None 259 # Scan ahead until we find a line that gives data about the 260 # next result. 261 while self.file: 262 # Read the next line of the file. 263 line = self.file.next() 264 # Each test result is printed on a line by itself, 265 # beginning with the DejaGNU outcome. For example: 266 # PASS: g++.dg/compat/eh/template1 cp_compat_y_tst.o compile 267 dejagnu_outcome = None 268 for o in DejaGNUTest.dejagnu_outcomes: 269 # Ignore WARNING and ERROR; those are not really test 270 # results. 271 if (o not in (DejaGNUTest.WARNING, 272 DejaGNUTest.ERROR) 273 and line.startswith(o)): 274 o_len = len(o) 275 if line[o_len:o_len + 2] == ": ": 276 dejagnu_outcome = o 277 break 278 if dejagnu_outcome: 279 break 280 # If we could not find any more result lines, then we have 281 # read all of the results in the file. 282 if not dejagnu_outcome: 283 return None, None, None 284 # Extract the name of the test. 285 if self.__UseCombinedMode(): 286 match = self.__id_regexp.search(line) 287 test_id = match.group("id") 288 else: 289 test_id = line[len(dejagnu_outcome) + 2:].strip() 290 # Extract the cause of faiulre. 291 cause = None 292 if "execution test" in line: 293 cause = "Compiled program behaved incorrectly." 294 elif dejagnu_outcome == DejaGNUTest.UNSUPPORTED: 295 cause = "Test is not applicable on this platform." 296 elif self.__UseCombinedMode(): 297 match = self.__cause_regexp.search(line) 298 if match: 299 cause = match.group("cause").capitalize() 300 if cause and cause[-1] != ".": 301 cause += "." 302 else: 303 cause = "" 304 return test_id, dejagnu_outcome, cause

305 306

307 - def __UpdateResult(self, result, dejagnu_outcome, cause):

308 """Update 'result' as indicated. 309 310 'result' -- A 'Result', which may contain information from 311 previous DejaGNU tests, in the combined mode. 312 313 'dejagnu_outcome' -- The DejaGNU outcome (one of the 314 'DejaGNUTest.dejagnu_outcomes') that applies to this 315 'result'. 316 317 'cause' -- The cause of failure, if known. 318 319 The 'result' is modified to reflect the new outcome and 320 cause. Results can only get worse, in the sense that if 321 reuslt has an outcome of 'Result.FAIL' upon entry to this 322 return, it will never have an outcome of 'Result.PASS' upon 323 return.""" 324 325 # Translate the DejaGNU outcome into a QMTest outcome. 326 if self.__GenerateExpectations(): 327 if dejagnu_outcome in (DejaGNUTest.KFAIL, 328 DejaGNUTest.KPASS, 329 DejaGNUTest.XFAIL, 330 DejaGNUTest.XPASS): 331 qmtest_outcome = Result.FAIL 332 elif dejagnu_outcome in (DejaGNUTest.UNSUPPORTED, 333 DejaGNUTest.UNRESOLVED): 334 qmtest_outcome = Result.UNTESTED 335 else: 336 qmtest_outcome = Result.PASS 337 else: 338 qmtest_outcome = DejaGNUTest.outcome_map[dejagnu_outcome] 339 # Update the QMTest result for this test, based on the 340 # DejaGNU result. 341 if qmtest_outcome == Result.ERROR: 342 result.SetOutcome(Result.ERROR) 343 elif (qmtest_outcome == Result.UNTESTED 344 and result.GetOutcome() != Result.ERROR): 345 result.SetOutcome(Result.UNTESTED) 346 elif (qmtest_outcome == Result.FAIL 347 and result.GetOutcome() not in (Result.ERROR, 348 Result.UNTESTED)): 349 result.SetOutcome(Result.FAIL) 350 if qmtest_outcome != Result.PASS and cause: 351 old_cause = result.GetCause() 352 if old_cause and cause in old_cause: 353 # Don't repeat the same cause multiple times. 354 pass 355 else: 356 if old_cause: 357 old_cause += " " 358 old_cause += cgi.escape(cause) 359 result.SetCause(old_cause)

360 361

362 - def __UseCombinedMode(self):

363 """Returns true in the combined mode. 364 365 returns -- True iff results should be read in the combined 366 mode.""" 367 368 return self.is_combined == "true"

369 370

371 - def __GenerateExpectations(self):

372 """Returns true if expected results should be generated. 373 374 returns -- True iff the results generated should reflect 375 expectations, rather than actual results.""" 376 377 return self.expectations == "true"

Source Code for Module qm.test.classes.dejagnu_stream