qm.test.result

31 """A 'Result' describes the outcome of a test. 32 33 A 'Result' contains two pieces of data: an outcome and a set 34 of annotations. The outcome indicates whether the test passed 35 or failed. More specifically, the outcome may be one of the 36 following constants: 37 38 'Result.PASS' -- The test passed. 39 40 'Result.FAIL' -- The test failed. 41 42 'Result.ERROR' -- Something went wrong in the process of trying to 43 execute the test. For example, if the Python code implementing 44 the 'Run' method in the test class raised an exception, the 45 outcome would be 'Result.ERROR'. 46 47 'Result.UNTESTED' -- QMTest did not even try to run the test. 48 For example, if a prerequiste was not satisfied, then this outcome 49 will be used.' 50 51 The annotations are a dictionary, mapping strings to strings. 52 53 The indices should be of the form 'class.name' where 'class' is 54 the name of the test class that created the annotation. Any 55 annotations created by QMTest, as opposed to the test class, will 56 have indices of the form 'qmtest.name'. 57 58 The annotation values are HTML. When displayed in the GUI, the 59 HTML is inserted directly into the result page; when the 60 command-line interface is used the HTML is converted to plain 61 text. 62 63 Currently, QMTest recognizes the following built-in annotations: 64 65 'Result.CAUSE' -- For results whose outcome is not 'FAIL', this 66 annotation gives a brief description of why the test failed. The 67 preferred form of this message is a phrase like "Incorrect 68 output." or "Exception thrown." The message should begin with a 69 capital letter and end with a period. Most results formatters 70 will display this information prominently. 71 72 'Result.EXCEPTION' -- If an exeption was thrown during the 73 test execution, a brief description of the exception. 74 75 'Result.TARGET' -- This annotation indicates on which target the 76 test was executed. 77 78 'Result.TRACEBACK' -- If an exeption was thrown during the test 79 execution, a representation of the traceback indicating where 80 the exception was thrown. 81 82 A 'Result' object has methods that allow it to act as a dictionary 83 from annotation names to annotation values. You can directly add 84 an annotation to a 'Result' by writing code of the form 85 'result[CAUSE] = "Exception thrown."'. 86 87 A 'Result' object is also used to describe the outcome of 88 executing either setup or cleanup phase of a 'Resource'.""" 89 90 # Constants for result kinds. 91 92 RESOURCE_SETUP = "resource_setup" 93 RESOURCE_CLEANUP = "resource_cleanup" 94 TEST = "test" 95 96 # Constants for outcomes. 97 98 FAIL = "FAIL" 99 ERROR = "ERROR" 100 UNTESTED = "UNTESTED" 101 PASS = "PASS" 102 103 # Constants for predefined annotations. 104 105 CAUSE = "qmtest.cause" 106 EXCEPTION = "qmtest.exception" 107 RESOURCE = "qmtest.resource" 108 TARGET = "qmtest.target" 109 TRACEBACK = "qmtest.traceback" 110 START_TIME = "qmtest.start_time" 111 END_TIME = "qmtest.end_time" 112 TIMEOUT_DETAIL = "qmtest.timeout_detail" 113 114 # Other class variables. 115 116 kinds = [ RESOURCE_SETUP, RESOURCE_CLEANUP, TEST ] 117 """A list of the possible kinds.""" 118 119 outcomes = [ ERROR, FAIL, UNTESTED, PASS ] 120 """A list of the possible outcomes. 121 122 The order of the 'outcomes' is significant; they are ordered from 123 most interesting to least interesting from the point of view of 124 someone browsing results.""" 125

126 - def __init__(self, kind, id, outcome=PASS, annotations={}):

127 """Construct a new 'Result'. 128 129 'kind' -- The kind of result. The value must be one of the 130 'Result.kinds'. 131 132 'id' -- The label for the test or resource to which this 133 result corresponds. 134 135 'outcome' -- The outcome associated with the test. The value 136 must be one of the 'Result.outcomes'. 137 138 'annotations' -- The annotations associated with the test.""" 139 140 assert kind in Result.kinds 141 assert outcome in Result.outcomes 142 143 self.__kind = kind 144 self.__id = id 145 self.__outcome = outcome 146 self.__annotations = annotations.copy()

147 148

149 - def __getstate__(self):

150 """Return a representation of this result for pickling. 151 152 By using an explicit tuple representation of 'Result's when 153 storing them in a pickle file, we decouple our storage format 154 from internal implementation details (e.g., the names of private 155 variables).""" 156 157 # A tuple containing the data needed to reconstruct a 'Result'. 158 # No part of this structure should ever be a user-defined type, 159 # because that will introduce interdependencies that we want to 160 # avoid. 161 return (self.__kind, 162 self.__id, 163 self.__outcome, 164 self.__annotations)

165 166

167 - def __setstate__(self, pickled_state):

168 """Construct a 'Result' from its pickled form.""" 169 170 if isinstance(pickled_state, dict): 171 # Old style pickle, from before we defined '__getstate__'. 172 # (Notionally, this is version "0".) The state is a 173 # dictionary containing the variables we used to have. 174 self.__kind = pickled_state["_Result__kind"] 175 self.__id = pickled_state["_Result__id"] 176 self.__outcome = pickled_state["_Result__outcome"] 177 self.__annotations = pickled_state["_Result__annotations"] 178 # Also has a key "_Result__context" containing a (probably 179 # invalid) context object, but we discard it. 180 else: 181 assert isinstance(pickled_state, tuple) \ 182 and len(pickled_state) == 4 183 # New style pickle, from after we defined '__getstate__'. 184 # (Notionally, this is version "1".) The state is a tuple 185 # containing the values of the variables we care about. 186 (self.__kind, 187 self.__id, 188 self.__outcome, 189 self.__annotations) = pickled_state

190 191

192 - def GetKind(self):

193 """Return the kind of result this is. 194 195 returns -- The kind of entity (one of the 'kinds') to which 196 this result corresponds.""" 197 198 return self.__kind

199 200

201 - def GetOutcome(self):

202 """Return the outcome associated with the test. 203 204 returns -- The outcome associated with the test. This value 205 will be one of the 'Result.outcomes'.""" 206 207 return self.__outcome

208 209

210 - def SetOutcome(self, outcome, cause = None, annotations = {}):

211 """Set the outcome associated with the test. 212 213 'outcome' -- One of the 'Result.outcomes'. 214 215 'cause' -- If not 'None', this value becomes the value of the 216 'Result.CAUSE' annotation. 217 218 'annotations' -- The annotations are added to the current set 219 of annotations.""" 220 221 assert outcome in Result.outcomes 222 self.__outcome = outcome 223 if cause: 224 self.SetCause(cause) 225 self.Annotate(annotations)

226 227

228 - def Annotate(self, annotations):

229 """Add 'annotations' to the current set of annotations.""" 230 self.__annotations.update(annotations)

231 232

233 - def Fail(self, cause = None, annotations = {}):

234 """Mark the test as failing. 235 236 'cause' -- If not 'None', this value becomes the value of the 237 'Result.CAUSE' annotation. 238 239 'annotations' -- The annotations are added to the current set 240 of annotations.""" 241 242 self.SetOutcome(Result.FAIL, cause, annotations)

243 244

245 - def GetId(self):

246 """Return the label for the test or resource. 247 248 returns -- A label indicating indicating to which test or 249 resource this result corresponds.""" 250 251 return self.__id

252 253

254 - def GetCause(self):

255 """Return the cause of failure, if the test failed. 256 257 returns -- If the test failed, return the cause of the 258 failure, if available.""" 259 260 if self.has_key(Result.CAUSE): 261 return self[Result.CAUSE] 262 else: 263 return ""

264 265

266 - def SetCause(self, cause):

267 """Set the cause of failure. 268 269 'cause' -- A string indicating the cause of failure. Like all 270 annotations, 'cause' will be interested as HTML.""" 271 272 self[Result.CAUSE] = cause

273 274

275 - def Quote(self, string):

276 """Return a version of string suitable for an annotation value. 277 278 Performs appropriate quoting for a string that should be taken 279 verbatim; this includes HTML entity escaping, and addition of 280 <pre> tags. 281 282 'string' -- The verbatim string to be quoted. 283 284 returns -- The quoted string.""" 285 286 return "<pre>%s</pre>" % cgi.escape(string)

287 288

289 - def NoteException(self, 290 exc_info = None, 291 cause = None, 292 outcome = ERROR):

293 """Note that an exception occurred during execution. 294 295 'exc_info' -- A triple, in the same form as that returned 296 from 'sys.exc_info'. If 'None', the value of 'sys.exc_info()' 297 is used instead. 298 299 'cause' -- The value of the 'Result.CAUSE' annotation. If 300 'None', a default message is used. 301 302 'outcome' -- The outcome of the test, now that the exception 303 has occurred. 304 305 A test class can call this method if an exception occurs while 306 the test is being run.""" 307 308 if not exc_info: 309 exc_info = sys.exc_info() 310 311 exception_type = exc_info[0] 312 313 # If no cause was specified, use an appropriate message. 314 if not cause: 315 if exception_type is ContextException: 316 cause = str(exc_info[1]) 317 else: 318 cause = "An exception occurred." 319 320 # For a 'ContextException', indicate which context variable 321 # was invalid. 322 if exception_type is ContextException: 323 self["qmtest.context_variable"] = exc_info[1].key 324 325 self.SetOutcome(outcome, cause) 326 self[Result.EXCEPTION] \ 327 = self.Quote("%s: %s" % exc_info[:2]) 328 self[Result.TRACEBACK] \ 329 = self.Quote(qm.format_traceback(exc_info))

330 331

332 - def CheckExitStatus(self, prefix, desc, status, non_zero_exit_ok = 0):

333 """Check the exit status from a command. 334 335 'prefix' -- The prefix that should be used when creating 336 result annotations. 337 338 'desc' -- A description of the executing program. 339 340 'status' -- The exit status, as returned by 'waitpid'. 341 342 'non_zero_exit_ok' -- True if a non-zero exit code is not 343 considered failure. 344 345 returns -- False if the test failed, true otherwise.""" 346 347 if sys.platform == "win32" or os.WIFEXITED(status): 348 # Obtain the exit code. 349 if sys.platform == "win32": 350 exit_code = status 351 else: 352 exit_code = os.WEXITSTATUS(status) 353 # If the exit code is non-zero, the test fails. 354 if exit_code != 0 and not non_zero_exit_ok: 355 self.Fail("%s failed with exit code %d." % (desc, exit_code)) 356 # Record the exit code in the result. 357 self[prefix + "exit_code"] = str(exit_code) 358 return False 359 360 elif os.WIFSIGNALED(status): 361 # Obtain the signal number. 362 signal = os.WTERMSIG(status) 363 # If the program gets a fatal signal, the test fails . 364 self.Fail("%s received fatal signal %d." % (desc, signal)) 365 self[prefix + "signal"] = str(signal) 366 return False 367 else: 368 # A process should only be able to stop by exiting, or 369 # by being terminated with a signal. 370 assert None 371 372 return True

373 374

375 - def MakeDomNode(self, document):

376 """Generate a DOM element node for this result. 377 378 Note that the context is not represented in the DOM node. 379 380 'document' -- The containing DOM document. 381 382 returns -- The element created.""" 383 384 # The node is a result element. 385 element = document.createElement("result") 386 element.setAttribute("id", self.GetId()) 387 element.setAttribute("kind", self.GetKind()) 388 element.setAttribute("outcome", str(self.GetOutcome())) 389 # Add an annotation element for each annotation. 390 keys = self.keys() 391 keys.sort() 392 for key in keys: 393 value = self[key] 394 annotation_element = document.createElement("annotation") 395 # The annotation name is an attribute. 396 annotation_element.setAttribute("name", str(key)) 397 # The annotation value is contained in a text node. The 398 # data is enclosed in quotes for robustness if the 399 # document is pretty-printed. 400 node = document.createTextNode('"' + str(value) + '"') 401 annotation_element.appendChild(node) 402 # Add the annotation element to the result node. 403 element.appendChild(annotation_element) 404 405 return element

406 407 # These methods allow 'Result' to act like a dictionary of 408 # annotations. 409

410 - def __getitem__(self, key):

411 assert type(key) in types.StringTypes 412 return self.__annotations[key]

413 414

415 - def __setitem__(self, key, value):

416 assert type(key) in types.StringTypes 417 assert type(value) in types.StringTypes 418 self.__annotations[key] = value

419 420

421 - def __delitem__(self, key):

422 assert type(key) in types.StringTypes 423 del self.__annotations[key]

424 425

426 - def get(self, key, default=None):

427 assert type(key) in types.StringTypes 428 return self.__annotations.get(key, default)

429 430

431 - def has_key(self, key):

432 assert type(key) in types.StringTypes 433 return self.__annotations.has_key(key)

434 435

436 - def keys(self):

437 return self.__annotations.keys()

438 439

440 - def items(self):

441 return self.__annotations.items()

Source Code for Module qm.test.result