qm.test.classes.command

1 ######################################################################## 2 # 3 # File: command.py 4 # Author: Alex Samuel 5 # Date: 2001-03-24 6 # 7 # Contents: 8 # Test classes for testing command-line programs. 9 # 10 # Copyright (c) 2001, 2002, 2003 by CodeSourcery, LLC. All rights reserved. 11 # 12 # For license terms see the file COPYING. 13 # 14 ######################################################################## 15 16 ######################################################################## 17 # Imports 18 ######################################################################## 19 20 import cPickle 21 import errno 22 import os 23 import qm.common 24 import qm.executable 25 import qm.fields 26 import qm.test.base 27 from qm.test.test import Test 28 from qm.test.result import Result 29 import string 30 import sys 31 import types 32 33 ######################################################################## 34 # Classes 35 ######################################################################## 36

37 -class ExecTestBase(Test):

38 """Check a program's output and exit code. 39 40 An 'ExecTestBase' runs a program and compares its standard output, 41 standard error, and exit code with expected values. The program 42 may be provided with command-line arguments and/or standard 43 input. 44 45 The test passes if the standard output, standard error, and 46 exit code are identical to the expected values.""" 47 48 arguments = [ 49 qm.fields.TextField( 50 name="stdin", 51 title="Standard Input", 52 verbatim="true", 53 multiline="true", 54 description="""The contents of the standard input stream. 55 56 If this field is left blank, the standard input stream will 57 contain no data.""" 58 ), 59 60 qm.fields.SetField(qm.fields.TextField( 61 name="environment", 62 title="Environment", 63 description="""Additional environment variables. 64 65 By default, QMTest runs tests with the same environment that 66 QMTest is running in. If you run tests in parallel, or 67 using a remote machine, the environment variables available 68 will be dependent on the context in which the remote test 69 is executing. 70 71 You may add variables to the environment. Each entry must 72 be of the form 'VAR=VAL'. The program will be run in an 73 environment where the environment variable 'VAR' has the 74 value 'VAL'. If 'VAR' already had a value in the 75 environment, it will be replaced with 'VAL'. 76 77 In addition, QMTest automatically adds an environment 78 variable corresponding to each context property. The name 79 of the environment variable is the name of the context 80 property, prefixed with 'QMV_'. For example, if the value 81 of the context property named 'target' is available in the 82 environment variable 'QMV_target'. Any dots in the context 83 key are replaced by a double-underscore; e.g., 84 "CompilerTable.c_path" will become 85 "QMV_CompilerTable__c_path".""" )), 86 87 qm.fields.IntegerField( 88 name="exit_code", 89 title="Exit Code", 90 description="""The expected exit code. 91 92 Most programs use a zero exit code to indicate success and a 93 non-zero exit code to indicate failure.""" 94 ), 95 96 qm.fields.TextField( 97 name="stdout", 98 title="Standard Output", 99 verbatim="true", 100 multiline="true", 101 description="""The expected contents of the standard output stream. 102 103 If the output written by the program does not match this 104 value, the test will fail.""" 105 ), 106 107 qm.fields.TextField( 108 name="stderr", 109 title="Standard Error", 110 verbatim="true", 111 multiline="true", 112 description="""The expected contents of the standard error stream. 113 114 If the output written by the program does not match this 115 value, the test will fail.""" 116 ), 117 118 qm.fields.IntegerField( 119 name="timeout", 120 title="Timeout", 121 description="""The number of seconds the child program will run. 122 123 If this field is non-negative, it indicates the number of 124 seconds the child program will be permitted to run. If this 125 field is not present, or negative, the child program will be 126 permitted to run for ever.""", 127 default_value = -1, 128 ), 129 ] 130 131

132 - def MakeEnvironment(self, context):

133 """Construct the environment for executing the target program.""" 134 135 # Start with any environment variables that are already present 136 # in the environment. 137 environment = os.environ.copy() 138 # Copy context variables into the environment. 139 for key, value in context.items(): 140 # If the value has unicode type, only transfer 141 # it if it can be cast to str. 142 if isinstance(value, unicode): 143 try: 144 value = str(value) 145 except UnicodeEncodeError: 146 continue 147 if isinstance(value, str): 148 name = "QMV_" + key.replace(".", "__") 149 environment[name] = value 150 # Extract additional environment variable assignments from the 151 # 'Environment' field. 152 for assignment in self.environment: 153 if "=" in assignment: 154 # Break the assignment at the first equals sign. 155 variable, value = string.split(assignment, "=", 1) 156 environment[variable] = value 157 else: 158 raise ValueError, \ 159 qm.error("invalid environment assignment", 160 assignment=assignment) 161 return environment

162 163

164 - def ValidateOutput(self, stdout, stderr, result):

165 """Validate the output of the program. 166 167 'stdout' -- A string containing the data written to the standard output 168 stream. 169 170 'stderr' -- A string containing the data written to the standard error 171 stream. 172 173 'result' -- A 'Result' object. It may be used to annotate 174 the outcome according to the content of stderr. 175 176 returns -- A list of strings giving causes of failure.""" 177 178 causes = [] 179 # Check to see if the standard output matches. 180 if not self.__CompareText(stdout, self.stdout): 181 causes.append("standard output") 182 result["ExecTest.expected_stdout"] = result.Quote(self.stdout) 183 # Check to see if the standard error matches. 184 if not self.__CompareText(stderr, self.stderr): 185 causes.append("standard error") 186 result["ExecTest.expected_stderr"] = result.Quote(self.stderr) 187 188 return causes

189 190

191 - def RunProgram(self, program, arguments, context, result):

192 """Run the 'program'. 193 194 'program' -- The path to the program to run. 195 196 'arguments' -- A list of the arguments to the program. This 197 list must contain a first argument corresponding to 'argv[0]'. 198 199 'context' -- A 'Context' giving run-time parameters to the 200 test. 201 202 'result' -- A 'Result' object. The outcome will be 203 'Result.PASS' when this method is called. The 'result' may be 204 modified by this method to indicate outcomes other than 205 'Result.PASS' or to add annotations.""" 206 207 # Construct the environment. 208 environment = self.MakeEnvironment(context) 209 # Create the executable. 210 if self.timeout >= 0: 211 timeout = self.timeout 212 else: 213 # If no timeout was specified, we sill run this process in a 214 # separate process group and kill the entire process group 215 # when the child is done executing. That means that 216 # orphaned child processes created by the test will be 217 # cleaned up. 218 timeout = -2 219 e = qm.executable.Filter(self.stdin, timeout) 220 # Run it. 221 status = e.Run(arguments, environment, path = program) 222 223 causes = [] 224 # Validate the exit status. 225 if not result.CheckExitStatus('ExecTest.', 'Program', 226 status, self.exit_code): 227 causes.append("exit_code") 228 result["ExecTest.expected_exit_code"] = str(self.exit_code) 229 230 result["ExecTest.stdout"] = result.Quote(e.stdout) 231 result["ExecTest.stderr"] = result.Quote(e.stderr) 232 233 # Validate the output. 234 causes += self.ValidateOutput(e.stdout, e.stderr, result) 235 # If anything went wrong, the test failed. 236 if causes: 237 result.Fail("Unexpected %s." % string.join(causes, ", "))

238 239

240 - def __CompareText(self, s1, s2):

241 """Compare 's1' and 's2', ignoring line endings. 242 243 's1' -- A string. 244 245 's2' -- A string. 246 247 returns -- True if 's1' and 's2' are the same, ignoring 248 differences in line endings.""" 249 250 # The "splitlines" method works independently of the line ending 251 # convention in use. 252 return s1.splitlines() == s2.splitlines()

253 254

255 -class ExecTest(ExecTestBase):

256 """Check a program's output and exit code. 257 258 An 'ExecTest' runs a program by using the 'exec' system call.""" 259 260 arguments = [ 261 qm.fields.TextField( 262 name="program", 263 title="Program", 264 not_empty_text=1, 265 description="""The path to the program. 266 267 This field indicates the path to the program. If it is not 268 an absolute path, the value of the 'PATH' environment 269 variable will be used to search for the program.""" 270 ), 271 272 qm.fields.SetField(qm.fields.TextField( 273 name="arguments", 274 title="Argument List", 275 description="""The command-line arguments. 276 277 If this field is left blank, the program is run without any 278 arguments. 279 280 An implicit 0th argument (the path to the program) is added 281 automatically.""" 282 ))] 283 284 _allow_arg_names_matching_class_vars = 1 285 286

287 - def Run(self, context, result):

288 """Run the test. 289 290 'context' -- A 'Context' giving run-time parameters to the 291 test. 292 293 'result' -- A 'Result' object. The outcome will be 294 'Result.PASS' when this method is called. The 'result' may be 295 modified by this method to indicate outcomes other than 296 'Result.PASS' or to add annotations.""" 297 298 # Was the program not specified? 299 if string.strip(self.program) == "": 300 result.Fail("No program specified.") 301 return 302 303 self.RunProgram(self.program, 304 [ self.program ] + self.arguments, 305 context, result)

306 307 308

309 -class ShellCommandTest(ExecTestBase):

310 """Check a shell command's output and exit code. 311 312 A 'ShellCommandTest' runs the shell and compares its standard 313 output, standard error, and exit code with expected values. The 314 shell may be provided with command-line arguments and/or standard 315 input. 316 317 QMTest determines which shell to use by the following method: 318 319 - If the context contains the property 320 'ShellCommandTest.command_shell', its value is split into 321 an argument list and used. 322 323 - Otherwise, if the '.qmrc' configuration file contains the common 324 property 'command_shell', its value is split into an argument 325 list and used. 326 327 - Otherwise, the default shell for the target system is used. 328 329 """ 330 331 arguments = [ 332 qm.fields.TextField( 333 name="command", 334 title="Command", 335 description="""The arguments to the shell. 336 337 This field contains the arguments that are passed to the 338 shell. It should not contain the path to the shell itself. 339 340 If this field is left blank, the shell is run without 341 arguments.""" 342 ) 343 ] 344 345

346 - def Run(self, context, result):

347 """Run the test. 348 349 'context' -- A 'Context' giving run-time parameters to the 350 test. 351 352 'result' -- A 'Result' object. The outcome will be 353 'Result.PASS' when this method is called. The 'result' may be 354 modified by this method to indicate outcomes other than 355 'Result.PASS' or to add annotations.""" 356 357 # If the context specifies a shell, use it. 358 if context.has_key("ShellCommandTest.command_shell"): 359 # Split the context value to build the argument list. 360 shell = qm.common.split_argument_list( 361 context["ShellCommandTest.command_shell"]) 362 else: 363 # Otherwise, use a platform-specific default. 364 shell = qm.platform.get_shell_for_command() 365 # Append the command at the end of the argument list. 366 arguments = shell + [ self.command ] 367 self.RunProgram(arguments[0], arguments, context, result)

368 369 370

371 -class ShellScriptTest(ExecTestBase):

372 """Check a shell script's output and exit code. 373 374 A 'ShellScriptTest' runs the shell script provided and compares its 375 standard output, standard error, and exit code with expected values. 376 The shell script may be provided with command-line arguments and/or 377 standard input. 378 379 QMTest determines which shell to use by the following method: 380 381 - If the context contains the property 382 'ShellScriptTest.script_shell', its value is split into an 383 argument list and used. 384 385 - Otherwise, if the '.qmrc' configuration file contains the common 386 property 'script_shell', its value is split into an argument 387 list and used. 388 389 - Otherwise, the default shell for the target system is used. 390 391 """ 392 393 arguments = [ 394 qm.fields.TextField( 395 name="script", 396 title="Script", 397 description="""The contents of the shell script. 398 399 Provide the entire shell script here. The script will be 400 written to a temporary file before it is executed. There 401 does not need to be an explicit '#! /path/to/shell' at 402 the beginning of the script because QMTest will not directly 403 invoke the script. Instead, it will run the shell, passing 404 it the name of the temporary file containing the script as 405 an argument.""", 406 verbatim="true", 407 multiline="true", 408 ), 409 qm.fields.SetField(qm.fields.TextField( 410 name="arguments", 411 title="Argument List", 412 description="""The command-line arguments. 413 414 If this field is left blank, the program is run without any 415 arguments. 416 417 An implicit 0th argument (the path to the program) is added 418 automatically.""" 419 )) 420 ] 421 422 _allow_arg_names_matching_class_vars = 1 423 424

425 - def Run(self, context, result):

426 """Run the test. 427 428 'context' -- A 'Context' giving run-time parameters to the 429 test. 430 431 'result' -- A 'Result' object. The outcome will be 432 'Result.PASS' when this method is called. The 'result' may be 433 modified by this method to indicate outcomes other than 434 'Result.PASS' or to add annotations.""" 435 436 # On Windows, batch files must end with a ".bat" suffix or the 437 # command shell will not execute them. 438 if sys.platform == "win32": 439 suffix = ".bat" 440 else: 441 suffix = "" 442 # Create a temporary file for the script. 443 self.__script_file_name, script_file \ 444 = qm.open_temporary_file("w+", suffix) 445 try: 446 # Write the script to the temporary file. 447 script_file.write(self.script) 448 script_file.close() 449 shell = self._GetShell(context) 450 # Construct the argument list. The argument list for the 451 # interpreter is followed by the name of the script 452 # temporary file, and then the arguments to the script. 453 arguments = shell \ 454 + [ self.__script_file_name ] \ 455 + self.arguments 456 self.RunProgram(arguments[0], arguments, context, result) 457 finally: 458 # Clean up the script file. 459 os.remove(self.__script_file_name)

460 461

462 - def _GetShell(self, context):

463 """Return the shell to use to run this test. 464 465 'context' -- As for 'Test.Run'. 466 467 returns -- A sequence of strings giving the path and arguments 468 to be supplied to the shell. The default implementation uses 469 the value of the context property 470 'ShellScriptTest.script_shell', or, if that is not defined, a 471 platform-specific default.""" 472 473 # If the context specifies a shell, use it. 474 if context.has_key("ShellScriptTest.script_shell"): 475 # Split the context value to build the argument list. 476 return qm.common.split_argument_list( 477 context["ShellScriptTest.script_shell"]) 478 479 # Otherwise, use a platform-specific default. 480 return qm.platform.get_shell_for_script()

481 482 483 484 ######################################################################## 485 # Local Variables: 486 # mode: python 487 # indent-tabs-mode: nil 488 # fill-column: 72 489 # End: 490

Source Code for Module qm.test.classes.command