1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 import os
21 import qm.common
22 import qm.queue
23 from qm.test.base import *
24 import qm.test.cmdline
25 import qm.test.database
26 from qm.test.expectation_database import ExpectationDatabase
27 from qm.test.context import *
28 import qm.xmlutil
29 from result import *
30 import select
31 import sys
32 import time
33
34
35
36
37
39 """A target requested termination of the test loop."""
40
41 pass
42
43
44
46 """A 'ExecutionEngine' executes tests.
47
48 A 'ExecutionEngine' object handles the execution of a collection
49 of tests.
50
51 This class schedules the tests across one or more targets.
52
53 The shedule is determined dynamically as the tests are executed
54 based on which targets are idle and which are not. Therefore, the
55 testing load should be reasonably well balanced, even across a
56 heterogeneous network of testing machines."""
57
58
60 """A '__TestStatus' indicates whether or not a test has been run.
61
62 The 'outcome' slot indicates whether the test has not be queued so
63 that it can be run, has completed, or has not been processed at all.
64
65 If there are tests that have this test as a prerequisite, they are
66 recorded in the 'dependants' slot.
67
68 Ever test passes through the following states, in the following
69 order:
70
71 1. Initial
72
73 A test in this state has not yet been processed. In this state,
74 the 'outcome' slot is 'None'.
75
76 2. Queued
77
78 A test in this state has been placed on the stack of tests
79 waiting to run. In this state, the 'outcome' slot is
80 'QUEUED'. Such a test may be waiting for prerequisites to
81 complete before it can run.
82
83 3. Ready
84
85 A test in this state is ready to run. All prerequisites have
86 completed, and their outcomes were as expected. In this
87 state, the 'outcome' slot is 'READY'.
88
89 4. Finished
90
91 A test in this state has finished running. In this state, the
92 'outcome' slot is one of the 'Result.outcomes'.
93
94 The only exception to this order is that when an error is noted
95 (like a failure to load a test from the database, or a
96 prerequisite has an unexpected outcome) a test may jump to the
97 "finished" state without passing through intermediate states."""
98
99 __slots__ = "outcome", "dependants"
100
101 QUEUED = "QUEUED"
102 READY = "READY"
103
105
106 self.outcome = None
107 self.dependants = None
108
109
111 """Return the state of this test.
112
113 returns -- The state of this test, using the representation
114 documented above."""
115
116 return self.outcome
117
118
120 """Place the test into the "queued" state."""
121
122 assert self.outcome is None
123 self.outcome = self.QUEUED
124
125
127 """Returns true if the test was ever queued.
128
129 returns -- True if the test has ever been on the queue.
130 Such a test may be ready to run, or may in fact have already
131 run to completion."""
132
133 return self.outcome == self.QUEUED or self.HasBeenReady()
134
135
137 """Place the test into the "ready" state."""
138
139 assert self.outcome is self.QUEUED
140 self.outcome = self.READY
141
142
144 """Returns true if the test was ever ready.
145
146 returns -- True if the test was every ready to run. Such a
147 test may have already run to completion."""
148
149 return self.outcome == self.READY or self.IsFinished()
150
151
153 """Returns true if the test is in the "finished" state.
154
155 returns -- True if this test is in the "finished" state."""
156
157 return not (self.outcome is None
158 or self.outcome is self.READY
159 or self.outcome is self.QUEUED)
160
161
163 """Note that 'test_id' depends on 'self'.
164
165 'test_id' -- The name of a test. That test has this test as a
166 prerequisite."""
167
168 if self.dependants is None:
169 self.dependants = [test_id]
170 else:
171 self.dependants.append(test_id)
172
173
174
175
176
177
178 __TARGET_IDLE = "IDLE"
179 __TARGET_BUSY = "BUSY"
180 __TARGET_STARVING = "STARVING"
181
182
183 - def __init__(self,
184 database,
185 test_ids,
186 context,
187 targets,
188 result_streams = None,
189 expectations = None):
190 """Set up a test run.
191
192 'database' -- The 'Database' containing the tests that will be
193 run.
194
195 'test_ids' -- A sequence of IDs of tests to run. Where
196 possible, the tests are started in the order specified.
197
198 'context' -- The context object to use when running tests.
199
200 'targets' -- A sequence of 'Target' objects, representing
201 targets on which tests may be run.
202
203 'result_streams' -- A sequence of 'ResultStream' objects. Each
204 stream will be provided with results as they are available.
205
206 'expectations' -- If not 'None', an ExpectationDatabase object."""
207
208 self.__database = database
209 self.__test_ids = test_ids
210 self.__context = context
211 self.__targets = targets
212 if result_streams is not None:
213 self.__result_streams = result_streams
214 else:
215 self.__result_streams = []
216 if expectations is not None:
217 self.__expectations = expectations
218 else:
219 self.__expectations = ExpectationDatabase(test_database = database)
220
221
222 self.__input_handlers = {}
223
224
225 self.__response_queue = qm.queue.Queue(0)
226
227 self.__running = 0
228
229 self.__any_unexpected_outcomes = 0
230
231
232 self.__terminated = 0
233
234
236 """Request that the execution engine stop executing tests.
237
238 Request that the execution thread be terminated. Termination
239 may take some time; tests that are already running will continue
240 to run, for example."""
241
242 self._Trace("Test loop termination requested.")
243 self.__terminated = 1
244
245
247 """Returns true if termination has been requested.
248
249 returns -- True if no further tests should be executed. If the
250 value is -1, the execution engine should simply terminate
251 gracefully."""
252
253 return self.__terminated
254
255
257 """Run the tests.
258
259 This method runs the tests specified in the __init__
260 function.
261
262 returns -- True if any tests had unexpected outcomes."""
263
264
265 self._WriteInitialAnnotations()
266
267
268 for target in self.__targets:
269 target.Start(self.__response_queue, self)
270
271
272 self._Trace("Starting test loop")
273 try:
274 try:
275 self._RunTests()
276 except:
277 self._Trace("Test loop exited with exception: %s"
278 % str(sys.exc_info()))
279 for rs in self.__result_streams:
280 rs.WriteAnnotation("qmtest.run.aborted", "true")
281 raise
282 finally:
283 self._Trace("Test loop finished.")
284
285
286 self._Trace("Stopping targets.")
287 for target in self.__targets:
288 target.Stop()
289
290
291 self._Trace("Checking for final responses.")
292 while self.__CheckForResponse(wait=0):
293 pass
294
295
296
297 end_time_str = qm.common.format_time_iso()
298 for rs in self.__result_streams:
299 rs.WriteAnnotation("qmtest.run.end_time", end_time_str)
300 rs.Summarize()
301
302 return self.__any_unexpected_outcomes
303
304
316
317
319
320 num_tests = len(self.__test_ids)
321
322
323 self.__num_tests_started = 0
324
325 self.__tests_iterator = iter(self.__test_ids)
326
327
328
329 self.__statuses = {}
330 for id in self.__test_ids:
331 self.__statuses[id] = self.__TestStatus()
332
333
334
335 self.__test_stack = []
336
337
338 self.__ids_on_stack = {}
339
340
341 self.__target_state = {}
342 for target in self.__targets:
343 self.__target_state[target] = self.__TARGET_IDLE
344 self.__has_idle_targets = 1
345
346
347 self.__target_groups = {}
348 for target in self.__targets:
349 self.__target_groups[target.GetGroup()] = None
350 self.__target_groups = self.__target_groups.keys()
351
352
353
354 self.__pattern_ok = {}
355
356 self.__patterns = {}
357
358
359 self.__target_pattern_queues = {}
360
361 while self.__num_tests_started < num_tests:
362
363 if self._IsTerminationRequested():
364 self._Trace("Terminating test loop as requested.")
365 raise TerminationRequested, "Termination requested."
366
367
368 while self.__CheckForResponse(wait=0):
369 pass
370
371
372 if not self.__has_idle_targets:
373
374 self._Trace("All targets are busy -- waiting.")
375 self.__CheckForResponse(wait=1)
376 self._Trace("Response received.")
377 continue
378
379
380
381 self.__has_idle_targets = 0
382 for target in self.__targets:
383 if self.__target_state[target] != self.__TARGET_IDLE:
384 continue
385
386
387 if not self.__FeedTarget(target):
388 self.__target_state[target] = self.__TARGET_STARVING
389 else:
390
391
392 if target.IsIdle():
393 self.__target_state[target] = self.__TARGET_IDLE
394 self.__has_idle_targets = 1
395 else:
396 self.__target_state[target] = self.__TARGET_BUSY
397
398
399
400 self._Trace("Waiting for remaining tests to finish.")
401 while self.__running:
402 self.__CheckForResponse(wait=1)
403
404
406 """Run a test on 'target'
407
408 'target' -- The 'Target' on which the test should be run.
409
410 returns -- True, iff a test could be found to run on 'target'.
411 False otherwise."""
412
413 self._Trace("Looking for a test for target %s" % target.GetName())
414
415
416 for pattern in self.__patterns.get(target.GetGroup(), []):
417 tests = self.__target_pattern_queues.get(pattern, [])
418 if tests:
419 descriptor = tests.pop()
420 break
421 else:
422
423
424 descriptor = self.__FindRunnableTest(target)
425 if descriptor is None:
426
427 return 0
428
429 target_name = target.GetName()
430 test_id = descriptor.GetId()
431 self._Trace("Running %s on %s" % (test_id, target_name))
432 assert self.__statuses[test_id].GetState() == self.__TestStatus.READY
433 self.__num_tests_started += 1
434 self.__running += 1
435 target.RunTest(descriptor, self.__context)
436 return 1
437
438
440 """Return a test that is ready to run.
441
442 'target' -- The 'Target' on which the test will run.
443
444 returns -- the 'TestDescriptor' for the next available ready
445 test, or 'None' if no test could be found that will run on
446 'target'.
447
448 If a test with unsatisfied prerequisites is encountered, the
449 test will be pushed on the stack and the prerequisites processed
450 recursively."""
451
452 while 1:
453 if not self.__test_stack:
454
455
456 try:
457 test_id = self.__tests_iterator.next()
458 except StopIteration:
459
460 return None
461 if self.__statuses[test_id].HasBeenQueued():
462
463
464 continue
465
466 if not self.__AddTestToStack(test_id):
467
468 continue
469 self._Trace("Added new test %s to stack" % test_id)
470
471 descriptor, prereqs = self.__test_stack[-1]
472
473 if prereqs:
474 new_test_id = prereqs.pop()
475
476
477
478
479
480 if self.__statuses[new_test_id].HasBeenQueued():
481
482
483
484 if new_test_id in self.__ids_on_stack:
485 self._Trace("Cycle detected (%s)"
486 % (new_test_id,))
487 self.__AddUntestedResult \
488 (new_test_id,
489 qm.message("dependency cycle"))
490 continue
491 else:
492 self.__AddTestToStack(new_test_id)
493 continue
494 else:
495
496 test_id = descriptor.GetId()
497 del self.__ids_on_stack[test_id]
498 self.__test_stack.pop()
499
500
501
502
503
504
505 if self.__statuses[test_id].HasBeenReady():
506 continue
507
508
509 prereqs = self.__GetPendingPrerequisites(descriptor)
510
511
512
513 if prereqs is None:
514 continue
515
516
517 if prereqs:
518 for p in prereqs:
519 self.__statuses[p].NoteDependant(test_id)
520
521 continue
522
523
524
525 if not target.IsInGroup(descriptor.GetTargetGroup()):
526
527
528 self.__AddToTargetPatternQueue(descriptor)
529 continue
530
531 self.__statuses[descriptor.GetId()].NoteReady()
532 return descriptor
533
534
536 """Adds 'test_id' to the stack of current tests.
537
538 returns -- True if the test was added to the stack; false if the
539 test could not be loaded. In the latter case, an 'UNTESTED'
540 result is recorded for the test."""
541
542 self._Trace("Trying to add %s to stack" % test_id)
543
544
545 self.__statuses[test_id].NoteQueued()
546
547
548 descriptor = self.__GetTestDescriptor(test_id)
549 if not descriptor:
550 return 0
551
552
553
554
555
556 for p in descriptor.GetPrerequisites():
557 if not self.__database.HasTest(p):
558 self.__AddUntestedResult(
559 test_id,
560 qm.message("prerequisite not in database",
561 prerequisite = p)
562 )
563 return 0
564
565
566 prereqs_iter = iter(descriptor.GetPrerequisites())
567 relevant_prereqs = filter(self.__statuses.has_key, prereqs_iter)
568
569
570 self.__ids_on_stack[test_id] = None
571 self.__test_stack.append((descriptor, relevant_prereqs))
572
573 return 1
574
575
577 """A a test to the appropriate target pattern queue.
578
579 'descriptor' -- A 'TestDescriptor'.
580
581 Adds the test to the target pattern queue indicated in the
582 descriptor."""
583
584 test_id = descriptor.GetId()
585 self.__statuses[test_id].NoteReady()
586
587 pattern = descriptor.GetTargetGroup()
588
589
590
591 if not self.__pattern_ok.has_key(pattern):
592 self.__pattern_ok[pattern] = 0
593 for group in self.__target_groups:
594 if re.match(pattern, group):
595 self.__pattern_ok[pattern] = 1
596 patterns = self.__patterns.setdefault(group, [])
597 patterns.append(pattern)
598
599 if not self.__pattern_ok[pattern]:
600 self.__AddUntestedResult(test_id,
601 "No target matching %s." % pattern)
602 return
603
604 queue = self.__target_pattern_queues.setdefault(pattern, [])
605 queue.append(descriptor)
606
607
609 """Return pending prerequisite tests for 'descriptor'.
610
611 'descriptor' -- A 'TestDescriptor'.
612
613 returns -- A list of prerequisite test ids that have to
614 complete, or 'None' if one of the prerequisites had an
615 unexpected outcome."""
616
617 needed = []
618
619 prereqs = descriptor.GetPrerequisites()
620 for prereq_id, outcome in prereqs.iteritems():
621 try:
622 prereq_status = self.__statuses[prereq_id]
623 except KeyError:
624
625
626 continue
627
628 if prereq_status.IsFinished():
629 prereq_outcome = prereq_status.outcome
630 if outcome != prereq_outcome:
631
632 self.__AddUntestedResult \
633 (descriptor.GetId(),
634 qm.message("failed prerequisite"),
635 {'qmtest.prequisite': prereq_id,
636 'qmtest.outcome': prereq_outcome,
637 'qmtest.expected_outcome': outcome })
638 return None
639 else:
640
641 needed.append(prereq_id)
642
643 return needed
644
645
723
724
726 """See if any of the targets have completed a task.
727
728 'wait' -- If false, this function returns immediately if there
729 is no available response. If 'wait' is true, this function
730 continues to wait until a response is available.
731
732 returns -- True iff a response was received."""
733
734 while 1:
735 try:
736
737 result = self.__response_queue.get(0)
738
739 self._Trace("Got %s result for %s from queue."
740 % (result.GetKind(), result.GetId()))
741
742 self.__AddResult(result)
743 if result.GetKind() == Result.TEST:
744 assert self.__running > 0
745 self.__running -= 1
746
747 self._Trace("Recorded result.")
748 return result
749 except qm.queue.Empty:
750
751
752 if not wait:
753 return None
754
755
756 if self.__input_handlers:
757
758
759 fds = self.__input_handlers.keys()
760 fds = select.select (fds, [], [], 0.1)[0]
761 for fd in fds:
762 self.__input_handlers[fd](fd)
763 else:
764 time.sleep(0.1)
765
766
767 continue
768
769
772 """Add a 'Result' indicating that 'test_name' was not run.
773
774 'test_name' -- The label for the test that could not be run.
775
776 'cause' -- A string explaining why the test could not be run.
777
778 'annotations' -- A map from strings to strings giving
779 additional annotations for the result.
780
781 'exc_info' -- If this test could not be tested due to a thrown
782 exception, 'exc_info' is the result of 'sys.exc_info()' when the
783 exception was caught. 'None' otherwise."""
784
785
786 self.__num_tests_started += 1
787
788
789 result = Result(Result.TEST, test_name, annotations = annotations)
790 if exc_info:
791 result.NoteException(exc_info, cause, Result.UNTESTED)
792 else:
793 result.SetOutcome(Result.UNTESTED, cause)
794 self.__AddResult(result)
795
796
797
798
800 """Return the 'TestDescriptor' for 'test_id'.
801
802 returns -- The 'TestDescriptor' for 'test_id', or 'None' if the
803 descriptor could not be loaded.
804
805 If the database cannot load the descriptor, an 'UNTESTED' result
806 is recorded for 'test_id'."""
807
808 try:
809 return self.__database.GetTest(test_id)
810 except:
811 self.__AddUntestedResult(test_id,
812 "Could not load test.",
813 exc_info = sys.exc_info())
814 return None
815
816
825
826
860
861
862
863
864
865
866
867
868