Source Code for Module qm.test.classes.pickle_result_stream

  1  ######################################################################## 
  2  # 
  3  # File:   pickle_result_stream.py 
  4  # Author: Mark Mitchell 
  5  # Date:   11/25/2002 
  6  # 
  7  # Contents: 
  8  #   PickleResultStream, PickleResultReader 
  9  # 
 10  # Copyright (c) 2002, 2003 by CodeSourcery, LLC.  All rights reserved.  
 11  # 
 12  ######################################################################## 
 13   
 14  ######################################################################## 
 15  # Imports 
 16  ######################################################################## 
 17   
 18  import types 
 19  import cPickle 
 20  import struct 
 21  import qm.fields 
 22  from   qm.test.file_result_stream import FileResultStream 
 23  from   qm.test.file_result_reader import FileResultReader 
 24   
 25  ######################################################################## 
 26  # Constants 
 27  ######################################################################## 
 28   
 29  # A subtlety is that because of how extension classes are loaded, we 
 30  # can't use the standard trick of using a nonce class for our sentinel, 
 31  # because the unpickler won't be able to find the class definition.  But 
 32  # 'None' has no other meaning in our format, so works fine. 
 33  _annotation_sentinel = None 
 34  """The sentinel value that marks the beginning of an annotation.""" 
 35   
 36  # Network byte order, 4 byte unsigned int 
 37  _int_format = "!I" 
 38  _int_size = struct.calcsize(_int_format) 
 39   
 40  ######################################################################## 
 41  # Classes 
 42  ######################################################################## 
 43   
 44 -class PickleResultStream(FileResultStream): 
 45      """A 'PickleResultStream' writes out results as Python pickles. 
 46   
 47      See also 'PickleResultReader', which does the reverse.""" 
 48   
 49      _max_pinned_results = 1000 
 50      """A limit on how many `Result's to pin in memory at once. 
 51   
 52      Pickling an object normally pins it in memory; this is necessary 
 53      to ensure correct behaviour when pickling multiple references to 
 54      the same object.  We know that `Result's can't refer to each 
 55      other, so this pinning is useless overhead; however, clearing the 
 56      cache at every call to `WriteResult' slows down both pickling and 
 57      unpickling by about a factor of two.  As a solution, any given 
 58      `PickleResultStream', will clear its cache after 
 59      `_max_pinned_results' calls to WriteResult.  This cache-clearing 
 60      technique causes a very minor slowdown on small result streams, 
 61      and a substantial speedup on large result streams.""" 
 62   
 63      _format_version = 1 
 64      """The version number of the format we write. 
 65   
 66      This is bumped every time the format is changed, to make sure that 
 67      we can retain backwards compatibility. 
 68   
 69      "Version 0" contains no version number, and is simply a bunch 
 70      of 'Result's pickled one after another. 
 71   
 72      "Version 1", and all later versions, contain a pickled version 
 73      number as the first thing in the file.  In version 1, this is 
 74      followed by a 4-byte unsigned integer in network byte order giving 
 75      the address of the first annotation, followed by the file proper. 
 76      The file proper is composed of a bunch of pickled 'Result's, 
 77      followed by a pickled sentinel value (None), followed by a 4-byte 
 78      unsigned integer in network-byte order, followed by the beginning of 
 79      a new pickle whose first item is a annotation tuple, and following 
 80      items are more 'Result's, and then another sentinel value, and so 
 81      on.  An annotation tuple is a tuple of n items, the first of which 
 82      is a string tagging the type of annotation, and the rest of which 
 83      have an interpretation that depends on the tag found.  The only tag 
 84      currently defined is "annotation", which is followed by two string 
 85      elements giving respectively the key and the value.  The 4-byte 
 86      integers always point to the file address of the next such integer, 
 87      except for the last, which has a value of 0; they are used to 
 88      quickly find all annotations.""" 
 89   
 90      arguments = [ 
 91          qm.fields.IntegerField( 
 92              name = "protocol_version", 
 93              description = """The pickle protocol version to use. 
 94   
 95              There are multiple versions of the pickle protocol; in 
 96              general, higher numbers correspond to faster operation and 
 97              more compact files, but may produce files that cannot be 
 98              understood by older versions of Python. 
 99   
100              As of 2003-06-20, the defined protocol versions are: 
101                 0: Traditional ASCII-only format. 
102                 1: Traditional binary format. 
103                 2: New binary format. 
104                -1: Equivalent to the highest version supported by your 
105                    Python release. 
106              Pickle versions 0 and 1 can be understood by any version 
107              of Python; version 2 pickles can only be created or 
108              understood by Python 2.3 and newer.  (See PEP 307 for 
109              details.) 
110   
111              Currently the default version is 1. 
112   
113              """, 
114              default_value = 1, 
115          ), 
116      ] 
117   
118      _is_binary_file = 1 
119   
120 -    def __init__(self, arguments = None, **args): 
121   
122          # Initialize the base class. 
123          super(PickleResultStream, self).__init__(arguments, **args) 
124          # Create initial pickler. 
125          self._ResetPickler() 
126          # We haven't processed any `Result's yet. 
127          self.__processed = 0 
128   
129          # Write out version number. 
130          self.__pickler.dump(self._format_version) 
131          # We have no previous annotations. 
132          self.__last_annotation = None 
133          # Write out annotation header. 
134          self._WriteAnnotationPtr() 
135   
136   
137 -    def _ResetPickler(self): 
138   
139          self.__pickler = cPickle.Pickler(self.file, self.protocol_version) 
140   
141   
142 -    def _WriteAnnotationPtr(self): 
143   
144          new_annotation = self.file.tell() 
145          if self.__last_annotation is not None: 
146              self.file.seek(self.__last_annotation) 
147              self.file.write(struct.pack(_int_format, new_annotation)) 
148              self.file.seek(new_annotation) 
149          self.file.write(struct.pack(_int_format, 0)) 
150          self.__last_annotation = new_annotation 
151          self._ResetPickler() 
152   
153   
154 -    def WriteAnnotation(self, key, value): 
155   
156          assert isinstance(key, types.StringTypes) 
157          assert isinstance(value, types.StringTypes) 
158          self.__pickler.dump(_annotation_sentinel) 
159          self._WriteAnnotationPtr() 
160          self.__pickler.dump(("annotation", key, value)) 
161   
162   
163 -    def WriteResult(self, result): 
164   
165          self.__pickler.dump(result) 
166          self.__processed += 1 
167          # If enough results have been pickeled, clear the pickling 
168          # cache. 
169          if not self.__processed % self._max_pinned_results: 
170              self.__pickler.clear_memo() 
171   
172   
173               
174 -class PickleResultReader(FileResultReader): 
175      """A 'PickleResultReader' reads in results from pickle files. 
176   
177      See also 'PickleResultStream', which does the reverse.""" 
178   
179 -    def __init__(self, arguments = None, **args): 
180   
181          super(PickleResultReader, self).__init__(arguments, **args) 
182          self._ResetUnpickler() 
183   
184          self._annotations = {} 
185   
186          # Check for a version number 
187          try: 
188              version = self.__unpickler.load() 
189          except (EOFError, cPickle.UnpicklingError): 
190              raise FileResultReader.InvalidFile, \ 
191                    "file is not a pickled result stream" 
192           
193          if not isinstance(version, int): 
194              # Version 0 file, no version number; in fact, we're 
195              # holding a 'Result'.  So we have no metadata to load and 
196              # should just rewind. 
197              self.file.seek(0) 
198              self._ResetUnpickler() 
199          elif version == 1: 
200              self._ReadMetadata() 
201          else: 
202              raise QMException, "Unknown format version %i" % (version,) 
203   
204   
205 -    def _ResetUnpickler(self): 
206   
207          self.__unpickler = cPickle.Unpickler(self.file) 
208   
209   
210 -    def _ReadAddress(self): 
211   
212          raw = self.file.read(_int_size) 
213          return struct.unpack(_int_format, raw)[0] 
214           
215   
216 -    def _ReadMetadata(self): 
217   
218          # We've read in the version number; next few bytes are the 
219          # address of the first annotation. 
220          addr = self._ReadAddress() 
221          # That advanced the read head to the first 'Result'; save this 
222          # spot to return to later. 
223          first_result_addr = self.file.tell() 
224          while addr: 
225              # Go the the address. 
226              self.file.seek(addr) 
227              # First four bytes are the next address. 
228              addr = self._ReadAddress() 
229              # Then we restart the pickle stream... 
230              self._ResetUnpickler() 
231              # ...and read in the annotation here. 
232              annotation_tuple = self.__unpickler.load() 
233              kind = annotation_tuple[0] 
234              if kind == "annotation": 
235                  (key, value) = annotation_tuple[1:] 
236                  self._annotations[key] = value 
237              else: 
238                  print "Unknown annotation type '%s'; ignoring" % (kind,) 
239              # Now loop back and jump to the next address. 
240   
241          # Finally, rewind back to the beginning for the reading of 
242          # 'Result's. 
243          self.file.seek(first_result_addr) 
244          self._ResetUnpickler() 
245   
246   
247 -    def GetAnnotations(self): 
248   
249          return self._annotations 
250   
251   
252 -    def GetResult(self): 
253   
254          while 1: 
255              try: 
256                  thing = self.__unpickler.load() 
257              except (EOFError, cPickle.UnpicklingError): 
258                  # When reading from a StringIO, no EOFError will be 
259                  # raised when the unpickler tries to read from the file. 
260                  # Instead, the unpickler raises UnpicklingError when it 
261                  # tries to unpickle the empty string. 
262                  return None 
263              else: 
264                  if thing is _annotation_sentinel: 
265                      # We're looking for results, but this is an annotation, 
266                      # so skip over it. 
267                      # By skipping past the address... 
268                      self.file.seek(_int_size, 1) 
269                      self._ResetUnpickler() 
270                      # ...and the annotation itself. 
271                      self.__unpickler.noload() 
272                      # Now loop. 
273                  else: 
274                      # We actually got a 'Result'. 
275                      return thing 
276   
277  ######################################################################## 
278  # Local Variables: 
279  # mode: python 
280  # indent-tabs-mode: nil 
281  # fill-column: 72 
282  # End: 
283