nodes: 0, module; 1, function_definition; 2, function_name:diffs2persistence; 3, parameters; 4, block; 5, identifier:rev_docs; 6, default_parameter; 7, default_parameter; 8, default_parameter; 9, default_parameter; 10, expression_statement; 11, expression_statement; 12, expression_statement; 13, expression_statement; 14, expression_statement; 15, comment:# Group the docs by page; 16, expression_statement; 17, for_statement; 18, identifier:window_size; 19, integer:50; 20, identifier:revert_radius; 21, integer:15; 22, identifier:sunset; 23, None; 24, identifier:verbose; 25, False; 26, comment:"""
      Processes a sorted and page-partitioned sequence of revision documents into
      and adds a 'persistence' field to them containing statistics about how each
      token "added" in the revision persisted through future revisions.
  
      :Parameters:
          rev_docs : `iterable` ( `dict` )
              JSON documents of revision data containing a 'diff' field as
              generated by ``dump2diffs``.  It's assumed that rev_docs are
              partitioned by page and otherwise in chronological order.
          window_size : `int`
              The size of the window of revisions from which persistence data
              will be generated.
          revert_radius : `int`
              The number of revisions back that a revert can reference.
          sunset : :class:`mwtypes.Timestamp`
              The date of the database dump we are generating from.  This is
              used to apply a 'time visible' statistic.  If not set, now() will
              be assumed.
          keep_diff : `bool`
              Do not drop the `diff` field from the revision document after
              processing is complete.
          verbose : `bool`
              Prints out dots and stuff to stderr
  
      :Returns:
          A generator of rev_docs with a 'persistence' field containing
          statistics about individual tokens.
      """; 27, assignment; 28, assignment; 29, assignment; 30, assignment; 31, assignment; 32, pattern_list; 33, identifier:page_docs; 34, block; 35, identifier:rev_docs; 36, call; 37, identifier:window_size; 38, call; 39, identifier:revert_radius; 40, call; 41, identifier:sunset; 42, conditional_expression:Timestamp(sunset) if sunset is not None \
                                 else Timestamp(time.time()); 43, identifier:page_docs; 44, call; 45, identifier:page_title; 46, identifier:rev_docs; 47, if_statement; 48, comment:# We need a look-ahead to know how long this revision was visible; 49, expression_statement; 50, comment:# The window allows us to manage memory; 51, expression_statement; 52, comment:# The state does the actual processing work; 53, expression_statement; 54, while_statement; 55, while_statement; 56, if_statement; 57, attribute; 58, argument_list; 59, identifier:int; 60, argument_list; 61, identifier:int; 62, argument_list; 63, call; 64, comparison_operator:sunset is not None; 65, line_continuation:\; 66, call; 67, identifier:groupby; 68, argument_list; 69, identifier:verbose; 70, block; 71, assignment; 72, assignment; 73, assignment; 74, identifier:rev_docs; 75, block; 76, comparison_operator:len(window) > 0; 77, block; 78, identifier:verbose; 79, block; 80, attribute; 81, identifier:normalize; 82, identifier:rev_docs; 83, identifier:window_size; 84, identifier:revert_radius; 85, identifier:Timestamp; 86, argument_list; 87, identifier:sunset; 88, None; 89, identifier:Timestamp; 90, argument_list; 91, identifier:rev_docs; 92, keyword_argument; 93, expression_statement; 94, identifier:rev_docs; 95, call; 96, identifier:window; 97, call; 98, identifier:state; 99, call; 100, expression_statement; 101, expression_statement; 102, if_statement; 103, if_statement; 104, expression_statement; 105, if_statement; 106, call; 107, integer:0; 108, expression_statement; 109, expression_statement; 110, expression_statement; 111, expression_statement; 112, if_statement; 113, expression_statement; 114, identifier:mwxml; 115, identifier:utilities; 116, identifier:sunset; 117, call; 118, identifier:key; 119, lambda; 120, call; 121, identifier:peekable; 122, argument_list; 123, identifier:deque; 124, argument_list; 125, identifier:DiffState; 126, argument_list; 127, assignment; 128, assignment; 129, comparison_operator:next_doc is not None; 130, block; 131, else_clause; 132, comparison_operator:seconds_visible < 0; 133, block; 134, assignment; 135, comparison_operator:len(window) == window_size; 136, comment:# Time to start writing some stats; 137, block; 138, else_clause; 139, identifier:len; 140, argument_list; 141, assignment; 142, assignment; 143, assignment; 144, yield; 145, identifier:verbose; 146, block; 147, call; 148, attribute; 149, argument_list; 150, lambda_parameters; 151, subscript; 152, attribute; 153, argument_list; 154, identifier:rev_docs; 155, keyword_argument; 156, keyword_argument; 157, identifier:rev_doc; 158, call; 159, identifier:next_doc; 160, call; 161, identifier:next_doc; 162, None; 163, expression_statement; 164, block; 165, identifier:seconds_visible; 166, integer:0; 167, expression_statement; 168, expression_statement; 169, pattern_list; 170, line_continuation:\; 171, call; 172, call; 173, identifier:window_size; 174, expression_statement; 175, expression_statement; 176, expression_statement; 177, expression_statement; 178, expression_statement; 179, if_statement; 180, block; 181, identifier:window; 182, pattern_list; 183, call; 184, identifier:persistence; 185, call; 186, subscript; 187, identifier:persistence; 188, identifier:old_doc; 189, expression_statement; 190, expression_statement; 191, attribute; 192, argument_list; 193, identifier:time; 194, identifier:time; 195, identifier:d; 196, subscript; 197, string; 198, attribute; 199, identifier:write; 200, binary_operator:page_title + ": "; 201, identifier:maxlen; 202, identifier:window_size; 203, identifier:revert_radius; 204, identifier:revert_radius; 205, identifier:next; 206, argument_list; 207, attribute; 208, argument_list; 209, assignment; 210, expression_statement; 211, call; 212, assignment; 213, identifier:_; 214, identifier:tokens_added; 215, identifier:_; 216, attribute; 217, argument_list; 218, identifier:len; 219, argument_list; 220, assignment; 221, call; 222, assignment; 223, assignment; 224, yield; 225, identifier:verbose; 226, block; 227, expression_statement; 228, identifier:old_doc; 229, identifier:old_added; 230, attribute; 231, argument_list; 232, identifier:token_persistence; 233, argument_list; 234, identifier:old_doc; 235, string; 236, call; 237, call; 238, attribute; 239, identifier:write; 240, string:"\n"; 241, identifier:d; 242, string; 243, string_content:title; 244, identifier:sys; 245, identifier:stderr; 246, identifier:page_title; 247, string:": "; 248, identifier:rev_docs; 249, identifier:rev_docs; 250, identifier:peek; 251, None; 252, identifier:seconds_visible; 253, binary_operator:Timestamp(next_doc['timestamp']) - \
                                    Timestamp(rev_doc['timestamp']); 254, assignment; 255, attribute; 256, argument_list; 257, identifier:seconds_visible; 258, integer:0; 259, identifier:state; 260, identifier:update_opdocs; 261, subscript; 262, subscript; 263, tuple; 264, identifier:window; 265, pattern_list; 266, subscript; 267, attribute; 268, argument_list; 269, identifier:persistence; 270, call; 271, subscript; 272, identifier:persistence; 273, identifier:old_doc; 274, expression_statement; 275, expression_statement; 276, call; 277, identifier:window; 278, identifier:popleft; 279, identifier:old_doc; 280, identifier:old_added; 281, identifier:window; 282, identifier:sunset; 283, string_content:persistence; 284, attribute; 285, argument_list; 286, attribute; 287, argument_list; 288, identifier:sys; 289, identifier:stderr; 290, string_content:page; 291, call; 292, line_continuation:\; 293, call; 294, identifier:seconds_visible; 295, binary_operator:sunset - Timestamp(rev_doc['timestamp']); 296, identifier:logger; 297, identifier:warn; 298, call; 299, identifier:rev_doc; 300, string; 301, subscript; 302, string; 303, subscript; 304, identifier:seconds_visible; 305, identifier:old_doc; 306, identifier:old_added; 307, identifier:window; 308, integer:0; 309, identifier:window; 310, identifier:append; 311, tuple; 312, identifier:token_persistence; 313, argument_list; 314, identifier:old_doc; 315, string; 316, call; 317, call; 318, attribute; 319, argument_list; 320, attribute; 321, identifier:write; 322, string:"_"; 323, attribute; 324, identifier:flush; 325, identifier:Timestamp; 326, argument_list; 327, identifier:Timestamp; 328, argument_list; 329, identifier:sunset; 330, call; 331, attribute; 332, argument_list; 333, string_content:sha1; 334, identifier:rev_doc; 335, string; 336, string_content:ops; 337, identifier:rev_doc; 338, string; 339, identifier:rev_doc; 340, identifier:tokens_added; 341, identifier:old_doc; 342, identifier:old_added; 343, identifier:window; 344, None; 345, string_content:persistence; 346, attribute; 347, argument_list; 348, attribute; 349, argument_list; 350, identifier:window; 351, identifier:append; 352, tuple; 353, identifier:sys; 354, identifier:stderr; 355, identifier:sys; 356, identifier:stderr; 357, subscript; 358, subscript; 359, identifier:Timestamp; 360, argument_list; 361, string:"Seconds visible {0} is less than zero."; 362, identifier:format; 363, identifier:seconds_visible; 364, string_content:diff; 365, string_content:user; 366, attribute; 367, identifier:write; 368, string:"."; 369, attribute; 370, identifier:flush; 371, identifier:rev_doc; 372, identifier:tokens_added; 373, identifier:next_doc; 374, string; 375, identifier:rev_doc; 376, string; 377, subscript; 378, identifier:sys; 379, identifier:stderr; 380, identifier:sys; 381, identifier:stderr; 382, string_content:timestamp; 383, string_content:timestamp; 384, identifier:rev_doc; 385, string; 386, string_content:timestamp
code: def diffs2persistence(rev_docs, window_size=50, revert_radius=15, sunset=None,
                        verbose=False):
      """
      Processes a sorted and page-partitioned sequence of revision documents into
      and adds a 'persistence' field to them containing statistics about how each
      token "added" in the revision persisted through future revisions.
  
      :Parameters:
          rev_docs : `iterable` ( `dict` )
              JSON documents of revision data containing a 'diff' field as
              generated by ``dump2diffs``.  It's assumed that rev_docs are
              partitioned by page and otherwise in chronological order.
          window_size : `int`
              The size of the window of revisions from which persistence data
              will be generated.
          revert_radius : `int`
              The number of revisions back that a revert can reference.
          sunset : :class:`mwtypes.Timestamp`
              The date of the database dump we are generating from.  This is
              used to apply a 'time visible' statistic.  If not set, now() will
              be assumed.
          keep_diff : `bool`
              Do not drop the `diff` field from the revision document after
              processing is complete.
          verbose : `bool`
              Prints out dots and stuff to stderr
  
      :Returns:
          A generator of rev_docs with a 'persistence' field containing
          statistics about individual tokens.
      """
      rev_docs = mwxml.utilities.normalize(rev_docs)
      window_size = int(window_size)
      revert_radius = int(revert_radius)
      sunset = Timestamp(sunset) if sunset is not None \
                                 else Timestamp(time.time())
  
      # Group the docs by page
      page_docs = groupby(rev_docs, key=lambda d: d['page']['title'])
  
      for page_title, rev_docs in page_docs:
  
          if verbose:
              sys.stderr.write(page_title + ": ")
  
          # We need a look-ahead to know how long this revision was visible
          rev_docs = peekable(rev_docs)
  
          # The window allows us to manage memory
          window = deque(maxlen=window_size)
  
          # The state does the actual processing work
          state = DiffState(revert_radius=revert_radius)
  
          while rev_docs:
              rev_doc = next(rev_docs)
              next_doc = rev_docs.peek(None)
  
              if next_doc is not None:
                  seconds_visible = Timestamp(next_doc['timestamp']) - \
                                    Timestamp(rev_doc['timestamp'])
              else:
                  seconds_visible = sunset - Timestamp(rev_doc['timestamp'])
  
              if seconds_visible < 0:
                  logger.warn("Seconds visible {0} is less than zero."
                              .format(seconds_visible))
                  seconds_visible = 0
  
              _, tokens_added, _ = \
                  state.update_opdocs(rev_doc['sha1'], rev_doc['diff']['ops'],
                                      (rev_doc['user'], seconds_visible))
  
              if len(window) == window_size:
                  # Time to start writing some stats
                  old_doc, old_added = window[0]
                  window.append((rev_doc, tokens_added))
                  persistence = token_persistence(old_doc, old_added, window,
                                                  None)
                  old_doc['persistence'] = persistence
                  yield old_doc
                  if verbose:
                      sys.stderr.write(".")
                      sys.stderr.flush()
              else:
                  window.append((rev_doc, tokens_added))
  
          while len(window) > 0:
              old_doc, old_added = window.popleft()
              persistence = token_persistence(old_doc, old_added, window, sunset)
              old_doc['persistence'] = persistence
              yield old_doc
              if verbose:
                  sys.stderr.write("_")
                  sys.stderr.flush()
  
          if verbose:
              sys.stderr.write("\n")