nodes: 0, module; 1, function_definition; 2, function_name:persistence2stats; 3, parameters; 4, block; 5, identifier:rev_docs; 6, default_parameter; 7, default_parameter; 8, default_parameter; 9, default_parameter; 10, default_parameter; 11, expression_statement; 12, expression_statement; 13, expression_statement; 14, expression_statement; 15, expression_statement; 16, expression_statement; 17, for_statement; 18, identifier:min_persisted; 19, integer:5; 20, identifier:min_visible; 21, integer:1209600; 22, identifier:include; 23, None; 24, identifier:exclude; 25, None; 26, identifier:verbose; 27, False; 28, comment:"""
      Processes a sorted and page-partitioned sequence of revision documents into
      and adds statistics to the 'persistence' field each token "added" in the
      revision persisted through future revisions.
  
      :Parameters:
          rev_docs : `iterable` ( `dict` )
              JSON documents of revision data containing a 'diff' field as
              generated by ``dump2diffs``.  It's assumed that rev_docs are
              partitioned by page and otherwise in chronological order.
          window_size : `int`
              The size of the window of revisions from which persistence data
              will be generated.
          min_persisted : `int`
              The minimum future revisions that a token must persist in order
              to be considered "persistent".
          min_visible : `int`
              The minimum number of seconds that a token must be visible in order
              to be considered "persistent".
          include : `func`
              A function that returns `True` when a token should be included in
              statistical processing
          exclude : `str` | `re.SRE_Pattern`
              A function that returns `True` when a token should *not* be
              included in statistical processing (Takes precedence over
              'include')
          verbose : `bool`
              Prints out dots and stuff to stderr
  
      :Returns:
          A generator of rev_docs with a 'persistence' field containing
          statistics about individual tokens.
      """; 29, assignment; 30, assignment; 31, assignment; 32, assignment; 33, assignment; 34, identifier:rev_doc; 35, identifier:rev_docs; 36, block; 37, identifier:rev_docs; 38, call; 39, identifier:min_persisted; 40, call; 41, identifier:min_visible; 42, call; 43, identifier:include; 44, conditional_expression:include if include is not None else lambda t: True; 45, identifier:exclude; 46, conditional_expression:exclude if exclude is not None else lambda t: False; 47, expression_statement; 48, expression_statement; 49, expression_statement; 50, for_statement; 51, if_statement; 52, expression_statement; 53, expression_statement; 54, attribute; 55, argument_list; 56, identifier:int; 57, argument_list; 58, identifier:int; 59, argument_list; 60, identifier:include; 61, comparison_operator:include is not None; 62, lambda; 63, identifier:exclude; 64, comparison_operator:exclude is not None; 65, lambda; 66, assignment; 67, assignment; 68, assignment; 69, identifier:token_doc; 70, identifier:filtered_docs; 71, block; 72, identifier:verbose; 73, block; 74, call; 75, yield; 76, attribute; 77, identifier:normalize; 78, identifier:rev_docs; 79, identifier:min_persisted; 80, identifier:min_visible; 81, identifier:include; 82, None; 83, lambda_parameters; 84, True; 85, identifier:exclude; 86, None; 87, lambda_parameters; 88, False; 89, identifier:persistence_doc; 90, subscript; 91, identifier:stats_doc; 92, dictionary; 93, identifier:filtered_docs; 94, generator_expression; 95, if_statement; 96, expression_statement; 97, expression_statement; 98, expression_statement; 99, expression_statement; 100, comment:# Look for time threshold; 101, if_statement; 102, expression_statement; 103, expression_statement; 104, attribute; 105, argument_list; 106, identifier:rev_doc; 107, identifier:mwxml; 108, identifier:utilities; 109, identifier:t; 110, identifier:t; 111, identifier:rev_doc; 112, string; 113, pair; 114, pair; 115, pair; 116, pair; 117, pair; 118, pair; 119, pair; 120, pair; 121, identifier:t; 122, for_in_clause; 123, if_clause; 124, identifier:verbose; 125, block; 126, augmented_assignment; 127, augmented_assignment; 128, augmented_assignment; 129, augmented_assignment; 130, comparison_operator:token_doc['seconds_visible'] >= min_visible; 131, block; 132, else_clause; 133, call; 134, call; 135, subscript; 136, identifier:update; 137, identifier:stats_doc; 138, string_content:persistence; 139, string; 140, integer:0; 141, string; 142, integer:0; 143, string; 144, integer:0; 145, string; 146, integer:0; 147, string; 148, integer:0; 149, string; 150, integer:0; 151, string; 152, False; 153, string; 154, False; 155, identifier:t; 156, subscript; 157, boolean_operator; 158, expression_statement; 159, expression_statement; 160, subscript; 161, integer:1; 162, subscript; 163, call; 164, subscript; 165, line_continuation:\; 166, call; 167, subscript; 168, line_continuation:\; 169, call; 170, subscript; 171, identifier:min_visible; 172, expression_statement; 173, expression_statement; 174, comment:# Look for review threshold; 175, block; 176, attribute; 177, argument_list; 178, attribute; 179, argument_list; 180, identifier:rev_doc; 181, string; 182, string_content:tokens_added; 183, string_content:persistent_tokens; 184, string_content:non_self_persistent_tokens; 185, string_content:sum_log_persisted; 186, string_content:sum_log_non_self_persisted; 187, string_content:sum_log_seconds_visible; 188, string_content:censored; 189, string_content:non_self_censored; 190, identifier:persistence_doc; 191, string; 192, call; 193, not_operator; 194, call; 195, call; 196, identifier:stats_doc; 197, string; 198, identifier:stats_doc; 199, string; 200, identifier:log; 201, argument_list; 202, identifier:stats_doc; 203, string; 204, identifier:log; 205, argument_list; 206, identifier:stats_doc; 207, string; 208, identifier:log; 209, argument_list; 210, identifier:token_doc; 211, string; 212, augmented_assignment; 213, augmented_assignment; 214, expression_statement; 215, expression_statement; 216, comment:# Check for censoring; 217, if_statement; 218, attribute; 219, identifier:write; 220, string:"\n"; 221, attribute; 222, identifier:flush; 223, string_content:persistence; 224, string_content:tokens; 225, identifier:include; 226, argument_list; 227, call; 228, attribute; 229, argument_list; 230, attribute; 231, argument_list; 232, string_content:tokens_added; 233, string_content:sum_log_persisted; 234, binary_operator:token_doc['persisted'] + 1; 235, string_content:sum_log_non_self_persisted; 236, binary_operator:token_doc['non_self_persisted'] + 1; 237, string_content:sum_log_seconds_visible; 238, binary_operator:token_doc['seconds_visible'] + 1; 239, string_content:seconds_visible; 240, subscript; 241, integer:1; 242, subscript; 243, integer:1; 244, augmented_assignment; 245, augmented_assignment; 246, comparison_operator:persistence_doc['seconds_possible'] < min_visible; 247, block; 248, else_clause; 249, identifier:sys; 250, identifier:stderr; 251, identifier:sys; 252, identifier:stderr; 253, subscript; 254, identifier:exclude; 255, argument_list; 256, attribute; 257, identifier:write; 258, string:"."; 259, attribute; 260, identifier:flush; 261, subscript; 262, integer:1; 263, subscript; 264, integer:1; 265, subscript; 266, integer:1; 267, identifier:stats_doc; 268, string; 269, identifier:stats_doc; 270, string; 271, subscript; 272, line_continuation:\; 273, comparison_operator:token_doc['persisted'] >= min_persisted; 274, subscript; 275, line_continuation:\; 276, comparison_operator:token_doc['non_self_persisted'] >= min_persisted; 277, subscript; 278, identifier:min_visible; 279, expression_statement; 280, expression_statement; 281, block; 282, identifier:t; 283, string; 284, subscript; 285, identifier:sys; 286, identifier:stderr; 287, identifier:sys; 288, identifier:stderr; 289, identifier:token_doc; 290, string; 291, identifier:token_doc; 292, string; 293, identifier:token_doc; 294, string; 295, string_content:persistent_tokens; 296, string_content:non_self_persistent_tokens; 297, identifier:stats_doc; 298, string; 299, subscript; 300, identifier:min_persisted; 301, identifier:stats_doc; 302, string; 303, subscript; 304, identifier:min_persisted; 305, identifier:persistence_doc; 306, string; 307, assignment; 308, assignment; 309, if_statement; 310, if_statement; 311, string_content:text; 312, identifier:t; 313, string; 314, string_content:persisted; 315, string_content:non_self_persisted; 316, string_content:seconds_visible; 317, string_content:persistent_tokens; 318, identifier:token_doc; 319, string; 320, string_content:non_self_persistent_tokens; 321, identifier:token_doc; 322, string; 323, string_content:seconds_possible; 324, subscript; 325, True; 326, subscript; 327, True; 328, comparison_operator:persistence_doc['revisions_processed'] < min_persisted; 329, block; 330, comparison_operator:persistence_doc['non_self_processed'] < min_persisted; 331, block; 332, string_content:text; 333, string_content:persisted; 334, string_content:non_self_persisted; 335, identifier:stats_doc; 336, string; 337, identifier:stats_doc; 338, string; 339, subscript; 340, identifier:min_persisted; 341, expression_statement; 342, subscript; 343, identifier:min_persisted; 344, expression_statement; 345, string_content:censored; 346, string_content:non_self_censored; 347, identifier:persistence_doc; 348, string; 349, assignment; 350, identifier:persistence_doc; 351, string; 352, assignment; 353, string_content:revisions_processed; 354, subscript; 355, True; 356, string_content:non_self_processed; 357, subscript; 358, True; 359, identifier:stats_doc; 360, string; 361, identifier:stats_doc; 362, string; 363, string_content:censored; 364, string_content:non_self_censored
code: def persistence2stats(rev_docs, min_persisted=5, min_visible=1209600,
                        include=None, exclude=None, verbose=False):
      """
      Processes a sorted and page-partitioned sequence of revision documents into
      and adds statistics to the 'persistence' field each token "added" in the
      revision persisted through future revisions.
  
      :Parameters:
          rev_docs : `iterable` ( `dict` )
              JSON documents of revision data containing a 'diff' field as
              generated by ``dump2diffs``.  It's assumed that rev_docs are
              partitioned by page and otherwise in chronological order.
          window_size : `int`
              The size of the window of revisions from which persistence data
              will be generated.
          min_persisted : `int`
              The minimum future revisions that a token must persist in order
              to be considered "persistent".
          min_visible : `int`
              The minimum number of seconds that a token must be visible in order
              to be considered "persistent".
          include : `func`
              A function that returns `True` when a token should be included in
              statistical processing
          exclude : `str` | `re.SRE_Pattern`
              A function that returns `True` when a token should *not* be
              included in statistical processing (Takes precedence over
              'include')
          verbose : `bool`
              Prints out dots and stuff to stderr
  
      :Returns:
          A generator of rev_docs with a 'persistence' field containing
          statistics about individual tokens.
      """
      rev_docs = mwxml.utilities.normalize(rev_docs)
  
      min_persisted = int(min_persisted)
      min_visible = int(min_visible)
      include = include if include is not None else lambda t: True
      exclude = exclude if exclude is not None else lambda t: False
  
      for rev_doc in rev_docs:
          persistence_doc = rev_doc['persistence']
          stats_doc = {
              'tokens_added': 0,
              'persistent_tokens': 0,
              'non_self_persistent_tokens': 0,
              'sum_log_persisted': 0,
              'sum_log_non_self_persisted': 0,
              'sum_log_seconds_visible': 0,
              'censored': False,
              'non_self_censored': False
          }
  
          filtered_docs = (t for t in persistence_doc['tokens']
                           if include(t['text']) and not exclude(t['text']))
          for token_doc in filtered_docs:
              if verbose:
                  sys.stderr.write(".")
                  sys.stderr.flush()
  
              stats_doc['tokens_added'] += 1
              stats_doc['sum_log_persisted'] += log(token_doc['persisted'] + 1)
              stats_doc['sum_log_non_self_persisted'] += \
                  log(token_doc['non_self_persisted'] + 1)
              stats_doc['sum_log_seconds_visible'] += \
                  log(token_doc['seconds_visible'] + 1)
  
              # Look for time threshold
              if token_doc['seconds_visible'] >= min_visible:
                  stats_doc['persistent_tokens'] += 1
                  stats_doc['non_self_persistent_tokens'] += 1
              else:
                  # Look for review threshold
                  stats_doc['persistent_tokens'] += \
                      token_doc['persisted'] >= min_persisted
  
                  stats_doc['non_self_persistent_tokens'] += \
                      token_doc['non_self_persisted'] >= min_persisted
  
                  # Check for censoring
                  if persistence_doc['seconds_possible'] < min_visible:
                      stats_doc['censored'] = True
                      stats_doc['non_self_censored'] = True
  
                  else:
                      if persistence_doc['revisions_processed'] < min_persisted:
                          stats_doc['censored'] = True
  
                      if persistence_doc['non_self_processed'] < min_persisted:
                          stats_doc['non_self_censored'] = True
  
          if verbose:
              sys.stderr.write("\n")
              sys.stderr.flush()
  
          rev_doc['persistence'].update(stats_doc)
  
          yield rev_doc