code: def persistence2stats(rev_docs, min_persisted=5, min_visible=1209600,
                        include=None, exclude=None, verbose=False):
      rev_docs = mwxml.utilities.normalize(rev_docs)
      min_persisted = int(min_persisted)
      min_visible = int(min_visible)
      include = include if include is not None else lambda t: True
      exclude = exclude if exclude is not None else lambda t: False
      for rev_doc in rev_docs:
          persistence_doc = rev_doc['persistence']
          stats_doc = {
              'tokens_added': 0,
              'persistent_tokens': 0,
              'non_self_persistent_tokens': 0,
              'sum_log_persisted': 0,
              'sum_log_non_self_persisted': 0,
              'sum_log_seconds_visible': 0,
              'censored': False,
              'non_self_censored': False
          }
          filtered_docs = (t for t in persistence_doc['tokens']
                           if include(t['text']) and not exclude(t['text']))
          for token_doc in filtered_docs:
              if verbose:
                  sys.stderr.write(".")
                  sys.stderr.flush()
              stats_doc['tokens_added'] += 1
              stats_doc['sum_log_persisted'] += log(token_doc['persisted'] + 1)
              stats_doc['sum_log_non_self_persisted'] += \
                  log(token_doc['non_self_persisted'] + 1)
              stats_doc['sum_log_seconds_visible'] += \
                  log(token_doc['seconds_visible'] + 1)
              if token_doc['seconds_visible'] >= min_visible:
                  stats_doc['persistent_tokens'] += 1
                  stats_doc['non_self_persistent_tokens'] += 1
              else:
                  stats_doc['persistent_tokens'] += \
                      token_doc['persisted'] >= min_persisted
                  stats_doc['non_self_persistent_tokens'] += \
                      token_doc['non_self_persisted'] >= min_persisted
                  if persistence_doc['seconds_possible'] < min_visible:
                      stats_doc['censored'] = True
                      stats_doc['non_self_censored'] = True
                  else:
                      if persistence_doc['revisions_processed'] < min_persisted:
                          stats_doc['censored'] = True
                      if persistence_doc['non_self_processed'] < min_persisted:
                          stats_doc['non_self_censored'] = True
          if verbose:
              sys.stderr.write("\n")
              sys.stderr.flush()
          rev_doc['persistence'].update(stats_doc)
          yield rev_doc