nodes: 0, module; 1, function_definition; 2, function_name:_frequency_order_transform; 3, parameters; 4, block; 5, identifier:sets; 6, expression_statement; 7, expression_statement; 8, expression_statement; 9, expression_statement; 10, expression_statement; 11, expression_statement; 12, return_statement; 13, comment:"""Transform tokens to integers according to global frequency order.
      This step replaces all original tokens in the sets with integers, and
      helps to speed up subsequent prefix filtering and similarity computation.
      See Section 4.3.2 in the paper "A Primitive Operator for Similarity Joins
      in Data Cleaning" by Chaudhuri et al..
  
      Args:
          sets (list): a list of sets, each entry is an iterable representing a
              set.
  
      Returns:
          sets (list): a list of sets, each entry is a sorted Numpy array with
              integer tokens replacing the tokens in the original set.
          order (dict): a dictionary that maps token to its integer representation
              in the frequency order.
      """; 14, call; 15, assignment; 16, assignment; 17, assignment; 18, call; 19, expression_list; 20, attribute; 21, argument_list; 22, identifier:counts; 23, call; 24, identifier:order; 25, call; 26, identifier:sets; 27, list_comprehension; 28, attribute; 29, argument_list; 30, identifier:sets; 31, identifier:order; 32, identifier:logging; 33, identifier:debug; 34, string:"Applying frequency order transform on tokens..."; 35, identifier:reversed; 36, argument_list; 37, identifier:dict; 38, generator_expression; 39, call; 40, for_in_clause; 41, identifier:logging; 42, identifier:debug; 43, string:"Done applying frequency order."; 44, call; 45, tuple; 46, for_in_clause; 47, attribute; 48, argument_list; 49, identifier:s; 50, identifier:sets; 51, attribute; 52, argument_list; 53, identifier:token; 54, identifier:i; 55, pattern_list; 56, call; 57, identifier:np; 58, identifier:sort; 59, list_comprehension; 60, call; 61, identifier:most_common; 62, identifier:i; 63, tuple_pattern; 64, identifier:enumerate; 65, argument_list; 66, subscript; 67, for_in_clause; 68, identifier:Counter; 69, generator_expression; 70, identifier:token; 71, identifier:_; 72, identifier:counts; 73, identifier:order; 74, identifier:token; 75, identifier:token; 76, identifier:s; 77, identifier:token; 78, for_in_clause; 79, for_in_clause; 80, identifier:s; 81, identifier:sets; 82, identifier:token; 83, identifier:s
code: def _frequency_order_transform(sets):
      """Transform tokens to integers according to global frequency order.
      This step replaces all original tokens in the sets with integers, and
      helps to speed up subsequent prefix filtering and similarity computation.
      See Section 4.3.2 in the paper "A Primitive Operator for Similarity Joins
      in Data Cleaning" by Chaudhuri et al..
  
      Args:
          sets (list): a list of sets, each entry is an iterable representing a
              set.
  
      Returns:
          sets (list): a list of sets, each entry is a sorted Numpy array with
              integer tokens replacing the tokens in the original set.
          order (dict): a dictionary that maps token to its integer representation
              in the frequency order.
      """
      logging.debug("Applying frequency order transform on tokens...")
      counts = reversed(Counter(token for s in sets for token in s).most_common())
      order = dict((token, i) for i, (token, _) in enumerate(counts))
      sets = [np.sort([order[token] for token in s]) for s in sets]
      logging.debug("Done applying frequency order.")
      return sets, order