nodes: 0, module; 1, function_definition; 2, function_name:dedupe; 3, parameters; 4, block; 5, identifier:contains_dupes; 6, default_parameter; 7, default_parameter; 8, expression_statement; 9, expression_statement; 10, comment:# iterate over items in *contains_dupes*; 11, for_statement; 12, comment:# uniquify *extractor* list; 13, expression_statement; 14, for_statement; 15, expression_statement; 16, comment:# check that extractor differs from contain_dupes (e.g. duplicates were found); 17, comment:# if not, then return the original list; 18, if_statement; 19, identifier:threshold; 20, integer:70; 21, identifier:scorer; 22, attribute; 23, comment:"""This convenience function takes a list of strings containing duplicates and uses fuzzy matching to identify
      and remove duplicates. Specifically, it uses the process.extract to identify duplicates that
      score greater than a user defined threshold. Then, it looks for the longest item in the duplicate list
      since we assume this item contains the most entity information and returns that. It breaks string
      length ties on an alphabetical sort.
  
      Note: as the threshold DECREASES the number of duplicates that are found INCREASES. This means that the
          returned deduplicated list will likely be shorter. Raise the threshold for fuzzy_dedupe to be less
          sensitive.
  
      Args:
          contains_dupes: A list of strings that we would like to dedupe.
          threshold: the numerical value (0,100) point at which we expect to find duplicates.
              Defaults to 70 out of 100
          scorer: Optional function for scoring matches between the query and
              an individual processed choice. This should be a function
              of the form f(query, choice) -> int.
              By default, fuzz.token_set_ratio() is used and expects both query and
              choice to be strings.
  
      Returns:
          A deduplicated list. For example:
  
              In: contains_dupes = ['Frodo Baggin', 'Frodo Baggins', 'F. Baggins', 'Samwise G.', 'Gandalf', 'Bilbo Baggins']
              In: fuzzy_dedupe(contains_dupes)
              Out: ['Frodo Baggins', 'Samwise G.', 'Bilbo Baggins', 'Gandalf']
          """; 24, assignment; 25, identifier:item; 26, identifier:contains_dupes; 27, comment:# return all duplicate matches found; 28, block; 29, assignment; 30, identifier:e; 31, identifier:extractor; 32, block; 33, assignment; 34, comparison_operator:len(extractor) == len(contains_dupes); 35, block; 36, else_clause; 37, identifier:fuzz; 38, identifier:token_set_ratio; 39, identifier:extractor; 40, list; 41, expression_statement; 42, comment:# filter matches based on the threshold; 43, expression_statement; 44, comment:# if there is only 1 item in *filtered*, no duplicates were found so append to *extracted*; 45, if_statement; 46, identifier:keys; 47, dictionary; 48, expression_statement; 49, identifier:extractor; 50, call; 51, call; 52, call; 53, return_statement; 54, block; 55, assignment; 56, assignment; 57, comparison_operator:len(filtered) == 1; 58, block; 59, else_clause; 60, assignment; 61, attribute; 62, argument_list; 63, identifier:len; 64, argument_list; 65, identifier:len; 66, argument_list; 67, identifier:contains_dupes; 68, return_statement; 69, identifier:matches; 70, call; 71, identifier:filtered; 72, list_comprehension; 73, call; 74, integer:1; 75, expression_statement; 76, comment:# alpha sort; 77, block; 78, subscript; 79, integer:1; 80, identifier:keys; 81, identifier:keys; 82, identifier:extractor; 83, identifier:contains_dupes; 84, identifier:extractor; 85, identifier:extract; 86, argument_list; 87, identifier:x; 88, for_in_clause; 89, if_clause; 90, identifier:len; 91, argument_list; 92, call; 93, expression_statement; 94, comment:# length sort; 95, expression_statement; 96, comment:# take first item as our 'canonical example'; 97, expression_statement; 98, identifier:keys; 99, identifier:e; 100, identifier:item; 101, identifier:contains_dupes; 102, keyword_argument; 103, keyword_argument; 104, identifier:x; 105, identifier:matches; 106, comparison_operator:x[1] > threshold; 107, identifier:filtered; 108, attribute; 109, argument_list; 110, assignment; 111, assignment; 112, call; 113, identifier:limit; 114, None; 115, identifier:scorer; 116, identifier:scorer; 117, subscript; 118, identifier:threshold; 119, identifier:extractor; 120, identifier:append; 121, subscript; 122, identifier:filtered; 123, call; 124, identifier:filter_sort; 125, call; 126, attribute; 127, argument_list; 128, identifier:x; 129, integer:1; 130, subscript; 131, integer:0; 132, identifier:sorted; 133, argument_list; 134, identifier:sorted; 135, argument_list; 136, identifier:extractor; 137, identifier:append; 138, subscript; 139, identifier:filtered; 140, integer:0; 141, identifier:filtered; 142, keyword_argument; 143, identifier:filtered; 144, keyword_argument; 145, keyword_argument; 146, subscript; 147, integer:0; 148, identifier:key; 149, lambda; 150, identifier:key; 151, lambda; 152, identifier:reverse; 153, True; 154, identifier:filter_sort; 155, integer:0; 156, lambda_parameters; 157, subscript; 158, lambda_parameters; 159, call; 160, identifier:x; 161, identifier:x; 162, integer:0; 163, identifier:x; 164, identifier:len; 165, argument_list; 166, subscript; 167, identifier:x; 168, integer:0
code: def dedupe(contains_dupes, threshold=70, scorer=fuzz.token_set_ratio):
      """This convenience function takes a list of strings containing duplicates and uses fuzzy matching to identify
      and remove duplicates. Specifically, it uses the process.extract to identify duplicates that
      score greater than a user defined threshold. Then, it looks for the longest item in the duplicate list
      since we assume this item contains the most entity information and returns that. It breaks string
      length ties on an alphabetical sort.
  
      Note: as the threshold DECREASES the number of duplicates that are found INCREASES. This means that the
          returned deduplicated list will likely be shorter. Raise the threshold for fuzzy_dedupe to be less
          sensitive.
  
      Args:
          contains_dupes: A list of strings that we would like to dedupe.
          threshold: the numerical value (0,100) point at which we expect to find duplicates.
              Defaults to 70 out of 100
          scorer: Optional function for scoring matches between the query and
              an individual processed choice. This should be a function
              of the form f(query, choice) -> int.
              By default, fuzz.token_set_ratio() is used and expects both query and
              choice to be strings.
  
      Returns:
          A deduplicated list. For example:
  
              In: contains_dupes = ['Frodo Baggin', 'Frodo Baggins', 'F. Baggins', 'Samwise G.', 'Gandalf', 'Bilbo Baggins']
              In: fuzzy_dedupe(contains_dupes)
              Out: ['Frodo Baggins', 'Samwise G.', 'Bilbo Baggins', 'Gandalf']
          """
  
      extractor = []
  
      # iterate over items in *contains_dupes*
      for item in contains_dupes:
          # return all duplicate matches found
          matches = extract(item, contains_dupes, limit=None, scorer=scorer)
          # filter matches based on the threshold
          filtered = [x for x in matches if x[1] > threshold]
          # if there is only 1 item in *filtered*, no duplicates were found so append to *extracted*
          if len(filtered) == 1:
              extractor.append(filtered[0][0])
  
          else:
              # alpha sort
              filtered = sorted(filtered, key=lambda x: x[0])
              # length sort
              filter_sort = sorted(filtered, key=lambda x: len(x[0]), reverse=True)
              # take first item as our 'canonical example'
              extractor.append(filter_sort[0][0])
  
      # uniquify *extractor* list
      keys = {}
      for e in extractor:
          keys[e] = 1
      extractor = keys.keys()
  
      # check that extractor differs from contain_dupes (e.g. duplicates were found)
      # if not, then return the original list
      if len(extractor) == len(contains_dupes):
          return contains_dupes
      else:
          return extractor