nodes: 0, module; 1, function_definition; 2, function_name:usearch_cluster_seqs_ref; 3, parameters; 4, block; 5, identifier:fasta_filepath; 6, default_parameter; 7, default_parameter; 8, default_parameter; 9, default_parameter; 10, default_parameter; 11, default_parameter; 12, default_parameter; 13, default_parameter; 14, default_parameter; 15, default_parameter; 16, default_parameter; 17, default_parameter; 18, default_parameter; 19, default_parameter; 20, default_parameter; 21, default_parameter; 22, default_parameter; 23, expression_statement; 24, if_statement; 25, expression_statement; 26, expression_statement; 27, expression_statement; 28, expression_statement; 29, if_statement; 30, if_statement; 31, expression_statement; 32, if_statement; 33, expression_statement; 34, expression_statement; 35, comment:# Need to create fasta file of all hits (with reference IDs),; 36, comment:# recluster failures if new clusters allowed, and create complete fasta; 37, comment:# file, with unique fasta label IDs.; 38, if_statement; 39, if_statement; 40, return_statement; 41, identifier:output_filepath; 42, None; 43, identifier:percent_id; 44, float:0.97; 45, identifier:sizein; 46, True; 47, identifier:sizeout; 48, True; 49, identifier:w; 50, integer:64; 51, identifier:slots; 52, integer:16769023; 53, identifier:maxrejects; 54, integer:64; 55, identifier:log_name; 56, string:"usearch_cluster_seqs.log"; 57, identifier:usersort; 58, True; 59, identifier:HALT_EXEC; 60, False; 61, identifier:save_intermediate_files; 62, False; 63, identifier:remove_usearch_logs; 64, False; 65, identifier:suppress_new_clusters; 66, False; 67, identifier:refseqs_fp; 68, None; 69, identifier:output_dir; 70, None; 71, identifier:working_dir; 72, None; 73, identifier:rev; 74, False; 75, comment:""" Cluster seqs at percent_id, output consensus fasta
  
      Also appends de novo clustered seqs if suppress_new_clusters is False.
      Forced to handle reference + de novo in hackish fashion as usearch does not
      work as listed in the helpstrings.  Any failures are clustered de novo,
      and given unique cluster IDs.
  
      fasta_filepath = input fasta file, generally a dereplicated fasta
      output_filepath = output reference clustered uc filepath
      percent_id = minimum identity percent.
      sizein = not defined in usearch helpstring
      sizeout = not defined in usearch helpstring
      w = Word length for U-sorting
      slots = Size of compressed index table. Should be prime, e.g. 40000003.
       Should also specify --w, typical is --w 16 or --w 32.
      maxrejects = Max rejected targets, 0=ignore, default 32.
      log_name = string specifying output log name
      usersort = Enable if input fasta not sorted by length purposefully, lest
       usearch will raise an error.  In post chimera checked sequences, the seqs
       are sorted by abundance, so this should be set to True.
      HALT_EXEC: Used for debugging app controller
      save_intermediate_files: Preserve all intermediate files created.
      suppress_new_clusters: Disables de novo OTUs when ref based OTU picking
       enabled.
      refseqs_fp: Filepath for ref based OTU picking
      output_dir: output directory
      rev = search plus and minus strands of sequences
      """; 76, not_operator; 77, block; 78, assignment; 79, assignment; 80, assignment; 81, assignment; 82, identifier:usersort; 83, block; 84, identifier:rev; 85, block; 86, assignment; 87, not_operator; 88, block; 89, assignment; 90, assignment; 91, identifier:suppress_new_clusters; 92, block; 93, else_clause; 94, not_operator; 95, block; 96, expression_list; 97, identifier:output_filepath; 98, expression_statement; 99, identifier:log_filepath; 100, call; 101, identifier:uc_filepath; 102, call; 103, identifier:params; 104, dictionary; 105, identifier:app; 106, call; 107, expression_statement; 108, expression_statement; 109, identifier:data; 110, dictionary; 111, identifier:remove_usearch_logs; 112, expression_statement; 113, identifier:app_result; 114, call; 115, identifier:files_to_remove; 116, list; 117, expression_statement; 118, expression_statement; 119, expression_statement; 120, comment:# Get fasta of successful ref based clusters; 121, block; 122, identifier:save_intermediate_files; 123, expression_statement; 124, identifier:app_result; 125, identifier:output_filepath; 126, assignment; 127, identifier:join; 128, argument_list; 129, identifier:join; 130, argument_list; 131, pair; 132, pair; 133, pair; 134, pair; 135, pair; 136, pair; 137, identifier:Usearch; 138, argument_list; 139, call; 140, call; 141, pair; 142, pair; 143, pair; 144, assignment; 145, identifier:app; 146, argument_list; 147, assignment; 148, assignment; 149, call; 150, expression_statement; 151, expression_statement; 152, comment:# get failures and recluster; 153, expression_statement; 154, expression_statement; 155, comment:# de novo cluster the failures; 156, expression_statement; 157, expression_statement; 158, expression_statement; 159, expression_statement; 160, expression_statement; 161, call; 162, pattern_list; 163, call; 164, identifier:working_dir; 165, identifier:log_name; 166, identifier:working_dir; 167, string:"clustered_seqs_post_chimera.uc"; 168, string; 169, identifier:sizein; 170, string; 171, identifier:sizeout; 172, string; 173, identifier:percent_id; 174, string; 175, identifier:w; 176, string; 177, identifier:slots; 178, string; 179, identifier:maxrejects; 180, identifier:params; 181, keyword_argument; 182, keyword_argument; 183, attribute; 184, argument_list; 185, attribute; 186, argument_list; 187, string; 188, identifier:fasta_filepath; 189, string; 190, identifier:uc_filepath; 191, string; 192, identifier:refseqs_fp; 193, subscript; 194, identifier:log_filepath; 195, identifier:data; 196, identifier:output_fna_filepath; 197, call; 198, pattern_list; 199, call; 200, attribute; 201, argument_list; 202, assignment; 203, assignment; 204, assignment; 205, assignment; 206, assignment; 207, assignment; 208, call; 209, call; 210, call; 211, identifier:remove_files; 212, argument_list; 213, identifier:_; 214, identifier:output_filepath; 215, identifier:mkstemp; 216, argument_list; 217, string_content:--sizein; 218, string_content:--sizeout; 219, string_content:--id; 220, string_content:--w; 221, string_content:--slots; 222, string_content:--maxrejects; 223, identifier:WorkingDir; 224, identifier:working_dir; 225, identifier:HALT_EXEC; 226, identifier:HALT_EXEC; 227, subscript; 228, identifier:on; 229, subscript; 230, identifier:on; 231, string_content:--query; 232, string_content:--uc; 233, string_content:--db; 234, identifier:data; 235, string; 236, identifier:join; 237, argument_list; 238, identifier:output_filepath; 239, identifier:labels_hits; 240, identifier:get_fasta_from_uc_file; 241, argument_list; 242, identifier:files_to_remove; 243, identifier:append; 244, identifier:uc_filepath; 245, identifier:output_fna_clustered; 246, call; 247, pattern_list; 248, line_continuation:\; 249, call; 250, identifier:output_fna_failures; 251, line_continuation:\; 252, call; 253, pattern_list; 254, line_continuation:\; 255, call; 256, pattern_list; 257, line_continuation:\; 258, call; 259, identifier:output_filepath; 260, call; 261, attribute; 262, argument_list; 263, attribute; 264, argument_list; 265, attribute; 266, argument_list; 267, identifier:files_to_remove; 268, keyword_argument; 269, keyword_argument; 270, attribute; 271, string; 272, attribute; 273, string; 274, string_content:--log; 275, identifier:output_dir; 276, string; 277, identifier:fasta_filepath; 278, identifier:uc_filepath; 279, keyword_argument; 280, keyword_argument; 281, keyword_argument; 282, identifier:join; 283, argument_list; 284, identifier:output_filepath_ref_clusters; 285, identifier:labels_hits; 286, identifier:get_fasta_from_uc_file; 287, argument_list; 288, identifier:join; 289, argument_list; 290, identifier:output_filepath_failures; 291, identifier:labels_hits; 292, identifier:get_fasta_from_uc_file; 293, argument_list; 294, identifier:app_result; 295, identifier:output_filepath_clustered_failures; 296, identifier:usearch_cluster_seqs; 297, argument_list; 298, identifier:concatenate_fastas; 299, argument_list; 300, identifier:files_to_remove; 301, identifier:append; 302, identifier:output_fna_clustered; 303, identifier:files_to_remove; 304, identifier:append; 305, identifier:output_fna_failures; 306, identifier:files_to_remove; 307, identifier:append; 308, identifier:output_filepath_clustered_failures; 309, identifier:prefix; 310, string; 311, identifier:suffix; 312, string; 313, identifier:app; 314, identifier:Parameters; 315, string_content:--usersort; 316, identifier:app; 317, identifier:Parameters; 318, string_content:--rev; 319, string_content:ref_clustered_seqs.fasta; 320, identifier:hit_type; 321, string:"H"; 322, identifier:output_dir; 323, identifier:output_dir; 324, identifier:output_fna_filepath; 325, identifier:output_fna_filepath; 326, identifier:output_dir; 327, string; 328, identifier:fasta_filepath; 329, identifier:uc_filepath; 330, keyword_argument; 331, keyword_argument; 332, keyword_argument; 333, identifier:output_dir; 334, string; 335, identifier:fasta_filepath; 336, identifier:uc_filepath; 337, keyword_argument; 338, keyword_argument; 339, keyword_argument; 340, identifier:output_fna_failures; 341, keyword_argument; 342, keyword_argument; 343, keyword_argument; 344, keyword_argument; 345, keyword_argument; 346, keyword_argument; 347, keyword_argument; 348, keyword_argument; 349, keyword_argument; 350, keyword_argument; 351, identifier:output_fna_clustered; 352, identifier:output_fna_failures; 353, keyword_argument; 354, string_content:usearch_cluster_ref_based; 355, string_content:.uc; 356, string_content:ref_clustered_seqs.fasta; 357, identifier:hit_type; 358, string:"H"; 359, identifier:output_dir; 360, identifier:output_dir; 361, identifier:output_fna_filepath; 362, identifier:output_fna_clustered; 363, string_content:ref_clustered_seqs_failures.fasta; 364, identifier:hit_type; 365, string:"N"; 366, identifier:output_dir; 367, identifier:output_dir; 368, identifier:output_fna_filepath; 369, identifier:output_fna_failures; 370, identifier:output_filepath; 371, call; 372, identifier:percent_id; 373, identifier:percent_id; 374, identifier:sizein; 375, identifier:sizein; 376, identifier:sizeout; 377, identifier:sizeout; 378, identifier:w; 379, identifier:w; 380, identifier:slots; 381, identifier:slots; 382, identifier:maxrejects; 383, identifier:maxrejects; 384, identifier:save_intermediate_files; 385, identifier:save_intermediate_files; 386, identifier:remove_usearch_logs; 387, identifier:remove_usearch_logs; 388, identifier:working_dir; 389, identifier:working_dir; 390, identifier:output_concat_filepath; 391, call; 392, identifier:join; 393, argument_list; 394, identifier:join; 395, argument_list; 396, identifier:output_dir; 397, string; 398, identifier:output_dir; 399, string; 400, string_content:clustered_seqs_reference_failures.fasta; 401, string_content:concatenated_reference_denovo_clusters.fasta
code: def usearch_cluster_seqs_ref(
          fasta_filepath,
          output_filepath=None,
          percent_id=0.97,
          sizein=True,
          sizeout=True,
          w=64,
          slots=16769023,
          maxrejects=64,
          log_name="usearch_cluster_seqs.log",
          usersort=True,
          HALT_EXEC=False,
          save_intermediate_files=False,
          remove_usearch_logs=False,
          suppress_new_clusters=False,
          refseqs_fp=None,
          output_dir=None,
          working_dir=None,
          rev=False):
      """ Cluster seqs at percent_id, output consensus fasta
  
      Also appends de novo clustered seqs if suppress_new_clusters is False.
      Forced to handle reference + de novo in hackish fashion as usearch does not
      work as listed in the helpstrings.  Any failures are clustered de novo,
      and given unique cluster IDs.
  
      fasta_filepath = input fasta file, generally a dereplicated fasta
      output_filepath = output reference clustered uc filepath
      percent_id = minimum identity percent.
      sizein = not defined in usearch helpstring
      sizeout = not defined in usearch helpstring
      w = Word length for U-sorting
      slots = Size of compressed index table. Should be prime, e.g. 40000003.
       Should also specify --w, typical is --w 16 or --w 32.
      maxrejects = Max rejected targets, 0=ignore, default 32.
      log_name = string specifying output log name
      usersort = Enable if input fasta not sorted by length purposefully, lest
       usearch will raise an error.  In post chimera checked sequences, the seqs
       are sorted by abundance, so this should be set to True.
      HALT_EXEC: Used for debugging app controller
      save_intermediate_files: Preserve all intermediate files created.
      suppress_new_clusters: Disables de novo OTUs when ref based OTU picking
       enabled.
      refseqs_fp: Filepath for ref based OTU picking
      output_dir: output directory
      rev = search plus and minus strands of sequences
      """
      if not output_filepath:
          _, output_filepath = mkstemp(prefix='usearch_cluster_ref_based',
                                       suffix='.uc')
  
      log_filepath = join(working_dir, log_name)
  
      uc_filepath = join(working_dir, "clustered_seqs_post_chimera.uc")
  
      params = {'--sizein': sizein,
                '--sizeout': sizeout,
                '--id': percent_id,
                '--w': w,
                '--slots': slots,
                '--maxrejects': maxrejects}
  
      app = Usearch(params, WorkingDir=working_dir, HALT_EXEC=HALT_EXEC)
  
      if usersort:
          app.Parameters['--usersort'].on()
      if rev:
          app.Parameters['--rev'].on()
  
      data = {'--query': fasta_filepath,
              '--uc': uc_filepath,
              '--db': refseqs_fp
              }
  
      if not remove_usearch_logs:
          data['--log'] = log_filepath
  
      app_result = app(data)
  
      files_to_remove = []
  
      # Need to create fasta file of all hits (with reference IDs),
      # recluster failures if new clusters allowed, and create complete fasta
      # file, with unique fasta label IDs.
  
      if suppress_new_clusters:
          output_fna_filepath = join(output_dir, 'ref_clustered_seqs.fasta')
          output_filepath, labels_hits = get_fasta_from_uc_file(fasta_filepath,
                                                                uc_filepath, hit_type="H", output_dir=output_dir,
                                                                output_fna_filepath=output_fna_filepath)
  
          files_to_remove.append(uc_filepath)
      else:
          # Get fasta of successful ref based clusters
          output_fna_clustered = join(output_dir, 'ref_clustered_seqs.fasta')
          output_filepath_ref_clusters,  labels_hits =\
              get_fasta_from_uc_file(fasta_filepath, uc_filepath, hit_type="H",
                                     output_dir=output_dir, output_fna_filepath=output_fna_clustered)
  
          # get failures and recluster
          output_fna_failures =\
              join(output_dir, 'ref_clustered_seqs_failures.fasta')
          output_filepath_failures, labels_hits =\
              get_fasta_from_uc_file(fasta_filepath,
                                     uc_filepath, hit_type="N", output_dir=output_dir,
                                     output_fna_filepath=output_fna_failures)
  
          # de novo cluster the failures
          app_result, output_filepath_clustered_failures =\
              usearch_cluster_seqs(output_fna_failures, output_filepath=
                                   join(
                                       output_dir,
                                       'clustered_seqs_reference_failures.fasta'),
                                   percent_id=percent_id, sizein=sizein, sizeout=sizeout, w=w,
                                   slots=slots, maxrejects=maxrejects,
                                   save_intermediate_files=save_intermediate_files,
                                   remove_usearch_logs=remove_usearch_logs, working_dir=working_dir)
  
          output_filepath = concatenate_fastas(output_fna_clustered,
                                               output_fna_failures, output_concat_filepath=join(
                                                   output_dir,
                                                   'concatenated_reference_denovo_clusters.fasta'))
  
          files_to_remove.append(output_fna_clustered)
          files_to_remove.append(output_fna_failures)
          files_to_remove.append(output_filepath_clustered_failures)
  
      if not save_intermediate_files:
          remove_files(files_to_remove)
  
      return app_result, output_filepath