nodes: 0, module; 1, function_definition; 2, function_name:wngram2idngram; 3, parameters; 4, block; 5, identifier:input_file; 6, identifier:vocab_file; 7, identifier:output_file; 8, default_parameter; 9, default_parameter; 10, default_parameter; 11, default_parameter; 12, default_parameter; 13, default_parameter; 14, default_parameter; 15, default_parameter; 16, expression_statement; 17, expression_statement; 18, if_statement; 19, if_statement; 20, if_statement; 21, if_statement; 22, if_statement; 23, if_statement; 24, if_statement; 25, if_statement; 26, comment:# Ensure that every parameter is of type 'str'; 27, expression_statement; 28, with_statement; 29, expression_statement; 30, expression_statement; 31, if_statement; 32, if_statement; 33, return_statement; 34, identifier:buffersize; 35, integer:100; 36, identifier:hashtablesize; 37, integer:2000000; 38, identifier:files; 39, integer:20; 40, identifier:compress; 41, False; 42, identifier:verbosity; 43, integer:2; 44, identifier:n; 45, integer:3; 46, identifier:write_ascii; 47, False; 48, identifier:fof_size; 49, integer:10; 50, comment:"""
          Takes a word N-gram file and a vocabulary file and lists every id n-gram which occurred in the text, along with its number of occurrences, in either ASCII or binary format.
  
          Note : It is important that the vocabulary file is in alphabetical order. If you are using vocabularies generated by wfreq2vocab then this should not be an issue, as they will already be alphabetically sorted.
      """; 51, assignment; 52, identifier:buffersize; 53, block; 54, identifier:hashtablesize; 55, block; 56, identifier:files; 57, block; 58, identifier:verbosity; 59, block; 60, identifier:n; 61, block; 62, identifier:fof_size; 63, block; 64, identifier:compress; 65, block; 66, identifier:write_ascii; 67, block; 68, assignment; 69, with_clause; 70, block; 71, assignment; 72, call; 73, comparison_operator:exitcode != 0; 74, block; 75, boolean_operator; 76, block; 77, call; 78, identifier:cmd; 79, list; 80, expression_statement; 81, expression_statement; 82, expression_statement; 83, expression_statement; 84, expression_statement; 85, expression_statement; 86, expression_statement; 87, expression_statement; 88, identifier:cmd; 89, list_comprehension; 90, with_item; 91, with_statement; 92, expression_statement; 93, identifier:logger; 94, call; 95, attribute; 96, argument_list; 97, identifier:exitcode; 98, integer:0; 99, raise_statement; 100, comparison_operator:sys.version_info >= (3,); 101, comparison_operator:type(output) is bytes; 102, expression_statement; 103, attribute; 104, argument_list; 105, string; 106, string; 107, call; 108, string; 109, call; 110, call; 111, call; 112, call; 113, call; 114, call; 115, call; 116, call; 117, call; 118, call; 119, for_in_clause; 120, as_pattern; 121, with_clause; 122, block; 123, assignment; 124, attribute; 125, argument_list; 126, identifier:logger; 127, identifier:debug; 128, binary_operator:"Command '%s' returned with exit code '%d'." % (' '.join(cmd), exitcode); 129, call; 130, attribute; 131, tuple; 132, call; 133, identifier:bytes; 134, assignment; 135, identifier:output; 136, identifier:strip; 137, string_content:wngram2idngram; 138, string_content:-vocab; 139, attribute; 140, argument_list; 141, string_content:-idngram; 142, attribute; 143, argument_list; 144, attribute; 145, argument_list; 146, attribute; 147, argument_list; 148, attribute; 149, argument_list; 150, attribute; 151, argument_list; 152, attribute; 153, argument_list; 154, attribute; 155, argument_list; 156, attribute; 157, argument_list; 158, attribute; 159, argument_list; 160, identifier:str; 161, argument_list; 162, identifier:x; 163, identifier:cmd; 164, call; 165, as_pattern_target; 166, with_item; 167, expression_statement; 168, expression_statement; 169, with_statement; 170, identifier:output; 171, call; 172, identifier:logging; 173, identifier:getLogger; 174, identifier:__name__; 175, string:"Command '%s' returned with exit code '%d'."; 176, tuple; 177, identifier:ConversionError; 178, argument_list; 179, identifier:sys; 180, identifier:version_info; 181, integer:3; 182, identifier:type; 183, argument_list; 184, identifier:output; 185, call; 186, attribute; 187, identifier:abspath; 188, identifier:vocab_file; 189, attribute; 190, identifier:abspath; 191, identifier:output_file; 192, identifier:cmd; 193, identifier:extend; 194, list; 195, identifier:cmd; 196, identifier:extend; 197, list; 198, identifier:cmd; 199, identifier:extend; 200, list; 201, identifier:cmd; 202, identifier:extend; 203, list; 204, identifier:cmd; 205, identifier:extend; 206, list; 207, identifier:cmd; 208, identifier:extend; 209, list; 210, identifier:cmd; 211, identifier:append; 212, string; 213, identifier:cmd; 214, identifier:append; 215, string; 216, identifier:x; 217, attribute; 218, argument_list; 219, identifier:output_f; 220, as_pattern; 221, call; 222, call; 223, with_clause; 224, block; 225, attribute; 226, argument_list; 227, call; 228, identifier:exitcode; 229, binary_operator:"'%r' returned with non-zero exit status '%s'" % (cmd, exitcode); 230, identifier:output; 231, attribute; 232, argument_list; 233, identifier:os; 234, identifier:path; 235, identifier:os; 236, identifier:path; 237, string; 238, identifier:buffersize; 239, string; 240, identifier:hashtablesize; 241, string; 242, identifier:files; 243, string; 244, identifier:verbosity; 245, string; 246, identifier:n; 247, string; 248, identifier:fof_size; 249, string_content:-compress; 250, string_content:-write_ascii; 251, identifier:tempfile; 252, identifier:SpooledTemporaryFile; 253, call; 254, as_pattern_target; 255, attribute; 256, argument_list; 257, attribute; 258, argument_list; 259, with_item; 260, with_statement; 261, identifier:output_f; 262, identifier:read; 263, attribute; 264, argument_list; 265, string:"'%r' returned with non-zero exit status '%s'"; 266, tuple; 267, identifier:output; 268, identifier:decode; 269, string; 270, string_content:-buffer; 271, string_content:-hash; 272, string_content:-files; 273, string_content:-verbosity; 274, string_content:-n; 275, string_content:-fof_size; 276, attribute; 277, argument_list; 278, identifier:input_f; 279, identifier:input_f; 280, identifier:write; 281, conditional_expression:text.encode('utf-8') if sys.version_info >= (3,) and type(text) is str else text; 282, identifier:input_f; 283, identifier:seek; 284, integer:0; 285, as_pattern; 286, with_clause; 287, block; 288, string; 289, identifier:join; 290, identifier:cmd; 291, identifier:cmd; 292, identifier:exitcode; 293, string_content:utf-8; 294, identifier:tempfile; 295, identifier:SpooledTemporaryFile; 296, call; 297, boolean_operator; 298, identifier:text; 299, call; 300, as_pattern_target; 301, with_item; 302, expression_statement; 303, string_content:; 304, attribute; 305, argument_list; 306, comparison_operator:sys.version_info >= (3,); 307, comparison_operator:type(text) is str; 308, identifier:output_to_debuglogger; 309, argument_list; 310, identifier:err_f; 311, call; 312, assignment; 313, identifier:text; 314, identifier:encode; 315, string; 316, attribute; 317, tuple; 318, call; 319, identifier:str; 320, identifier:do_in_tempdir; 321, argument_list; 322, identifier:exitcode; 323, call; 324, string_content:utf-8; 325, identifier:sys; 326, identifier:version_info; 327, integer:3; 328, identifier:type; 329, argument_list; 330, attribute; 331, argument_list; 332, identifier:text; 333, identifier:subprocess; 334, identifier:call; 335, identifier:cmd; 336, keyword_argument; 337, keyword_argument; 338, keyword_argument; 339, identifier:stdin; 340, identifier:input_f; 341, identifier:stdout; 342, identifier:output_f; 343, identifier:stderr; 344, identifier:err_f
code: def wngram2idngram(input_file, vocab_file, output_file, buffersize=100, hashtablesize=2000000, files=20, compress=False, verbosity=2, n=3, write_ascii=False, fof_size=10):
      """
          Takes a word N-gram file and a vocabulary file and lists every id n-gram which occurred in the text, along with its number of occurrences, in either ASCII or binary format.
  
          Note : It is important that the vocabulary file is in alphabetical order. If you are using vocabularies generated by wfreq2vocab then this should not be an issue, as they will already be alphabetically sorted.
      """
      cmd = ['wngram2idngram', '-vocab', os.path.abspath(vocab_file),
                               '-idngram', os.path.abspath(output_file)]
      if buffersize:
          cmd.extend(['-buffer', buffersize])
  
      if hashtablesize:
          cmd.extend(['-hash', hashtablesize])
  
      if files:
          cmd.extend(['-files', files])
  
      if verbosity:
          cmd.extend(['-verbosity', verbosity])
  
      if n:
          cmd.extend(['-n', n])
  
      if fof_size:
          cmd.extend(['-fof_size', fof_size])
  
      if compress:
          cmd.append('-compress')
      if write_ascii:
          cmd.append('-write_ascii')
  
      # Ensure that every parameter is of type 'str'
      cmd = [str(x) for x in cmd]
      
      with tempfile.SpooledTemporaryFile() as output_f:
          with tempfile.SpooledTemporaryFile() as input_f:
              input_f.write(text.encode('utf-8') if sys.version_info >= (3,) and type(text) is str else text)
              input_f.seek(0)
              with  output_to_debuglogger() as err_f:
                  with do_in_tempdir():
                      exitcode = subprocess.call(cmd, stdin=input_f, stdout=output_f, stderr=err_f)
          output = output_f.read()
      
      logger = logging.getLogger(__name__)
      logger.debug("Command '%s' returned with exit code '%d'." % (' '.join(cmd), exitcode))
  
      if exitcode != 0:
          raise ConversionError("'%r' returned with non-zero exit status '%s'" % (cmd, exitcode))
  
      if sys.version_info >= (3,) and type(output) is bytes:
          output = output.decode('utf-8')
  
      return output.strip()