nodes: 0, module; 1, function_definition; 2, function_name:chooseBestDuplicates; 3, parameters; 4, block; 5, identifier:tped; 6, identifier:samples; 7, identifier:oldSamples; 8, identifier:completion; 9, identifier:concordance_all; 10, identifier:prefix; 11, expression_statement; 12, comment:# The output files; 13, expression_statement; 14, try_statement; 15, print_statement; 16, expression_statement; 17, try_statement; 18, print_statement; 19, comment:# For each duplicated sample; 20, expression_statement; 21, expression_statement; 22, for_statement; 23, comment:# Closing the output files; 24, expression_statement; 25, expression_statement; 26, return_statement; 27, comment:"""Choose the best duplicates according to the completion rate.
  
      :param tped: the ``tped`` containing the duplicated samples.
      :param samples: the updated position of the samples in the tped containing
                      only duplicated samples.
      :param oldSamples: the original duplicated sample positions.
      :param completion: the completion of each of the duplicated samples.
      :param concordance_all: the concordance of every duplicated samples.
      :param prefix: the prefix of all the files.
  
      :type tped: :py:class:`numpy.array`
      :type samples: dict
      :type oldSamples: dict
      :type completion: :py:class:`numpy.array`
      :type concordance_all: dict
      :type prefix: str
  
      :returns: a tuple where the first element is a list of the chosen samples'
                indexes, the second on is the completion and the last one is the
                concordance (a map).
  
      These are the steps to find the best duplicated sample:
  
      1. Sort the list of concordances.
      2. Sort the list of completions.
      3. Choose the best of the concordance and put in a set.
      4. Choose the best of the completion and put it in a set.
      5. Compute the intersection of the two sets. If there is one sample or
         more, then randomly choose one sample.
      6. If the intersection doesn't contain at least one sample, redo steps 3
         and 4, but increase the number of chosen best by one. Redo step 5 and 6
         (if required).
  
      The chosen samples are written in ``prefix.chosen_samples.info``. The rest
      are written in ``prefix.excluded_samples.info``.
  
      """; 28, assignment; 29, block; 30, except_clause; 31, chevron; 32, call; 33, assignment; 34, block; 35, except_clause; 36, chevron; 37, call; 38, assignment; 39, assignment; 40, pattern_list; 41, call; 42, comment:# Getting the completion for those duplicated samples; 43, block; 44, call; 45, call; 46, expression_list; 47, identifier:chosenFile; 48, None; 49, expression_statement; 50, identifier:IOError; 51, block; 52, identifier:chosenFile; 53, attribute; 54, argument_list; 55, identifier:excludedFile; 56, None; 57, expression_statement; 58, identifier:IOError; 59, block; 60, identifier:excludedFile; 61, attribute; 62, argument_list; 63, identifier:chosenIndexes; 64, dictionary; 65, identifier:sampleConcordance; 66, dictionary; 67, identifier:sample; 68, identifier:indexes; 69, attribute; 70, argument_list; 71, expression_statement; 72, comment:# Sorting those completion; 73, expression_statement; 74, comment:# Getting the concordance; 75, expression_statement; 76, expression_statement; 77, for_statement; 78, expression_statement; 79, if_statement; 80, comment:# Sorting the concordance; 81, expression_statement; 82, comment:# Trying to find the best duplicate to keep; 83, expression_statement; 84, expression_statement; 85, while_statement; 86, if_statement; 87, comment:# Printing the chosen samples; 88, print_statement; 89, comment:# Printing the excluded samples; 90, for_statement; 91, expression_statement; 92, attribute; 93, argument_list; 94, attribute; 95, argument_list; 96, identifier:chosenIndexes; 97, identifier:completion; 98, identifier:sampleConcordance; 99, assignment; 100, expression_statement; 101, raise_statement; 102, string:"\t"; 103, identifier:join; 104, list; 105, assignment; 106, expression_statement; 107, raise_statement; 108, string:"\t"; 109, identifier:join; 110, list; 111, identifier:samples; 112, identifier:iteritems; 113, assignment; 114, assignment; 115, assignment; 116, assignment; 117, identifier:i; 118, call; 119, block; 120, assignment; 121, comparison_operator:sample not in sampleConcordance; 122, block; 123, assignment; 124, assignment; 125, assignment; 126, comparison_operator:nbToCheck <= len(indexes); 127, comment:# Getting the `nbToCheck` best value (higher to lower); 128, block; 129, comparison_operator:chosenIndex is None; 130, block; 131, chevron; 132, call; 133, pattern_list; 134, call; 135, block; 136, assignment; 137, identifier:chosenFile; 138, identifier:close; 139, identifier:excludedFile; 140, identifier:close; 141, identifier:chosenFile; 142, call; 143, assignment; 144, call; 145, string:"origIndex"; 146, string:"dupIndex"; 147, string:"famID"; 148, string:"indID"; 149, identifier:excludedFile; 150, call; 151, assignment; 152, call; 153, string:"origIndex"; 154, string:"dupIndex"; 155, string:"famID"; 156, string:"indID"; 157, identifier:currCompletion; 158, subscript; 159, identifier:sortedCompletionIndexes; 160, call; 161, identifier:concordance; 162, subscript; 163, identifier:currConcordance; 164, list_comprehension; 165, identifier:xrange; 166, argument_list; 167, expression_statement; 168, expression_statement; 169, identifier:currConcordance; 170, call; 171, identifier:sample; 172, identifier:sampleConcordance; 173, expression_statement; 174, identifier:sortedConcordanceIndexes; 175, call; 176, identifier:nbToCheck; 177, integer:1; 178, identifier:chosenIndex; 179, None; 180, identifier:nbToCheck; 181, call; 182, expression_statement; 183, expression_statement; 184, comment:# Getting the indexes to consider; 185, expression_statement; 186, expression_statement; 187, comment:# Getting the intersection of the indexes; 188, expression_statement; 189, if_statement; 190, expression_statement; 191, identifier:chosenIndex; 192, None; 193, expression_statement; 194, raise_statement; 195, identifier:chosenFile; 196, attribute; 197, argument_list; 198, identifier:i; 199, identifier:index; 200, identifier:enumerate; 201, argument_list; 202, if_statement; 203, subscript; 204, subscript; 205, identifier:open; 206, argument_list; 207, identifier:msg; 208, binary_operator:"%(prefix)s.chosen_samples.info: can't write file" % locals(); 209, identifier:ProgramError; 210, argument_list; 211, identifier:open; 212, argument_list; 213, identifier:msg; 214, binary_operator:"%(prefix)s.excluded_samples.info: can't write file" % locals(); 215, identifier:ProgramError; 216, argument_list; 217, identifier:completion; 218, identifier:indexes; 219, attribute; 220, argument_list; 221, identifier:concordance_all; 222, identifier:sample; 223, list; 224, for_in_clause; 225, call; 226, assignment; 227, assignment; 228, attribute; 229, argument_list; 230, assignment; 231, attribute; 232, argument_list; 233, identifier:len; 234, argument_list; 235, assignment; 236, assignment; 237, assignment; 238, assignment; 239, assignment; 240, comparison_operator:len(toConsider) >= 1; 241, block; 242, augmented_assignment; 243, assignment; 244, call; 245, string:"\t"; 246, identifier:join; 247, list; 248, identifier:indexes; 249, comparison_operator:i != chosenIndex; 250, block; 251, identifier:chosenIndexes; 252, identifier:sample; 253, identifier:indexes; 254, identifier:chosenIndex; 255, binary_operator:prefix + ".chosen_samples.info"; 256, string:"w"; 257, string:"%(prefix)s.chosen_samples.info: can't write file"; 258, call; 259, identifier:msg; 260, binary_operator:prefix + ".excluded_samples.info"; 261, string:"w"; 262, string:"%(prefix)s.excluded_samples.info: can't write file"; 263, call; 264, identifier:msg; 265, identifier:np; 266, identifier:argsort; 267, identifier:currCompletion; 268, identifier:i; 269, call; 270, identifier:len; 271, argument_list; 272, identifier:indexToKeep; 273, call; 274, subscript; 275, call; 276, identifier:np; 277, identifier:array; 278, identifier:currConcordance; 279, subscript; 280, identifier:currConcordance; 281, identifier:np; 282, identifier:argsort; 283, identifier:currConcordance; 284, identifier:indexes; 285, identifier:completionValue; 286, subscript; 287, identifier:concordanceValue; 288, subscript; 289, identifier:completionToConsider; 290, call; 291, identifier:concordanceToConsider; 292, call; 293, identifier:toConsider; 294, binary_operator:concordanceToConsider & completionToConsider; 295, call; 296, integer:1; 297, expression_statement; 298, break_statement; 299, identifier:nbToCheck; 300, integer:1; 301, identifier:msg; 302, call; 303, identifier:ProgramError; 304, argument_list; 305, call; 306, call; 307, subscript; 308, subscript; 309, identifier:i; 310, identifier:chosenIndex; 311, print_statement; 312, identifier:prefix; 313, string:".chosen_samples.info"; 314, identifier:locals; 315, argument_list; 316, identifier:prefix; 317, string:".excluded_samples.info"; 318, identifier:locals; 319, argument_list; 320, identifier:xrange; 321, argument_list; 322, identifier:indexes; 323, identifier:list; 324, argument_list; 325, identifier:currConcordance; 326, identifier:i; 327, attribute; 328, argument_list; 329, identifier:sampleConcordance; 330, identifier:sample; 331, identifier:currCompletion; 332, subscript; 333, identifier:currConcordance; 334, subscript; 335, identifier:set; 336, argument_list; 337, identifier:set; 338, argument_list; 339, identifier:concordanceToConsider; 340, identifier:completionToConsider; 341, identifier:len; 342, argument_list; 343, assignment; 344, attribute; 345, argument_list; 346, identifier:msg; 347, identifier:str; 348, argument_list; 349, identifier:str; 350, argument_list; 351, identifier:sample; 352, integer:0; 353, identifier:sample; 354, integer:1; 355, chevron; 356, call; 357, call; 358, binary_operator:set(range(len(indexes))) - set([i]); 359, identifier:np; 360, identifier:mean; 361, subscript; 362, identifier:sortedCompletionIndexes; 363, binary_operator:nbToCheck*-1; 364, identifier:sortedConcordanceIndexes; 365, binary_operator:nbToCheck*-1; 366, subscript; 367, subscript; 368, identifier:toConsider; 369, identifier:chosenIndex; 370, call; 371, string:"Could not choose the best sample ID for {}"; 372, identifier:format; 373, identifier:sample; 374, binary_operator:oldSamples[sample][chosenIndex]+1; 375, binary_operator:indexes[chosenIndex]+1; 376, identifier:excludedFile; 377, attribute; 378, argument_list; 379, identifier:len; 380, argument_list; 381, call; 382, call; 383, identifier:concordance; 384, identifier:i; 385, identifier:indexToKeep; 386, identifier:nbToCheck; 387, unary_operator; 388, identifier:nbToCheck; 389, unary_operator; 390, call; 391, integer:0; 392, call; 393, integer:0; 394, attribute; 395, argument_list; 396, subscript; 397, integer:1; 398, subscript; 399, integer:1; 400, string:"\t"; 401, identifier:join; 402, list; 403, identifier:indexes; 404, identifier:set; 405, argument_list; 406, identifier:set; 407, argument_list; 408, integer:1; 409, integer:1; 410, attribute; 411, argument_list; 412, attribute; 413, argument_list; 414, identifier:random; 415, identifier:choice; 416, call; 417, subscript; 418, identifier:chosenIndex; 419, identifier:indexes; 420, identifier:chosenIndex; 421, call; 422, call; 423, subscript; 424, subscript; 425, call; 426, list; 427, identifier:np; 428, identifier:where; 429, comparison_operator:currCompletion >= completionValue; 430, identifier:np; 431, identifier:where; 432, comparison_operator:currConcordance >= concordanceValue; 433, identifier:list; 434, argument_list; 435, identifier:oldSamples; 436, identifier:sample; 437, identifier:str; 438, argument_list; 439, identifier:str; 440, argument_list; 441, identifier:sample; 442, integer:0; 443, identifier:sample; 444, integer:1; 445, identifier:range; 446, argument_list; 447, identifier:i; 448, identifier:currCompletion; 449, identifier:completionValue; 450, identifier:currConcordance; 451, identifier:concordanceValue; 452, identifier:toConsider; 453, binary_operator:oldSamples[sample][i]+1; 454, binary_operator:index+1; 455, call; 456, subscript; 457, integer:1; 458, identifier:index; 459, integer:1; 460, identifier:len; 461, argument_list; 462, subscript; 463, identifier:i; 464, identifier:indexes; 465, identifier:oldSamples; 466, identifier:sample
code: def chooseBestDuplicates(tped, samples, oldSamples, completion,
                           concordance_all, prefix):
      """Choose the best duplicates according to the completion rate.
  
      :param tped: the ``tped`` containing the duplicated samples.
      :param samples: the updated position of the samples in the tped containing
                      only duplicated samples.
      :param oldSamples: the original duplicated sample positions.
      :param completion: the completion of each of the duplicated samples.
      :param concordance_all: the concordance of every duplicated samples.
      :param prefix: the prefix of all the files.
  
      :type tped: :py:class:`numpy.array`
      :type samples: dict
      :type oldSamples: dict
      :type completion: :py:class:`numpy.array`
      :type concordance_all: dict
      :type prefix: str
  
      :returns: a tuple where the first element is a list of the chosen samples'
                indexes, the second on is the completion and the last one is the
                concordance (a map).
  
      These are the steps to find the best duplicated sample:
  
      1. Sort the list of concordances.
      2. Sort the list of completions.
      3. Choose the best of the concordance and put in a set.
      4. Choose the best of the completion and put it in a set.
      5. Compute the intersection of the two sets. If there is one sample or
         more, then randomly choose one sample.
      6. If the intersection doesn't contain at least one sample, redo steps 3
         and 4, but increase the number of chosen best by one. Redo step 5 and 6
         (if required).
  
      The chosen samples are written in ``prefix.chosen_samples.info``. The rest
      are written in ``prefix.excluded_samples.info``.
  
      """
      # The output files
      chosenFile = None
      try:
          chosenFile = open(prefix + ".chosen_samples.info", "w")
      except IOError:
          msg = "%(prefix)s.chosen_samples.info: can't write file" % locals()
          raise ProgramError(msg)
      print >>chosenFile, "\t".join(["origIndex", "dupIndex", "famID", "indID"])
  
      excludedFile = None
      try:
          excludedFile = open(prefix + ".excluded_samples.info", "w")
      except IOError:
          msg = "%(prefix)s.excluded_samples.info: can't write file" % locals()
          raise ProgramError(msg)
      print >>excludedFile, "\t".join(["origIndex", "dupIndex", "famID",
                                       "indID"])
  
      # For each duplicated sample
      chosenIndexes = {}
      sampleConcordance = {}
      for sample, indexes in samples.iteritems():
          # Getting the completion for those duplicated samples
          currCompletion = completion[indexes]
  
          # Sorting those completion
          sortedCompletionIndexes = np.argsort(currCompletion)
  
          # Getting the concordance
          concordance = concordance_all[sample]
          currConcordance = [[] for i in xrange(len(indexes))]
          for i in xrange(len(indexes)):
              indexToKeep = list(set(range(len(indexes))) - set([i]))
              currConcordance[i] = np.mean(concordance[i, indexToKeep])
          currConcordance = np.array(currConcordance)
          if sample not in sampleConcordance:
              sampleConcordance[sample] = currConcordance
  
          # Sorting the concordance
          sortedConcordanceIndexes = np.argsort(currConcordance)
  
          # Trying to find the best duplicate to keep
          nbToCheck = 1
          chosenIndex = None
          while nbToCheck <= len(indexes):
              # Getting the `nbToCheck` best value (higher to lower)
              completionValue = currCompletion[
                  sortedCompletionIndexes[nbToCheck*-1]
              ]
              concordanceValue = currConcordance[
                  sortedConcordanceIndexes[nbToCheck*-1]
              ]
  
              # Getting the indexes to consider
              completionToConsider = set(
                  np.where(currCompletion >= completionValue)[0]
              )
              concordanceToConsider = set(
                  np.where(currConcordance >= concordanceValue)[0])
  
              # Getting the intersection of the indexes
              toConsider = concordanceToConsider & completionToConsider
              if len(toConsider) >= 1:
                  chosenIndex = random.choice(list(toConsider))
                  break
              nbToCheck += 1
  
          if chosenIndex is None:
              msg = "Could not choose the best sample ID for {}".format(sample)
              raise ProgramError(msg)
  
          # Printing the chosen samples
          print >>chosenFile, "\t".join([str(oldSamples[sample][chosenIndex]+1),
                                         str(indexes[chosenIndex]+1), sample[0],
                                         sample[1]])
  
          # Printing the excluded samples
          for i, index in enumerate(indexes):
              if i != chosenIndex:
                  print >>excludedFile, "\t".join([str(oldSamples[sample][i]+1),
                                                   str(index+1), sample[0],
                                                   sample[1]])
  
          chosenIndexes[sample] = indexes[chosenIndex]
  
      # Closing the output files
      chosenFile.close()
      excludedFile.close()
  
      return chosenIndexes, completion, sampleConcordance