Based on the student's answer (input_lines), we grab each associated
        message if its corresponding misunderstanding's count is above the threshold
171133
def show_guidance_msg(self, unique_id, input_lines, hash_key,
                          guidance_flag=False):
        
        if self.load_error:
            print(GUIDANCE_DEFAULT_MSG)
            return EMPTY_MISUCOUNT_TGID_PRNTEDMSG

        response = repr(input_lines)
        self.set_tg()
        log.info("Guidance TG is %d", self.tg_id)

        if self.tg_id == TG_ERROR_VALUE:
            # If self.tg_id == -1, there was an error when trying to access the server
            log.warning("Error when trying to access server. TG == -1")
            print(GUIDANCE_DEFAULT_MSG)
            return EMPTY_MISUCOUNT_TGID_PRNTEDMSG

        lambda_string_key = self.guidance_json[
            'dictTg2Func'].get(str(self.tg_id))

        if not lambda_string_key:
            log.info("Cannot find the correct lambda in the dictionary.")
            print(GUIDANCE_DEFAULT_MSG)
            return EMPTY_MISUCOUNT_TGID_PRNTEDMSG
        log.info("Lambda Group: %s", lambda_string_key)

        lambda_info_misu = lambda_string_key_to_func.get(lambda_string_key)
        if not lambda_info_misu:
            log.info("Cannot find info misU given the lambda string key.")
            print(GUIDANCE_DEFAULT_MSG)
            return EMPTY_MISUCOUNT_TGID_PRNTEDMSG

        shorten_unique_id = assess_id_util.canonicalize(unique_id)
        # Try to get the info dictionary for this question. Maps wrong answer
        # to dictionary
        assess_dict_info = self.guidance_json[
            'dictAssessId2Info'].get(shorten_unique_id)
        if not assess_dict_info:
            log.info("shorten_unique_id %s is not in dictAssessId2Info", repr(shorten_unique_id))
            print(GUIDANCE_DEFAULT_MSG)
            return EMPTY_MISUCOUNT_TGID_PRNTEDMSG

        wa_details = assess_dict_info['dictWA2DictInfo'].get(response)
        if not wa_details:
            log.info("Cannot find the wrong answer in the WA2Dict for this assesment.")
            lst_mis_u = []
        else:
            lst_mis_u = wa_details.get('lstMisU', [])

        # No list of misunderstandings for this wrong answer, default message
        if not lst_mis_u:
            log.info("Cannot find the list of misunderstandings.")

        wa_count_threshold = self.guidance_json['wrongAnsThresh']
        wa_lst_assess_num = assess_dict_info['dictWA2LstAssessNum_WA']
        msg_id_set = set()
        should_skip_propagation = self.tg_id == 3 or self.tg_id == 4

        answerDict, countData = self.get_misUdata()
        prev_responses = answerDict.get(shorten_unique_id, [])

        # Confirm that this WA has not been given before
        seen_before = response in prev_responses
        if seen_before:
            log.info("Answer has been seen before: {}".format(response))
        else:
            answerDict[shorten_unique_id] = prev_responses + [response]
            self.save_misUdata(answerDict, countData)

            # Lookup the list of assessNum and WA related to this wrong answer
            # in the question's dictWA2LstAssessNum_WA
            lst_assess_num = wa_lst_assess_num.get(response, [])
            if not lst_assess_num:
                log.info("Cannot get the lst of assess nums given this reponse.")
            log.debug("Related LST_ASSESS_NUM: %s", lst_assess_num)

            # Check if the current wrong answer is in the question's dictWA2DictInfo
            if wa_details:
                log.info("The current wrong answer (%s) is in dictWA2DictInfo", response)
                # Check in answerDict to see if the student has ever given
                # any of these wrong answers (sourced from dictWA2LstAssessNum_WA)
                num_prev_responses = 1

                for other_num, other_resp in lst_assess_num:
                    # Get assess_id
                    other_id = self.get_aid_from_anum(other_num)
                    log.info("Checking if %s is in answerDict[%s]", other_resp, repr(other_id))
                    if other_resp in answerDict.get(other_id, []):
                        log.debug("%s is in answerDict[%s]", other_resp, repr(other_id))
                        num_prev_responses += 1

                log.info("Has given %d previous responses in lst_assess_num", num_prev_responses)

                if not should_skip_propagation:
                    # Increment countDict by the number of wrong answers seen
                    # for each tag assoicated with this wrong answerDict
                    increment = num_prev_responses
                    for misu in lst_mis_u:
                        log.info("Updating the count of misu: %s by %s", misu, increment)
                        countData[misu] = countData.get(misu, 0) + increment

                for misu in lst_mis_u:
                    log.debug("Misu: %s has count %s", misu, countData.get(misu, 0))
                    if countData.get(misu, 0) >= wa_count_threshold:
                        msg_info = lambda_info_misu(wa_details, misu)
                        if msg_info:
                            msg_id_set.add(msg_info)

            elif not should_skip_propagation:
                # Lookup the lst_mis_u of each wrong answer in the list of wrong
                # answers related to the current wrong answer (lst_assess_num),
                # using dictAssessNum2AssessId
                assess_num_to_aid = self.guidance_json['dictAssessNum2AssessId']
                log.debug("Looking up the lst_misu_u of all related WA")

                # misu -> list of wrong answers for that
                related_misu_tags_dict = {}

                for related_num, related_resp in lst_assess_num:
                    related_aid = assess_num_to_aid.get(related_num)
                    log.info("Getting related resp %s for AID %s", repr(related_aid), related_resp)
                    resp_seen_before = related_resp in answerDict.get(related_aid, [])

                    if not resp_seen_before:
                      continue

                    # Get the lst_misu for this asssigmment
                    related_info = self.guidance_json['dictAssessId2Info'].get(related_aid)
                    if not related_info:
                        log.info("Could not find related id: %s in info dict",
                                 related_aid)
                        continue
                    related_wa_info = related_info['dictWA2DictInfo'].get(related_resp)

                    if not related_info:
                        log.info("Could not find response %s in %s info dict",
                                 related_resp, related_aid)
                        continue

                    related_misu_list = related_wa_info.get('lstMisU', [])
                    log.info("The related MISU list is %s", related_misu_list)

                    for misu in related_misu_list:
                        existing_resps = related_misu_tags_dict.get(misu, [])
                        # Add dictWA2DictInfo to list of responses for this misunderstanding.
                        related_misu_tags_dict[misu] = existing_resps + [related_wa_info]
                        # Increment countDict for each tag in the set of tags for each related resp
                        countData[misu] = countData.get(misu, 0) + 1

                    for misu, lst_wa_info in related_misu_tags_dict.items():
                        if countData.get(misu, 0) >= wa_count_threshold:
                            for wa_info in lst_wa_info:
                                msg_id_set.add(lambda_info_misu(wa_info, misu))
                        else:
                            log.info("misu %s seen %s/%s times",
                                     misu, countData.get(misu, 0), wa_count_threshold)

        self.save_misUdata(answerDict, countData)

        wa_lst_explain_responses = assess_dict_info.get('lstWrongAnsWatch', [])
        if response in wa_lst_explain_responses:
            rationale = self.prompt_with_prob(orig_response=input_lines, prob=1.0)
        else:
            rationale = self.prompt_with_prob(orig_response=input_lines)

        if len(msg_id_set) == 0:
            log.info("No messages to display.")
            print(GUIDANCE_DEFAULT_MSG)
            return (countData, self.tg_id, [], rationale)

        print("\n-- Helpful Hint --")

        printed_out_msgs = []
        for message_id in msg_id_set:
            msg = self.guidance_json['dictId2Msg'].get(str(message_id))
            if msg:
                printed_out_msgs.append(msg)
                print(msg)
                print("-"*18)
            else:
                log.info("{} did not have a message".format(message_id))
        print()
        print(GUIDANCE_DEFAULT_MSG)

        return (countData, self.tg_id, printed_out_msgs, rationale)
353630
def validate_words(word_list):
    '''
    
    '''
    if word_list is None:
        return {}
    elif isinstance(word_list, list):
        if not word_list:
            return {}
        else:
            return set(word for word in word_list if word in WORD_DISTRIBUTION)
    else:
        raise InputError("list variable not passed as argument to validate_words")
955735
function _parseSmartSelfAssessment(lines) {
        /*
          Example 1:

          === START OF READ SMART DATA SECTION ===
          SMART overall-health self-assessment test result: PASSED

          Example 2:

          === START OF READ SMART DATA SECTION ===
          SMART Health Status: OK
        */
        var result = {};
        //ignore the first line
        for (var i = 1; i < lines.length; i+=1) {
            _parseSmartColonLine(lines[i], result);
        }
        return result;
    }
1217820
def assessments(self, words=[], negation=True):
        
        a = []
        m = None # Preceding modifier (i.e., adverb or adjective).
        n = None # Preceding negation (e.g., "not beautiful").
        for w, pos in words:
            # Only assess known words, preferably by part-of-speech tag.
            # Including unknown words (polarity 0.0 and subjectivity 0.0) lowers the average.
            if w is None:
                continue
            if w in self and pos in self[w]:
                p, s, i = self[w][pos]
                # Known word not preceded by a modifier ("good").
                if m is None:
                    a.append(dict(w=[w], p=p, s=s, i=i, n=1, x=self.labeler.get(w)))
                # Known word preceded by a modifier ("really good").
                if m is not None:
                    a[-1]["w"].append(w)
                    a[-1]["p"] = max(-1.0, min(p * a[-1]["i"], +1.0))
                    a[-1]["s"] = max(-1.0, min(s * a[-1]["i"], +1.0))
                    a[-1]["i"] = i
                    a[-1]["x"] = self.labeler.get(w)
                # Known word preceded by a negation ("not really good").
                if n is not None:
                    a[-1]["w"].insert(0, n)
                    a[-1]["i"] = 1.0 / a[-1]["i"]
                    a[-1]["n"] = -1
                # Known word may be a negation.
                # Known word may be modifying the next word (i.e., it is a known adverb).
                m = None
                n = None
                if pos and pos in self.modifiers or any(map(self[w].__contains__, self.modifiers)):
                    m = (w, pos)
                if negation and w in self.negations:
                    n = w
            else:
                # Unknown word may be a negation ("not good").
                if negation and w in self.negations:
                    n = w
                # Unknown word. Retain negation across small words ("not a good").
                elif n and len(w.strip("'")) > 1:
                    n = None
                # Unknown word may be a negation preceded by a modifier ("really not good").
                if n is not None and m is not None and (pos in self.modifiers or self.modifier(m[0])):
                    a[-1]["w"].append(n)
                    a[-1]["n"] = -1
                    n = None
                # Unknown word. Retain modifier across small words ("really is a good").
                elif m and len(w) > 2:
                    m = None
                # Exclamation marks boost previous word.
                if w == "!" and len(a) > 0:
                    a[-1]["w"].append("!")
                    a[-1]["p"] = max(-1.0, min(a[-1]["p"] * 1.25, +1.0))
                # Exclamation marks in parentheses indicate sarcasm.
                if w == "(!)":
                    a.append(dict(w=[w], p=0.0, s=1.0, i=1.0, n=1, x=IRONY))
                # EMOTICONS: {("grin", +1.0): set((":-D", ":D"))}
                if w.isalpha() is False and len(w) <= 5 and w not in PUNCTUATION: # speedup
                    for (type, p), e in EMOTICONS.items():
                        if w in map(lambda e: e.lower(), e):
                            a.append(dict(w=[w], p=p, s=1.0, i=1.0, n=1, x=MOOD))
                            break
        for i in range(len(a)):
            w = a[i]["w"]
            p = a[i]["p"]
            s = a[i]["s"]
            n = a[i]["n"]
            x = a[i]["x"]
            # "not good" = slightly bad, "not bad" = slightly good.
            a[i] = (w, p * -0.5 if n < 0 else p, s, x)
        return a
323048
def _evaluate(dataset, predictions):
    '''
    
    '''
    f1_result = exact_match = total = 0
    count = 0
    for article in dataset:
        for paragraph in article['paragraphs']:
            for qa_pair in paragraph['qas']:
                total += 1
                if qa_pair['id'] not in predictions:
                    count += 1
                    continue
                ground_truths = list(map(lambda x: x['text'], qa_pair['answers']))
                prediction = predictions[qa_pair['id']]
                exact_match += metric_max_over_ground_truths(
                    exact_match_score, prediction, ground_truths)
                f1_result += metric_max_over_ground_truths(
                    f1_score, prediction, ground_truths)
    print('total', total, 'exact_match', exact_match, 'unanswer_question ', count)
    exact_match = 100.0 * exact_match / total
    f1_result = 100.0 * f1_result / total
    return {'exact_match': exact_match, 'f1': f1_result}
1241916
def infer_msg(self, tts, rsp):
        
        # rsp should be <requests.Response>
        # http://docs.python-requests.org/en/master/api/
        status = rsp.status_code
        reason = rsp.reason

        cause = "Unknown"
        if status == 403:
            cause = "Bad token or upstream API changes"
        elif status == 404 and not tts.lang_check:
            cause = "Unsupported language '%s'" % self.tts.lang
        elif status >= 500:
            cause = "Uptream API error. Try again later."

        return "%i (%s) from TTS API. Probable cause: %s" % (
            status, reason, cause)
331647
def check(sentence):
    """"""

    # How we decide what to put as the human readable feedback
    #
    # Our order of prefence is,
    #
    # 1. Spelling errors.
    #   - A spelling error can change the sentence meaning
    # 2. Subject-verb agreement errors
    # 3. Subordinate conjunction starting a sentence
    # 4. Participle phrase fragment
    # 5. Other errors

    result = Feedback()
    is_missing_verb = detect_missing_verb(sentence)
    is_infinitive = detect_infinitive_phrase(sentence)
    is_participle = is_participle_clause_fragment(sentence)
    lang_tool_feedback = get_language_tool_feedback(sentence)
    subject_and_verb_agree = get_subject_verb_agreement_feedback(sentence)
    ####
    if is_missing_verb: # Lowest priority
        result.matches['missing_verb'] = True 
        result.human_readable = MISSING_VERB_ADVICE.replace('\n', '') 
        result.primary_error = 'MISSING_VERB_ERROR'
        result.specific_error = 'MISSING_VERB'
    if is_participle > .5: 
        result.matches['participle_phrase'] = is_participle
        result.human_readable = PARTICIPLE_FRAGMENT_ADVICE.replace('\n', '')
        result.primary_error = 'FRAGMENT_ERROR'
        result.specific_error = 'PARTICIPLE_PHRASE'
    if lang_tool_feedback:
        result.matches['lang_tool'] = lang_tool_feedback
        for ltf in lang_tool_feedback:
            if ltf['rule']['id'] == 'SENTENCE_FRAGMENT':
                result.human_readable = lang_tool_feedback[0]['message']
                result.primary_error = 'FRAGMENT_ERROR'
                result.specific_error = 'SUBORDINATE_CLAUSE'
    if is_infinitive:
        result.matches['infinitive_phrase'] = True 
        result.human_readable = INFINITIVE_PHRASE_ADVICE.replace('\n', '') 
        result.primary_error = 'INFINITIVE_PHRASE_ERROR'
        result.specific_error = 'INFINITIVE_PHRASE'
    if not subject_and_verb_agree:
        result.matches['subject_verb_agreement'] = subject_and_verb_agree
        result.human_readable = SUBJECT_VERB_AGREEMENT_ADVICE.replace('\n', '') 
        result.primary_error = 'SUBJECT_VERB_AGREEMENT_ERROR'
        result.specific_error = 'SUBJECT_VERB_AGREEMENT'
    if lang_tool_feedback: # Highest priority (spelling, other lang tool errors)
        result.matches['lang_tool'] = lang_tool_feedback
        for ltf in lang_tool_feedback:
            if ltf['rule']['id'] == 'MORFOLOGIK_RULE_EN_US':
                result.human_readable = ltf['message']
                result.primary_error = 'SPELLING_ERROR'
                result.specific_error = 'SPELLING_ERROR'
        if not result.primary_error:
            result.human_readable = ltf['message']
            result.primary_error = 'OTHER_ERROR'
            result.specific_error = ltf['rule']['id']

         
    ####
    if not result.matches:
        result.human_readable = STRONG_SENTENCE_ADVICE.replace('\n', '')

    return result
898822
def _check_neg(self, level, *tokens):
        """"""
        for record in self.records:
            if level is not None and record.levelno != level:
                continue
            if all(token in record.message for token in tokens):
                break
        else:
            return

        # didn't exit, all tokens found in the same record
        msg = "Tokens {} found in the following record:  {}  {!r}".format(
            tokens, record.levelname, record.message)
        self.test_instance.fail(msg)
1229335
def segment_messages_from_magnitudes(magnitudes: np.ndarray, noise_threshold: float):
    
    return c_auto_interpretation.segment_messages_from_magnitudes(magnitudes, noise_threshold)
682882
def _fix_evidence_text(txt):
    """"""
    txt = re.sub('[ ]?\( xref \)', '', txt)
    # This is to make [ xref ] become [] to match the two readers
    txt = re.sub('\[ xref \]', '[]', txt)
    txt = re.sub('[\(]?XREF_BIBR[\)]?[,]?', '', txt)
    txt = re.sub('[\(]?XREF_FIG[\)]?[,]?', '', txt)
    txt = re.sub('[\(]?XREF_SUPPLEMENT[\)]?[,]?', '', txt)
    txt = txt.strip()
    return txt
1137388
def validate(self, corpus):
        
        invalid_utterances = {}

        for utterance in corpus.utterances.values():
            if self.label_list_idx in utterance.label_lists.keys():
                ll = utterance.label_lists[self.label_list_idx]

                if len(ll) < self.min_number_of_labels:
                    invalid_utterances[utterance.idx] = 'Only {} labels'.format(len(ll))
            else:
                invalid_utterances[utterance.idx] = 'No label-list {}'.format(self.label_list_idx)

        passed = len(invalid_utterances) <= 0
        info = {
            'Min. number of labels': str(self.min_number_of_labels),
            'Label-List ID': self.label_list_idx
        }

        return base.InvalidUtterancesResult(passed, invalid_utterances, name=self.name(), info=info)
1202186
def _check_validity(cls, text):
        

        if not text[0].lstrip().startswith('1 ') or not text[1].lstrip().startswith('2 '):
            raise ValueError("Line number check failed")

        for line in text:
            line = line.strip()
            if str(cls._checksum(line)) != line[-1]:
                raise ValueError("Checksum validation failed")
884087
def validate(self):
        
        try:
            assert self.question_type == exercises.SINGLE_SELECTION, "Assumption Failed: Question should be single selection type"
            assert len(self.answers) > 0, "Assumption Failed: Multiple selection question should have answers"
            correct_answers = 0
            for a in self.answers:
                assert 'answer' in a and isinstance(a['answer'], str), "Assumption Failed: Answer in answer list is not a string"
                assert 'correct' in a and isinstance(a['correct'], bool), "Assumption Failed: Correct indicator is not a boolean in answer list"
                correct_answers += 1 if a['correct'] else 0
            assert correct_answers == 1, "Assumption Failed: Single selection question should have only one correct answer"
            for h in self.hints:
                assert isinstance(h, str), "Assumption Failed: Hint in hints list is not a string"
            return super(SingleSelectQuestion, self).validate()
        except AssertionError as ae:
            raise InvalidQuestionException("Invalid question: {0}".format(self.__dict__))
977729
def text2sentences(text, labels):
    '''
    
    '''
    sentence = ''
    for i, label in enumerate(labels):
        if label == '1':
            if sentence:
                yield sentence
            sentence = ''
        else:
            sentence += text[i]
    if sentence:
        yield sentence
1344183
def check_denotation(target_values, predicted_values):
    
    # Check size
    if len(target_values) != len(predicted_values):
        return False
    # Check items
    for target in target_values:
        if not any(target.match(pred) for pred in predicted_values):
            return False
    return True
612117
def check_all(self):
        
        self.file_errors = 0
        self.line_number = 0
        self.indent_char = None
        self.indent_level = 0
        self.previous_logical = ''
        self.blank_lines = 0
        self.tokens = []
        parens = 0
        for token in tokenize.generate_tokens(self.readline_check_physical):
            # print tokenize.tok_name[token[0]], repr(token)
            self.tokens.append(token)
            token_type, text = token[0:2]
            if token_type == tokenize.OP and text in '([{':
                parens += 1
            if token_type == tokenize.OP and text in '}])':
                parens -= 1
            if token_type == tokenize.NEWLINE and not parens:
                self.check_logical()
                self.blank_lines = 0
                self.tokens = []
            if token_type == tokenize.NL and not parens:
                if len(self.tokens) <= 1:
                    # The physical line contains only this token.
                    self.blank_lines += 1
                self.tokens = []
            if token_type == tokenize.COMMENT:
                source_line = token[4]
                token_start = token[2][1]
                if source_line[:token_start].strip() == '':
                    self.blank_lines = 0
                if text.endswith('\n') and not parens:
                    # The comment also ends a physical line.  This works around
                    # Python < 2.6 behaviour, which does not generate NL after
                    # a comment which is on a line by itself.
                    self.tokens = []
        return self.file_errors
881655
def check(text):
    """"""
    err = "strunk_white.composition"
    msg = "Try '{}' instead of '{}'."

    bad_forms = [
        # Put statements in positive form
        ["dishonest",               ["not honest"]],
        ["trifling",                ["not important"]],
        ["forgot",                  ["did not remember"]],
        ["ignored",                 ["did not pay (any )?attention to"]],
        ["distrusted",              ["did not have much confidence in"]],

        # Omit needless words
        ["whether",                 ["the question as to whether"]],
        ["no doubt",                ["there is no doubt but that"]],
        ["used for fuel",           ["used for fuel purposes"]],
        ["he",                      ["he is a man who"]],
        ["hastily",                 ["in a hasty manner"]],
        ["this subject",            ["this is a subject that"]],
        ["Her story is strange.",   ["Her story is a strange one."]],
        ["because",                 ["the reason why is that"]],
        ["because / since",         ["owing to the fact that"]],
        ["although / though",       ["in spite of the fact that"]],
        ["remind you / notify you",
            ["call your attention to the fact that"]],
        ["I did not know that / I was unaware that",
            ["I was unaware of the fact that"]],
        ["his failure",             ["the fact that he had not succeeded"]],
        ["my arrival",              ["the fact that i had arrived"]]
    ]

    return preferred_forms_check(text, bad_forms, err, msg)