Extract EDUs from DPLPs .merge output files.

        Returns
        -------
        edus : dict from EDU IDs (int) to words (list(str))
505774
def extract_edus(merge_file_str):
        
        lines = merge_file_str.splitlines()

        edus = defaultdict(list)
        for line in lines:
            if line.strip():  # ignore empty lines
                token = line.split('\t')[2]
                edu_id = int(line.split('\t')[9])
                edus[edu_id].append(token)
        return edus
976348
def outputs_of(self, partition_index):
        
        partition = self.partition[partition_index]
        outputs = set(partition).intersection(self.output_indices)
        return tuple(sorted(outputs))
741793
def get_outdegrees(self):
        
        return {vertex: len(self.adj_dict[vertex]) for vertex in self.adj_dict}
203529
def _extract_features(self):
        
        for parsed_line in self.parsed_lines:
            result = {'raw': parsed_line}

            if 'ip' in parsed_line:
                result['ip'] = parsed_line['ip']
                if result['ip'] not in self.features['ips']:
                    self.features['ips'].append(result['ip'])
947067
def get_or_generate_vocab(data_dir, tmp_dir, vocab_filename, vocab_size,
                          sources, file_byte_budget=1e6,
                          max_subtoken_length=None):
  """"""

  vocab_generator = generate_lines_for_vocab(tmp_dir, sources, file_byte_budget)
  return get_or_generate_vocab_inner(data_dir, vocab_filename, vocab_size,
                                     vocab_generator, max_subtoken_length)
349218
def text_to_data(self, text, elt, ps):
        '''
        '''
        if self.pyclass is not None:
            v = self.pyclass(text) 
        else:
            try:
                v = int(text)
            except:
                try:
                    v = long(text)
                except:
                    raise EvaluateException('Unparseable integer', 
                        ps.Backtrace(elt))
        return v
1031845
def extract_words(files):
    '''  '''
    output_directory = 'extracted_words'
    if not os.path.exists(output_directory):
        os.makedirs(output_directory)

    for f in files:
        file_format = None
        source_segment = None
        if f.lower().endswith('.mp3'):
            file_format = 'mp3'
            source_segment = AudioSegment.from_mp3(f)
        elif f.lower().endswith('.wav'):
            file_format = 'wav'
            source_segment = AudioSegment.from_wav(f)
        if not file_format or source_segment:
            print('Unsupported audio format for ' + f)
        sentences = convert_timestamps(files)
        for s in sentences:
            for word in s['words']:
                start = float(word[1]) * 1000
                end = float(word[2]) * 1000
                word = word[0]
                total_time = end - start
                audio = AudioSegment.silent(duration=total_time)
                audio = audio.overlay(source_segment[start:end])
                number = 0
                output_path = None
                while True:
                    output_filename = word
                    if number:
                        output_filename += "_" + str(number)
                    output_filename = output_filename + '.' + file_format
                    output_path = os.path.join(output_directory, output_filename)
                    if not os.path.exists(output_path):
                        # this file doesn't exist, so we can continue
                        break
                    # file already exists, increment name and try again
                    number += 1
                print('Exporting to: ' + output_path)
                audio.export(output_path, format=file_format)
753758
def _extract(self, stim):
        '''  '''
        props = [(e.text, e.onset, e.duration) for e in stim.elements]
        vals, onsets, durations = map(list, zip(*props))
        return ExtractorResult(vals, stim, self, ['word'], onsets, durations)
223017
def process_document(self, doc):
        

        descriptions = doc.select_segments("projects[*].description")
        projects = doc.select_segments("projects[*]")

        for d, p in zip(descriptions, projects):

            # First phase of extraction
            names = doc.extract(self.name_extractor, d)
            p.store(names, "members")

            # Second phase of extraction
            students = []
            for name_extraction in names:
                students += doc.extract(self.student_extractor, name_extraction)
            p.store(students, "students")
        doc.kg.add_value("developer", json_path="projects[*].members[*]")
        doc.kg.add_value("owner", json_path="projects[*].students[*]")
        return list()
355891
def split_class_dir_fixed(class_dir, output, fixed, seed):
    
    files = setup_files(class_dir, seed)

    assert len(files) > sum(fixed)

    split_train = len(files) - sum(fixed)
    split_val = split_train + fixed[0]

    li = split_files(files, split_train, split_val, len(fixed) == 2)
    copy_files(li, class_dir, output)
    return len(files)
321676
def get_word_level_vocab(self):
        

        def simple_tokenize(source_str, token_delim=' ', seq_delim='\n'):
            return list(filter(None, re.split(token_delim + '|' + seq_delim, source_str)))

        return VocabProvider._create_squad_vocab(simple_tokenize, self._dataset)
361375
def envdict2listdict(envdict):
    """"""
    sep = os.path.pathsep
    for key in envdict:
        if sep in envdict[key]:
            envdict[key] = [path.strip() for path in envdict[key].split(sep)]
    return envdict
55449
def text_to_data(self, text, elt, ps):
        '''
        '''
        val = b64decode(text.replace(' ', '').replace('\n','').replace('\r',''))
        if self.pyclass is not None:
            return self.pyclass(val)
        return val
1276586
def get_readable_tasks(self, course):
        
        course_fs = self._filesystem.from_subfolder(course.get_id())
        tasks = [
            task[0:len(task)-1]  # remove trailing /
            for task in course_fs.list(folders=True, files=False, recursive=False)
            if self._task_file_exists(course_fs.from_subfolder(task))]
        return tasks
751742
def text_to_data(self, text, elt, ps):
        '''
        '''
        if text is None:
            return None
        
        m = self.lex_pattern.match(text)
        if not m:
            raise EvaluateException('Bad Gregorian: %s' %text, ps.Backtrace(elt))
        try:
            retval = _dict_to_tuple(m.groupdict())
        except ValueError, e:
            #raise EvaluateException(str(e))
            raise
        
        if self.pyclass is not None:
            return self.pyclass(retval)
        return retval
158532
def _word_to_score(self, ids, scores):
        
        # should be 1-D vectors
        assert len(ids.shape) == 1
        assert ids.shape == scores.shape

        w2s = {}
        for i in range(len(ids)):
            w2s[self.vocab.index2word(ids[i])] = scores[i]
        return w2s
754034
def _extract_notebook_text(self, element_id):
        
        headers = self._auth_headers_with_json()
        data = {'courseId': self._course_id,
                'learnerId': self._user_id, 'itemId': element_id}
        dom = get_page(self._session, OPENCOURSE_NOTEBOOK_LAUNCHES,
                       post=True,
                       json=True,
                       user_id=self._user_id,
                       course_id=self._course_id,
                       headers=headers,
                       element_id=element_id,
                       data=json.dumps(data)
                       )

        # Return authorization id. This id changes on each request
        return dom['elements'][0]['authorizationId']