def tokenizer(text):
    stream = deque(text.split("\n"))
    while len(stream) > 0:
        line = stream.popleft()
        if line.startswith("
            yield KeyValue("
            yield KeyValue("HEADER", line)
            for identifier in line.split(" "):
                if ":" in identifier:
                    key, value = identifier.split(":")
                    yield KeyValue(key, value)
        elif line.startswith("
            yield KeyValue("HEADER", line)
        elif line.startswith("
            yield KeyValue("
            yield KeyValue("
        elif line.startswith("
            yield KeyValue("
            yield KeyValue(line.strip(), "\n")
        elif line.startswith("SUBJECT_SAMPLE_FACTORS"):
            key, subject_type, local_sample_id, factors, additional_sample_data = line.split("\t")
            yield SubjectSampleFactors(key.strip(), subject_type, local_sample_id, factors, additional_sample_data)
        elif line.endswith("_START"):
            yield KeyValue(line, "\n")
            while not line.endswith("_END"):
                line = stream.popleft()
                if line.endswith("_END"):
                    yield KeyValue(line.strip(), "\n")
                else:
                    data = line.split("\t")
                    yield KeyValue(data[0], tuple(data))
        else:
            if line:
                if line.startswith("MS:MS_RESULTS_FILE") or line.startswith("NM:NMR_RESULTS_FILE"):
                    try:
                        key, value, extra = line.split("\t")
                        extra_key, extra_value = extra.strip().split(":")
                        yield KeyValueExtra(key.strip()[3:], value, extra_key, extra_value)
                    except ValueError:
                        key, value = line.split("\t")
                        yield KeyValue(key.strip()[3:], value)
                else:
                    try:
                        key, value = line.split("\t")
                        if ":" in key:
                            if key.startswith("MS_METABOLITE_DATA:UNITS"):
                                yield KeyValue(key.strip(), value)
                            else:
                                yield KeyValue(key.strip()[3:], value)
                        else:
                            yield KeyValue(key.strip(), value)
                    except ValueError:
                        print("LINE WITH ERROR:\n\t", repr(line))
                        raise
    yield KeyValue("
    yield KeyValue("!