["task_categories:text-generation", "task_categories:feature-extraction", "language:en", "license:odc-by", "size_categories:1M<n<10M", "format:parquet", "modality:text", "library:datasets", "library:dask", "library:mlcroissant", "library:polars", "region:us"]
[{"name": "text", "dtype": "string"}, {"name": "meta", "struct": [{"name": "cc-path", "dtype": "string"}, {"name": "corpusid", "dtype": "int64"}, {"name": "dup_signals", "struct": [{"name": "dup_details", "struct": [{"name": "2013-20", "dtype": "int64"}, {"name": "2013-48", "dtype": "int64"}, {"name": "2014-10", "dtype": "int64"}, {"name": "2014-15", "dtype": "int64"}, {"name": "2014-23", "dtype": "int64"}, {"name": "2014-35", "dtype": "int64"}, {"name": "2014-41", "dtype": "int64"}, {"name": "2014-42", "dtype": "int64"}, {"name": "2014-49", "dtype": "null"}, {"name": "2014-52", "dtype": "int64"}, {"name": "2015-06", "dtype": "int64"}, {"name": "2015-11", "dtype": "int64"}, {"name": "2015-14", "dtype": "int64"}, {"name": "2015-18", "dtype": "int64"}, {"name": "2015-22", "dtype": "int64"}, {"name": "2015-27", "dtype": "int64"}, {"name": "2015-32", "dtype": "int64"}, {"name": "2015-35", "dtype": "int64"}, {"name": "2015-40", "dtype": "int64"}, {"name": "2016-07", "dtype": "int64"}, {"name": "2016-18", "dtype": "int64"}, {"name": "2016-22", "dtype": "int64"}, {"name": "2016-26", "dtype": "int64"}, {"name": "2016-30", "dtype": "int64"}, {"name": "2016-40", "dtype": "int64"}, {"name": "2016-44", "dtype": "int64"}, {"name": "2016-50", "dtype": "int64"}, {"name": "2017-04", "dtype": "int64"}, {"name": "2017-09", "dtype": "int64"}, {"name": "2017-13", "dtype": "int64"}, {"name": "2017-17", "dtype": "int64"}, {"name": "2017-22", "dtype": "int64"}, {"name": "2017-26", "dtype": "int64"}, {"name": "2017-30", "dtype": "int64"}, {"name": "2017-34", "dtype": "int64"}, {"name": "2017-39", "dtype": "int64"}, {"name": "2017-43", "dtype": "int64"}, {"name": "2017-47", "dtype": "int64"}, {"name": "2017-51", "dtype": "int64"}, {"name": "2018-05", "dtype": "int64"}, {"name": "2018-09", "dtype": "int64"}, {"name": "2018-13", "dtype": "int64"}, {"name": "2018-17", "dtype": "int64"}, {"name": "2018-22", "dtype": "int64"}, {"name": "2018-26", "dtype": "int64"}, {"name": "2018-30", "dtype": "int64"}, {"name": "2018-34", "dtype": "int64"}, {"name": "2018-39", "dtype": "int64"}, {"name": "2018-43", "dtype": "int64"}, {"name": "2018-47", "dtype": "int64"}, {"name": "2018-51", "dtype": "int64"}, {"name": "2019-04", "dtype": "int64"}, {"name": "2019-09", "dtype": "int64"}, {"name": "2019-13", "dtype": "int64"}, {"name": "2019-18", "dtype": "int64"}, {"name": "2019-22", "dtype": "int64"}, {"name": "2019-26", "dtype": "int64"}, {"name": "2019-30", "dtype": "int64"}, {"name": "2019-35", "dtype": "int64"}, {"name": "2019-39", "dtype": "int64"}, {"name": "2019-43", "dtype": "int64"}, {"name": "2019-47", "dtype": "int64"}, {"name": "2019-51", "dtype": "int64"}, {"name": "2020-05", "dtype": "int64"}, {"name": "2020-10", "dtype": "int64"}, {"name": "2020-16", "dtype": "int64"}, {"name": "2020-24", "dtype": "int64"}, {"name": "2020-29", "dtype": "int64"}, {"name": "2020-34", "dtype": "int64"}, {"name": "2020-40", "dtype": "int64"}, {"name": "2020-45", "dtype": "int64"}, {"name": "2020-50", "dtype": "int64"}, {"name": "2021-04", "dtype": "int64"}, {"name": "2021-10", "dtype": "int64"}, {"name": "2021-17", "dtype": "int64"}, {"name": "2021-21", "dtype": "int64"}, {"name": "2021-25", "dtype": "int64"}, {"name": "2021-31", "dtype": "int64"}, {"name": "2021-39", "dtype": "int64"}, {"name": "2021-43", "dtype": "int64"}, {"name": "2021-49", "dtype": "int64"}, {"name": "2022-05", "dtype": "int64"}, {"name": "2022-21", "dtype": "int64"}, {"name": "2022-27", "dtype": "int64"}, {"name": "2022-33", "dtype": "int64"}, {"name": "2022-40", "dtype": "int64"}, {"name": "2022-49", "dtype": "int64"}, {"name": "2023-06", "dtype": "int64"}, {"name": "2023-14", "dtype": "int64"}, {"name": "2023-23", "dtype": "int64"}, {"name": "2023-40", "dtype": "int64"}, {"name": "2023-50", "dtype": "int64"}, {"name": "2024-10", "dtype": "int64"}, {"name": "2024-18", "dtype": "int64"}, {"name": "2024-22", "dtype": "int64"}, {"name": "2024-26", "dtype": "int64"}, {"name": "2024-30", "dtype": "int64"}, {"name": "curated_sources", "dtype": "int64"}, {"name": "unknown", "dtype": "int64"}]}, {"name": "dup_doc_count", "dtype": "int64"}, {"name": "dup_dump_count", "dtype": "int64"}]}, {"name": "id", "dtype": "int64"}, {"name": "lang", "dtype": "string"}, {"name": "lang_score", "dtype": "float64"}, {"name": "language", "dtype": "string"}, {"name": "openaccessinfo", "struct": [{"name": "externalids", "struct": [{"name": "ACL", "dtype": "string"}, {"name": "ArXiv", "dtype": "string"}, {"name": "DOI", "dtype": "string"}, {"name": "MAG", "dtype": "string"}, {"name": "PubMedCentral", "dtype": "string"}]}, {"name": "license", "dtype": "string"}, {"name": "status", "dtype": "string"}, {"name": "url", "dtype": "string"}]}, {"name": "pmid", "dtype": "int64"}, {"name": "quality_signals", "struct": [{"name": "fraction_of_characters_in_duplicate_lines", "dtype": "float64"}, {"name": "fraction_of_characters_in_duplicate_ngrams", "sequence": {"sequence": "float64"}}, {"name": "fraction_of_characters_in_duplicate_paragraphs", "dtype": "float64"}, {"name": "fraction_of_characters_in_most_common_ngram", "sequence": {"sequence": "float64"}}, {"name": "fraction_of_duplicate_lines", "dtype": "float64"}, {"name": "fraction_of_duplicate_paragraphs", "dtype": "float64"}, {"name": "fraction_of_lines_ending_with_ellipsis", "dtype": "float64"}, {"name": "fraction_of_lines_starting_with_bullet_point", "dtype": "float64"}, {"name": "fraction_of_lines_with_toxic_words", "dtype": "float64"}, {"name": "fraction_of_words_corrected_in_lines", "dtype": "float64"}, {"name": "fraction_of_words_with_alpha_character", "dtype": "float64"}, {"name": "has_curly_bracket", "dtype": "bool"}, {"name": "has_lorem_ipsum", "dtype": "bool"}, {"name": "mean_word_length", "dtype": "float64"}, {"name": "num_of_lines_with_toxic_words", "dtype": "int64"}, {"name": "num_of_paragraphs", "dtype": "int64"}, {"name": "num_of_sentences", "dtype": "int64"}, {"name": "num_of_stop_words", "dtype": "int64"}, {"name": "num_of_toxic_words", "dtype": "int64"}, {"name": "orig_text_has_dup_lines", "dtype": "bool"}, {"name": "symbol_to_word_ratio", "dtype": "float64"}, {"name": "url_score", "dtype": "float64"}, {"name": "word_count", "dtype": "int64"}]}, {"name": "timestamp", "dtype": "timestamp[us]"}, {"name": "title", "dtype": "string"}, {"name": "url", "dtype": "string"}]}, {"name": "subset", "dtype": "string"}]