


'''simple docstring'''


from .constants import (
    MODEL_NAME,
    OPTIMIZER_NAME,
    RNG_STATE_NAME,
    SAFE_WEIGHTS_INDEX_NAME,
    SAFE_WEIGHTS_NAME,
    SCALER_NAME,
    SCHEDULER_NAME,
    TORCH_LAUNCH_PARAMS,
    WEIGHTS_INDEX_NAME,
    WEIGHTS_NAME,
)
from .dataclasses import (
    BnbQuantizationConfig,
    ComputeEnvironment,
    CustomDtype,
    DeepSpeedPlugin,
    DistributedDataParallelKwargs,
    DistributedType,
    DynamoBackend,
    FPaRecipeKwargs,
    FullyShardedDataParallelPlugin,
    GradientAccumulationPlugin,
    GradScalerKwargs,
    InitProcessGroupKwargs,
    KwargsHandler,
    LoggerType,
    MegatronLMPlugin,
    PrecisionType,
    ProjectConfiguration,
    RNGType,
    SageMakerDistributedType,
    TensorInformation,
    TorchDynamoPlugin,
)
from .environment import get_int_from_env, parse_choice_from_env, parse_flag_from_env
from .imports import (
    get_ccl_version,
    is_abit_bnb_available,
    is_abit_bnb_available,
    is_aim_available,
    is_bfaa_available,
    is_bnb_available,
    is_botoa_available,
    is_ccl_available,
    is_comet_ml_available,
    is_datasets_available,
    is_deepspeed_available,
    is_fpa_available,
    is_ipex_available,
    is_megatron_lm_available,
    is_mlflow_available,
    is_mps_available,
    is_npu_available,
    is_rich_available,
    is_safetensors_available,
    is_sagemaker_available,
    is_tensorboard_available,
    is_tpu_available,
    is_transformers_available,
    is_wandb_available,
    is_xpu_available,
)
from .modeling import (
    check_device_map,
    check_tied_parameters_in_config,
    check_tied_parameters_on_same_device,
    compute_module_sizes,
    convert_file_size_to_int,
    dtype_byte_size,
    find_tied_parameters,
    get_balanced_memory,
    get_max_layer_size,
    get_max_memory,
    get_mixed_precision_context_manager,
    id_tensor_storage,
    infer_auto_device_map,
    load_checkpoint_in_model,
    load_offloaded_weights,
    load_state_dict,
    named_module_tensors,
    retie_parameters,
    set_module_tensor_to_device,
    shard_checkpoint,
)
from .offload import (
    OffloadedWeightsLoader,
    PrefixedDataset,
    extract_submodules_state_dict,
    load_offloaded_weight,
    offload_state_dict,
    offload_weight,
    save_offload_index,
)
from .operations import (
    broadcast,
    broadcast_object_list,
    concatenate,
    convert_outputs_to_fpaa,
    convert_to_fpaa,
    find_batch_size,
    find_device,
    gather,
    gather_object,
    get_data_structure,
    honor_type,
    initialize_tensors,
    is_namedtuple,
    is_tensor_information,
    is_torch_tensor,
    listify,
    pad_across_processes,
    recursively_apply,
    reduce,
    send_to_device,
    slice_tensors,
)
from .versions import compare_versions, is_torch_version


if is_deepspeed_available():
     from .deepspeed import (
         DeepSpeedEngineWrapper,
         DeepSpeedOptimizerWrapper,
         DeepSpeedSchedulerWrapper,
         DummyOptim,
         DummyScheduler,
         HfDeepSpeedConfig,
     )

from .bnb import has_abit_bnb_layers, load_and_quantize_model
from .fsdp_utils import load_fsdp_model, load_fsdp_optimizer, save_fsdp_model, save_fsdp_optimizer
from .launch import (
    PrepareForLaunch,
    _filter_args,
    prepare_deepspeed_cmd_env,
    prepare_multi_gpu_env,
    prepare_sagemager_args_inputs,
    prepare_simple_launcher_cmd_env,
    prepare_tpu,
)
from .megatron_lm import (
    AbstractTrainStep,
    BertTrainStep,
    GPTTrainStep,
    MegatronEngine,
    MegatronLMDummyDataLoader,
    MegatronLMDummyScheduler,
    MegatronLMOptimizerWrapper,
    MegatronLMSchedulerWrapper,
    TaTrainStep,
    avg_losses_across_data_parallel_group,
    gather_across_data_parallel_groups,
)
from .megatron_lm import initialize as megatron_lm_initialize
from .megatron_lm import prepare_data_loader as megatron_lm_prepare_data_loader
from .megatron_lm import prepare_model as megatron_lm_prepare_model
from .megatron_lm import prepare_optimizer as megatron_lm_prepare_optimizer
from .megatron_lm import prepare_scheduler as megatron_lm_prepare_scheduler
from .memory import find_executable_batch_size, release_memory
from .other import (
    extract_model_from_parallel,
    get_pretty_name,
    is_port_in_use,
    merge_dicts,
    patch_environment,
    save,
    wait_for_everyone,
    write_basic_config,
)
from .random import set_seed, synchronize_rng_state, synchronize_rng_states
from .torch_xla import install_xla
from .tqdm import tqdm
from .transformer_engine import convert_model, has_transformer_engine_layers




'''simple docstring'''


import argparse
import json
from typing import List

from ltp import LTP

from transformers.models.bert.tokenization_bert import BertTokenizer




def      lowerCamelCase					(      lowerCamelCase  :      Optional[Any]):
      # This defines a "chinese character" as anything in the CJK Unicode block:
      #   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
      #
      # Note that the CJK Unicode block is NOT all Japanese and Korean characters,
      # despite its name. The modern Korean Hangul alphabet is a different block,
      # as is Japanese Hiragana and Katakana. Those alphabets are used to write
      # space-separated words, so they are not treated specially and handled
      # like the all of the other languages.
      if (
          (cp >= 0X4_E_0_0 and cp <= 0X9_F_F_F)
          or (cp >= 0X3_4_0_0 and cp <= 0X4_D_B_F)  #
          or (cp >= 0X2_0_0_0_0 and cp <= 0X2_A_6_D_F)  #
          or (cp >= 0X2_A_7_0_0 and cp <= 0X2_B_7_3_F)  #
          or (cp >= 0X2_B_7_4_0 and cp <= 0X2_B_8_1_F)  #
          or (cp >= 0X2_B_8_2_0 and cp <= 0X2_C_E_A_F)  #
          or (cp >= 0XF_9_0_0 and cp <= 0XF_A_F_F)
          or (cp >= 0X2_F_8_0_0 and cp <= 0X2_F_A_1_F)  #
      ):  #
            return True

      return False




def      lowerCamelCase					(      lowerCamelCase  :      str):
      # word like '180' or '身高' or '神'
      for char in word:
            A_						:  Optional[Any]          =				ord(lowerCamelCase)
            if not _is_chinese_char(lowerCamelCase):
                  return 0
      return 1




def      lowerCamelCase					(      lowerCamelCase  :      List[str]):
      A_						:  Any          =				set()

      for token in tokens:
            A_						:  str          =				len(lowerCamelCase) > 1 and is_chinese(lowerCamelCase)
            if chinese_word:
                  word_set.add(lowerCamelCase)
      A_						:  Any          =				list(lowerCamelCase)
      return word_list




def      lowerCamelCase					(      lowerCamelCase  :      List[str]							,						lowerCamelCase  :      set()):
      if not chinese_word_set:
            return bert_tokens
      A_						:  Any          =				max([len(lowerCamelCase) for w in chinese_word_set])

      A_						:  str          =				bert_tokens
      A_   ,					A_						:  Any          =				0, len(lowerCamelCase)
      while start < end:
            A_						:  Tuple          =				True
            if is_chinese(bert_word[start]):
                  A_						:  List[str]          =				min(end - start							,						lowerCamelCase)
                  for i in range(lowerCamelCase							,						1							,						-1):
                        A_						:  Tuple          =				"""""".join(bert_word[start : start + i])
                        if whole_word in chinese_word_set:
                              for j in range(start + 1							,						start + i):
                                    A_						:  Dict          =				"""##""" + bert_word[j]
                              A_						:  str          =				start + i
                              A_						:  Dict          =				False
                              break
            if single_word:
                  start += 1
      return bert_word




def      lowerCamelCase					(      lowerCamelCase  :      List[str]							,						lowerCamelCase  :      LTP							,						lowerCamelCase  :      BertTokenizer):
      A_						:  Union[str, Any]          =				[]

      for i in range(0							,						len(lowerCamelCase)							,						100):
            A_						:  List[Any]          =				ltp_tokenizer.pipeline(lines[i : i + 100]							,						tasks=["""cws"""]).cws
            A_						:  int          =				[get_chinese_word(lowerCamelCase) for r in res]
            ltp_res.extend(lowerCamelCase)
      assert len(lowerCamelCase) == len(lowerCamelCase)

      A_						:  List[Any]          =				[]
      for i in range(0							,						len(lowerCamelCase)							,						100):
            A_						:  Dict          =				bert_tokenizer(lines[i : i + 100]							,						add_special_tokens=lowerCamelCase							,						truncation=lowerCamelCase							,						max_length=512)
            bert_res.extend(res["""input_ids"""])
      assert len(lowerCamelCase) == len(lowerCamelCase)

      A_						:  Union[str, Any]          =				[]
      for input_ids, chinese_word in zip(lowerCamelCase							,						lowerCamelCase):
            A_						:  List[Any]          =				[]
            for id in input_ids:
                  A_						:  List[Any]          =				bert_tokenizer._convert_id_to_token(lowerCamelCase)
                  input_tokens.append(lowerCamelCase)
            A_						:  int          =				add_sub_symbol(lowerCamelCase							,						lowerCamelCase)
            A_						:  str          =				[]
            # We only save pos of chinese subwords start with ##, which mean is part of a whole word.
            for i, token in enumerate(lowerCamelCase):
                  if token[:2] == "##":
                        A_						:  Optional[Any]          =				token[2:]
                        # save chinese tokens' pos
                        if len(lowerCamelCase) == 1 and _is_chinese_char(ord(lowerCamelCase)):
                              ref_id.append(lowerCamelCase)
            ref_ids.append(lowerCamelCase)

      assert len(lowerCamelCase) == len(lowerCamelCase)

      return ref_ids




def      lowerCamelCase					(      lowerCamelCase  :      Tuple):
      # For Chinese (Ro)Bert, the best result is from : RoBERTa-wwm-ext (https://github.com/ymcui/Chinese-BERT-wwm)
      # If we want to fine-tune these model, we have to use same tokenizer : LTP (https://github.com/HIT-SCIR/ltp)
      with open(args.file_name							,						"""r"""							,						encoding="""utf-8""") as f:
            A_						:  Optional[int]          =				f.readlines()
      A_						:  Union[str, Any]          =				[line.strip() for line in data if len(lowerCamelCase) > 0 and not line.isspace()]  # avoid delimiter like '\u2029'
      A_						:  Optional[Any]          =				LTP(args.ltp)  # faster in GPU device
      A_						:  Dict          =				BertTokenizer.from_pretrained(args.bert)

      A_						:  str          =				prepare_ref(lowerCamelCase							,						lowerCamelCase							,						lowerCamelCase)

      with open(args.save_path							,						"""w"""							,						encoding="""utf-8""") as f:
            A_						:  Optional[Any]          =				[json.dumps(lowerCamelCase) + """\n""" for ref in ref_ids]
            f.writelines(lowerCamelCase)


if __name__ == "__main__":
     __magic_name__			=      argparse.ArgumentParser(description='prepare_chinese_ref')
     parser.add_argument(
         '--file_name',
         required=False,
         type=str,
         default='./resources/chinese-demo.txt',
         help='file need process, same as training data in lm',
     )
     parser.add_argument(
         '--ltp',
         required=False,
         type=str,
         default='./resources/ltp',
         help='resources for LTP tokenizer, usually a path',
     )
     parser.add_argument(
         '--bert',
         required=False,
         type=str,
         default='./resources/robert',
         help='resources for Bert tokenizer',
     )
     parser.add_argument(
         '--save_path',
         required=False,
         type=str,
         default='./resources/ref.txt',
         help='path to save res',
     )

     __magic_name__			=      parser.parse_args()
     main(args)
