Copy disabled (too large)
Download .txt
Showing preview only (35,434K chars total). Download the full file to get everything.
Repository: Ablustrund/LoRAMoE
Branch: main
Commit: 117728885a40
Files: 1362
Total size: 33.5 MB
Directory structure:
gitextract_holo9v9h/
├── .gitignore
├── README.md
├── __init__.py
├── build_dataset.py
├── data/
│ └── tiny_data/
│ ├── test.json
│ └── train/
│ └── train.json
├── ds_zero2_no_offload.json
├── ds_zero3_nvme_offload.json
├── ds_zero3_offload.json
├── environment.yml
├── flash_attn_patch.py
├── peft/
│ ├── __init__.py
│ ├── mapping.py
│ ├── peft_model.py
│ ├── tuners/
│ │ ├── __init__.py
│ │ ├── lora.py
│ │ ├── p_tuning.py
│ │ ├── prefix_tuning.py
│ │ └── prompt_tuning.py
│ └── utils/
│ ├── __init__.py
│ ├── adapters_utils.py
│ ├── config.py
│ ├── other.py
│ └── save_and_load.py
├── requirements.txt
├── run_loramoe.py
├── run_loramoe.sh
└── transformers/
├── __init__.py
├── activations.py
├── activations_tf.py
├── audio_utils.py
├── benchmark/
│ ├── __init__.py
│ ├── benchmark.py
│ ├── benchmark_args.py
│ ├── benchmark_args_tf.py
│ ├── benchmark_args_utils.py
│ ├── benchmark_tf.py
│ └── benchmark_utils.py
├── commands/
│ ├── __init__.py
│ ├── add_new_model.py
│ ├── add_new_model_like.py
│ ├── convert.py
│ ├── download.py
│ ├── env.py
│ ├── lfs.py
│ ├── pt_to_tf.py
│ ├── run.py
│ ├── serving.py
│ ├── train.py
│ ├── transformers_cli.py
│ └── user.py
├── configuration_utils.py
├── convert_graph_to_onnx.py
├── convert_pytorch_checkpoint_to_tf2.py
├── convert_slow_tokenizer.py
├── convert_slow_tokenizers_checkpoints_to_fast.py
├── convert_tf_hub_seq_to_seq_bert_to_pytorch.py
├── data/
│ ├── __init__.py
│ ├── data_collator.py
│ ├── datasets/
│ │ ├── __init__.py
│ │ ├── glue.py
│ │ ├── language_modeling.py
│ │ └── squad.py
│ ├── metrics/
│ │ ├── __init__.py
│ │ └── squad_metrics.py
│ └── processors/
│ ├── __init__.py
│ ├── glue.py
│ ├── squad.py
│ ├── utils.py
│ └── xnli.py
├── debug_utils.py
├── deepspeed.py
├── dependency_versions_check.py
├── dependency_versions_table.py
├── dynamic_module_utils.py
├── feature_extraction_sequence_utils.py
├── feature_extraction_utils.py
├── file_utils.py
├── generation/
│ ├── __init__.py
│ ├── beam_constraints.py
│ ├── beam_search.py
│ ├── configuration_utils.py
│ ├── flax_logits_process.py
│ ├── flax_utils.py
│ ├── logits_process.py
│ ├── stopping_criteria.py
│ ├── streamers.py
│ ├── tf_logits_process.py
│ ├── tf_utils.py
│ └── utils.py
├── generation_flax_utils.py
├── generation_tf_utils.py
├── generation_utils.py
├── hf_argparser.py
├── image_processing_utils.py
├── image_transforms.py
├── image_utils.py
├── integrations.py
├── keras_callbacks.py
├── kernels/
│ ├── deformable_detr/
│ │ ├── cpu/
│ │ │ ├── ms_deform_attn_cpu.cpp
│ │ │ └── ms_deform_attn_cpu.h
│ │ ├── cuda/
│ │ │ ├── ms_deform_attn_cuda.cu
│ │ │ ├── ms_deform_attn_cuda.cuh
│ │ │ ├── ms_deform_attn_cuda.h
│ │ │ └── ms_deform_im2col_cuda.cuh
│ │ ├── ms_deform_attn.h
│ │ └── vision.cpp
│ ├── rwkv/
│ │ ├── wkv_cuda.cu
│ │ ├── wkv_cuda_bf16.cu
│ │ └── wkv_op.cpp
│ └── yoso/
│ ├── common.h
│ ├── common_cuda.h
│ ├── common_cuda_device.h
│ ├── fast_lsh_cumulation.cu
│ ├── fast_lsh_cumulation.h
│ ├── fast_lsh_cumulation_cuda.cu
│ ├── fast_lsh_cumulation_cuda.h
│ └── fast_lsh_cumulation_torch.cpp
├── modelcard.py
├── modeling_flax_outputs.py
├── modeling_flax_pytorch_utils.py
├── modeling_flax_utils.py
├── modeling_outputs.py
├── modeling_tf_outputs.py
├── modeling_tf_pytorch_utils.py
├── modeling_tf_utils.py
├── modeling_utils.py
├── models/
│ ├── __init__.py
│ ├── albert/
│ │ ├── __init__.py
│ │ ├── configuration_albert.py
│ │ ├── convert_albert_original_tf_checkpoint_to_pytorch.py
│ │ ├── modeling_albert.py
│ │ ├── modeling_flax_albert.py
│ │ ├── modeling_tf_albert.py
│ │ ├── tokenization_albert.py
│ │ └── tokenization_albert_fast.py
│ ├── align/
│ │ ├── __init__.py
│ │ ├── configuration_align.py
│ │ ├── convert_align_tf_to_hf.py
│ │ ├── modeling_align.py
│ │ └── processing_align.py
│ ├── altclip/
│ │ ├── __init__.py
│ │ ├── configuration_altclip.py
│ │ ├── modeling_altclip.py
│ │ └── processing_altclip.py
│ ├── audio_spectrogram_transformer/
│ │ ├── __init__.py
│ │ ├── configuration_audio_spectrogram_transformer.py
│ │ ├── convert_audio_spectrogram_transformer_original_to_pytorch.py
│ │ ├── feature_extraction_audio_spectrogram_transformer.py
│ │ └── modeling_audio_spectrogram_transformer.py
│ ├── auto/
│ │ ├── __init__.py
│ │ ├── auto_factory.py
│ │ ├── configuration_auto.py
│ │ ├── feature_extraction_auto.py
│ │ ├── image_processing_auto.py
│ │ ├── modeling_auto.py
│ │ ├── modeling_flax_auto.py
│ │ ├── modeling_tf_auto.py
│ │ ├── processing_auto.py
│ │ └── tokenization_auto.py
│ ├── autoformer/
│ │ ├── __init__.py
│ │ ├── configuration_autoformer.py
│ │ └── modeling_autoformer.py
│ ├── bart/
│ │ ├── __init__.py
│ │ ├── configuration_bart.py
│ │ ├── convert_bart_original_pytorch_checkpoint_to_pytorch.py
│ │ ├── modeling_bart.py
│ │ ├── modeling_flax_bart.py
│ │ ├── modeling_tf_bart.py
│ │ ├── tokenization_bart.py
│ │ └── tokenization_bart_fast.py
│ ├── barthez/
│ │ ├── __init__.py
│ │ ├── tokenization_barthez.py
│ │ └── tokenization_barthez_fast.py
│ ├── bartpho/
│ │ ├── __init__.py
│ │ └── tokenization_bartpho.py
│ ├── beit/
│ │ ├── __init__.py
│ │ ├── configuration_beit.py
│ │ ├── convert_beit_unilm_to_pytorch.py
│ │ ├── feature_extraction_beit.py
│ │ ├── image_processing_beit.py
│ │ ├── modeling_beit.py
│ │ └── modeling_flax_beit.py
│ ├── bert/
│ │ ├── __init__.py
│ │ ├── configuration_bert.py
│ │ ├── convert_bert_original_tf2_checkpoint_to_pytorch.py
│ │ ├── convert_bert_original_tf_checkpoint_to_pytorch.py
│ │ ├── convert_bert_pytorch_checkpoint_to_original_tf.py
│ │ ├── convert_bert_token_dropping_original_tf2_checkpoint_to_pytorch.py
│ │ ├── modeling_bert.py
│ │ ├── modeling_flax_bert.py
│ │ ├── modeling_tf_bert.py
│ │ ├── tokenization_bert.py
│ │ ├── tokenization_bert_fast.py
│ │ └── tokenization_bert_tf.py
│ ├── bert_generation/
│ │ ├── __init__.py
│ │ ├── configuration_bert_generation.py
│ │ ├── modeling_bert_generation.py
│ │ └── tokenization_bert_generation.py
│ ├── bert_japanese/
│ │ ├── __init__.py
│ │ └── tokenization_bert_japanese.py
│ ├── bertweet/
│ │ ├── __init__.py
│ │ └── tokenization_bertweet.py
│ ├── big_bird/
│ │ ├── __init__.py
│ │ ├── configuration_big_bird.py
│ │ ├── convert_bigbird_original_tf_checkpoint_to_pytorch.py
│ │ ├── modeling_big_bird.py
│ │ ├── modeling_flax_big_bird.py
│ │ ├── tokenization_big_bird.py
│ │ └── tokenization_big_bird_fast.py
│ ├── bigbird_pegasus/
│ │ ├── __init__.py
│ │ ├── configuration_bigbird_pegasus.py
│ │ ├── convert_bigbird_pegasus_tf_to_pytorch.py
│ │ └── modeling_bigbird_pegasus.py
│ ├── biogpt/
│ │ ├── __init__.py
│ │ ├── configuration_biogpt.py
│ │ ├── convert_biogpt_original_pytorch_checkpoint_to_pytorch.py
│ │ ├── modeling_biogpt.py
│ │ └── tokenization_biogpt.py
│ ├── bit/
│ │ ├── __init__.py
│ │ ├── configuration_bit.py
│ │ ├── convert_bit_to_pytorch.py
│ │ ├── image_processing_bit.py
│ │ └── modeling_bit.py
│ ├── blenderbot/
│ │ ├── __init__.py
│ │ ├── configuration_blenderbot.py
│ │ ├── convert_blenderbot_original_pytorch_checkpoint_to_pytorch.py
│ │ ├── modeling_blenderbot.py
│ │ ├── modeling_flax_blenderbot.py
│ │ ├── modeling_tf_blenderbot.py
│ │ ├── tokenization_blenderbot.py
│ │ └── tokenization_blenderbot_fast.py
│ ├── blenderbot_small/
│ │ ├── __init__.py
│ │ ├── configuration_blenderbot_small.py
│ │ ├── modeling_blenderbot_small.py
│ │ ├── modeling_flax_blenderbot_small.py
│ │ ├── modeling_tf_blenderbot_small.py
│ │ ├── tokenization_blenderbot_small.py
│ │ └── tokenization_blenderbot_small_fast.py
│ ├── blip/
│ │ ├── __init__.py
│ │ ├── configuration_blip.py
│ │ ├── convert_blip_original_pytorch_to_hf.py
│ │ ├── image_processing_blip.py
│ │ ├── modeling_blip.py
│ │ ├── modeling_blip_text.py
│ │ ├── modeling_tf_blip.py
│ │ ├── modeling_tf_blip_text.py
│ │ └── processing_blip.py
│ ├── blip_2/
│ │ ├── __init__.py
│ │ ├── configuration_blip_2.py
│ │ ├── convert_blip_2_original_to_pytorch.py
│ │ ├── modeling_blip_2.py
│ │ └── processing_blip_2.py
│ ├── bloom/
│ │ ├── __init__.py
│ │ ├── configuration_bloom.py
│ │ ├── convert_bloom_original_checkpoint_to_pytorch.py
│ │ ├── modeling_bloom.py
│ │ └── tokenization_bloom_fast.py
│ ├── bort/
│ │ ├── __init__.py
│ │ └── convert_bort_original_gluonnlp_checkpoint_to_pytorch.py
│ ├── bridgetower/
│ │ ├── __init__.py
│ │ ├── configuration_bridgetower.py
│ │ ├── image_processing_bridgetower.py
│ │ ├── modeling_bridgetower.py
│ │ └── processing_bridgetower.py
│ ├── byt5/
│ │ ├── __init__.py
│ │ ├── convert_byt5_original_tf_checkpoint_to_pytorch.py
│ │ └── tokenization_byt5.py
│ ├── camembert/
│ │ ├── __init__.py
│ │ ├── configuration_camembert.py
│ │ ├── modeling_camembert.py
│ │ ├── modeling_tf_camembert.py
│ │ ├── tokenization_camembert.py
│ │ └── tokenization_camembert_fast.py
│ ├── canine/
│ │ ├── __init__.py
│ │ ├── configuration_canine.py
│ │ ├── convert_canine_original_tf_checkpoint_to_pytorch.py
│ │ ├── modeling_canine.py
│ │ └── tokenization_canine.py
│ ├── chinese_clip/
│ │ ├── __init__.py
│ │ ├── configuration_chinese_clip.py
│ │ ├── convert_chinese_clip_original_pytorch_to_hf.py
│ │ ├── feature_extraction_chinese_clip.py
│ │ ├── image_processing_chinese_clip.py
│ │ ├── modeling_chinese_clip.py
│ │ └── processing_chinese_clip.py
│ ├── clap/
│ │ ├── __init__.py
│ │ ├── configuration_clap.py
│ │ ├── convert_clap_original_pytorch_to_hf.py
│ │ ├── feature_extraction_clap.py
│ │ ├── modeling_clap.py
│ │ └── processing_clap.py
│ ├── clip/
│ │ ├── __init__.py
│ │ ├── configuration_clip.py
│ │ ├── convert_clip_original_pytorch_to_hf.py
│ │ ├── feature_extraction_clip.py
│ │ ├── image_processing_clip.py
│ │ ├── modeling_clip.py
│ │ ├── modeling_flax_clip.py
│ │ ├── modeling_tf_clip.py
│ │ ├── processing_clip.py
│ │ ├── tokenization_clip.py
│ │ └── tokenization_clip_fast.py
│ ├── clipseg/
│ │ ├── __init__.py
│ │ ├── configuration_clipseg.py
│ │ ├── convert_clipseg_original_pytorch_to_hf.py
│ │ ├── modeling_clipseg.py
│ │ └── processing_clipseg.py
│ ├── codegen/
│ │ ├── __init__.py
│ │ ├── configuration_codegen.py
│ │ ├── modeling_codegen.py
│ │ ├── tokenization_codegen.py
│ │ └── tokenization_codegen_fast.py
│ ├── conditional_detr/
│ │ ├── __init__.py
│ │ ├── configuration_conditional_detr.py
│ │ ├── convert_conditional_detr_original_pytorch_checkpoint_to_pytorch.py
│ │ ├── feature_extraction_conditional_detr.py
│ │ ├── image_processing_conditional_detr.py
│ │ └── modeling_conditional_detr.py
│ ├── convbert/
│ │ ├── __init__.py
│ │ ├── configuration_convbert.py
│ │ ├── convert_convbert_original_tf1_checkpoint_to_pytorch_and_tf2.py
│ │ ├── modeling_convbert.py
│ │ ├── modeling_tf_convbert.py
│ │ ├── tokenization_convbert.py
│ │ └── tokenization_convbert_fast.py
│ ├── convnext/
│ │ ├── __init__.py
│ │ ├── configuration_convnext.py
│ │ ├── convert_convnext_to_pytorch.py
│ │ ├── feature_extraction_convnext.py
│ │ ├── image_processing_convnext.py
│ │ ├── modeling_convnext.py
│ │ └── modeling_tf_convnext.py
│ ├── convnextv2/
│ │ ├── __init__.py
│ │ ├── configuration_convnextv2.py
│ │ ├── convert_convnextv2_to_pytorch.py
│ │ └── modeling_convnextv2.py
│ ├── cpm/
│ │ ├── __init__.py
│ │ ├── tokenization_cpm.py
│ │ └── tokenization_cpm_fast.py
│ ├── cpmant/
│ │ ├── __init__.py
│ │ ├── configuration_cpmant.py
│ │ ├── modeling_cpmant.py
│ │ └── tokenization_cpmant.py
│ ├── ctrl/
│ │ ├── __init__.py
│ │ ├── configuration_ctrl.py
│ │ ├── modeling_ctrl.py
│ │ ├── modeling_tf_ctrl.py
│ │ └── tokenization_ctrl.py
│ ├── cvt/
│ │ ├── __init__.py
│ │ ├── configuration_cvt.py
│ │ ├── convert_cvt_original_pytorch_checkpoint_to_pytorch.py
│ │ ├── modeling_cvt.py
│ │ └── modeling_tf_cvt.py
│ ├── data2vec/
│ │ ├── __init__.py
│ │ ├── configuration_data2vec_audio.py
│ │ ├── configuration_data2vec_text.py
│ │ ├── configuration_data2vec_vision.py
│ │ ├── convert_data2vec_audio_original_pytorch_checkpoint_to_pytorch.py
│ │ ├── convert_data2vec_text_original_pytorch_checkpoint_to_pytorch.py
│ │ ├── convert_data2vec_vision_original_pytorch_checkpoint_to_pytorch.py
│ │ ├── modeling_data2vec_audio.py
│ │ ├── modeling_data2vec_text.py
│ │ ├── modeling_data2vec_vision.py
│ │ └── modeling_tf_data2vec_vision.py
│ ├── deberta/
│ │ ├── __init__.py
│ │ ├── configuration_deberta.py
│ │ ├── modeling_deberta.py
│ │ ├── modeling_tf_deberta.py
│ │ ├── tokenization_deberta.py
│ │ └── tokenization_deberta_fast.py
│ ├── deberta_v2/
│ │ ├── __init__.py
│ │ ├── configuration_deberta_v2.py
│ │ ├── modeling_deberta_v2.py
│ │ ├── modeling_tf_deberta_v2.py
│ │ ├── tokenization_deberta_v2.py
│ │ └── tokenization_deberta_v2_fast.py
│ ├── decision_transformer/
│ │ ├── __init__.py
│ │ ├── configuration_decision_transformer.py
│ │ └── modeling_decision_transformer.py
│ ├── deformable_detr/
│ │ ├── __init__.py
│ │ ├── configuration_deformable_detr.py
│ │ ├── convert_deformable_detr_to_pytorch.py
│ │ ├── feature_extraction_deformable_detr.py
│ │ ├── image_processing_deformable_detr.py
│ │ ├── load_custom.py
│ │ └── modeling_deformable_detr.py
│ ├── deit/
│ │ ├── __init__.py
│ │ ├── configuration_deit.py
│ │ ├── convert_deit_timm_to_pytorch.py
│ │ ├── feature_extraction_deit.py
│ │ ├── image_processing_deit.py
│ │ ├── modeling_deit.py
│ │ └── modeling_tf_deit.py
│ ├── deta/
│ │ ├── __init__.py
│ │ ├── configuration_deta.py
│ │ ├── convert_deta_resnet_to_pytorch.py
│ │ ├── convert_deta_swin_to_pytorch.py
│ │ ├── image_processing_deta.py
│ │ └── modeling_deta.py
│ ├── detr/
│ │ ├── __init__.py
│ │ ├── configuration_detr.py
│ │ ├── convert_detr_original_pytorch_checkpoint_to_pytorch.py
│ │ ├── convert_detr_to_pytorch.py
│ │ ├── feature_extraction_detr.py
│ │ ├── image_processing_detr.py
│ │ └── modeling_detr.py
│ ├── dialogpt/
│ │ ├── __init__.py
│ │ └── convert_dialogpt_original_pytorch_checkpoint_to_pytorch.py
│ ├── dinat/
│ │ ├── __init__.py
│ │ ├── configuration_dinat.py
│ │ └── modeling_dinat.py
│ ├── distilbert/
│ │ ├── __init__.py
│ │ ├── configuration_distilbert.py
│ │ ├── modeling_distilbert.py
│ │ ├── modeling_flax_distilbert.py
│ │ ├── modeling_tf_distilbert.py
│ │ ├── tokenization_distilbert.py
│ │ └── tokenization_distilbert_fast.py
│ ├── dit/
│ │ ├── __init__.py
│ │ └── convert_dit_unilm_to_pytorch.py
│ ├── donut/
│ │ ├── __init__.py
│ │ ├── configuration_donut_swin.py
│ │ ├── convert_donut_to_pytorch.py
│ │ ├── feature_extraction_donut.py
│ │ ├── image_processing_donut.py
│ │ ├── modeling_donut_swin.py
│ │ └── processing_donut.py
│ ├── dpr/
│ │ ├── __init__.py
│ │ ├── configuration_dpr.py
│ │ ├── convert_dpr_original_checkpoint_to_pytorch.py
│ │ ├── modeling_dpr.py
│ │ ├── modeling_tf_dpr.py
│ │ ├── tokenization_dpr.py
│ │ └── tokenization_dpr_fast.py
│ ├── dpt/
│ │ ├── __init__.py
│ │ ├── configuration_dpt.py
│ │ ├── convert_dpt_hybrid_to_pytorch.py
│ │ ├── convert_dpt_to_pytorch.py
│ │ ├── feature_extraction_dpt.py
│ │ ├── image_processing_dpt.py
│ │ └── modeling_dpt.py
│ ├── efficientformer/
│ │ ├── __init__.py
│ │ ├── configuration_efficientformer.py
│ │ ├── convert_efficientformer_original_pytorch_checkpoint_to_pytorch.py
│ │ ├── image_processing_efficientformer.py
│ │ ├── modeling_efficientformer.py
│ │ └── modeling_tf_efficientformer.py
│ ├── efficientnet/
│ │ ├── __init__.py
│ │ ├── configuration_efficientnet.py
│ │ ├── convert_efficientnet_to_pytorch.py
│ │ ├── image_processing_efficientnet.py
│ │ └── modeling_efficientnet.py
│ ├── electra/
│ │ ├── __init__.py
│ │ ├── configuration_electra.py
│ │ ├── convert_electra_original_tf_checkpoint_to_pytorch.py
│ │ ├── modeling_electra.py
│ │ ├── modeling_flax_electra.py
│ │ ├── modeling_tf_electra.py
│ │ ├── tokenization_electra.py
│ │ └── tokenization_electra_fast.py
│ ├── encoder_decoder/
│ │ ├── __init__.py
│ │ ├── configuration_encoder_decoder.py
│ │ ├── modeling_encoder_decoder.py
│ │ ├── modeling_flax_encoder_decoder.py
│ │ └── modeling_tf_encoder_decoder.py
│ ├── ernie/
│ │ ├── __init__.py
│ │ ├── configuration_ernie.py
│ │ └── modeling_ernie.py
│ ├── ernie_m/
│ │ ├── __init__.py
│ │ ├── configuration_ernie_m.py
│ │ ├── modeling_ernie_m.py
│ │ └── tokenization_ernie_m.py
│ ├── esm/
│ │ ├── __init__.py
│ │ ├── configuration_esm.py
│ │ ├── convert_esm.py
│ │ ├── modeling_esm.py
│ │ ├── modeling_esmfold.py
│ │ ├── modeling_tf_esm.py
│ │ ├── openfold_utils/
│ │ │ ├── __init__.py
│ │ │ ├── chunk_utils.py
│ │ │ ├── data_transforms.py
│ │ │ ├── feats.py
│ │ │ ├── loss.py
│ │ │ ├── protein.py
│ │ │ ├── residue_constants.py
│ │ │ ├── rigid_utils.py
│ │ │ └── tensor_utils.py
│ │ └── tokenization_esm.py
│ ├── flaubert/
│ │ ├── __init__.py
│ │ ├── configuration_flaubert.py
│ │ ├── modeling_flaubert.py
│ │ ├── modeling_tf_flaubert.py
│ │ └── tokenization_flaubert.py
│ ├── flava/
│ │ ├── __init__.py
│ │ ├── configuration_flava.py
│ │ ├── convert_dalle_to_flava_codebook.py
│ │ ├── convert_flava_original_pytorch_to_hf.py
│ │ ├── feature_extraction_flava.py
│ │ ├── image_processing_flava.py
│ │ ├── modeling_flava.py
│ │ └── processing_flava.py
│ ├── fnet/
│ │ ├── __init__.py
│ │ ├── configuration_fnet.py
│ │ ├── convert_fnet_original_flax_checkpoint_to_pytorch.py
│ │ ├── modeling_fnet.py
│ │ ├── tokenization_fnet.py
│ │ └── tokenization_fnet_fast.py
│ ├── focalnet/
│ │ ├── __init__.py
│ │ ├── configuration_focalnet.py
│ │ ├── convert_focalnet_to_hf_format.py
│ │ └── modeling_focalnet.py
│ ├── fsmt/
│ │ ├── __init__.py
│ │ ├── configuration_fsmt.py
│ │ ├── convert_fsmt_original_pytorch_checkpoint_to_pytorch.py
│ │ ├── modeling_fsmt.py
│ │ └── tokenization_fsmt.py
│ ├── funnel/
│ │ ├── __init__.py
│ │ ├── configuration_funnel.py
│ │ ├── convert_funnel_original_tf_checkpoint_to_pytorch.py
│ │ ├── modeling_funnel.py
│ │ ├── modeling_tf_funnel.py
│ │ ├── tokenization_funnel.py
│ │ └── tokenization_funnel_fast.py
│ ├── git/
│ │ ├── __init__.py
│ │ ├── configuration_git.py
│ │ ├── convert_git_to_pytorch.py
│ │ ├── modeling_git.py
│ │ └── processing_git.py
│ ├── glpn/
│ │ ├── __init__.py
│ │ ├── configuration_glpn.py
│ │ ├── convert_glpn_to_pytorch.py
│ │ ├── feature_extraction_glpn.py
│ │ ├── image_processing_glpn.py
│ │ └── modeling_glpn.py
│ ├── gpt2/
│ │ ├── __init__.py
│ │ ├── configuration_gpt2.py
│ │ ├── convert_gpt2_original_tf_checkpoint_to_pytorch.py
│ │ ├── modeling_flax_gpt2.py
│ │ ├── modeling_gpt2.py
│ │ ├── modeling_tf_gpt2.py
│ │ ├── tokenization_gpt2.py
│ │ ├── tokenization_gpt2_fast.py
│ │ └── tokenization_gpt2_tf.py
│ ├── gpt_bigcode/
│ │ ├── __init__.py
│ │ ├── configuration_gpt_bigcode.py
│ │ └── modeling_gpt_bigcode.py
│ ├── gpt_neo/
│ │ ├── __init__.py
│ │ ├── configuration_gpt_neo.py
│ │ ├── convert_gpt_neo_mesh_tf_to_pytorch.py
│ │ ├── modeling_flax_gpt_neo.py
│ │ └── modeling_gpt_neo.py
│ ├── gpt_neox/
│ │ ├── __init__.py
│ │ ├── configuration_gpt_neox.py
│ │ ├── modeling_gpt_neox.py
│ │ └── tokenization_gpt_neox_fast.py
│ ├── gpt_neox_japanese/
│ │ ├── __init__.py
│ │ ├── configuration_gpt_neox_japanese.py
│ │ ├── modeling_gpt_neox_japanese.py
│ │ └── tokenization_gpt_neox_japanese.py
│ ├── gpt_sw3/
│ │ ├── __init__.py
│ │ ├── convert_megatron_to_pytorch.py
│ │ └── tokenization_gpt_sw3.py
│ ├── gptj/
│ │ ├── __init__.py
│ │ ├── configuration_gptj.py
│ │ ├── modeling_flax_gptj.py
│ │ ├── modeling_gptj.py
│ │ └── modeling_tf_gptj.py
│ ├── gptsan_japanese/
│ │ ├── __init__.py
│ │ ├── configuration_gptsan_japanese.py
│ │ ├── convert_gptsan_tf_checkpoint_to_pytorch.py
│ │ ├── modeling_gptsan_japanese.py
│ │ └── tokenization_gptsan_japanese.py
│ ├── graphormer/
│ │ ├── __init__.py
│ │ ├── algos_graphormer.pyx
│ │ ├── collating_graphormer.py
│ │ ├── configuration_graphormer.py
│ │ └── modeling_graphormer.py
│ ├── groupvit/
│ │ ├── __init__.py
│ │ ├── configuration_groupvit.py
│ │ ├── convert_groupvit_nvlab_to_hf.py
│ │ ├── modeling_groupvit.py
│ │ └── modeling_tf_groupvit.py
│ ├── herbert/
│ │ ├── __init__.py
│ │ ├── tokenization_herbert.py
│ │ └── tokenization_herbert_fast.py
│ ├── hubert/
│ │ ├── __init__.py
│ │ ├── configuration_hubert.py
│ │ ├── convert_distilhubert_original_s3prl_checkpoint_to_pytorch.py
│ │ ├── convert_hubert_original_pytorch_checkpoint_to_pytorch.py
│ │ ├── convert_hubert_original_s3prl_checkpoint_to_pytorch.py
│ │ ├── modeling_hubert.py
│ │ └── modeling_tf_hubert.py
│ ├── ibert/
│ │ ├── __init__.py
│ │ ├── configuration_ibert.py
│ │ ├── modeling_ibert.py
│ │ └── quant_modules.py
│ ├── imagegpt/
│ │ ├── __init__.py
│ │ ├── configuration_imagegpt.py
│ │ ├── convert_imagegpt_original_tf2_to_pytorch.py
│ │ ├── feature_extraction_imagegpt.py
│ │ ├── image_processing_imagegpt.py
│ │ └── modeling_imagegpt.py
│ ├── informer/
│ │ ├── __init__.py
│ │ ├── configuration_informer.py
│ │ └── modeling_informer.py
│ ├── jukebox/
│ │ ├── __init__.py
│ │ ├── configuration_jukebox.py
│ │ ├── convert_jukebox.py
│ │ ├── modeling_jukebox.py
│ │ └── tokenization_jukebox.py
│ ├── layoutlm/
│ │ ├── __init__.py
│ │ ├── configuration_layoutlm.py
│ │ ├── modeling_layoutlm.py
│ │ ├── modeling_tf_layoutlm.py
│ │ ├── tokenization_layoutlm.py
│ │ └── tokenization_layoutlm_fast.py
│ ├── layoutlmv2/
│ │ ├── __init__.py
│ │ ├── configuration_layoutlmv2.py
│ │ ├── feature_extraction_layoutlmv2.py
│ │ ├── image_processing_layoutlmv2.py
│ │ ├── modeling_layoutlmv2.py
│ │ ├── processing_layoutlmv2.py
│ │ ├── tokenization_layoutlmv2.py
│ │ └── tokenization_layoutlmv2_fast.py
│ ├── layoutlmv3/
│ │ ├── __init__.py
│ │ ├── configuration_layoutlmv3.py
│ │ ├── feature_extraction_layoutlmv3.py
│ │ ├── image_processing_layoutlmv3.py
│ │ ├── modeling_layoutlmv3.py
│ │ ├── modeling_tf_layoutlmv3.py
│ │ ├── processing_layoutlmv3.py
│ │ ├── tokenization_layoutlmv3.py
│ │ └── tokenization_layoutlmv3_fast.py
│ ├── layoutxlm/
│ │ ├── __init__.py
│ │ ├── processing_layoutxlm.py
│ │ ├── tokenization_layoutxlm.py
│ │ └── tokenization_layoutxlm_fast.py
│ ├── led/
│ │ ├── __init__.py
│ │ ├── configuration_led.py
│ │ ├── modeling_led.py
│ │ ├── modeling_tf_led.py
│ │ ├── tokenization_led.py
│ │ └── tokenization_led_fast.py
│ ├── levit/
│ │ ├── __init__.py
│ │ ├── configuration_levit.py
│ │ ├── convert_levit_timm_to_pytorch.py
│ │ ├── feature_extraction_levit.py
│ │ ├── image_processing_levit.py
│ │ └── modeling_levit.py
│ ├── lilt/
│ │ ├── __init__.py
│ │ ├── configuration_lilt.py
│ │ └── modeling_lilt.py
│ ├── llama/
│ │ ├── __init__.py
│ │ ├── configuration_llama.py
│ │ ├── convert_llama_weights_to_hf.py
│ │ ├── modeling_llama.py
│ │ ├── tokenization_llama.py
│ │ └── tokenization_llama_fast.py
│ ├── longformer/
│ │ ├── __init__.py
│ │ ├── configuration_longformer.py
│ │ ├── convert_longformer_original_pytorch_lightning_to_pytorch.py
│ │ ├── modeling_longformer.py
│ │ ├── modeling_tf_longformer.py
│ │ ├── tokenization_longformer.py
│ │ └── tokenization_longformer_fast.py
│ ├── longt5/
│ │ ├── __init__.py
│ │ ├── configuration_longt5.py
│ │ ├── convert_longt5x_checkpoint_to_flax.py
│ │ ├── modeling_flax_longt5.py
│ │ └── modeling_longt5.py
│ ├── luke/
│ │ ├── __init__.py
│ │ ├── configuration_luke.py
│ │ ├── convert_luke_original_pytorch_checkpoint_to_pytorch.py
│ │ ├── modeling_luke.py
│ │ └── tokenization_luke.py
│ ├── lxmert/
│ │ ├── __init__.py
│ │ ├── configuration_lxmert.py
│ │ ├── convert_lxmert_original_tf_checkpoint_to_pytorch.py
│ │ ├── modeling_lxmert.py
│ │ ├── modeling_tf_lxmert.py
│ │ ├── tokenization_lxmert.py
│ │ └── tokenization_lxmert_fast.py
│ ├── m2m_100/
│ │ ├── __init__.py
│ │ ├── configuration_m2m_100.py
│ │ ├── convert_m2m100_original_checkpoint_to_pytorch.py
│ │ ├── modeling_m2m_100.py
│ │ └── tokenization_m2m_100.py
│ ├── marian/
│ │ ├── __init__.py
│ │ ├── configuration_marian.py
│ │ ├── convert_marian_tatoeba_to_pytorch.py
│ │ ├── convert_marian_to_pytorch.py
│ │ ├── modeling_flax_marian.py
│ │ ├── modeling_marian.py
│ │ ├── modeling_tf_marian.py
│ │ └── tokenization_marian.py
│ ├── markuplm/
│ │ ├── __init__.py
│ │ ├── configuration_markuplm.py
│ │ ├── feature_extraction_markuplm.py
│ │ ├── modeling_markuplm.py
│ │ ├── processing_markuplm.py
│ │ ├── tokenization_markuplm.py
│ │ └── tokenization_markuplm_fast.py
│ ├── mask2former/
│ │ ├── __init__.py
│ │ ├── configuration_mask2former.py
│ │ ├── convert_mask2former_original_pytorch_checkpoint_to_pytorch.py
│ │ ├── image_processing_mask2former.py
│ │ └── modeling_mask2former.py
│ ├── maskformer/
│ │ ├── __init__.py
│ │ ├── configuration_maskformer.py
│ │ ├── configuration_maskformer_swin.py
│ │ ├── convert_maskformer_original_pytorch_checkpoint_to_pytorch.py
│ │ ├── convert_maskformer_resnet_to_pytorch.py
│ │ ├── convert_maskformer_swin_to_pytorch.py
│ │ ├── feature_extraction_maskformer.py
│ │ ├── image_processing_maskformer.py
│ │ ├── modeling_maskformer.py
│ │ └── modeling_maskformer_swin.py
│ ├── mbart/
│ │ ├── __init__.py
│ │ ├── configuration_mbart.py
│ │ ├── convert_mbart_original_checkpoint_to_pytorch.py
│ │ ├── modeling_flax_mbart.py
│ │ ├── modeling_mbart.py
│ │ ├── modeling_tf_mbart.py
│ │ ├── tokenization_mbart.py
│ │ └── tokenization_mbart_fast.py
│ ├── mbart50/
│ │ ├── __init__.py
│ │ ├── tokenization_mbart50.py
│ │ └── tokenization_mbart50_fast.py
│ ├── mctct/
│ │ ├── __init__.py
│ │ ├── configuration_mctct.py
│ │ ├── feature_extraction_mctct.py
│ │ ├── modeling_mctct.py
│ │ └── processing_mctct.py
│ ├── mega/
│ │ ├── __init__.py
│ │ ├── configuration_mega.py
│ │ ├── convert_mega_original_pytorch_checkpoint_to_pytorch.py
│ │ └── modeling_mega.py
│ ├── megatron_bert/
│ │ ├── __init__.py
│ │ ├── configuration_megatron_bert.py
│ │ ├── convert_megatron_bert_checkpoint.py
│ │ └── modeling_megatron_bert.py
│ ├── megatron_gpt2/
│ │ ├── __init__.py
│ │ ├── checkpoint_reshaping_and_interoperability.py
│ │ └── convert_megatron_gpt2_checkpoint.py
│ ├── mgp_str/
│ │ ├── __init__.py
│ │ ├── configuration_mgp_str.py
│ │ ├── modeling_mgp_str.py
│ │ ├── processing_mgp_str.py
│ │ └── tokenization_mgp_str.py
│ ├── mluke/
│ │ ├── __init__.py
│ │ ├── convert_mluke_original_pytorch_checkpoint_to_pytorch.py
│ │ └── tokenization_mluke.py
│ ├── mmbt/
│ │ ├── __init__.py
│ │ ├── configuration_mmbt.py
│ │ └── modeling_mmbt.py
│ ├── mobilebert/
│ │ ├── __init__.py
│ │ ├── configuration_mobilebert.py
│ │ ├── convert_mobilebert_original_tf_checkpoint_to_pytorch.py
│ │ ├── modeling_mobilebert.py
│ │ ├── modeling_tf_mobilebert.py
│ │ ├── tokenization_mobilebert.py
│ │ └── tokenization_mobilebert_fast.py
│ ├── mobilenet_v1/
│ │ ├── __init__.py
│ │ ├── configuration_mobilenet_v1.py
│ │ ├── convert_original_tf_checkpoint_to_pytorch.py
│ │ ├── feature_extraction_mobilenet_v1.py
│ │ ├── image_processing_mobilenet_v1.py
│ │ └── modeling_mobilenet_v1.py
│ ├── mobilenet_v2/
│ │ ├── __init__.py
│ │ ├── configuration_mobilenet_v2.py
│ │ ├── convert_original_tf_checkpoint_to_pytorch.py
│ │ ├── feature_extraction_mobilenet_v2.py
│ │ ├── image_processing_mobilenet_v2.py
│ │ └── modeling_mobilenet_v2.py
│ ├── mobilevit/
│ │ ├── __init__.py
│ │ ├── configuration_mobilevit.py
│ │ ├── convert_mlcvnets_to_pytorch.py
│ │ ├── feature_extraction_mobilevit.py
│ │ ├── image_processing_mobilevit.py
│ │ ├── modeling_mobilevit.py
│ │ └── modeling_tf_mobilevit.py
│ ├── mobilevitv2/
│ │ ├── __init__.py
│ │ ├── configuration_mobilevitv2.py
│ │ ├── convert_mlcvnets_to_pytorch.py
│ │ └── modeling_mobilevitv2.py
│ ├── mpnet/
│ │ ├── __init__.py
│ │ ├── configuration_mpnet.py
│ │ ├── modeling_mpnet.py
│ │ ├── modeling_tf_mpnet.py
│ │ ├── tokenization_mpnet.py
│ │ └── tokenization_mpnet_fast.py
│ ├── mt5/
│ │ ├── __init__.py
│ │ ├── configuration_mt5.py
│ │ ├── modeling_flax_mt5.py
│ │ ├── modeling_mt5.py
│ │ └── modeling_tf_mt5.py
│ ├── mvp/
│ │ ├── __init__.py
│ │ ├── configuration_mvp.py
│ │ ├── modeling_mvp.py
│ │ ├── tokenization_mvp.py
│ │ └── tokenization_mvp_fast.py
│ ├── nat/
│ │ ├── __init__.py
│ │ ├── configuration_nat.py
│ │ └── modeling_nat.py
│ ├── nezha/
│ │ ├── __init__.py
│ │ ├── configuration_nezha.py
│ │ └── modeling_nezha.py
│ ├── nllb/
│ │ ├── __init__.py
│ │ ├── tokenization_nllb.py
│ │ └── tokenization_nllb_fast.py
│ ├── nllb_moe/
│ │ ├── __init__.py
│ │ ├── configuration_nllb_moe.py
│ │ ├── convert_nllb_moe_sharded_original_checkpoint_to_pytorch.py
│ │ └── modeling_nllb_moe.py
│ ├── nystromformer/
│ │ ├── __init__.py
│ │ ├── configuration_nystromformer.py
│ │ ├── convert_nystromformer_original_pytorch_checkpoint_to_pytorch.py
│ │ └── modeling_nystromformer.py
│ ├── oneformer/
│ │ ├── __init__.py
│ │ ├── configuration_oneformer.py
│ │ ├── convert_to_hf_oneformer.py
│ │ ├── image_processing_oneformer.py
│ │ ├── modeling_oneformer.py
│ │ └── processing_oneformer.py
│ ├── open_llama/
│ │ ├── __init__.py
│ │ ├── configuration_open_llama.py
│ │ └── modeling_open_llama.py
│ ├── openai/
│ │ ├── __init__.py
│ │ ├── configuration_openai.py
│ │ ├── convert_openai_original_tf_checkpoint_to_pytorch.py
│ │ ├── modeling_openai.py
│ │ ├── modeling_tf_openai.py
│ │ ├── tokenization_openai.py
│ │ └── tokenization_openai_fast.py
│ ├── opt/
│ │ ├── __init__.py
│ │ ├── configuration_opt.py
│ │ ├── convert_opt_original_pytorch_checkpoint_to_pytorch.py
│ │ ├── modeling_flax_opt.py
│ │ ├── modeling_opt.py
│ │ └── modeling_tf_opt.py
│ ├── owlvit/
│ │ ├── __init__.py
│ │ ├── configuration_owlvit.py
│ │ ├── convert_owlvit_original_flax_to_hf.py
│ │ ├── feature_extraction_owlvit.py
│ │ ├── image_processing_owlvit.py
│ │ ├── modeling_owlvit.py
│ │ └── processing_owlvit.py
│ ├── pegasus/
│ │ ├── __init__.py
│ │ ├── configuration_pegasus.py
│ │ ├── convert_pegasus_tf_to_pytorch.py
│ │ ├── modeling_flax_pegasus.py
│ │ ├── modeling_pegasus.py
│ │ ├── modeling_tf_pegasus.py
│ │ ├── tokenization_pegasus.py
│ │ └── tokenization_pegasus_fast.py
│ ├── pegasus_x/
│ │ ├── __init__.py
│ │ ├── configuration_pegasus_x.py
│ │ └── modeling_pegasus_x.py
│ ├── perceiver/
│ │ ├── __init__.py
│ │ ├── configuration_perceiver.py
│ │ ├── convert_perceiver_haiku_to_pytorch.py
│ │ ├── feature_extraction_perceiver.py
│ │ ├── image_processing_perceiver.py
│ │ ├── modeling_perceiver.py
│ │ └── tokenization_perceiver.py
│ ├── phobert/
│ │ ├── __init__.py
│ │ └── tokenization_phobert.py
│ ├── pix2struct/
│ │ ├── __init__.py
│ │ ├── configuration_pix2struct.py
│ │ ├── convert_pix2struct_original_pytorch_to_hf.py
│ │ ├── image_processing_pix2struct.py
│ │ ├── modeling_pix2struct.py
│ │ └── processing_pix2struct.py
│ ├── plbart/
│ │ ├── __init__.py
│ │ ├── configuration_plbart.py
│ │ ├── convert_plbart_original_checkpoint_to_torch.py
│ │ ├── modeling_plbart.py
│ │ └── tokenization_plbart.py
│ ├── poolformer/
│ │ ├── __init__.py
│ │ ├── configuration_poolformer.py
│ │ ├── convert_poolformer_original_to_pytorch.py
│ │ ├── feature_extraction_poolformer.py
│ │ ├── image_processing_poolformer.py
│ │ └── modeling_poolformer.py
│ ├── prophetnet/
│ │ ├── __init__.py
│ │ ├── configuration_prophetnet.py
│ │ ├── convert_prophetnet_original_pytorch_checkpoint_to_pytorch.py
│ │ ├── modeling_prophetnet.py
│ │ └── tokenization_prophetnet.py
│ ├── qdqbert/
│ │ ├── __init__.py
│ │ ├── configuration_qdqbert.py
│ │ └── modeling_qdqbert.py
│ ├── rag/
│ │ ├── __init__.py
│ │ ├── configuration_rag.py
│ │ ├── modeling_rag.py
│ │ ├── modeling_tf_rag.py
│ │ ├── retrieval_rag.py
│ │ └── tokenization_rag.py
│ ├── realm/
│ │ ├── __init__.py
│ │ ├── configuration_realm.py
│ │ ├── modeling_realm.py
│ │ ├── retrieval_realm.py
│ │ ├── tokenization_realm.py
│ │ └── tokenization_realm_fast.py
│ ├── reformer/
│ │ ├── __init__.py
│ │ ├── configuration_reformer.py
│ │ ├── convert_reformer_trax_checkpoint_to_pytorch.py
│ │ ├── modeling_reformer.py
│ │ ├── tokenization_reformer.py
│ │ └── tokenization_reformer_fast.py
│ ├── regnet/
│ │ ├── __init__.py
│ │ ├── configuration_regnet.py
│ │ ├── convert_regnet_seer_10b_to_pytorch.py
│ │ ├── convert_regnet_to_pytorch.py
│ │ ├── modeling_flax_regnet.py
│ │ ├── modeling_regnet.py
│ │ └── modeling_tf_regnet.py
│ ├── rembert/
│ │ ├── __init__.py
│ │ ├── configuration_rembert.py
│ │ ├── convert_rembert_tf_checkpoint_to_pytorch.py
│ │ ├── modeling_rembert.py
│ │ ├── modeling_tf_rembert.py
│ │ ├── tokenization_rembert.py
│ │ └── tokenization_rembert_fast.py
│ ├── resnet/
│ │ ├── __init__.py
│ │ ├── configuration_resnet.py
│ │ ├── convert_resnet_to_pytorch.py
│ │ ├── modeling_flax_resnet.py
│ │ ├── modeling_resnet.py
│ │ └── modeling_tf_resnet.py
│ ├── retribert/
│ │ ├── __init__.py
│ │ ├── configuration_retribert.py
│ │ ├── modeling_retribert.py
│ │ ├── tokenization_retribert.py
│ │ └── tokenization_retribert_fast.py
│ ├── roberta/
│ │ ├── __init__.py
│ │ ├── configuration_roberta.py
│ │ ├── convert_roberta_original_pytorch_checkpoint_to_pytorch.py
│ │ ├── modeling_flax_roberta.py
│ │ ├── modeling_roberta.py
│ │ ├── modeling_tf_roberta.py
│ │ ├── tokenization_roberta.py
│ │ └── tokenization_roberta_fast.py
│ ├── roberta_prelayernorm/
│ │ ├── __init__.py
│ │ ├── configuration_roberta_prelayernorm.py
│ │ ├── convert_roberta_prelayernorm_original_pytorch_checkpoint_to_pytorch.py
│ │ ├── modeling_flax_roberta_prelayernorm.py
│ │ ├── modeling_roberta_prelayernorm.py
│ │ └── modeling_tf_roberta_prelayernorm.py
│ ├── roc_bert/
│ │ ├── __init__.py
│ │ ├── configuration_roc_bert.py
│ │ ├── modeling_roc_bert.py
│ │ └── tokenization_roc_bert.py
│ ├── roformer/
│ │ ├── __init__.py
│ │ ├── configuration_roformer.py
│ │ ├── convert_roformer_original_tf_checkpoint_to_pytorch.py
│ │ ├── modeling_flax_roformer.py
│ │ ├── modeling_roformer.py
│ │ ├── modeling_tf_roformer.py
│ │ ├── tokenization_roformer.py
│ │ ├── tokenization_roformer_fast.py
│ │ └── tokenization_utils.py
│ ├── rwkv/
│ │ ├── __init__.py
│ │ ├── configuration_rwkv.py
│ │ ├── convert_rwkv_checkpoint_to_hf.py
│ │ └── modeling_rwkv.py
│ ├── sam/
│ │ ├── __init__.py
│ │ ├── configuration_sam.py
│ │ ├── convert_sam_original_to_hf_format.py
│ │ ├── image_processing_sam.py
│ │ ├── modeling_sam.py
│ │ ├── modeling_tf_sam.py
│ │ └── processing_sam.py
│ ├── segformer/
│ │ ├── __init__.py
│ │ ├── configuration_segformer.py
│ │ ├── convert_segformer_original_to_pytorch.py
│ │ ├── feature_extraction_segformer.py
│ │ ├── image_processing_segformer.py
│ │ ├── modeling_segformer.py
│ │ └── modeling_tf_segformer.py
│ ├── sew/
│ │ ├── __init__.py
│ │ ├── configuration_sew.py
│ │ ├── convert_sew_original_pytorch_checkpoint_to_pytorch.py
│ │ └── modeling_sew.py
│ ├── sew_d/
│ │ ├── __init__.py
│ │ ├── configuration_sew_d.py
│ │ ├── convert_sew_d_original_pytorch_checkpoint_to_pytorch.py
│ │ └── modeling_sew_d.py
│ ├── speech_encoder_decoder/
│ │ ├── __init__.py
│ │ ├── configuration_speech_encoder_decoder.py
│ │ ├── convert_mbart_wav2vec2_seq2seq_original_to_pytorch.py
│ │ ├── convert_speech_to_text_wav2vec2_seq2seq_original_to_pytorch.py
│ │ ├── modeling_flax_speech_encoder_decoder.py
│ │ └── modeling_speech_encoder_decoder.py
│ ├── speech_to_text/
│ │ ├── __init__.py
│ │ ├── configuration_speech_to_text.py
│ │ ├── convert_s2t_fairseq_to_tfms.py
│ │ ├── feature_extraction_speech_to_text.py
│ │ ├── modeling_speech_to_text.py
│ │ ├── modeling_tf_speech_to_text.py
│ │ ├── processing_speech_to_text.py
│ │ └── tokenization_speech_to_text.py
│ ├── speech_to_text_2/
│ │ ├── __init__.py
│ │ ├── configuration_speech_to_text_2.py
│ │ ├── modeling_speech_to_text_2.py
│ │ ├── processing_speech_to_text_2.py
│ │ └── tokenization_speech_to_text_2.py
│ ├── speecht5/
│ │ ├── __init__.py
│ │ ├── configuration_speecht5.py
│ │ ├── convert_hifigan.py
│ │ ├── convert_speecht5_original_pytorch_checkpoint_to_pytorch.py
│ │ ├── feature_extraction_speecht5.py
│ │ ├── modeling_speecht5.py
│ │ ├── processing_speecht5.py
│ │ └── tokenization_speecht5.py
│ ├── splinter/
│ │ ├── __init__.py
│ │ ├── configuration_splinter.py
│ │ ├── modeling_splinter.py
│ │ ├── tokenization_splinter.py
│ │ └── tokenization_splinter_fast.py
│ ├── squeezebert/
│ │ ├── __init__.py
│ │ ├── configuration_squeezebert.py
│ │ ├── modeling_squeezebert.py
│ │ ├── tokenization_squeezebert.py
│ │ └── tokenization_squeezebert_fast.py
│ ├── swiftformer/
│ │ ├── __init__.py
│ │ ├── configuration_swiftformer.py
│ │ ├── convert_swiftformer_original_to_hf.py
│ │ └── modeling_swiftformer.py
│ ├── swin/
│ │ ├── __init__.py
│ │ ├── configuration_swin.py
│ │ ├── convert_swin_simmim_to_pytorch.py
│ │ ├── convert_swin_timm_to_pytorch.py
│ │ ├── modeling_swin.py
│ │ └── modeling_tf_swin.py
│ ├── swin2sr/
│ │ ├── __init__.py
│ │ ├── configuration_swin2sr.py
│ │ ├── convert_swin2sr_original_to_pytorch.py
│ │ ├── image_processing_swin2sr.py
│ │ └── modeling_swin2sr.py
│ ├── swinv2/
│ │ ├── __init__.py
│ │ ├── configuration_swinv2.py
│ │ ├── convert_swinv2_timm_to_pytorch.py
│ │ └── modeling_swinv2.py
│ ├── switch_transformers/
│ │ ├── __init__.py
│ │ ├── configuration_switch_transformers.py
│ │ ├── convert_big_switch.py
│ │ ├── convert_switch_transformers_original_flax_checkpoint_to_pytorch.py
│ │ └── modeling_switch_transformers.py
│ ├── t5/
│ │ ├── __init__.py
│ │ ├── configuration_t5.py
│ │ ├── convert_t5_original_tf_checkpoint_to_pytorch.py
│ │ ├── convert_t5x_checkpoint_to_flax.py
│ │ ├── convert_t5x_checkpoint_to_pytorch.py
│ │ ├── modeling_flax_t5.py
│ │ ├── modeling_t5.py
│ │ ├── modeling_tf_t5.py
│ │ ├── tokenization_t5.py
│ │ └── tokenization_t5_fast.py
│ ├── table_transformer/
│ │ ├── __init__.py
│ │ ├── configuration_table_transformer.py
│ │ ├── convert_table_transformer_original_pytorch_checkpoint_to_pytorch.py
│ │ └── modeling_table_transformer.py
│ ├── tapas/
│ │ ├── __init__.py
│ │ ├── configuration_tapas.py
│ │ ├── convert_tapas_original_tf_checkpoint_to_pytorch.py
│ │ ├── modeling_tapas.py
│ │ ├── modeling_tf_tapas.py
│ │ └── tokenization_tapas.py
│ ├── tapex/
│ │ ├── __init__.py
│ │ └── tokenization_tapex.py
│ ├── time_series_transformer/
│ │ ├── __init__.py
│ │ ├── configuration_time_series_transformer.py
│ │ └── modeling_time_series_transformer.py
│ ├── timesformer/
│ │ ├── __init__.py
│ │ ├── configuration_timesformer.py
│ │ ├── convert_timesformer_to_pytorch.py
│ │ └── modeling_timesformer.py
│ ├── timm_backbone/
│ │ ├── __init__.py
│ │ ├── configuration_timm_backbone.py
│ │ └── modeling_timm_backbone.py
│ ├── trajectory_transformer/
│ │ ├── __init__.py
│ │ ├── configuration_trajectory_transformer.py
│ │ ├── convert_trajectory_transformer_original_pytorch_checkpoint_to_pytorch.py
│ │ └── modeling_trajectory_transformer.py
│ ├── transfo_xl/
│ │ ├── __init__.py
│ │ ├── configuration_transfo_xl.py
│ │ ├── convert_transfo_xl_original_tf_checkpoint_to_pytorch.py
│ │ ├── modeling_tf_transfo_xl.py
│ │ ├── modeling_tf_transfo_xl_utilities.py
│ │ ├── modeling_transfo_xl.py
│ │ ├── modeling_transfo_xl_utilities.py
│ │ └── tokenization_transfo_xl.py
│ ├── trocr/
│ │ ├── __init__.py
│ │ ├── configuration_trocr.py
│ │ ├── convert_trocr_unilm_to_pytorch.py
│ │ ├── modeling_trocr.py
│ │ └── processing_trocr.py
│ ├── tvlt/
│ │ ├── __init__.py
│ │ ├── configuration_tvlt.py
│ │ ├── feature_extraction_tvlt.py
│ │ ├── image_processing_tvlt.py
│ │ ├── modeling_tvlt.py
│ │ └── processing_tvlt.py
│ ├── unispeech/
│ │ ├── __init__.py
│ │ ├── configuration_unispeech.py
│ │ ├── convert_unispeech_original_pytorch_checkpoint_to_pytorch.py
│ │ └── modeling_unispeech.py
│ ├── unispeech_sat/
│ │ ├── __init__.py
│ │ ├── configuration_unispeech_sat.py
│ │ ├── convert_unispeech_original_s3prl_checkpoint_to_pytorch.py
│ │ ├── convert_unispeech_sat_original_pytorch_checkpoint_to_pytorch.py
│ │ └── modeling_unispeech_sat.py
│ ├── upernet/
│ │ ├── __init__.py
│ │ ├── configuration_upernet.py
│ │ ├── convert_convnext_upernet_to_pytorch.py
│ │ ├── convert_swin_upernet_to_pytorch.py
│ │ └── modeling_upernet.py
│ ├── van/
│ │ ├── __init__.py
│ │ ├── configuration_van.py
│ │ ├── convert_van_to_pytorch.py
│ │ └── modeling_van.py
│ ├── videomae/
│ │ ├── __init__.py
│ │ ├── configuration_videomae.py
│ │ ├── convert_videomae_to_pytorch.py
│ │ ├── feature_extraction_videomae.py
│ │ ├── image_processing_videomae.py
│ │ └── modeling_videomae.py
│ ├── vilt/
│ │ ├── __init__.py
│ │ ├── configuration_vilt.py
│ │ ├── convert_vilt_original_to_pytorch.py
│ │ ├── feature_extraction_vilt.py
│ │ ├── image_processing_vilt.py
│ │ ├── modeling_vilt.py
│ │ └── processing_vilt.py
│ ├── vision_encoder_decoder/
│ │ ├── __init__.py
│ │ ├── configuration_vision_encoder_decoder.py
│ │ ├── modeling_flax_vision_encoder_decoder.py
│ │ ├── modeling_tf_vision_encoder_decoder.py
│ │ └── modeling_vision_encoder_decoder.py
│ ├── vision_text_dual_encoder/
│ │ ├── __init__.py
│ │ ├── configuration_vision_text_dual_encoder.py
│ │ ├── modeling_flax_vision_text_dual_encoder.py
│ │ ├── modeling_tf_vision_text_dual_encoder.py
│ │ ├── modeling_vision_text_dual_encoder.py
│ │ └── processing_vision_text_dual_encoder.py
│ ├── visual_bert/
│ │ ├── __init__.py
│ │ ├── configuration_visual_bert.py
│ │ ├── convert_visual_bert_original_pytorch_checkpoint_to_pytorch.py
│ │ └── modeling_visual_bert.py
│ ├── vit/
│ │ ├── __init__.py
│ │ ├── configuration_vit.py
│ │ ├── convert_dino_to_pytorch.py
│ │ ├── convert_vit_timm_to_pytorch.py
│ │ ├── feature_extraction_vit.py
│ │ ├── image_processing_vit.py
│ │ ├── modeling_flax_vit.py
│ │ ├── modeling_tf_vit.py
│ │ └── modeling_vit.py
│ ├── vit_hybrid/
│ │ ├── __init__.py
│ │ ├── configuration_vit_hybrid.py
│ │ ├── convert_vit_hybrid_timm_to_pytorch.py
│ │ ├── image_processing_vit_hybrid.py
│ │ └── modeling_vit_hybrid.py
│ ├── vit_mae/
│ │ ├── __init__.py
│ │ ├── configuration_vit_mae.py
│ │ ├── convert_vit_mae_to_pytorch.py
│ │ ├── modeling_tf_vit_mae.py
│ │ └── modeling_vit_mae.py
│ ├── vit_msn/
│ │ ├── __init__.py
│ │ ├── configuration_vit_msn.py
│ │ ├── convert_msn_to_pytorch.py
│ │ └── modeling_vit_msn.py
│ ├── wav2vec2/
│ │ ├── __init__.py
│ │ ├── configuration_wav2vec2.py
│ │ ├── convert_wav2vec2_original_pytorch_checkpoint_to_pytorch.py
│ │ ├── convert_wav2vec2_original_s3prl_checkpoint_to_pytorch.py
│ │ ├── feature_extraction_wav2vec2.py
│ │ ├── modeling_flax_wav2vec2.py
│ │ ├── modeling_tf_wav2vec2.py
│ │ ├── modeling_wav2vec2.py
│ │ ├── processing_wav2vec2.py
│ │ └── tokenization_wav2vec2.py
│ ├── wav2vec2_conformer/
│ │ ├── __init__.py
│ │ ├── configuration_wav2vec2_conformer.py
│ │ ├── convert_wav2vec2_conformer_original_pytorch_checkpoint_to_pytorch.py
│ │ └── modeling_wav2vec2_conformer.py
│ ├── wav2vec2_phoneme/
│ │ ├── __init__.py
│ │ └── tokenization_wav2vec2_phoneme.py
│ ├── wav2vec2_with_lm/
│ │ ├── __init__.py
│ │ └── processing_wav2vec2_with_lm.py
│ ├── wavlm/
│ │ ├── __init__.py
│ │ ├── configuration_wavlm.py
│ │ ├── convert_wavlm_original_pytorch_checkpoint_to_pytorch.py
│ │ ├── convert_wavlm_original_s3prl_checkpoint_to_pytorch.py
│ │ └── modeling_wavlm.py
│ ├── whisper/
│ │ ├── __init__.py
│ │ ├── configuration_whisper.py
│ │ ├── convert_openai_to_hf.py
│ │ ├── english_normalizer.py
│ │ ├── feature_extraction_whisper.py
│ │ ├── modeling_flax_whisper.py
│ │ ├── modeling_tf_whisper.py
│ │ ├── modeling_whisper.py
│ │ ├── processing_whisper.py
│ │ ├── tokenization_whisper.py
│ │ └── tokenization_whisper_fast.py
│ ├── x_clip/
│ │ ├── __init__.py
│ │ ├── configuration_x_clip.py
│ │ ├── convert_x_clip_original_pytorch_to_hf.py
│ │ ├── modeling_x_clip.py
│ │ └── processing_x_clip.py
│ ├── xglm/
│ │ ├── __init__.py
│ │ ├── configuration_xglm.py
│ │ ├── convert_xglm_original_ckpt_to_trfms.py
│ │ ├── modeling_flax_xglm.py
│ │ ├── modeling_tf_xglm.py
│ │ ├── modeling_xglm.py
│ │ ├── tokenization_xglm.py
│ │ └── tokenization_xglm_fast.py
│ ├── xlm/
│ │ ├── __init__.py
│ │ ├── configuration_xlm.py
│ │ ├── convert_xlm_original_pytorch_checkpoint_to_pytorch.py
│ │ ├── modeling_tf_xlm.py
│ │ ├── modeling_xlm.py
│ │ └── tokenization_xlm.py
│ ├── xlm_prophetnet/
│ │ ├── __init__.py
│ │ ├── configuration_xlm_prophetnet.py
│ │ ├── modeling_xlm_prophetnet.py
│ │ └── tokenization_xlm_prophetnet.py
│ ├── xlm_roberta/
│ │ ├── __init__.py
│ │ ├── configuration_xlm_roberta.py
│ │ ├── modeling_flax_xlm_roberta.py
│ │ ├── modeling_tf_xlm_roberta.py
│ │ ├── modeling_xlm_roberta.py
│ │ ├── tokenization_xlm_roberta.py
│ │ └── tokenization_xlm_roberta_fast.py
│ ├── xlm_roberta_xl/
│ │ ├── __init__.py
│ │ ├── configuration_xlm_roberta_xl.py
│ │ ├── convert_xlm_roberta_xl_original_pytorch_checkpoint_to_pytorch.py
│ │ └── modeling_xlm_roberta_xl.py
│ ├── xlnet/
│ │ ├── __init__.py
│ │ ├── configuration_xlnet.py
│ │ ├── convert_xlnet_original_tf_checkpoint_to_pytorch.py
│ │ ├── modeling_tf_xlnet.py
│ │ ├── modeling_xlnet.py
│ │ ├── tokenization_xlnet.py
│ │ └── tokenization_xlnet_fast.py
│ ├── xmod/
│ │ ├── __init__.py
│ │ ├── configuration_xmod.py
│ │ ├── convert_xmod_original_pytorch_checkpoint_to_pytorch.py
│ │ └── modeling_xmod.py
│ ├── yolos/
│ │ ├── __init__.py
│ │ ├── configuration_yolos.py
│ │ ├── convert_yolos_to_pytorch.py
│ │ ├── feature_extraction_yolos.py
│ │ ├── image_processing_yolos.py
│ │ └── modeling_yolos.py
│ └── yoso/
│ ├── __init__.py
│ ├── configuration_yoso.py
│ ├── convert_yoso_pytorch_to_pytorch.py
│ └── modeling_yoso.py
├── onnx/
│ ├── __init__.py
│ ├── __main__.py
│ ├── config.py
│ ├── convert.py
│ ├── features.py
│ └── utils.py
├── optimization.py
├── optimization_tf.py
├── pipelines/
│ ├── __init__.py
│ ├── audio_classification.py
│ ├── audio_utils.py
│ ├── automatic_speech_recognition.py
│ ├── base.py
│ ├── conversational.py
│ ├── depth_estimation.py
│ ├── document_question_answering.py
│ ├── feature_extraction.py
│ ├── fill_mask.py
│ ├── image_classification.py
│ ├── image_segmentation.py
│ ├── image_to_text.py
│ ├── mask_generation.py
│ ├── object_detection.py
│ ├── pt_utils.py
│ ├── question_answering.py
│ ├── table_question_answering.py
│ ├── text2text_generation.py
│ ├── text_classification.py
│ ├── text_generation.py
│ ├── token_classification.py
│ ├── video_classification.py
│ ├── visual_question_answering.py
│ ├── zero_shot_audio_classification.py
│ ├── zero_shot_classification.py
│ ├── zero_shot_image_classification.py
│ └── zero_shot_object_detection.py
├── processing_utils.py
├── pytorch_utils.py
├── sagemaker/
│ ├── __init__.py
│ ├── trainer_sm.py
│ └── training_args_sm.py
├── testing_utils.py
├── tf_utils.py
├── time_series_utils.py
├── tokenization_utils.py
├── tokenization_utils_base.py
├── tokenization_utils_fast.py
├── tools/
│ ├── __init__.py
│ ├── agents.py
│ ├── base.py
│ ├── document_question_answering.py
│ ├── evaluate_agent.py
│ ├── image_captioning.py
│ ├── image_question_answering.py
│ ├── image_segmentation.py
│ ├── prompts.py
│ ├── python_interpreter.py
│ ├── speech_to_text.py
│ ├── text_classification.py
│ ├── text_question_answering.py
│ ├── text_summarization.py
│ ├── text_to_speech.py
│ └── translation.py
├── trainer.py
├── trainer_callback.py
├── trainer_pt_utils.py
├── trainer_seq2seq.py
├── trainer_tf.py
├── trainer_utils.py
├── training_args.py
├── training_args_seq2seq.py
├── training_args_tf.py
└── utils/
├── __init__.py
├── backbone_utils.py
├── bitsandbytes.py
├── constants.py
├── doc.py
├── dummy_detectron2_objects.py
├── dummy_flax_objects.py
├── dummy_keras_nlp_objects.py
├── dummy_pt_objects.py
├── dummy_sentencepiece_and_tokenizers_objects.py
├── dummy_sentencepiece_objects.py
├── dummy_speech_objects.py
├── dummy_tensorflow_text_objects.py
├── dummy_tf_objects.py
├── dummy_tokenizers_objects.py
├── dummy_vision_objects.py
├── fx.py
├── generic.py
├── hp_naming.py
├── hub.py
├── import_utils.py
├── logging.py
├── model_parallel_utils.py
├── notebook.py
├── quantization_config.py
├── sentencepiece_model_pb2.py
└── versions.py
================================================
FILE CONTENTS
================================================
================================================
FILE: .gitignore
================================================
output/
wandb/
run_moe.sh
run_multi_node.sh
single_lora_run_sft.sh
run_loramoe_bak.sh
data/tiny_data/test_1024
data/tiny_data/train/train_1024
================================================
FILE: README.md
================================================
# LoRAMoE: Revolutionizing Mixture of Experts for Maintaining World Knowledge in Language Model Alignment
This is the repository for [LoRAMoE: Revolutionizing Mixture of Experts for Maintaining World Knowledge in Language Model Alignment](https://arxiv.org/abs/2312.09979).
## Implementation
You can quickly export the environment by using the follow command:
```bash
conda env create -f environment.yml
```
or
```bash
conda create -n loramoe python=3.10 -y
pip install -r requirements.txt
```
We *do not* install the `peft` to avoid the conflicts with the local `peft` package.
## Usage
### Data Format
We construct a tiny dataset to demonstrate the data format during the training and inference phase and evaluate the correct of code.
```
data/
|--tiny_data/
|--train/train.json
|--test.json
```
### Train LoRAMoE on Single Node
```bash
bash run_loramoe.sh
```
### Explanations of Hyper-parameters
| blc weight | blc alpha | LoRA rank | LoRA alpha | LoRA trainable |LoRA dropout |LoRA num |
|---------------|---------------|---------------|------------|----------------|---------------| --------|
| the strength of localized balance constraints |degree of imbalance | rank of LoRA experts | LoRA scale | where the LoRA layers are added | dropout rate in LoRA|number of experts|
## Note: Our main changes to `transformers` and `peft`
In `transformers`, we mainly change `modeling_llama.py` to introduce new para `task_types`.
In `peft`, we replace the original LoRA class with the mixtures of experts architecture.
## How to Evaluate
We use [opencompass](https://github.com/open-compass/opencompass/tree/main) for evaluation. To run LoRAMoE on opencompass:
- In `opencompass/opencompass/models/huggingface.py`, add:
```python
import sys
sys.path.insert(0, 'path_to_your_current_dir_containing_changed_peft&transformers')
```
- In the config file
```python
models = [
dict(
type=HuggingFaceCausalLM,
abbr='',
path="path_to_base_model",
tokenizer_path='path_to_tokenizer',
peft_path='path_to_loramoe',
...
)
]
```
## Citation
If you find this useful in your research, please consider citing
```
@misc{dou2024loramoe,
title={LoRAMoE: Revolutionizing Mixture of Experts for Maintaining World Knowledge in Language Model Alignment},
author={Shihan Dou and Enyu Zhou and Yan Liu and Songyang Gao and Jun Zhao and Wei Shen and Yuhao Zhou and Zhiheng Xi and Xiao Wang and Xiaoran Fan and Shiliang Pu and Jiang Zhu and Rui Zheng and Tao Gui and Qi Zhang and Xuanjing Huang},
year={2023},
eprint={2312.09979},
archivePrefix={arXiv},
primaryClass={cs.CL}
}
```
================================================
FILE: __init__.py
================================================
================================================
FILE: build_dataset.py
================================================
import logging
import os
from dataclasses import dataclass
from typing import Dict, Sequence, Union, List
import datasets
import torch
from datasets import load_dataset, concatenate_datasets
import transformers
IGNORE_INDEX = -100
logger = logging.getLogger('__name__')
PROMPT_TEMPLATE = (
"{instruction}</s>"
)
def build_instruction_dataset(data_path: Union[List[str],str],
tokenizer: transformers.PreTrainedTokenizer,
max_seq_length: int, data_cache_dir = None,
preprocessing_num_workers = None,
):
def tokenization(examples):
sources = []
targets = []
task_types = []
prompt = PROMPT_TEMPLATE
for instruction, input, output, task_type in zip(examples['instruction'], examples['input'], examples['output'], examples['task_type']):
if input is not None and input !="":
instruction = instruction+'\n'+input
source = prompt.format_map({'instruction':instruction})
target = f"{output}{tokenizer.eos_token}"
sources.append(source)
targets.append(target)
task_types.append(task_type)
tokenized_sources = tokenizer(sources,return_attention_mask=False)
tokenized_targets = tokenizer(targets,return_attention_mask=False,add_special_tokens=False)
all_input_ids = []
all_labels = []
for s,t in zip(tokenized_sources['input_ids'],tokenized_targets['input_ids']):
input_ids = torch.LongTensor(s + t)[:max_seq_length]
labels = torch.LongTensor([IGNORE_INDEX] * len(s) + t)[:max_seq_length]
assert len(input_ids) == len(labels)
all_input_ids.append(input_ids)
all_labels.append(labels)
results = {'input_ids':all_input_ids, 'labels': all_labels, 'task_types': task_types}
return results
logging.warning("building dataset...")
all_datasets = []
if not isinstance(data_path,(list,tuple)):
data_path = [data_path]
for file in data_path:
if data_cache_dir is None:
data_cache_dir = str(os.path.dirname(file))
cache_path = os.path.join(data_cache_dir,os.path.basename(file).split('.')[0]+f"_{max_seq_length}")
os.makedirs(cache_path, exist_ok=True)
try:
processed_dataset = datasets.load_from_disk(cache_path)
logger.info(f'training datasets-{file} has been loaded from disk')
except Exception:
raw_dataset = load_dataset("json", data_files=file, cache_dir=cache_path)
tokenization_func = tokenization
tokenized_dataset = raw_dataset.map(
tokenization_func,
batched=True,
num_proc=preprocessing_num_workers,
remove_columns=["instruction","input","output","task_type"],
keep_in_memory=False,
desc="preprocessing on dataset",
)
processed_dataset = tokenized_dataset
processed_dataset.save_to_disk(cache_path)
processed_dataset.set_format('torch')
all_datasets.append(processed_dataset['train'])
all_datasets = concatenate_datasets(all_datasets)
return all_datasets
@dataclass
class DataCollatorForSupervisedDataset(object):
"""Collate examples for supervised fine-tuning."""
tokenizer: transformers.PreTrainedTokenizer
def __call__(self, instances: Sequence[Dict]) -> Dict[str, torch.Tensor]:
input_ids, labels, task_types = tuple([instance[key] for instance in instances] for key in ("input_ids", "labels", "task_types"))
input_ids = torch.nn.utils.rnn.pad_sequence(
input_ids, batch_first=True, padding_value=self.tokenizer.pad_token_id
)
labels = torch.nn.utils.rnn.pad_sequence(labels, batch_first=True, padding_value=-100)
task_types = torch.tensor(task_types)
return dict(
input_ids=input_ids,
labels=labels,
attention_mask=input_ids.ne(self.tokenizer.pad_token_id),
task_types=task_types
)
================================================
FILE: data/tiny_data/test.json
================================================
[
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nSo often repeated are the words of the Swedish diplomat Dag Hammarskjold, the organisation's most beloved secretary general, they have come to serve as a mission statement of sorts.\nAdditionally, they function as a crude benchmark against which the work of the United Nations can be judged.\nWhen the organisation was formed in 1945, in the aftermath of World War Two and the atomic bombings of Hiroshima and Nagasaki, \"hell\" would have been the outbreak of a third global conflict and nuclear Armageddon, neither of which has come to pass.\nBut in those post-War years, as the full horror of the Holocaust was uncovered, \"hell\" also meant genocide, a word which had only just been coined: the systematic massacre of thousands of people because of their ethnicity, religion, race or nationality.\nHere, the UN has not always been able to halt the descent into the abyss.\nTo its member states' eternal shame, on some occasions it has been a bystander to genocide.\nIn any historical ledger, Rwanda and Srebrenica stand out as ghastly failures.\nDuring the Rwanda genocide, UN peacekeepers deployed in the country concentrated on evacuating expatriates and government officials, failing to intervene as 800,000 Tutsis and sympathetic Hutus were slaughtered.\nIn Srebrenica in July 1995, more than 8,000 Muslims, mainly men and boys, were massacred by Bosnian Serb forces, which barged past Dutch soldiers wearing the distinctive blue helmet of the UN peacekeepers as if they weren't there.\nWhat made the massacre all the more horrifying was that the Muslims had sheltered in enclaves deemed \"safe areas\" under the protections of the UN.\nIn some conflicts, such as Yugoslavia, the UN was slow to respond.\nIn others, such as Vietnam and the Iraq war, it was sidelined.\nIts efforts to broker peace talks during Syria's civil war, now in its fifth year, have always ended in failure.\nNow, a third UN envoy, the Italian diplomat Staffan de Mistura, is trying, without success so far, to break the impasse.\nPeace has also proved elusive in the Israeli-Palestinian conflict, one of the UN's first major dilemmas following its formation in 1945 and a long-running bugbear.\nSometimes the UN has been part of the problem rather than the solution.\nBlue-helmeted peacekeepers have been accused of a litany of sexual abuses, most recently in the Central African Republic.\nIn Haiti, peacekeepers from Nepal were the source, the evidence suggests, of a cholera outbreak that has killed more than 8,000 people - though the UN refuses to accept any legal liability.\nWhile the UN sees itself as a force for democratisation, it is a ridiculously undemocratic organisation.\nIts locus of power is the Security Council, where the five permanent members - the US, Britain, France, China and Russia - still wield crippling vetoes.\nThe Security Council, like amber encasing an extinct insect, preserves the influence of the victors from World War Two, freezing a moment in a time.\nTellingly, when Franklin Delano Roosevelt coined the phrase the \"United Nations,\" he was referring not to the countries of the world but rather the Allied powers.\nGermany and Japan do not have permanent seats on the Security Council, nor do India or Brazil.\nThough every country has a vote in the General Assembly, a less powerful body, almost 75% of the world's population is effectively disenfranchised in the Security Council.\nThere, preposterously, one veto-wielding power can thwart the will of the other 192 members.\nAll five veto powers have to agree, for instance, on the appointment of secretaries general, enabling weak, if well-intentioned, compromise candidates, like the present leader Ban Ki-moon, to reach the top.\nFor all its shortcomings, however, the United Nations can look back on much of the past 70 years with pride.\nIt is credited with brokering more than 170 peace settlements, though it has proved better at preventing nation-to-nation conflicts rather than civil wars.\nThe Cold War never turned hot, although there were Cold War proxy conflicts in Korea, Vietnam, Nicaragua, Angola, Afghanistan and elsewhere.\nThe number of people killed in conflicts has declined since 1945.\nFewer died in the first decade of this century than in any decade during the last.\nIts peacekeeping operations, which began during the Suez crisis in 1956, have expanded to 16 missions around the world, keeping and monitoring the peace in Haiti to Darfur, Cyprus to the Golan Heights.\nThe UN has codified a panoply of international laws and forged the Universal Declaration of Human Rights in 1948.\nIt has helped broker major international treaties, such as the landmark, nuclear non-proliferation treaty that came into force in 1970, and helped organise historic elections, such as the first presidential contest in Afghanistan in 2004.\nThe work of UN agencies, much of it unnoticed and unsung, has been impressive, partly because they employ some of the world's leading experts in disaster relief, public health and economic development.\nNot only are their staff expert, but often incredibly brave and dedicated.\nPartly because of the efforts of Unicef, deaths of children under the age of five have declined from nearly 12 million in 1990 to 6.9 million in 2011.\nThe UN's refugee agency, the UNHCR, has helped 17 million asylum seekers and refugees, picking up two Nobel peace prizes, in 1954 and 1981, for its efforts.\nThe World Food Programme each year gives assistance to 80 million people in 75 countries.\nThe preservation of 1,031 of the world's most beautiful sites, from the Serengeti National Park to Machu Picchu, is partly thanks to the cultural agency Unesco.\nIts Millennium Development Goals, which will soon be superseded by the Sustainable Development Goals, have been described as the greatest anti-poverty drive in history.\nOften, however, the work of agencies is hindered by a lack of funding from member states, which is often called donor fatigue.\nThere is a chronic shortfall in Syria, for instance, where only 38% of the funding requirements have been met.\nOf the $4.5bn needed by the UN to confront the Syrian refugee crisis, only $1.8bn has been contributed.\nThe UN's convening power, the simple fact that it brings together 193 members, is obviously unique.\nStill, all too often its members deliberately hamper its work.\nUN agencies would like to deliver humanitarian aid to Syria, to use one example, but for much of the past five years the Security Council has not mandated them to do so.\nRussia, with the backing usually of China, has used its veto as a block to protect its ally, Syria's President Bashar al-Assad.\nThis kind of obstructionism and negative statecraft is common at the UN.\nCritics of Israel, a country that regularly complains of being unfairly vilified at the UN, bemoan the regular use of US vetoes to shield it from international criticism.\nOften the UN doesn't solve the world's problems, but merely reflects them.\nAs the late US diplomat Richard Holbrooke once memorably said, blaming the UN is rather like blaming Madison Square Garden when the New York Knicks lose.\nIt is the players that count, and they who ultimately decide on whether the UN succeeds or fails as a multinational problem-solver.\nSyria offers a case study of the UN at its best and its worst.\nThe deadlock in Security Council has stymied peace efforts, but the UN runs the massive Zaatari refugee camp in Jordan, the home to almost 80,000 displaced people.\nThat camp could never in any way be described as \"heaven\".\nBut it has saved those seeking shelter from \"hell\".",
"input": "",
"output": "The summary is: \n\"The UN was not created to take mankind to heaven, but to save humanity from hell.\"",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nScottish winger Matt Williams' early touchdown caused shudders round a soaking wet Sixways.\nBut the home side responded with four tries, allied to 18 points from the boot of stand-off Ryan Lamb.\nFlanker Sam Lewis scored a hat-trick of tries, while winger Dean Hammond also crossed the whitewash.\nLamb kicked three of his four conversion attempts, as well as two key first-half penalties - and two more late in the game.\nFull-back Peter Lydon got the Exiles' other try, which he converted, along with a first-half penalty for a 10-point haul.\nScottish trailed by five points from Saturday's first leg, only for that advantage to be wiped out inside the first six minutes.\nJamie Stevenson's blindside run set up right wing Williams to score in the corner.\nBut that turned out to be the nearest this contest got to a Scottish gain as Warriors eventually rallied and started to tick the right boxes.\nTwo Lamb penalties in the space of three minutes were followed by a Sam Lewis pushover try in the left corner, from which Lamb was also successful with the conversion.\nPeter Lydon did reduce the deficit at the interval to 13-8 with a penalty, but two tries in four minutes at the start of the second half killed the contest.\nAll Blacks winger Cooper Vuna, switched to full-back following an early injury to Ben Howard, set up Hammond, converted by Lamb before Lewis crashed over in the right corner, from which Lamb missed his first kick of the night.\nLydon converted his own try to bring it back to 25-15 on the night, before Lewis's third try, again converted by Lamb.\nLamb landed two more penalties before injury-weakened Warriors brought the biggest roar of the night with the late introduction of 17-year-old schoolboy Jamie Shillcock at scrum-half.\nWarriors: Howard; Hammond, Grove, Mills, Vuna; Lamb, Bruzulier; Rapava Ruskin, Creevy, Schonert, Percival, Thomas, Mike Williams, Lewis, van Velze (capt).\nReplacements: Annett, Fainga'anuku, Rees, Cox, Shillcock, Fatiaki, Biggs.\nLondon Scottish: Lydon; Matt Williams, Moffat, Gidlow, Doneghan; Newton, Stevenson; Lilley, Kwasnicki, Prescott, Phillips, Thomas Brown, Gillanders, Best, Bright (capt).\nReplacements: Hallam, Stephenson, Rae, Chisholm, Walker, Heeks, Millar.\nAttendance: 6,658",
"input": "",
"output": "The summary is: \nWorcester Warriors booked their place in the Championship play-off final, but they had to come from behind to beat London Scottish on the night.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nSwansea's 1-0 win over Everton, after Hull had lost 2-0 at Sunderland, saw the Welsh side climb out of the bottom three with two games remaining.\nBut the Swansea boss says there is still work to do and his side must remain focused.\n\"It can swing so quickly the other way,\" Clement said.\n\"We have to really focus on making sure we do a good job at Sunderland.\n\"We know that Hull have got a difficult game with Crystal Palace still not out of it and that's going to be hard.\n\"But the most important thing is to do a good job when we go to Sunderland.\"\nThe Swans were bottom with only 12 points from 19 games when Clement was appointed in January.\nClement says keeping Swansea City in the Premier League would be the highlight of his career and eclipse winning the Champions League.\nMedia playback is not supported on this device\nClement was Carlo Ancelotti's assistant when Real Madrid won the Champions League in 2014.\nHe was also the Italian's number two when Chelsea won the Premier League and FA Cup double in 2010.\n\"I've been in a very privileged position in the past to have worked with some fantastic teams and different players and got my hands on some unbelievable silverware,\" Clement said.\n\"But this will be the best by far if we manage to stay in this league, because I'm the one making the decisions.\n\"I'm the one in charge and because of the position when I came into this club.\n\"It was difficult for the supporters and for the players. I was the third coach in one season, so it will be a fantastic achievement if we do it.\"\nFernando Llorente scored the only goal against Everton as Swansea's win combined with Hull's defeat against already-relegated Sunderland saw Clement's side move out of the bottom three.\nSwansea travel to Sunderland next Saturday and the club's players will cover the cost of 3,000 away tickets.\n\"We have picked up seven points from games against Stoke, Manchester United and Everton and that's a tough run,\" Clement added.\n\"Now we go to Sunderland and I am glad they won.\n\"One because it helped us, but also because it shows we can not underestimate them.\"",
"input": "",
"output": "The summary is: \nPaul Clement says Swansea City cannot waste their opportunity after moving out of the Premier League relegation zone.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nThe aircraft have brought in medical equipment and food and water supplies from the Red Cross and the UN children's fund (Unicef).\nThe UN has warned that basic services are unravelling in Yemen, with widespread food and fuel shortages.\nMeanwhile, Pakistan has ruled itself out of joining the Saudi-led coalition fighting Houthi rebels in Yemen.\nPakistan's parliament voted against joining its ally, Saudi Arabia, saying in a resolution that it should \"maintain neutrality\" in Yemen.\nThe International Committee of the Red Cross (ICRC) said its plane landed in Sanaa carrying 16 tonnes of medical aid, including drugs and surgical equipment.\nUnicef's plane has flown in the same amount of aid - bringing food supplements for 20,000 children as well as medical supplies.\n\"The supplies we have managed to bring in today can make the difference between life and death for children and their families,\" said Unicef's Julien Harneis.\nThe arrival of the flights comes after days of delays while both organisations waited for clearance from all sides in the conflict to land in Yemen.\nThe UN's humanitarian co-ordinator for Yemen has called for a humanitarian \"pause\" in the bombardment and fighting on the ground to allow the aid to be delivered.\nJohannes van der Klaauw told reporters in Geneva that the conflict has now spread to 15 of Yemen's 22 provinces.\nHe described the situation in Aden in particular as \"catastrophic\", a descent into urban warfare, with control of the air and seaports shifting daily between rival groups.\nA million people in the city risk being cut off from access to clean water within a matter of days unless additional fuel is brought in, he said.\nThe World Health Organisation (WHO) says almost 650 people have been killed and more than 2,200 have been injured since 19 March, but Mr van der Klaauw said the actual number of casualties is likely to be far higher because many are not being brought to hospital or are being buried immediately.\nYemen has been in chaos since Houthi rebels, backed by army units loyal to the ousted former President Ali Abdullah Saleh, took full control of Sanaa in January and placed current President Abdrabbuh Mansour Hadi under house arrest.\nMr Hadi escaped and took refuge in Aden in February, but left the country at the end of March when the Houthis reached the outskirts of the southern port city.\nSaudi Arabia began air strikes two weeks ago against the Houthis, a Zaidi Shia rebel movement that the US and Saudi Arabia allege is receiving military assistance from regional Shia power Iran.\nBut they have failed to halt the Houthi advance into Aden, as well as neighbouring southern and eastern provinces. Overnight, coalition aircraft targeted the defence ministry building in Sanaa and weapons storage sites.\nSaudi Arabia asked Pakistan last month to contribute ships, aircraft and troops to the campaign to restore Mr Hadi to power.\nBut after days of debate, Pakistan's parliament voted to \"maintain neutrality in the Yemen conflict so as to be able to play a proactive diplomatic role to end the crisis\".\nAnalysts say Pakistan, which has a Sunni majority but also a sizeable Shia minority, fears being caught between the two if it sends troops to Yemen.\nWho is fighting whom in Yemen?\nHouthis - The Zaidi Shia Muslim rebels from the north overran Sanaa last year and then expanded their control. They want to replace Mr Hadi, whose government they say is corrupt. The US alleges Iran is providing military assistance to the rebels.\nAli Abdullah Saleh - Military units loyal to the former president - forced to hand over power in 2011 after mass protests - are fighting alongside the Houthis.\nAbdrabbuh Mansour Hadi - The president fled abroad in March as the rebels advanced on Aden, where he had taken refuge in February. Sunni Muslim tribesmen and Southern separatists have formed militia to fight the rebels.\nSaudi-led coalition - A US-backed coalition of nine, mostly Sunni Arab states says it is seeking to \"defend the legitimate government\" of Mr Hadi.\nAl-Qaeda in the Arabian Peninsula - AQAP opposes both the Houthis and President Hadi. A rival affiliate of Islamic State has also recently emerged.",
"input": "",
"output": "The summary is: \nTwo planes carrying much-needed relief supplies have arrived in the Yemeni capital Sanaa.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nOther highlights include a stage version of Hancock's Half Hour and the Berliner Ensemble's Waiting for Godot.\nThe annual event in Enniskillen, Northern Ireland, is now in its fourth year.\nIt boasts performances in multiple locations, including the school where Beckett was a pupil in the early 1920s.\nThe lights will be turned out for All That Fall, in a staging by former Royal Court artistic director Max Stafford-Clark.\nHe said: \"I was asked for my vision for the play and my response was that there is absolutely no vision at all - the whole play takes place in the dark.\"\nThe drama, co-produced with the Out of Joint Theatre Company, will star Irish actress Rosaleen Linehan.\n\"It will be as dark as we can make it, the audience won't be invited to see anything,\" Stafford-Clark told the BBC. \"It will be a bit spooky I imagine but that's the effect that Beckett wanted.\"\nAll That Fall was previously staged in London in 2012 with a cast including Dame Eileen Atkins and Michael Gambon.\nThe radio play, first broadcast in 1957, tells of an elderly woman's journey to a railway station to meet her blind husband.\nThe Hancock play is based on several \"lost\" radio scripts - by Ray Galton and Alan Simpson - which were revived on Radio 4 last year.\n\"Hancock is the perfect Beckett character. He is the small man shaking his fist as a universe that doesn't care,\" said Drop The Dead Donkey star Neil Pearson, who will direct the show.\n\"I think we are habitually rather too po-faced about Beckett. He's a funny writer. I don't know whether he knew of Hancock but I'm pretty sure he would have approved of the uncaring way the world treats him.\"\nTheatre director Sophie Hunter - who recently married Sherlock star Benedict Cumberbatch - is putting on Benjamin Britten's Phaedra - the composer's final work - inside the ruined Necarne Castle.\nShe said her concept was to create \"an intimate experience in an epic space\".\n\"At the heart of it is the story of a woman who has taken in poison and is dying over 15 minutes - the music mimics the effect of the poison that is coursing through her veins.\"\nThe Enniskillen International Beckett Festival, Happy Days, will take place over two long weekends, between 23 July and 3 August 2015.\nThe full line-up is on the Happy Days website.",
"input": "",
"output": "The summary is: \nA play performed in complete darkness is among this year's line up for a summer festival celebrating writer Samuel Beckett.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nThe man, who has not been named, was with a friend on Scales Fell, one of the Blencathra peaks, on Thursday.\nThe terrain meant an air ambulance was unable to land at the scene, so members of the Keswick mountain rescue team took the decision to carry him to a waiting helicopter.\nThe man was taken to hospital in Newcastle with severe head injuries.\nA spokesman for the rescue team said: \"Two walkers on their way down Blencathra spotted something blue on a lower path.\n\"As they watched, they saw an arm move, and realised to their horror that it was a man in distress.\n\"One of them got down to him, and realised that he had fallen some considerable distance from the path above, and had suffered serious head injuries.\n\"A decision was taken to carry the patient down. This was achieved successfully and the casualty was then airlifted to Newcastle's Royal Victoria Infirmary.\nA spokesman for the Great North Air Ambulance added: \"The helicopter took our doctor and paramedic as close to the scene as was safe, before landing at the base of Scales Fell.\n\"The patient was assessed and treated before being carried three miles down to the helicopter.\"",
"input": "",
"output": "The summary is: \nRescuers carried an injured man for three miles after he fell more than 130ft (40m) down a Cumbrian mountain.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nThe Reds impressed in last weekend's 2-0 win over Ballinamallard, leaving then eight points behind leaders Crusaders.\n\"We were fantastic against the Mallards and we have to continue that when we play Portadown,\" said Lyttle.\n\"We'll never give up on going for the title - our squad is in good shape and we'll just keep going for it.\"\nHe added: \"We are not worried about Crusaders, our focus is only on what we do.\n\"We have brought in quality players and we have quality players coming back from injury so the squad if getting a bit bigger now.\n\"Our aim on Saturday is simple - three points and a clean sheet.\"\nThe Ports have bolstered their attack by signing striker Mikey Withers from Lisburn Distillery on an 18-month deal.\nCrusaders visit Coleraine while third-placed Linfield, who have brought in forward Michael McLellan from H&W Welders, welcome Carrick Rangers to Windsor Park.\nBallymena United will be without the suspended Tony Kane for the Ferney Park clash against a Ballinamallard side sitting just one point above the bottom.\nThe Mallards are under pressure from inform Warrenpoint Town, who remain the basement team but are eyeing safety after an unbeaten run of six league games.\nHowever, Warrenpoint's game at Dungannon has been called off because of snow while Glenavon's contest with Glentoran has also falling victim to the winter weather.",
"input": "",
"output": "The summary is: \nCliftonville manager Gerard Lyttle hopes to maintain their Premiership title push with victory over Portadown at Solitude on Saturday.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nCurrently, they can start school at the beginning of the term in which they have their fourth birthday.\nBut Powys council's cabinet approved the plans on Tuesday, which will see children start school in the September.\nThe change will be introduced from September 2017 and will save £1.2m a year.\nThe council also voted to increase the hours of free pre-school provision from 10 hours per week to 12.5 hours.\nCouncillor Arwel Jones, cabinet member for schools, said: \"There's no secret that we are proposing this revised policy to help in our bid to meet the £27m budget savings target over the next three financial years.\"\nHe added: \"Today's decision will bring us in line with the majority of other councils in England and Wales where children start school in the September after their fourth birthday.\"",
"input": "",
"output": "The summary is: \nChildren in Powys will only be able to start primary school after turning four years old, it has been decided.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nThe former England international, who switched his allegiance to The Elephants ahead of their Nations Cup defence, came off the bench in Abu Dhabi to provide the perfect cross for Giovanni Sio who headed home a winner.\nAn own-goal from Wilfried Kanon had put Sweden ahead, with Yao Serge Nguessan equalising on the stroke of half-time.\n24-year-old Zaha was born in Ivory Coast but has two England caps having played against Sweden in November 2012 and Scotland the following year.\nAs both were friendly matches, he was permitted to commit his international future to his country of birth.\nThe Ivorians have been preparing for the Nations Cup in the United Arab Emirates.\nThey will be heading to Gabon on Thursday and will play their opening Group C game on 16 January against Togo.\nIn other friendly internationals this weekend, Algeria were 3-1 winners over Mauritania; Burkina Faso beat Mali 2-1; Uganda defeated Slovakia 3-1; Senegal were 2-1 winners over Libya and Egypt recorded a rare victory over North African rivals Tunisia, winning 1-0.",
"input": "",
"output": "The summary is: \nCrystal Palace winger Wilfried Zaha provided the decisive cross on his Ivory Coast debut as the African champions beat Sweden 2-1 in an Africa Cup of Nations warm-up match on Sunday.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nIn the three months to August, output fell by 0.8%, the biggest such decline since March 2013.\nMeanwhile, the UK trade deficit was £3.3bn in August, a narrowing of £1.2bn from July, it said.\nBut the deficit was larger than expected and is set to weigh on growth, the ONS added.\nAn ONS official said the weak figures for construction in August may have been linked to wet weather during the month.\nHousebuilding fell by 3% from July and output in other parts of the sector also contracted for the first across-the-board decline since 2010.\nThe trade figures showed the UK's deficit in its trade in goods narrowed to £11.1bn in August compared with £12.2bn in July, although some analysts had expected it to shrink further.\nThe deficit of £11.1bn on goods was partly offset by a £7.9bn surplus on services. Exports increased by £0.8bn, boosted by cars.\nThe combined goods deficit for July and August is already twice that of the previous quarter, and is likely to have a negative effect on overall GDP growth.\nThe UK's economy grew by 0.7% in the second quarter of the year, but Howard Archer of IHS Global Insight said overall growth prospects for the third quarter had received a \"double blow\" from the construction and trade data, which was \"seriously bad news overall\".\n\"Overall, the data reinforce our belief that GDP growth is likely be no better than 0.5% quarter-on-quarter in the third quarter, and there is now a significant risk that it could have been weaker still.\"\nDavid Kern, chief economist of the British Chambers of Commerce, said: \"The large trade deficit remains a major national problem. Greater efforts are needed to support our exporters and to secure a long-term improvement in our trading position.\"",
"input": "",
"output": "The summary is: \nConstruction output fell 4.3% in August, its sharpest drop since late 2012, the Office for National Statistics (ONS) has said.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nHe is credited with the BJP's win in the recent election in the politically crucial state of Uttar Pradesh.\nHe replaces Rajnath Singh, who is the home minister in the new government.\nA controversial politician, Mr Shah is accused of sanctioning the killing of a Muslim civilian in 2005, when he was the home minister of Gujarat state.\nIn 2010, Mr Shah resigned after he was charged with murder and kidnapping of Sohrabuddin Sheikh and arrested in connection with the killing.\nHe spent more than three months in jail after which he was released on bail. Mr Shah denies the charges.\nMr Shah, a general secretary of the BJP, was chosen its new president by the parliamentary board consisting of top party leadership on Wednesday.\nAnnouncing his appointment, the outgoing chief Rajnath Singh said Mr Shah was chosen unanimously by all members of the board.\nThe 49-year-old is reported to be one of the youngest presidents of the party.\nHe has a reputation for being a good organiser - in the run up to the general election, he was appointed to head the BJP's campaign in the most populous state of Uttar Pradesh where he helped the party win an unprecedented 71 of the 80 seats for the party.\nDuring the campaign, the Election Commission barred him from addressing rallies after finding him guilty of giving \"hate speeches\" against the Muslim community.\nThe ban was lifted after Mr Shah apologised and promised not to \"use abusive or derogatory language\".\nA long-time member of the Hindu nationalist Rashtriya Swayamsevak Sangh (RSS), the ideological fountainhead of the BJP, Mr Shah has known Mr Modi for more than three decades.\nCorrespondents say his appointment to the top post will give Mr Modi complete control over the party and the government.",
"input": "",
"output": "The summary is: \nAmit Shah, a close aide of Prime Minister Narendra Modi, has been appointed the chief of India's governing Bharatiya Janata Party (BJP).",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nBut there have been many such agreements in the past and the omens for peace in the region are not good.\nRussian-backed fighters and the Ukrainian army have clashed almost daily for the last 30 months.\nAt the beginning of January there was a serious escalation in the violence.\nUkraine said two of its soldiers had been killed and 16 injured in fighting over the weekend.\nIn theory the two sides will this week pull back heavy weaponry from areas near the front line.\nBut a source at the Munich talks over the weekend told the BBC that no progress had been made in reaching a political solution.",
"input": "",
"output": "The summary is: \nA ceasefire is due to come into effect in eastern Ukraine following a deal in Munich over the weekend to halt fighting and withdraw heavy weapons from the front line.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nThe Torquay United goalkeeper has yet to make his debut for the National League side, but no-one else at the club can claim to have faced the kind of players the Gibraltar international has.\nIn the eight caps he has won for the fledgling football nation, Robba has come up against world champions Germany, Euro 2016 winners Portugal as well as Scotland and the Republic of Ireland.\n\"It's hard to get my head around it. How the hell am I playing against the European champions? It doesn't make sense,\" he tells BBC Sport.\nBut since Gibraltar's acceptance as an international footballing nation in 2013, Robba and his team-mates, who had previously relied on the Island Games to give them 'international' matches against the likes of Jersey, the Isle of Man and Shetland, now find themselves on the same stage as World Cup golden boot winner Thomas Muller.\n\"The whole team pinches themselves while we're there, because most of them are semi-pros at most,\" says Robba.\n\"We've got a few professionals, but even the professionals think 'what are we doing playing against the European champions?'\"\nBut from the heights of having four put past him by Portugal in Porto and 'narrowly' losing 4-1 to Greece in midweek, Robba will come back down to earth with a bump on Saturday as he returns to the bench for Torquay United when they host York City in the National League.\nIt is the 24-year-old's first professional deal and he is understudy to the club's United States goalkeeper Brendan Moore.\nBut Robba hopes that playing full-time will help ensure that he is the one who gets the nod for the Rock's big matches.\n\"It's my first year that I've tried to be a professional, so I have to adapt and I'm still adapting and still trying to get used to it,\" he says of the switch from part-time to full-time football.\n\"I'm hoping that this can cement my number one place in the long term in the Gibraltar squad and the line-up.\n\"It's between me and Jordan Perez, who played against Greece, who's a good goalkeeper himself and played well against Greece.\n\"I've come to Torquay to try and do my best. Brendan's a really good keeper and a really good guy - I try and push him and he tries and push me. That's football, that's what you do.\n\"Good competition makes us all better and it's the manager's choice who plays.\"\nHow long Robba has to wait to make his debut in England is unclear, but his manager at Torquay, Kevin Nicholson, says he has potential.\n\"Jamie's come in done great for us, trained well and he's keen to show what he can do,\" Nicholson said.\n\"Brendan's been doing very well, so he'll keep his spot, but I know that Jamie won't give up.\"",
"input": "",
"output": "The summary is: \nIn the space of 10 days one footballer will go from taking on the European champions to warming the bench in the fifth tier of English football - welcome to the world of Jamie Robba.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nOn Wednesday the Egyptian Competition Authority said it had referred the Caf president Issa Hayatou to prosecutors for investigation.\nIt said he was suspected of not opening up the tender to free and fair competition as required by Egyptian law.\nCaf is based in Cairo so the authorities say it must follow their laws.\nCaf said the reports were false, adding that in the letter it was sent by the competition authority there was no mention of any prosecution against Mr Hayatou.\n\"False information, published in the Egyptian press since yesterday and widely reported around the world, indicates a recommendation for prosecution of the president of the Caf to the Attorney General of Egypt on corruption charges,\" said the statement.\n\"It should be noted that in the letter sent to Caf by the Egyptian Competition Authority, there is no mention of any prosecution against the president of Caf, whether for acts of corruption or something else.\"\n\"Caf wishes to point out that the contract with Lagardère Sports does not contravene national or supranational legislation, as established by categorical legal opinions in this regard.\"\nIn June 2015, Caf signed a deal with Lagardere which gave the French media company broadcasting rights to a variety of African football competitions, including the flagship Africa Cup of Nations, from 2017 until 2028.\nThe deal was valued at $1 billion by African football's ruling body and followed on from a previous contract with Lagardere, which had run from 2008 to 2016.\nThe French company is not the subject of the referral, but has denied any wrongdoing.\n\"Any allegations that the agreement breaches local Egyptian competition laws are wholly unfounded and we have clear and categorical legal advice to that effect,\" it said in a statement to the BBC.\nThe Egyptian Competition Authority's complaint comes days after the new deal took effect and a week before the Nations Cup gets underway.\nThe continent's largest sporting event kicks off in Gabon on 14 January, with the final taking place on 5 February.",
"input": "",
"output": "The summary is: \nThe Confederation of African Football (Caf) has strongly denied that a deal with a media company for broadcast rights to several African football tournaments has broken any laws.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nFrancis Cadell's depiction of ''George Street and Charlotte Square'' was whitewashed over and the canvas was reused by the son of another colourist, Samuel Peploe.\nThe Scottish Gallery in Edinburgh discovered the missing Cadell during conservation.\nIt is estimated that the painting could sell for more than £50,000.\nThe Scottish Colourists were four post-impressionist painters, Peploe and Cadell, along with John Leslie Hunter and John Duncan Fergusson.\nThey absorbed and reworked the strong and vibrant colours of contemporary French painting into a distinctive Scottish idiom during the 1920s and 1930s.\nThe lost Cadell work was painted around 1909 from his studio at 112 George Street, Edinburgh, and looks across the street to Charlotte Square.\nWhen the artist died in 1937, his sister Jean Percival Clark, well-known as the actress Jean Cadell, came up to Edinburgh to sort out his affairs.\nShe was helped by Denis Peploe, son of Samuel, who was a student at Edinburgh College of Art.\nShe gifted him some of her brother's art material and included among the canvases, probably including \"George Street and Charlotte Square\", taken off its stretcher, turned and re-stretched ready to be used again.\nIt is not known why Cadell abandoned the painting, which is finished and bears a strong signature.\nYears later, Denis Peploe painted his own picture, Begonias, a still life on a trestle table and whitewashed over the Cadell exposed on the other side.\nThe Scottish Gallery acquired the Denis Peploe and in the process of conservation discovered the Cadell on the reverse.\nDenis's son, Guy, who is director of the Scottish Gallery, told BBC Scotland that he had bought the painting at auction and was shocked when he got a call from the picture conservator.\n\"He said 'I think there's something interesting on the other side of the picture'.\n\"I said go-ahead take it off its stretcher and see what we can see. He called back a few minutes later and said 'bottom left hand corner, signature FCB Cadell.'\n\"I think I choked on my morning roll.\"\nTommy Zyw from the Scottish Gallery said: \"It is heard of to have paintings on either side of a canvas.\n\"Occasionally if an artist is struggling, he flips it over and tries again.\n\"But in this case this is quite unusual to have two paintings by two different artists - linked by a family friendship.\"",
"input": "",
"output": "The summary is: \nA lost painting by one of the Scottish Colourists has been discovered on the reverse of another artwork.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nBut this wasn't any ordinary political programme. To investigate what the women of Scotland want from the independence debate - never mind the outcome - we decided to have an all-female cast of contributors.\nThis wasn't to alienate the good men of Scotland but to ensure we crammed the programme with as many disparate female voices as possible.\nThe spark for \"What Women Want\" was a poll by the influential Scottish Social Attitudes Survey that highlighted a sustained gender gap in voting intentions.\nWho? Voters in Scotland will go to the polls to vote on their country's future.\nWhat? They will be asked the yes/no question: \"Should Scotland be an independent country?\"\nWhen? The vote takes place on Thursday, 18 September, 2014.\nFor as long as the organisation has asked a question about independence - about 15 years - women have been less keen on voting Yes by a consistent six or seven percent margin. Combine that with the disproportionate number of women who say they are undecided, and you have a societal grouping that could prove pivotal for September's outcome.\nWe wanted to get to the root of why so many women felt unable to make a decision and why, among those who'd professed an opinion, so many felt independence wasn't the road to travel.\nIt's easy for people like me who've been interviewing politicians on the referendum debate since Adam was a boy to assume every word of it is fed into Scotland's homes to be analysed and debated, but that's clearly nonsense.\nNormal people have other priorities, and it could be argued that because so many women have families or are carers, our lives tend to be more fragmented. It's not that we work any harder, let's just say we often have more plates spinning than men.\nOn the search for women from all walks of life a night at the bingo proved an eye-opener and a humiliation. You would imagine that hearing a number and marking a card is fairly straightforward. It's not. Trailing three numbers behind the caller was a bit disconcerting; having the elderly lady next to me take over my card while marking three of her own, was downright embarrassing.\nWhen I wasn't interrupting their games the women at the bingo were keen to tell me how they felt they weren't being served by the Yes and No campaign. Time and time again I heard a plea for transparency in the debate and a hope that the politicians would stop shouting at each other and provide some facts.\nWhether or not there are facts to be had in this most crucial of decisions is debatable in itself, but as far as the women we spoke to are concerned, at least some straight talking wouldn't go amiss.\nThe programme also decided to tackle head on the hypothesis that women are more risk averse and therefore prone to sticking with the status quo, i.e. the UK. Try telling the ladies of Edinburgh's Auld Reekie Roller Girls, who meet on a Friday night and knock lumps out of each other on roller skates, that they're a bunch of fearties.\nThe fact that the paramedics arrived and hauled one of the roller ladies off for a night in the infirmary was passed off as a regular, but mild inconvenience.\nWere women simply afraid of independence and its leap into the unknown? The general consensus at the roller derby seemed to be no, but as the Roller Girls are on first name terms with the staff at the fracture clinic, perhaps they weren't an entirely representative bunch.\nThere was certainly more time for considered thinking at the huge wedding fair that took over much of the SECC.\nOnce the brides-to-be and their assorted mums and pals realised that my camera crew and I were interrupting their day to talk politics rather than place settings they were surprisingly forthcoming. I suppose you can't prepare for a life of wedded bliss and even plan a family without taking cognisance of the sort of country you want to live in.\nIt's clear to me, having ambushed women to carry out similar interviews six months ago, that there has been a big shift. From general apathy back then there is now a widespread hunger for information and for certainties.\nAs I've said, the cynic in me very much doubts either side can offer the copper-bottomed facts at least this half of the electorate so badly wants, but I hope I'm proved wrong.\nAway from the civilians in this battle, the seasoned political warriors we spoke to did try to convince us of the surety of their positions, as you'd expect.\nNicola Sturgeon chatted to us in her kitchen which she revealed, with disarming honesty, she rarely visited. She also conceded that unless women could be persuaded to favour independence the referendum would not be won.\nWe spoke to powerful women in the unionist camp too, as well as historians, writers, professors - and even a neuroscientist. We cleaned gutters - or rather I did - we shopped for clothes and put a focus group of undecided women under the spotlight and gave them a grilling.\nOverall, we tried to represent some of the diverse roles Scotswomen in the 21st century find themselves in while they ponder their country's future. And did we eventually discover what women want? Of course I'm not telling you, you'll have to watch the programme to find out.",
"input": "",
"output": "The summary is: \nWhen I started making a documentary about women and the independence referendum little did I know that Maw Broon, a night at the bingo, paramedics at the roller derby, cleaning out my gutters and learning of Nicola Sturgeon's unfamiliarity with her kitchen would all play a part.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nThe Foreign Office sought to identify countries that could pose \"similar risks\" as Afghanistan, a senior former diplomat told the Iraq Inquiry.\nStephen Pattison said the phrase was dropped after it emerged it had been taken from a magazine article.\nMr Pattison told the inquiry the process led to Iraq moving up the political agenda after 9/11.\nMr Pattison, who oversaw the Foreign Office's dealings with the United Nations in the run-up to the 2003 invasion said: \"After 9/11, there was obviously considerable concern about areas of the world which might pose a threat.\n\"There was a phrase that was current at that time which was 'draining the swamp'.\n\"It was the title of a paper put up by our planning department about how we could address areas of the world which post 9/11 might fit into the same pattern - areas like Afghanistan - which frankly we had not paid enough attention to before 9/11 and which had resulted in an attack on the US.\n\"The 'draining the swamp' line was a bit about: 'let's look around and see where there might be other places that could pose similar risks'.\"\nAlthough the phrase \"encapsulated\" the post 9/11 security approach, he said it was soon dropped \"partly because I think it actually came from a published magazine article\".\nPanel member Sir Roderic Lyne, a former Ambassador to Russia, asked him whether the phrase \"didn't originally come from Chairman Mao\".\nHe replied: \"I think it originally did but then I think it was taken up by the Economist in the context of the post 9/11 stance. The thought behind it, I think, was the thought which drove the then British government into focusing very hard on Iraq.\"\nThe phrase \"draining the swamp\" is commonly attributed to Chairman Mao - but Dr Yiyi Lu, a Chinese expert at Chatham House, said she did not immediately recognise the phrase and that its meaning seemed \"too vague\".\nAsked about the decision to go to war, Mr Pattison said the UK was driven by rather \"idealistic\" motives rather than direct concerns about its own security or a decision to alter the balance of power in the region.\n\"I think Tony Blair's view was always that - the idealists' view that we were doing this to make the world a safer place. We were not doing this because there was a direct threat to the UK; we were doing this because it was in the interests of the international community and because it was in the interests of the international community he expected the international community to step up to the plate and do it.\"\nThe Iraq Inquiry is concluding its latest round of hearings - expected to be the last before it starts the process of compiling its report.\nFormer foreign secretary Jack Straw will give evidence for the third time on Wednesday.",
"input": "",
"output": "The summary is: \nThe UK drew up a list of countries seen as potential threats after 9/11 in a process known as \"draining the swamp\".",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nThe children, aged between three and seven, were being driven to their kindergarten in the city of Weihai when the bus burst into flames in a tunnel.\nThe driver was angry that his overtime and night shift pay had been cut, police told Xinhua news agency.\nThe children's teacher and the driver were also killed.\nThe fire was started on the bus floor near the driver's seat. Part of a lighter was discovered nearby and petrol traces were found on the bus, Xinhua said.\nElectrical faults and traffic accidents had been ruled out as possible causes, police said.",
"input": "",
"output": "The summary is: \nA fire on a bus in China that killed five South Korean and six Chinese children was started deliberately by the driver, Chinese state media say.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nThe Insolvency Service said enough coal would be bought to keep the ovens alight as talks with \"interested parties\" continue about their future.\nA spokesman said a further decision would be made next week.\nThe steelworks are with receivers after owners SSI UK went into liquidation, blaming a slump in global steel prices.\nA statement from the Insolvency Service said: \"A decision has been made to buy sufficient coal to keep the Redcar Coke Ovens going until the weekend, enabling the Official Receiver to continue discussions with interested parties about purchasing assets in working order.\n\"A decision about purchasing further coal to keep the ovens operational beyond the weekend will be taken at the end of this week.\"\nThe government has promised an £80m aid package to help the 1,700 workers who have lost their jobs.",
"input": "",
"output": "The summary is: \nCoke ovens at the SSI Steelworks in Redcar will remain lit until at the least the weekend, the site's receivers have confirmed.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\n20 December 2015 Last updated at 13:02 GMT\nAndrew Russell, 36, was found unconscious in a car park on Bradshaw Way, in Derby, shortly before 02:00 on 16 November. He was taken to hospital but later died.\nDet Insp Graham Prince, of Derbyshire Police, said: \"We are trying to trace a number of cars seen driving along London Road between 1.30am and 2.15am that morning, along with several lone people walking down the road during the same time period.\n\"These people have yet to come forward and they could have information which may help with the inquiry.\"\nA 41-year-old man has been charged in connection with the alleged attack.",
"input": "",
"output": "The summary is: \nDetectives investigating a fatal assault have released CCTV footage of potential witnesses who have yet to come forward.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nThe 28-year-old MTN-Qhubeka rider maintained his 13-second lead over Team Sky's Wouter Poels on Sunday's 14-lap final stage around central London.\nTeam Sky's Elia Viviani was awarded the stage win after Andre Greipel, who crossed the line first, was penalised.\nOwain Doull, riding for Team Wiggins, was the highest placed Briton in third.\nThe Welshman finished 10th on the stage but picked up bonus seconds in the intermediate sprint to leapfrog Rasmus Guldhammer to end the race 42 seconds behind Boasson Hagen and also win the points classification.\nGermany's Griepel beat Viviani by milimetres in Saturday's penultimate stage and was again first over the finish line on Sunday.\nHowever, the Lotto-Soudal rider was adjudged to have impeded Viviani in the sprint for the line and was relegated to the back of the bunch by race officials.\n\"I didn't see Viviani coming,\" said Greipel.\n\"Everybody was on the limit on the final corner. I didn't do anything for purpose that's for sure. That's sprinting.\"\nAfter winning his third stage of the race, Italian Viviani, who crossed the finish line with his hand in the air in complaint, said: \"He came across a little bit and that edged me towards the barriers.\n\"I'm disappointed because it is better to win without this but we won in London and that is the main thing.\"\nStage eight result:\n1. Elia Viviani (Ita/Team Sky) 1hr 50mins 16secs,\n2. Juan Jose Lobato Del Valle (Esp/Movistar) same time\n3. Matteo Trentin (Ita/Etixx-Quickstep)\n4. Edvald Boasson Hagen (Nor/MTN-Qhubeka)\n5. Jens Debusschere (Bel/Lotto-Soudal)\n6. Sondre Holst Enger (Nor/IAM)\n7. Mark Renshaw (Aus/Etixx-Quickstep)\n8. Graham Briggs (GB/JLT Condor)\n9. Ruben Zepuntke (Ger/Cannondale-Garmin)\n10. Owain Doull (GB/Team Wiggins)\nGeneral classification:\n1. Edvald Boasson Hagen (Nor/MTN-Qhubeka) 34hrs 52mins 52secs,\n2. Wouter Poels (Ned/Team Sky) +13 secs,\n3. Owain Doull (GB/Team Wiggins) +42secs\n4. Rasmus Guldhammer (Den/Cult Energy Pro Cycling) +43secs\n5. Zdenek Stybar (Cze/Etixx-Quick-Step) +51secs\n6. Ruben Fernandez (Spa/Movistar) same time\n7. Steven Kruijswijk (Ned/Team LottoNL-Jumbo)\n8. Dylan van Baarle (Ned/Cannondale-Garmin) +53secs\n9. Chris Anker Sorensen (Den/Tinkoff-Saxo) +59secs\n10. Xandro Meurisse (Bel/An Post - Chainreaction) +1:02\nSelected others:\n18. Peter Kennaugh (GB/Team Sky) +2:51\n24. Ian Stannard (GB/Team Sky) +38:36\n87. Bradley Wiggins (GB/Team WIGGINS) +1.31:03",
"input": "",
"output": "The summary is: \nEdvald Boasson Hagen became the first rider to win the Tour of Britain twice since its return to the professional cycling calendar in 2004.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nA small group of people ran on to the M4 spur road and laid down in front of oncoming traffic on Saturday, causing temporary disruption.\nPeople aged between 21 and 67 have been charged with wilful obstruction of the highway, according to the Met Police.\nThey have all been bailed to appear at Ealing Magistrates' Court on 22 December.\nAmong the accused are seven Londoners. They are: Isabelle Anderson, 30, of Stratford; Madeleine Allis-Petersen, 24, of Ealing; Joanne Louise Bodimeade, 28, of Lambeth; Alexis Delage, 25, of Lewisham; Sophia Lysaczanko, 28, of Haringey; Tom Venner-Woodcock, 29, of Southwark; and Tess Lotter, 30, of Camden.\nThe others charged are: Antoine Thalmann, 25, and Henry Owen, 23, both of Oxford; Simon Bramwell, 44, of Stroud, Gloucestershire; Ian Bray, 49, of Kirklees, West Yorkshire; Graham Lewis, 53, of Wells, Somerset; Thomas Harford, 26, and Margaret Charnley, 67, both of Bristol; and Sibi Moore, 21, of Sidmouth, Devon.",
"input": "",
"output": "The summary is: \nFifteen people have been charged after campaigners against airport expansion staged a protest near Heathrow Airport.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nGreybull Capital, a UK-based investment firm, is to plough up to £400m into the plant - but workers had to accept a pay cut and smaller pension contributions.\nIn an earlier ballot, Community, Unite and GMB members accepted temporary changes to their terms and conditions.\nOfficials said it was a positive step in \"securing a sustainable future\" for the plant.\nMore on this and other local stories in Lincolnshire\nUnite's National Officer, Harish Patel, said: \"This will have been a difficult decision to take for many, but by agreeing to make these short-term sacrifices, members have secured a future for steelmaking in Scunthorpe and the long product division's other sites.\n\"Government ministers need to make sure that the sacrifices are not being made in vain by taking decisive action to support the steel industry and allowing steelworkers to compete on a level playing field with their global competitors.\"\nSteve McCool, from the union Community, echoed calls for Government action.\n\"He said: \"The steel made at Scunthorpe and across the north of England is some of the best in the world and is absolutely vital to the infrastructure and construction industries.\"\n\"When Britain builds, we must always build using British steel,\" he said.\nThe Government is yet to respond to a request to comment on what the union leaders have said.\nAhead of the vote, steelworker Charlotte Upton said the proposed deal meant \"job security for me so I can stay where my family is, near my home\".\n\"It means I can continue to be a steelworker, I love my job.\"\nThe proposed temporary changes to terms and conditions include a one year, 3% reduction in pay, and a one year, 3% reduction (from 6%) in both employer's and employee's pension contributions.\nThe Tata signs will be also removed and replaced with ones saying British Steel.\nThe Scunthorpe steelworks is part of Tata Steel's long products division, which was put up for sale in 2014.",
"input": "",
"output": "The summary is: \nWorkers have voted to accept a deal which will safeguard about 4,400 jobs at Tata's steelworks at Scunthorpe.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nClifton, 18, spent the 2016-17 season on loan with Grantham Town in the Evo-Stik League Northern Premier division.\nHe turned professional in July 2015 after coming through the academy, but is yet to play a game for the Mariners, who finished 14th this season.\nHowever, Clifton has earned a new deal after impressing manager Russell Slade, who replaced Marcus Bignot in April.",
"input": "",
"output": "The summary is: \nMidfielder Harry Clifton has signed a one-year contract extension with League Two side Grimsby Town.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nIn Drug and Therapeutics Bulletin, researchers say they looked at all evidence and found supplements did not boost the health of mothers and babies.\nBut pregnant women should make sure they take folic acid and vitamin D, as well as eating a well-balanced diet, as per NHS guidelines, they add.\nSupplements-makers said some women were not getting enough nutrients.\nThe researchers said folic acid had the strongest evidence to support its use - taking 400 micrograms a day can protect against abnormalities called neural tube defects in the developing baby.\nVitamin D - 10 micrograms a day - is recommended for healthy bones in the mother and baby.\nSome women can get these two pills for free on the Healthy Start scheme.\nA supplement that can be dangerous in pregnancy is vitamin A. Too much can harm the baby.\nThe researchers said pregnant women might feel coerced into buying expensive multivitamins in order to give their baby the best start in life.\nBut they would do well to resist the marketing claims, which did not seem to translate into better outcomes for mother or baby, they said.\n\"The only supplements recommended for all women during pregnancy are folic acid and vitamin D, which are available at relatively low cost,\" they said.\nJanet Fyle, from the Royal College of Midwives, said: \"We would encourage women who are pregnant or are thinking of becoming pregnant to have a healthy, varied diet including fresh fruit and vegetables, alongside taking folic acid supplements.\n\"We would also stress that there is no need for pregnant women to 'eat for two'.\n\"This is a myth, and all that is required is a normal balanced amount of food.\"\nThe Health Food Manufacturers' Association, which represents the food supplements industry, insists that a substantial proportion of women of child-bearing age are not getting enough nutrients from diet alone.\nThe industry-funded Health Supplements Information Service said food supplements could help plug dietary gaps.",
"input": "",
"output": "The summary is: \nPregnancy multivitamins are a waste of money because most mothers-to-be do not need them, according to researchers.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nIts design is centred around patients and their families and it has taken ideas from Norway.\nExperts say the design of a building can have a very real impact on patient care.\n\"The environment can actually improve a patient's symptoms because if you have an anxiety quite often that can heighten a patient's physical pain,\" says Rhona Baillie, chief executive officer of the Prince and Princess of Wales Hospice.\nShe has had more than 20 years experience as a palliative care nurse.\n\"If we can help to control that anxiety and make that patient feel more relaxed in an environment that they're very comfortable in, that can help their overall physical state.\"\nAt the moment the site for the new hospice building in Bellahouston Park is full of scrubby grasses and drying out mud, but building work is scheduled to start soon.\nThe idea is to create a homely atmosphere for patients and their families who have often been on a very hard clinical journey.\nThey found a model for this in Scandinavia, with beds clustered around a courtyard. In the middle of that there is space for tables and chairs, where families can eat together, just like at home with soft furnishing and beautiful lighting.\nThe clinical equipment will be there, but very much in the background.\n\"It's got a domestic-sized front door,\" explains Ms Baillie.\n\"As soon as you walk in there will be no barriers of having a desk you have to walk up to and the first thing you'll see is a fireplace, so all of that signifies home.\"\nThe hospice has been housed in its present building on the banks of the Clyde for more than 30 years.\nThe architects spent time understanding how it worked before they started on the new building.\n\"This project has influenced me hugely,\" says Alastair Forbes, architectural director with Ryder Architecture.\nPart of what he was trying to do was use the layout of the building to limit the time staff would spend away from patients and to break down the scale of the place.\nIt is designed to look like four interlinked houses.\nIt also meant a first for him in his career as an architect as he spent time examining how a room looks when you are lying in bed. It is something which he says has become a \"touchstone\" for him in the project.\n\"What a patient sees, how much they see the ceiling, how much they don't actually see out the window,\" he explains.\n\"It's a very clinical environment, it's the smells, it's the noises, the proximity to staff, to other patients, personalisation you can see there is quite difficult.\"\n\"Everything's about people.\"\nThe design of the new hospice building also considers its parkland setting with rooms which allow patients to see the sky and the gardens from their bed, as well as giving them and their families the opportunity to eat together.\n\"Coming from a Pakistani background, you cook for your family and friends all the time,\" says Saf Akram.\nHis sister spent time at the hospice towards the end of her life and his mum cooked at home for her and brought the food in.\n\"Then we'd come in from work, and get a little nibble of whatever she had for the day. I'm sure if they'd had a kitchen there my mum would have cooked for everybody, she's just that kind of person.\"\nHe adds: \"That's exactly what they need down here, somewhere you can join in as a family, continue your family life in a place where you're getting the clinical care you need.\"",
"input": "",
"output": "The summary is: \nWork is beginning shortly on a new hospice building in Glasgow.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\n\"This battle is ours... and I promise you victory,\" he said in a TV address.\nSyrian rebels in the besieged town of Qusair say they are under heavy bombardment from Hezbollah combatants.\nThe town is close to the Lebanese border, a conduit for both the government and rebels to get weapons.\nIn the speech from an undisclosed location, Mr Hasrallah said if Sunni Islamists took over in Syria, they would pose a threat to the entire Lebanese population - Shia and Sunni Muslims, as well as Christians.\nHe said his movement could never be aligned with Syrian rebels who, in his view, were supported by the United States and Israel.\nHezbollah plunges deep into Syria conflict\nDozens of Hezbollah militants are said to have been killed fighting alongside Syrian troops in Qusair since 19 May, when government forces launched an offensive to recapture the rebel-held town.\nLast week, US Secretary of State John Kerry said thousands of Hezbollah fighters were contributing significantly to the violence in Syria.\nHe added that Iran was actively supporting Hezbollah's involvement - a claim denied by Tehran.\nIran and Hezbollah are predominantly Shia, while Mr Assad's Alawite sect is an offshoot of Shia Islam.\nThe week-long fighting in Qusair intensified early on Saturday, when activists reported heavy bombardments, including two ground-to-ground missiles and an air strike as well as artillery and rocket fire.\nSyrian state media said the army had launched a three-pronged offensive in the north, centre and south of Qusair, and was making big advances after \"killing large numbers\" of fighters.\nQusair is important for the Syrian government because it links the capital, Damascus, with the Alawite heartland on the Mediterranean coast. However, official media made no mention of the part played by Hezbollah.\nThe Lebanese group is also known to have lost a number of fighters in Qusair, prompting Lebanese President Michel Suleiman to warn the Shia militia against getting \"bogged down in the sands of discord\".\nThe Syrian Observatory for Human Rights, a UK-based activist group that monitors the conflict, said at least 22 people including 18 rebels had been killed in the latest fighting in Qusair. Dozens had been wounded, it added.",
"input": "",
"output": "The summary is: \nThe leader of the Lebanese Shia militant Hezbollah movement, Hassan Nasrallah, has promised his supporters they will prevail in Syria, where they are backing President Bashar al-Assad.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nIf given the go-ahead, new premises would be built at the police headquarters site in Ripley.\nBoth organisations said the move would enable them to work better together and be more effective.\nThe costs would be met from police capital reserves and from the sale of the old fire service headquarters.\n\"Two of the current buildings on the site are more than 40-years-old and have increasing maintenance and heating costs,\" said police and crime commissioner Alan Charles.\n\"We have looked at all the options from repair and refurbishment to new build and it is clear that over the lifetime of the building the new build represents best value for the taxpayer.\n\"At the same time we are seeking to undertake a collaborative building project with the fire and rescue service, which will reduce costs still further.\"\nHe added: \"Importantly, we are able to fund this from our capital reserves and it will not impact negatively on our current resources for frontline policing.\"\nOther buildings will remain on the site.\nThe fire service said its share of the costs would be largely met through the sale of its 19th Century building in Littleover.\nAdditional funding would be sought from government transformation grants for \"joint blue light\" schemes.",
"input": "",
"output": "The summary is: \nThe police and fire service in Derbyshire are considering plans to share headquarters in a bid to improve working practices and save money.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nThe 87-year-old went to hospital following the fall and pulled out of presenting his BBC variety show Bruce's Hall Of Fame.\nSpeaking after his fall, Sir Bruce said he was \"really sad\" not to be part of the programme.\nPointless presenter Alexander Armstrong will take over as the show's host.\nSir Bruce said: \"I was really looking forward to this show and working with such a talented cast, and I am really sad not to be part of it.\n\"It is now in the most capable hands of Alexander Armstrong and I would like to wish him, the guests and the whole production team good luck on Sunday.\"\nIn a statement, the show's production company Kalooki Pictures said: \"This morning, Sir Bruce Forsyth slipped and fell at his home resulting in facial abrasions and minor concussion.\n\"He attended hospital and had a series of scans and tests all of which happily proved negative.\n\"However, because of his injury, he has been told by doctors he must have complete rest for at least seven days.\"\nSir Bruce had to pull out of hosting Strictly Come Dancing after being taken ill with flu in October 2013.\nHe announced he was leaving Strictly Come Dancing in April last year and Claudia Winkleman took over his role, alongside his regular co-host Tess Daly.\nBruce's Hall Of Fame, to be filmed in London's Dominion Theatre, is expected to be screened in the new year.",
"input": "",
"output": "The summary is: \nSir Bruce Forsyth has been told by doctors to have complete rest for at least a week after suffering a fall at his home.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nThe girls, aged 13 and 14, were charged with the murder of Angela Wrightson, 39, whose body was discovered at her home in Stephen Street on Tuesday.\nThey appeared separately, with the younger girl wiping away a tear and the older one weeping throughout.\nNo pleas were entered and both were remanded to youth custody, with a preliminary hearing on 18 December.\nOne of the girls' mothers wept as they appeared at an earlier hearing at Teesside Youth Court.\nThe 13-year-old's parents were present at the hearing, with her mother sobbing throughout, and the older girl was watched by her father.\nAt the crown court, no wigs were worm by the judge or prosecution and defence barristers due to the age of the defendants.\nA post-mortem examination found Ms Wrightson died as a result of blood loss and substantial injuries.",
"input": "",
"output": "The summary is: \nTwo teenage girls accused of murdering a woman in Hartlepool have appeared at Teesside Crown Court.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nThe veterans have set off from the Canary Islands.\nThey aim to row the 3,000 miles in under 55 days as part of the Talisker Whisky Atlantic Challenge.\nThe Row2Recovery group are competing against 25 other teams in the race, which will end in Antigua in the Caribbean.\nThe race was scheduled to start on 15 December, but stormy weather postponed the challenge.\nOne of the rowers, Lee Spencer, said: \"We'll literally be on our own.\n\"We have a life raft and personal location devices and if we end up in the water swimming is not a big deal. We only have three legs between us.\n\"But the day-to-day chores are the things we'll struggle with.\"\nMr Spencer said they had been training their upper bodies to compensate for having lost lower limbs.\n\"We want to be an example to all people; we're just normal guys who have suffered some misfortune, but life carries on,\" he said.\nCayle Royce - 29, from Dartmouth. Suffered serious injuries serving in Afghanistan\nPaddy Gallagher - 30, from Cambridgeshire. He was injured in Afghanistan while serving with the Irish Guards\nNigel Rogof - 56, from Hereford, who lost his leg while taking part in an RAF parachuting display\nLee Spencer - 46, from Yelverton in Devon. He lost a leg when he was struck by debris when he stopped to rescue a seriously injured motorist on the M3\nAlso competing are a group of four women aged 44 to 51 from Yorkshire, and 20-year-old University of Bristol aerospace engineering student Callum Gathercole, who is a solo competitor.\nCrews will spend at least a month at sea, living on freeze-dried food, while raising money for charity.\nCarsten Heron Olsen, race organiser, said: \"This year we have 26 teams from the US, Italy, the UK, Antigua, Australia and South Africa, and there are 62 rowers in total.\n\"Winds look extremely favourable for the rowers for the first few days at sea, and alongside the high level of professionalism of the participants, we're anticipating a quick and competitive race and hopefully break some records.\n\"The race was planned to start on Tuesday, but due to strong winds going in the wrong direction we had to delay the race for a few days.\"",
"input": "",
"output": "The summary is: \nFour ex-servicemen are rowing across the Atlantic in a bid to become what is believed to be the first all-amputee team to make the trip.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nSpanish official David Fernandez Borbalan ruled out a late Shane Duffy winner and waved away penalty appeals.\nWest Brom's McClean described the referee as \"Austria's 12th man\" while O'Neill said the Spanish official was \"very poor\" in the Aviva Stadium game.\nFifa has begun disciplinary processes.\nA Fifa spokesman told the BBC they are probing remarks made by both men and it is understood manager and player have until Friday to respond to the charges.\nAs well as ruling out Duffy's header, Borbalan also decided against giving the Republic a penalty when Jon Walters went down under a challenge from Stefan Lainer.\n\"It should count, the referee should have given the goal,\" the manager said of Duffy's header.\n\"I personally think it typified the referee's performance.\n\"The lineman thinks he has given a goal and he's almost up at the halfway line before he is called back.\"\nThe Football Association of Ireland declined to make any comment when contacted on Wednesday.\nFifa stated: \"We can confirm that disciplinary proceedings have been opened.\n\"Be informed that two cases were opened: one against James McClean and another one against the coach Martin O'Neill.\"\nA spokesman for the world governing body said there will be no further comment as the matter is ongoing.",
"input": "",
"output": "The summary is: \nRepublic of Ireland boss Martin O'Neill and winger James McClean face punishment from Fifa for criticising the referee after the 1-1 draw in their World Cup qualifier with Austria.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nThe ivory, with a black market value of $30m (£19.4m), is the largest consignment to be destroyed in Kenya.\n\"Many of these tusks belonged to elephants which were wantonly slaughtered by criminals,\" he said at the ceremony in Nairobi National Park.\nElephant ivory is often smuggled to Asia for use in ornaments.\nRhinos are also poached for their horns for use in traditional medicine.\nConservationists have warned that elephants could be wiped out in some parts of Africa in the next few years.\n\"Twenty-five years after the historic banning of the ivory trade, demand from the emerging markets once again threatens Africa's elephants and rhinos,\" President Kenyatta said.\nThe burning of the ivory was to show that wildlife trophies must be put \"beyond economic use\", he said.\n\"We want future generations of Kenyans, Africans and indeed the entire world to experience the majesty and beauty of these magnificent animals.\n\"Poachers and their enablers will not have the last word in Kenya.\"\nMr Kenyatta promised that his government would destroy the country's entire stockpile of ivory - thought to be another 115 tonnes - by the end of the year.\n\"We are committed to combating the menace robustly and persistently until we dismantle the entire vile economy,\" the president said, adding that Interpol's new regional office on environmental crime in Kenya was a significant boost in the battle.\nLast month, China imposed a one-year ban on the import of ivory, amid criticism that demand from its consumers was fuelling poaching in Africa.",
"input": "",
"output": "The summary is: \nKenyan President Uhuru Kenyatta has set fire to 15 tonnes of elephant ivory as part of the East African nation's efforts to curb poaching.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nBen Purdy, 18, was shot in the head in Bermondsey after confronting his girlfriend's ex, Michael Bagnall, about \"threatening\" messages.\nBagnall, 22, of Hospital Way, Lewisham, must serve at least 28 years.\nHis uncle Andrew Bayne, 37, of Trundleys Terrace, Lewisham was sentenced to at least 30 years.\nDuring the trial at the Old Bailey, jurors heard that they both suffered from learning disabilities, but Judge Christopher Moss said they were still \"able to organise and carry out this dreadful murder\".\nAs they were led from the dock, Bagnall launched into a tirade of abuse at the judge and shouted out \"I'm glad he's dead\" in the courtroom packed with Ben's family.\nIn a statement, Ben's mother, Joanne Treston-Smith, said: \"The void that Ben's death has left will never be filled. We will always have him missing in our lives.\"\nThe court heard that the murder followed a feud between Michael Bagnall and the victim, who was going out with the Bagnall's ex-girlfriend.\nBagnall was said to have resented the new relationship and sent the girl \"threatening messages\" by phone or Facebook.\nOn the day of his murder, Ben Purdy decided to confront Bagnall, tracking him down in which led to a \"skirmish\" in Bermondsey, south London.\nAfter that encounter, the jury heard Bagnall and and his uncle Andrew Bayne decided Mr Purdy needed to be \"taught a lesson\".\nThe pair armed themselves with a self loading pistol and an array of other weapons and got in a car to scour the streets for Mr Purdy and his friends.\nWhen they caught up with them, Bayne shot Mr Purdy in the head on 23 November 2014. He died in hospital the next day.",
"input": "",
"output": "The summary is: \nA man and his uncle have received life sentences after being found guilty of shooting dead a teenager in south London to \"teach him a lesson\".",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nMLAs have spent most of Tuesday debating the welfare reform bill, which has reached the consideration stage in Stormont's legislative process.\nThe day's proceedings adjourned at 22:00 GMT but will resume on Wednesday.\nWelfare reform had caused an impasse until a deal was reached at December's inter-party talks in Stormont House.\nPoliticians agreed to set aside tens of millions of pounds for a fund designed to provide financial support for those adversely impacted by welfare changes.\nMr Robinson said the financial cost of not implementing welfare reform would have been at such a level that \"we could not have sustained an executive\".\nHe said that other parties could not have an \"a la carte\" approach to the Stormont House Agreement.\n\"If people genuinely want to move forward in Northern Ireland, then it is important this legislation goes through. It's important that the parties uphold the agreement that all of us reached,\" he said.\nAt the start of the debate, the DUP was accused of \"killing off discussion\" of the bill.\nUlster Unionist Roy Beggs said the DUP had done so by tabling a series of petitions of concern against amendments to the bill.\n\"They have displayed the undemocratic nature of their attitudes as MLAs and the undemocratic nature of their party, which of course has the word democracy in their name,\" he said.\nMr Robinson rejected Mr Beggs' claim that his party's actions were \"shameful\".\nThe measure was designed as a way to safeguard minority rights in Stormont's power-sharing assembly.\nIf a petition of concern is presented to the assembly speaker, any motion or amendment will need cross-community support.\nIn such cases, a vote on proposed legislation will only pass if supported by a weighted majority (60%) of members voting, including at least 40% of each of the nationalist and unionist designations present and voting.\nEffectively this means that, provided enough MLAs from a particular community agree, that community can exercise a veto over the assembly's decisions.\nApart from the amendments tabled by Social Development Minister Mervyn Storey of the DUP, only two others - put forward by the UUP - survived.\nHowever, Mr Robinson, the first minister, said assembly members were still capable of discussing the bill as well as the amendments.\nThe SDLP's Alex Attwood accused the DUP of trying to run a \"coach and horses\" through the amendments.\n\"Never before in the life of the chamber has there been such a swingeing attempt through petitions of concern to shut down what might be good law for the people of this part of the world,\" he said.\nSinn Féin's Martin McGuinness said some SDLP assembly members were defying their party leader Alasdair McDonnell by tabling the amendments in the chamber.\n\"The SDLP dissidents are clearly now in charge of the party and are prepared to risk the collapse of the Stormont House Agreement - and thereby the power-sharing institutions themselves - for the sake of party political grandstanding,\" he said.\nGenerally, Northern Ireland Assembly bills reach their consideration stage a few months after MLAs first debate their principles.\nThe fact that two years and four months have passed since this bill was last on the floor of the chamber shows just how difficult the arguments over welfare reform have been.\nHowever, the essential deal was struck before Christmas, with the parties accepting the introduction of universal credit and personal independence payments, to replace benefits such as jobseeker's allowance, tax credits and disability living allowance.\nStormont is providing support for those adversely impacted financially by the changes to the welfare system by setting aside nearly £30m to assist claimants who lose out in 2015/16.\nSinn Féin has claimed the fund will amount to more than £500m over the next six years.",
"input": "",
"output": "The summary is: \nFirst Minister Peter Robinson has told MLAs the Northern Ireland Assembly would have \"gone down\" if there had been no agreement on welfare reform.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nProduction would never reach that level again, with the strike heralding the long slow decline of an industry they once called King Coal.\nThirty years later, China's growth in coal consumption - just its growth - was not far off the UK's 1983 total output.\nIn 2013, China consumed an extra 93 million tonnes of the stuff.\nThat amount - a mountain of the black fuel that would at one time have kept the best part of a quarter of a million British miners in work - represented only a 2.6% increase in China's seemingly insatiable appetite for coal.\nLike Britain, China's industrial revolution has been coal-powered, but it has been on a scale and speed like nothing else in world history, bringing with it serious environmental implications.\nChina surpassed the United States to become the biggest emitter of greenhouse gases in 2007 and, if that trajectory is followed, it is well on track to double US emission levels within the next few years.\nFor anyone, anywhere worried about climate change, China has become the problem, and with the country opening a new coal-fired power station on average every week, it is a problem that has looked likely to simply grow and grow.\nExcept that the recently released figures for 2014 suggest that something very interesting may now be happening.\nRather than another giant increase in coal consumption, for the first time in 15 years, government data shows that China's annual coal consumption declined by 2.9%, with an accompanying 1% fall in carbon dioxide emissions.\nA series of articles looking at how the world will meet increasing demand for energy and the need to cut CO2 emissions linked to global warming, using old and new technologies\nRather than never-ending growth, all the talk now is of \"peak coal\", the moment when China begins to wean itself off fossil fuels.\nAnd some analysts believe, on the basis of that 2014 figure, the moment may well have already arrived.\n\"It's quite possible,\" says Wang Tao, an expert on climate and energy policy at the Carnegie-Tsinghua Centre for Global Policy in Beijing.\n\"I wouldn't say 100% sure, but given what we're seeing in the heavy industries and the direction that China is trying to drive its economy, I don't think we're going to see a dramatic change and coal consumption back up again.\"\nOther analysts are a little more cautious, but almost all agree that peak coal, if it hasn't yet arrived, is closer than anyone previously thought.\nAnd while some of it may be down to simple economic factors - the now well-documented slowdown in Chinese growth in recent years - there is wide recognition that a significant shift in Chinese environmental policy is also playing a part.\nChina used to argue that it was unfair for developed countries to lecture as, just as they had in the course of their industrialisation, it had the \"right to pollute\".\nIf it had to choose between its economy or its environment, the old orthodoxy used to go, the economy would win every time.\n\"There are priorities driving Chinese policy makers to move faster than they are used to,\" says Li Yan, head of climate and energy campaign for Greenpeace East Asia.\n\"I think that the environmental crisis we're facing right now, especially the air pollution - no-one expected this to be a top political priority four years ago but look at where we are now,\" she says.\n\"The issue is shaping energy policy, economic policy and even local agendas in the most polluted regions.\"\nHere, she says, the public simply \"cannot bear the air quality the way it is any longer\".\nChina is now the world's biggest investor in renewable energy, particularly in power generation. In fact, the country has seen more than $400bn (£267bn) invested in clean energy in the past 10 years, and is ranked number one in the world in consultancy EY's renewable energy country attractiveness index.\nAccording to Wang Tao, one in every four units of power generated now comes from wind, solar or hydro plants, and a new debate has begun, focusing not on the need to build more renewable energy plants, but on how to best utilise this new and still rapidly growing resource.\n\"We have to make sure that people have the incentives to continue to invest in these renewables, and also that consumers will be able to know and to choose wisely in terms of what kind of electricity they consume, and also change their behaviour,\" he says.\nAnd where once everyone spoke about the huge vested interests in China's fossil fuel-powered sectors, many believe the government is starting to take them on.\n\"In Hubei Province,\" Li Yan says, \"we are observing very bold and firm action to close down the dirtiest fleet of the iron, steel and cement sector, even at the cost of temporary job losses.\n\"I think that's a painful process, but it's also a demonstration of how important the air pollution agenda is in this region.\"\nGreenpeace's great fear had once been that China was preparing for a huge shift towards coal gasification projects - rather than using coal directly to fuel power plants, using it to produce natural gas.\nWhile the end product may be cleaner, critics argue that the industrial processes involved in the conversion emit more greenhouse gases and have other serious environmental impacts, like the huge amount of water consumed.\nBut even here, there appear to be signs of a bit of a rethink going on.\nChina's state-run media has cited an unnamed policymaker as saying that while the country will complete the construction of already approved coal-to-natural-gas plants, it will not approve new ones, at least until 2020.\nIt is of course much too early to suggest that China is turning its back on King Coal.\nThe fuel will make up the majority of its energy sector well into the next decade, a period over which it will continue to burn well over 3 billion tonnes of it every year.\nBut even as new power plants come on stream, it seems likely that - if it hasn't already happened - very soon the overall reliance on coal will begin to decrease and more and more of those new plants will be forced to operate below capacity.\nIf the slowdown in economic growth becomes more serious and sustained, then some environmentalists believe we could yet see the Chinese government lurch for another bout of stimulus spending, pouring money into the big energy-intensive industries and sparking another coal boom.\nBut for now, there are signs that China's unbearable air has become the catalyst for at least the beginnings of a fundamental change in direction.",
"input": "",
"output": "The summary is: \nIn 1983, the year before the coal miners' strike - one of the most bitter industrial disputes in British history - the UK produced 119 million tonnes of coal.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nChief executive Véronique Laury said the aim was to \"leverage the scale of the business by becoming a single, unified company\".\nDetails of the \"ONE Kingfisher\" plan came ahead of an investor day.\nInvestors reacted negatively to the move, sending Kingfisher shares down 6.1% to 324p in afternoon trading.\nThe slide made it the biggest faller on the FTSE 100 on Monday.\nThe retailer, which also owns Screwfix as well as Castorama in France, will face more competition following the sale of Homebase to Wesfarmers.\nThe Australian company plans to rebrand the DIY chain as Bunnings and revamp stores.\nMs Laury said improving Kingfisher's digital capability was one of its priorities.\nClive Black, head of research at Shore Capital, said: \"It looks like Kingfisher is coming to terms with the realities of the limitations of large shops, so a focus upon the digital age. We think shareholders will welcome the focus on digital over stores and the return of cash, albeit the exceptional costs are substantial.\"\nIndependent retail analyst Nick Bubb said that the plan's goals would involve costs of up to £800m.\n\"The benefits aren't as clear-cut as you might think, although the news that Kingfisher also intend to return about £600m of capital to shareholders over the next three years (via share buybacks) will provide some comfort,\" he said.\nInvestec analyst Kate Calvert said the potential returns for shareholders outlined in the plan did not outweigh the risks involved.\n\"There are a lot of moving parts and no guarantee that all the costs will fall out and the profits come through,\" she said.\nKingfisher also said Rakhi Parekh, a former Amazon UK executive, had been appointed a non-executive director.\nMs Laury said Ms Parekh's extensive experience in digital and multichannel retailing would be vital to the company's plans.\nKingfisher said in November that profit for the 13 weeks to 1 November fell 11.8% to £225m, with total sales down 3.6%.\nIn France, sales slid by 9.3%, but the poor performance was partially offset by a 4.8% rise in the UK.",
"input": "",
"output": "The summary is: \nKingfisher, which owns B&Q, has announced a push to increase annual pre-tax profits by £500m within five years and return £600m to shareholders.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nNile Ranger was involved in the first meaningful chance as his neat flick found Anthony Wordsworth on the edge of the box, but the midfielder's low shot was well saved by James Shea.\nRanger then got a brilliant opener for Southend in the 35th minute, shuffling past three players and prodding a shot into the right-hand corner while off balance to score his third goal in as many games.\nThe Dons had been wasteful in possession but Lyle Taylor nearly latched onto Tom Soares' through-ball just before half-time.\nSimon Cox doubled Southend's lead shortly after the hour mark after taking advantage of a fortunate rebound in midfield and lashing past Shea from 20 yards.\nShrimpers substitute Theo Robinson hit the post after latching on to a ball over the top, but the away side were rarely troubled at the other end as they maintained their play-off hopes.\nMatch report supplied by the Press Association.\nMatch ends, AFC Wimbledon 0, Southend United 2.\nSecond Half ends, AFC Wimbledon 0, Southend United 2.\nFoul by Darius Charles (AFC Wimbledon).\n(Southend United) wins a free kick in the attacking half.\nDean Parrett (AFC Wimbledon) wins a free kick in the attacking half.\nFoul by Theo Robinson (Southend United).\nWill Nightingale (AFC Wimbledon) wins a free kick on the left wing.\nFoul by Nile Ranger (Southend United).\nAttempt blocked. Darius Charles (AFC Wimbledon) right footed shot from the left side of the six yard box is blocked.\nFoul by Andy Barcham (AFC Wimbledon).\nWill Atkinson (Southend United) wins a free kick in the defensive half.\nPaul Robinson (AFC Wimbledon) wins a free kick in the defensive half.\nFoul by Nile Ranger (Southend United).\nSubstitution, AFC Wimbledon. Dominic Poleon replaces Lyle Taylor.\nSubstitution, AFC Wimbledon. Tyrone Barnett replaces Tom Elliott.\nCorner, Southend United. Conceded by Darius Charles.\nTheo Robinson (Southend United) hits the right post with a right footed shot from the centre of the box.\nSubstitution, Southend United. Theo Robinson replaces Simon Cox.\nDelay over. They are ready to continue.\nDelay in match Simon Cox (Southend United) because of an injury.\nLyle Taylor (AFC Wimbledon) is shown the yellow card.\nAttempt blocked. Dean Parrett (AFC Wimbledon) right footed shot from outside the box is blocked.\nDelay in match Andy Barcham (AFC Wimbledon) because of an injury.\nCorner, AFC Wimbledon. Conceded by Jason Demetriou.\nSubstitution, AFC Wimbledon. Dean Parrett replaces Tom Soares.\nAttempt missed. Lyle Taylor (AFC Wimbledon) left footed shot from outside the box is close, but misses to the left.\nGoal! AFC Wimbledon 0, Southend United 2. Simon Cox (Southend United) right footed shot from the centre of the box to the top right corner.\nCorner, AFC Wimbledon. Conceded by Anton Ferdinand.\nAttempt missed. Jake Reeves (AFC Wimbledon) right footed shot from outside the box misses to the left.\nWill Nightingale (AFC Wimbledon) wins a free kick on the left wing.\nFoul by Ben Coker (Southend United).\nCorner, Southend United. Conceded by James Shea.\nAttempt saved. Will Atkinson (Southend United) left footed shot from the right side of the box is saved in the bottom left corner.\nFoul by Tom Soares (AFC Wimbledon).\nAnthony Wordsworth (Southend United) wins a free kick in the defensive half.\nAttempt missed. Tom Elliott (AFC Wimbledon) left footed shot from the left side of the box is close, but misses to the right.\nSecond Half begins AFC Wimbledon 0, Southend United 1.\nFirst Half ends, AFC Wimbledon 0, Southend United 1.\nFoul by Tom Soares (AFC Wimbledon).\nAnthony Wordsworth (Southend United) wins a free kick in the defensive half.",
"input": "",
"output": "The summary is: \nPhil Brown marked his fourth anniversary in charge of Southend with a third consecutive win as his side beat Wimbledon in League One.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nMore than 5,500 people signed a petition against plans to build a five-metre embankment along the waterfront.\nHowever, the council has admitted that there will never be a consensus on any flood protection proposal.\nA report to a meeting of Dumfries and Galloway Council's environment committee next week will attempt to find a way forward.\nWhat's happening in Scotland? Keep in touch through our live page.\nChairman Colin Smyth said: \"What we are now able to do is focus on what I think is the biggest issue as far as the public is concerned. In the draft proposal, the height of the embankment and the walls were simply too high and the public did not support that.\n\"What we now need to do is make sure that we find a solution that deals with the flooding, regenerates the Whitesands, solves the car parking issues, but also reduces the height of any proposed flood protection scheme.\"\nWater from the River Nith regularly spills over into the Whitesands, flooding a major town centre car park and nearby business premises.\nCampaigners against the £15m proposal to build an embankment claimed it would have a detrimental effect on the town's main beauty spots.\nThey also raised concerns that the move would lead to the loss of about 200 waterfront car parking spaces.\nDavid Slater, a local businessman who has been one of the project's most vocal objectors, said: \"However many other consultations they do now, public opinion will not change at this stage.\n\"It will be interesting to see how they can agree with the public to reduce the height of the bunds. There has to be better ideas because we can't put that in our town.\"\nEarlier this year MSPs called for the row over the flood protection plans to be brought to a \"positive conclusion\".",
"input": "",
"output": "The summary is: \nSenior councillors in Dumfries have pledged to find a compromise solution to the Whitesands flooding problem.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nMonsignor Angel Lucio Vallejo Balda has said he was manipulated by a woman co-defendant with whom he was romantically entangled.\nHe was questioned as the so-called Vatileaks II trial resumed.\nIt centres on two books that depict a Vatican plagued by graft and where Pope Francis faces resistance to his agenda.\nThe books came out last year and were based on the leaked information. The five people on trial face jail terms of up to eight years.\nLeaks lift lid on Pope Francis's financial fight\nVatican reforms may be starting to bite\nMr Vallejo Balda, 54, was questioned for three hours and most of his testimony revolved around his relationship with Francesca Chaouqui, 35, a married public relations consultant.\nThey were members of a now-defunct commission appointed by Pope Francis to tackle the Vatican's financial holdings and propose reforms to improve cash flow to the poor.\n\"Yes, I passed documents,\" Mr Vallejo Balda told the court in Spanish.\nHe also admitted to giving one of the authors some 87 passwords to access electronic documents and email accounts in the Vatican.\nThe priest said his actions were the result of a combination of sexual tension and blackmail by Ms Chaouqui, who claimed she was a spy with Italy's secret services.\nSaying he felt \"compromised\" as a priest, Mr Vallejo Balda recounted how she once entered his room in a Florence hotel.\nThe priest, at one point, described the feeling of being \"in a situation with no way out\".\nIn the testimony, he also said he received threatening messages from Ms Chaouqui and her husband, especially after the commission's work was over.\nMs Chaouqui, who is in late pregnancy, attended the hearing and is expected to give evidence next week.\nShe denies accusations of conspiring with Mr Vallejo Balda and his assistant Nicola Maio to leak information they had access to as members of the commission.\nThe two journalists on trial, Gianluigi Nuzzi and Emiliano Fittipaldi, wrote the books Avarice and Merchants in the Temple.\nThey are accused of putting pressure on the priest and Ms Chaouqui to get the documents, allegation both journalists deny.\nThe five are on trial under a legislation criminalising the leaking of documents, introduced in 2013 after a scandal known as the first Vatileaks.",
"input": "",
"output": "The summary is: \nA Spanish priest has admitted to leaking classified Vatican documents to journalists, saying he had felt intimidated.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nA BBC investigation reveals Southampton docks does not monitor its air pollution rates, despite the city being among the most polluted in the UK.\nSouthampton City Council estimates the port contributes up to 23 per cent of air pollution in the city.\nShips can use a \"plug-in\" system to reduce emissions but not in the UK.\nIt comes after the World Health Organisation (WHO) called Southampton one of the most polluted cities in the UK.\nThe city's port welcomes thousands of ships a year, including some of the biggest cruise liners and container ships in the world.\nThe vessels leave their engines running while docked to power their electrics, but elsewhere in the world ships use a shore-based electricity supply, virtually eliminating their emissions.\nCargo and cruise ships, including the Queen Mary 2 and Britannia which regularly dock in Southampton, use the method - just not when visiting the British port.\nPort Director Alastair Welch from ABP said: \"The challenge has been in particular there is no one standard for shore power. I'd like it in place as soon as possible.\n\"I should emphasise shore power is not the only answer and that's why we're working with solar power and hybrid ships now, because all of them have a part to play for the future.\"\nA review of Air Quality in Southampton in 2015 by the local authority showed the port is believed to contribute between seven and 23 per cent of the air pollution, while cars contribute 18 per cent and HGVs 12 per cent.\nThe government has since told Southampton to implement clean air zones by 2020 and the council is implementing a Clean Air Strategy to meet national goals.",
"input": "",
"output": "The summary is: \nSouthampton, the biggest cruise port in Britain, has no way of monitoring air pollution generated by emissions from the largest ships in the world.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nMs Wood, whose comments come after the Grenfell Tower fire, said a review into product safety she carried out for the government had been ignored.\nShe said she felt she had been offered an MBE last year, which she turned down, to \"stop her nagging\" officials.\nThe government said it took consumer product safety \"extremely seriously\".\nMs Wood, who presented consumer affairs show Watchdog from 1985 to 1993, and who has long campaigned on consumer rights, was asked to look into the product recall system by the coalition government.\nIt followed concerns the system was not working properly, leading to avoidable injuries, accidents, and even deaths.\nThe review was published in February 2016, but Ms Wood said the government had not acted on any of its findings.\nThe the review's key recommendations included:\nSpeaking to Radio 5 Live's Wake Up To Money, Ms Wood said following the review she was invited to meet a government minister who had not \"read a word of it\".\n\"I said, actually minister I have been invited along to talk to you about something I've spent the last nine months doing.\n\"I thought it was shocking. It made me feel they had wasted my time and a lot of other people's time.\"\nMs Wood said the UK was importing appliances from many more countries than it used to, and that while most would be safe, greater vigilance was needed against unscrupulous suppliers.\nShe cited incidents including last year's Shepherd's Bush tower block fire, believed to have been caused by a faulty tumble dryer, as well blazes linked to Beko fridge freezers.\nShe said Trading Standards departments in local authorities were struggling to police companies because of budget cuts, and businesses had become bolder about cutting corners.\n\"We do not know what caused the Grenfell Tower fire, but what we do know is that we are putting people at risk because we don't have a good enough system,\" she said.\nA business department spokeswoman said a working group had been established in October 2016 to look at product recalls and safety \"to ensure the products we all use are as safe as possible\".\nShe said the group, led by fire safety expert Neil Gibbins, was exploring Ms Faulds Wood's recommendations and \"developing options\" for improvement.\nShe added that the group had commissioned the British Standards Institute to develop a code of practice on recalls.",
"input": "",
"output": "The summary is: \nThe government is not doing enough to protect consumers from faulty products that can cause fires, former BBC presenter Lynn Faulds-Wood has said.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nHenry, a replacement for Zach Clough 10 minutes earlier, struck from 18 yards in the third of six minutes of added time with a shot that looked to deflect off Iron defender Murray Wallace.\nBolton's win took them into the automatic promotion places while Scunthorpe, top at the start of the day, slipped to third after Sheffield United's 1-0 win over Northampton earlier in the afternoon.\nUntil Henry's intervention Neal Bishop's fourth goal of the season appeared to have earned Graham Alexander's side a deserved draw.\nBolton, watched by the Macron Stadium's second biggest attendance of the season - 17,062 - led 1-0 at half-time.\nJosh Vela's half volley from the edge of the area after 17 minutes was his eighth goal of the campaign.\nScunthorpe had a goal disallowed for offside but Clough wasted a good opportunity to double Wanderers' advantage and felt he should have had a penalty after Jordan Clarke's challenge.\nScunthorpe were much the better side after the break and they deservedly drew level when Bolton failed to clear a 62nd-minute corner and Bishop swivelled on the loose ball to rifle home from 10 yards. But their endeavours drew a blank courtesy of Henry's hammer blow.\nReport supplied by the Press Association.\nMatch ends, Bolton Wanderers 2, Scunthorpe United 1.\nSecond Half ends, Bolton Wanderers 2, Scunthorpe United 1.\nKevin van Veen (Scunthorpe United) is shown the yellow card for a bad foul.\nFoul by Kevin van Veen (Scunthorpe United).\nZach Clough (Bolton Wanderers) wins a free kick in the defensive half.\nStephen Dawson (Scunthorpe United) wins a free kick in the attacking half.\nFoul by Gary Madine (Bolton Wanderers).\nAttempt saved. James Henry (Bolton Wanderers) right footed shot from the centre of the box is saved in the centre of the goal.\nGoal! Bolton Wanderers 2, Scunthorpe United 1. James Henry (Bolton Wanderers) right footed shot from outside the box to the bottom left corner.\nSubstitution, Scunthorpe United. Tom Hopper replaces Paddy Madden.\nAttempt missed. Sam Mantom (Scunthorpe United) right footed shot from outside the box is close, but misses the top right corner.\nAttempt missed. Jay Spearing (Bolton Wanderers) right footed shot from outside the box is just a bit too high from a direct free kick.\nSam Mantom (Scunthorpe United) is shown the yellow card for a bad foul.\nFoul by Sam Mantom (Scunthorpe United).\nTom Thorpe (Bolton Wanderers) wins a free kick in the defensive half.\nFoul by Sam Mantom (Scunthorpe United).\nDerik (Bolton Wanderers) wins a free kick on the left wing.\nAttempt missed. Tom Thorpe (Bolton Wanderers) header from the centre of the box is close, but misses to the right following a corner.\nCorner, Bolton Wanderers. Conceded by Stephen Dawson.\nFoul by Charlie Goode (Scunthorpe United).\nGary Madine (Bolton Wanderers) wins a free kick in the defensive half.\nAttempt missed. Gary Madine (Bolton Wanderers) right footed shot from the right side of the box is just a bit too high.\nAttempt blocked. Tom Thorpe (Bolton Wanderers) right footed shot from outside the box is blocked.\nStephen Dawson (Scunthorpe United) wins a free kick in the defensive half.\nFoul by Sammy Ameobi (Bolton Wanderers).\nSubstitution, Bolton Wanderers. James Henry replaces Zach Clough.\nAttempt blocked. Jay Spearing (Bolton Wanderers) right footed shot from outside the box is blocked.\nSam Mantom (Scunthorpe United) wins a free kick in the attacking half.\nFoul by Sammy Ameobi (Bolton Wanderers).\nJordan Clarke (Scunthorpe United) is shown the yellow card for a bad foul.\nFoul by Jordan Clarke (Scunthorpe United).\nJay Spearing (Bolton Wanderers) wins a free kick in the attacking half.\nSubstitution, Scunthorpe United. Kevin van Veen replaces Hakeeb Adelakun.\nAndrew Taylor (Bolton Wanderers) is shown the yellow card for a bad foul.\nHakeeb Adelakun (Scunthorpe United) wins a free kick on the left wing.\nFoul by Andrew Taylor (Bolton Wanderers).\nStephen Dawson (Scunthorpe United) wins a free kick in the defensive half.\nFoul by David Wheater (Bolton Wanderers).\nCorner, Bolton Wanderers. Conceded by Luke Daniels.\nAttempt saved. Gary Madine (Bolton Wanderers) right footed shot from the right side of the six yard box is saved in the bottom right corner.",
"input": "",
"output": "The summary is: \nSubstitute James Henry's injury time winner earned Bolton a dramatic victory over former leaders Scunthorpe.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nNothing unusual there, you might think.\nBut one writes about Allah and her mosque, the other about her synagogue and the Star of David.\nThe two met at a Washington DC youth slam poetry event and write and communicate about their faiths in ways that aim to engage and provoke their audiences.\nIt's a type of dialogue that will be much needed now, after the killings in Paris and the attack on a kosher supermarket there.\nThe young poets were taking part in a session at Limmud, which has been likened to a Jewish Edinburgh festival, Hay and Glastonbury rolled into one and is held on Warwick University campus just before New Year.\nLimmud (from the Hebrew word for \"learning\") began life in 1980 and now has some 50 offshoots around the world.\nIn the UK this year, it drew over 2,500 people from 27 countries - all the more remarkable given the relatively small size of the UK's Jewish community, which numbers under 300,000 people, a community with close links to Jewish families in Paris and elsewhere in France.\nThe question of identity and security was tackled in many of Limmud's discussions, as Judaism and the role, lives and place of Jewish people in Europe and across the world was examined from every perspective, from the historical to the feminist, and from the religious to the secular.\nEven before the Paris attacks, there were worries over a sharp rise in anti-Semitism in the UK and mainland Europe in 2014, during and after the latest conflict in Gaza.\nIn France, the killing of Jewish schoolchildren in Toulouse in 2012 by a Frenchman of Algerian descent, and the murders of four people inside the Jewish museum in Brussels by another Frenchman of Algerian descent in May 2014 heightened fears amongst the Jewish community in France and elsewhere.\nThe uneasy mood was articulated by BBC's director of television, Danny Cohen, at a conference in Jerusalem last year.\nHe said he had \"never felt so uncomfortable as a Jew in the UK\", as figures showed that anti-Semitic incidents in Britain also rose to record annual levels in 2014.\nMr Cohen said levels of hatred were on the rise across Europe.\n\"You've seen the number of attacks rise, you've seen murders in France, you've seen murders in Belgium. It's been pretty grim actually.\"\nThe killings at Charlie Hebdo in Paris last week have further focused attention on radical Islamism in Europe, and the safety of Jewish communities in France, Germany and the UK.\nAt one Limmud seminar, Michael Whine, of the Community Security Trust (CST), which seeks to protect Jewish communities and offer advice on security, sought to place last summer's figure in a wider European perspective, citing research by the Institute for Jewish Policy Research showing that Jewish people in the UK felt safer than those in France.\nHowever, CST figures still showed a significant rise in both verbal and physical attacks on Jewish people in the UK in 2014 compared with 2013, with 543 recorded in July and August 2014 alone - although the percentage of violent attacks - 7% - did not rise in the UK, unlike France.\nOver the weekend, Ephraim Mirvis, Chief Rabbi of the United Hebrew Congregations of the Commonwealth, joined the march in Paris, and the memorial service at the Great Synagogue there.\nHe said he was pleased to have the opportunity \"to express solidarity with the people of France and the French Jewish Community\".\n\"We stand together at this time challenging time.\"\nThe Senior Rabbi to the Movement for Reform Judaism in the UK, Laura Janner-Klausner, also went to Paris.\nShe travelled with a Muslim colleague to join the march with well over a million others, to show solidarity with the French, of all faiths and none.\nShe said they wanted to demonstrate that all communities could work together to counter the hatred and extremism manifested in the attacks last week.\n\"Some parts of the Jewish community in Paris were still suffering from the trauma of what happened there,\" she said.\n\"Some had heard the shots being fired at Charlie Hebdo, so they were frightened.\n\"Some were tearful, and emotional. But they were not saying, 'Now we are going to pack our bags and leave.'\n\"They were saying,' This is our home, and we don't want the narrative to be that we are leaving because of this.'\"\nNonetheless, she said that anti-Semitism in France was more obvious and public than in the UK.\n\"There, the anti-Semitism can be palpable, whereas in Britain what we generally experience is a wonderful sense of integration.\n\"This is where we live, this is where we want our children to go to school and grow up. If the worst happened in the UK, which it could, this is still our home.\"\nMany believe that the legacy of France's colonial history in north Africa still drives much of the anti-Semitism evident there.\nSome Muslim immigrants from Algeria, Morocco and Tunisia - even now in the second and third generation - continue to feel excluded from the mainstream in France.\nMany live very separate lives in the grimmer suburbs around Paris and other major cities, where unemployment is high and prospects are often limited.\nJewish immigrants to France from north Africa often integrated faster and have sometimes enjoyed better economic prospects, leading to tensions between the two.\nAt the same time, both communities often live side by side, but remain in many cases divided by the perception amongst some younger Muslims that while they struggle to be accepted in France as truly French, Jewish families have not had as much of an uphill task.\nYet some 7,000 French Jewish people chose to leave for Israel last year, a record number, thanks partly to a rise in anti-Semitic attacks, although a dearth of jobs and economic stagnation may also have played its part.\nSome French Jewish people have also moved to London, to seek work or education there.\nIn the UK this week, the CST said there would be increased police patrols in Jewish areas, as a visible reassurance to the community.\nThe hope is that it is only a precaution, not a necessity, with Jewish-Muslim relations in the UK enjoying a more harmonious history than in France.\nIn the UK, Justin Cohen, news editor of the Jewish News, says that nonetheless: \"British Jews are still on edge following the huge increase in anti-Semitism last summer, and the horrific events in Paris have heightened tensions.\n\"Still, there is a clear difference between that and the levels of hate faced by the community in France.\n\"And it is absolutely not the case that British Jews are packing their bags to leave, as is the case across the Channel.\n\"Nevertheless, members of the British Jewish community are all too aware that, alongside US, British and Israeli sites, Jewish ones are high up in the target list for Islamist fundamentalists.\n\"Jewish schools and institutions will be going about their usual business today, but with this and fresh calls for vigilance uppermost in their minds.\"\nThe worry remains that the virulent strain of Islamist extremism that exploded on to the world stage again so violently in France last week is an ideology that can be used by extremists anywhere to seek to divide communities, different faiths and societies across the world.\nThe challenge now for many is how to strengthen interfaith ties to resist that attempt.",
"input": "",
"output": "The summary is: \nTwo young college student poets, Hannah Halpern and Amina Iro, are talking about their faith.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nFilming for Knights of the Roundtable: King Arthur will begin on Monday and last for six days, Snowdonia National Park Authority has confirmed.\nThe Guy Ritchie film will star Charlie Hunnam as King Arthur and Jude Law as the villain Vortigern.\nVelocity Productions will be filming in and around Capel Curig, Nant Gwynant and Pen y Gwryd.\nMeanwhile, Bangor University has just extended its archive about Arthur after Flintshire council donated its Arthurian Collection.",
"input": "",
"output": "The summary is: \nSnowdonia's renowned natural beauty is to play a starring role in a Hollywood film featuring Jude Law.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nThe zero-fuel aircraft arrived in Dayton at 21:56 local time (01:56 GMT) having flown from Tulsa, Oklahoma.\nThe 1,113km journey took pilot Andre Borschberg about 16 hours to complete, a relatively short hop for the plane.\nSolar Impulse is aiming to get to New York in the next couple of weeks before it crosses the Atlantic - the last big leg in its global endeavour.\nTo complete the circumnavigation, the aeroplane needs to get to Abu Dhabi in the United Arab Emirates where the journey started in March last year.\nAs well as setting new aviation milestones, the stated purpose of the project is to demonstrate the capability of clean technologies.\nThe plane gets all its energy from the sun, captured by 17,000 photovoltaic cells on its top surfaces. These power the craft's propellers during the day but also charge batteries that the vehicle's motors can then call on during the night.\nThe craft is wider than a 747 jumbo jet but weighs just 2.3 tonnes. Low flight speed means mission legs can take several days and nights of continuous flight.\nThe pilot is permitted only catnaps of up to 20 minutes, and the cockpit is little bigger than a public telephone box.\nLEG 1: 9 March. Abu Dhabi (UAE) to Muscat (Oman) - 772km; 13 Hours 1 Minute\nLEG 2: 10 March. Muscat (Oman) to Ahmedabad (India) - 1,593km; 15 Hours 20 Minutes\nLEG 3: 18 March. Ahmedabad (India) to Varanasi (India) - 1,170km; 13 Hours 15 Minutes\nLEG 4: 18 March. Varanasi (India) to Mandalay (Myanmar) - 1,536km; 13 Hours 29 Minutes\nLEG 5: 29 March. Mandalay (Myanmar) to Chongqing (China) - 1,636km; 20 Hours 29 Minutes\nLEG 6: 21 April. Chongqing (China) to Nanjing (China) - 1,384km; 17 Hours 22 Minutes\nLEG 7: 30 May. Nanjing (China) to Nagoya (Japan) - 2,942km; 1 Day 20 Hours 9 Minutes\nLEG 8: 28 June. Nagoya (Japan) to Kalaeloa, Hawaii (US) - 8,924km; 4 Days 21 Hours 52 Minutes\nLEG 9: 21 April. Kalaeloa, Hawaii (US) to Mountain View, California (US) - 4,523km; 2 Days 17 Hours 29 Minutes\nLEG 10: 2 May. Mountain View, California (US) to Phoenix, Arizona (US) - 1,199km; 15 Hours 52 Minutes\nLEG 11: 12 May. Phoenix, Arizona (US) to Tulsa, Oklahoma (US) - 1,570 km; 18 Hours 10 Minutes\nLEG 12: 21 May. Tulsa, Oklahoma (US) to Dayton, Ohio (US) - 1,113 km; 16 Hours 34 Minutes",
"input": "",
"output": "The summary is: \nSolar Impulse has landed in the US state of Ohio following the 12th stage of its circumnavigation of the globe.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nNew York City's new Mayor Bill de Blasio refused to join the city's annual parade, and Guinness has withdrawn its sponsorship.\nParade organisers said gay groups are not prohibited but may not carry identifying banners.\nBoston Mayor Martin Walsh also skipped his city's parade on Sunday.\nMr de Blasio has said he would not join the parade in protest against its long-standing policy of excluding gay Irish groups from marching openly.\nThe New York mayor played host to Irish Prime Minister Enda Kenny for a St Patrick's Day breakfast before the parade.\nMr Kenny joined the procession down Fifth Avenue in New York City's Manhattan borough on Monday, after saying the holiday was about Irishness, not sexuality.\nOn Sunday, Irish beer brand Guinness said it was dropping its participation in the New York parade.\n\"Guinness has a strong history of supporting diversity and being an advocate for equality for all,\" the brand said in a statement issued by parent company, Diageo.\n\"We were hopeful that the policy of exclusion would be reversed for this year's parade.\"\nThe firm pulled any promotional materials that were not already printed, although the beer maker had already made a payment to parade organisers, spokeswoman Alix Dunn said.\nSome gay and lesbian groups protested along the parade route on Monday, while a plan to dump Guinness beer from the shelves of the Stonewall Inn, the birthplace of the gay rights movement in New York City, was cancelled after the company pulled out of the parade.\nNew York's parade draws more than one million spectators and about 200,000 participants during the St Patrick's Day holiday.\nOn Friday, two other major beer brands, Boston-based Sam Adams and Heineken, also dropped parade sponsorships.\nIn Boston, Mr Walsh, the first Irish-American Boston mayor in 20 years, said: \"So much of our Irish history has been shaped by the fight against oppression.\n\"As mayor of the city of Boston, I have to do my best to ensure that all Bostonians are free to participate fully in the civic life of our city.\"",
"input": "",
"output": "The summary is: \nThe world's largest St Patrick's Day parade has kicked off under a cloud of protest against the organisers' refusal to allow gay groups to march openly.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nAnd 15 other forces are looking at following Nottinghamshire Police's lead in treating incidents against women in this way, the Victoria Derbyshire programme has learned.\nOne expert said harassment by men was an \"everyday experience\" for many.\nHarassment on grounds of race, religion and sexuality is already illegal.\nNottinghamshire Police introduced its reclassification of harassment by men against women in July. Its figures, however, cover incidents dating back to April.\nThe force found that there had been 11 misogynistic \"hate crimes\" - offences including harassment, kidnapping, possession of weapons and causing public fear, alarm or distress.\nThere were also 19 \"hate incidents\" - behaviour also motivated by misogyny but falling short of criminal acts, such as name-calling and offensive jokes.\n\"We're not saying all men engage in this behaviour, but for some women this has become an everyday experience. A lot of men are not aware of the implications it has on women,\" said Loretta Trickett, a criminologist at Nottingham Trent University.\n\"Up until now, women have largely not reported this. Women put up with it because it is trivialised in society. People say it's complimentary to be wolf-whistled.\n\"I think the new recording will give women reassurance that if they call the police, their incident will be registered and they will do something.\"\nMartha Jephcott, who has trained Nottinghamshire police officers on how to deal with misogyny as a hate crime, said: \"Recognising misogyny as a hate crime is important because it acknowledges the world in which women live and the everyday nature of these sorts of incidents.\"\nFifteen police forces will attend a conference in Nottingham on Wednesday, looking at the possibility of adopting similar schemes, which they hope will increase the reporting of harassment.\nMs Jephcott said: \"I want forces across the country to adopt this. I think it's a matter of equality.\n\"UK-wide, racist and homophobic hate crimes take place and are recognised as such. Women should have that too because, wherever they are, they probably will have experienced this.\"\nNottinghamshire Police define misogynistic hate crime as \"incidents against women that are motivated by an attitude of a man towards a woman and includes behaviour targeted towards a woman by men simply because they are a woman\".\nThe classification means people can report incidents which might not otherwise be considered to be a crime and the police will investigate.\nDomestic abuse will not be recorded as a misogyny hate crime because it has its own procedure.\nA crime that the victim or any other person perceives to be motivated by hostility or prejudice towards any aspect of a person's identity.\nPolice forces in England, Wales and Northern Ireland annually monitor five strands of hate crime:\nForces can include their own definition of a hate crime with several recently adding sub-cultures.\nThe Victoria Derbyshire programme is broadcast on weekdays between 09:00 and 11:00 on BBC Two and the BBC News channel.",
"input": "",
"output": "The summary is: \nA police force that reclassified wolf-whistling, cat-calling and other misogynistic harassment as hate crimes has handled 30 cases in five months.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nTwo years ago, that border comprised two parallel dirt tracks, one on the Bulgarian, one on the Turkish side.\nNow a razor-wire fence, 1.5m (5ft) thick, welcomes would-be migrants. Thirty kilometres is already completed, while 100km more remains under construction.\n\"The purpose of the fence,\" says Philip Gunev, deputy interior minister, \"is to divert flows of migrants towards border crossing points where our limited financial resources allow us to protect European borders in a more efficient way\".\nSuch official checkpoints, he insists, are safer for asylum seekers than trudging long distances, often with small children, over the rough, hilly terrain the fence now cuts across.\nFixed and mobile cameras, mounted on four-by-four vehicles, complete the picture along the whole length of the border.\nIn the past eight years, since joining the European Union, Bulgaria has spent €300m (£215m) of mostly EU money, reinforcing this border. Another €100m is available to complete the job until 2020. Only €2m will be received for the better integration of refugees in the same period.\nKrassimir Kanev, director of the Bulgarian Helsinki Committee, a human rights advocacy groups, is unhappy with the checkpoints.\n\"The only way to get through is to pay smugglers,\" he says, arguing that only the richer migrants get a chance to try. \"And there's nothing safe about being cramped in the hidden compartment of a truck in the heat of summer.\"\nAbout one third of Bulgaria's migrants are caught on the border with Turkey, another third as they head north or west inside Bulgaria, and the rest on the Serbian or Romanian borders, as they try to continue their journey towards Hungary on their way to Germany.\nBulgaria has one of the highest rates of granting refugee status in the EU. Refugee status means they also receive a Convention Travel Document (CTD) and under the 1951 Refugee Convention they can travel on to anywhere in the EU and stay for up to 90 days.\nIn practice, few ever come back, travelling to Germany or elsewhere.\nAt the Harmanli refugee camp in south-eastern Bulgaria, two police buses bring more asylum seekers.\nChildren wave happily, adults look more concerned.\nConditions here are much better than they were in 2013, when overcrowding, appalling sanitary conditions, and the alleged cruelty of guards gave Bulgaria a bad name.\nSome asylum seekers still express frustration at delays with their applications.\nA group of men hold up a snake they killed in the camp the day before. But most acknowledge a big improvement in conditions for the 1,600 refugees here.\nBulgaria is facing growing pressure from Western governments to identify exactly who they do let in.\nNinety percent arrive with no documents whatsoever because they were taken by the smugglers who brought them this far.\nIn an upstairs room at Harmanli, officers from the Bulgarian intelligence services cross-examine the refugees, most of whom are Syrian Kurds.\nWhile Harmanli is an open camp, those deemed suspect are taken to a prison at Busmantsi, near Sofia, where they can be detained for up to a year, while more investigations are carried out.\n\"The most frustrating thing about life there was the waiting,\" said one former Busmantsi inmate, who asked not to be named.\n\"Your whole life is waiting. You know there will be an end to all this, and one day you will be out, but at this moment you have nothing to do but wait.\"\nAre there radical Islamists inside the prison? I ask.\n\"People keep themselves to themselves. They only share what they have to,\" he tells me. \"But the radical mood among my friends is all about money, which comes mostly from Saudi Arabia. It has nothing to do with political or religious beliefs.\"\n\"Don't link those fleeing terror with those who would like to create it,\" says Boris Cheshirkov, a UN refugee agency spokesman in Bulgaria. \"States can protect refugees, and address security concerns too, by screening and registering them early on.\"",
"input": "",
"output": "The summary is: \nIn the control room of the Bulgarian border police at Elhovo, set back from the country's 270km (165 mile) long border with Turkey, officers control banks of CCTV screens.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nThe 27-year-old left the Blues for French Top 14 side Toulon in 2014 but his contract expires in the summer.\nThe Wales international full-back was sat in the stands at the Cardiff Arms Park as the Blues beat the Dragons 27-16 in the Welsh derby on Boxing Day.\n\"It was great to see him and I know he was quickly back on the plane to his duties in France,\" Wilson said.\nBlues chief executive Richard Holland said earlier in December that they offered Halfpenny a deal to bring him back to Wales.\nSpeaking after Toulon beat Scarlets in France in the European Champions Cup earlier in December, Halfpenny said he was \"weighing up\" his options.\n\"Leigh's obviously got a lot of colleagues and friends from his time with the Blues at the Arms Park,\" Wilson continued.\n\"Being home for Christmas I'd imagine with the derby being on his doorstep it was a natural game for him to go and watch.\"\nHalfpenny, who has won 66 caps for Wales, played for Blues for six years before his move to France two years ago.\n\"I saw him briefly after the game and had a catch up. It's been well documented and I think everybody would like to see Leigh back in Wales,\" said Wilson.\n\"Those things are very much ongoing.\"",
"input": "",
"output": "The summary is: \nCardiff Blues head coach Danny Wilson says it would be good to see Leigh Halfpenny return to Wales",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nChinese electronics firm Hangzhou Xiongmai issued the recall soon after its cameras were identified as aiding the massive web attacks.\nThey made access to popular websites, such as Reddit, Twitter, Spotify and many other sites, intermittent.\nSecurity experts said easy-to-guess default passwords, used on Xiongmai webcams, aided the hijacking.\nThe web attack enrolled thousands of devices that make up the internet of things - smart devices used to oversee homes and which can be controlled remotely.\nIn a statement, Hangzhou Xiongmai said hackers were able to take over the cameras because users had not changed the devices' default passwords.\nXiongmai rejected suggestions that its webcams made up the bulk of the devices used in the attacks.\n\"Security issues are a problem facing all mankind,\" it said. \"Since industry giants have experienced them, Xiongmai is not afraid to experience them once, too.\"\nIt has also pledged to improve the way it uses passwords on its products and will send customers a software patch to harden devices against attack.\nThe recall affects all the circuit boards and components made by Hangzhou Xiongmai that go into webcams. It is not clear how effective the recall will be in reducing the numbers of vulnerable devices hackers can call on to mount attacks.\nYes, and it probably will. The smart devices making up the IoT are proving very popular with the malicious hackers who make their living by selling attack services or extorting cash by threatening firms with devastating attacks.\nBefore the rise of the IoT it was tricky to set up a network of hijacked machines as most would be PCs that, generally, are more secure. Running such a network is hard and often machines had to be rented for a few hours just to carry out attacks. Now anyone can scan the net for vulnerable cameras, DVRs and other gadgets, take them over and start bombarding targets whenever they want.\nFor the same reason you would care if your car was stolen and used by bank robbers as a getaway vehicle.\nAnd because if your webcam, printer or DVR is hijacked you have, in effect, allowed a stranger to enter your home. Hackers are likely to start using these gadgets to spy on you and scoop up valuable data. It's worth taking steps to shut out the intruders.\nNot easily. Many of the devices being targeted are hard to update and the passwords on some, according to one report, are hard-coded which means they cannot be changed.\nThere is also the difficulty of identifying whether you are using a vulnerable product. A lot of IoT devices are built from components sourced from lots of different places. Finding out what software is running on them can be frustrating.\nAlso, even if recalls and updates are massively successful there will still be plenty of unpatched devices available for malicious hackers to use. Some manufacturers of cheaper devices have refused to issue updates meaning there is a ready population of vulnerable gadgets available.\nBecause security costs money and electronics firms want to make their IoT device as cheap as possible. Paying developers to write secure code might mean a gadget is late to market and is more expensive. Plus enforcing good security on these devices can make them harder to use - again that might hit sales.\nDespite this, many industry bodies are trying to draw up standards that enforce good security habits. Unfortunately, these initiatives are taking time to have any impact, meaning there are millions of insecure devices already installed and working.\nRight now, we don't know. Some hacker groups have claimed responsibility but none of their claims are credible. We might never know because the vulnerable devices making up the IoT attack network are changing hands regularly as rivals scramble to gain control of as many as they can.\nIn one sense the large web attacks are marketing exercises which show how effective a particular network of bots can be when turned against a target. Competition among rival bot operators is ferocious so a successful attack can be a good way to impress potential customers. It might also persuade victims of extortion emails to pay up rather than risk being knocked out.",
"input": "",
"output": "The summary is: \nHome webcams that were hijacked to help knock popular websites offline last week are being recalled in the US.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nMost swimmers will be take to the waters of Windermere to swim a mile (1.6Km) with some longer swims scheduled for Sunday.\nThe event is expected to attract 10,000 swimmers, organisers said.\nA 10k marathon distance has also been introduced and is expected to take experienced swimmers four hours to complete.\nGreat Swim Director Alex Jackson said: \"The Great North Swim is proving to be as popular as ever, with 10,000 expected in Windermere for our ninth event here.\"\nIntroducing the \"10k event will provide a new challenge\" he added.",
"input": "",
"output": "The summary is: \nThousands of swimmers have headed to the Lake District this weekend to take part in the Great North Swim.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nCryne joined Barnsley's board in 2003 as part of a consortium led by ex-Leeds chairman Peter Ridsdale.\nRidsdale left the Championship club in December 2004, leaving Cryne and former chairman Gordon Shepherd in control.\nIn August, Cryne told the Barnsley Chronicle that he would welcome takeover offers from fans' groups.",
"input": "",
"output": "The summary is: \nBarnsley owner Patrick Cryne will not be involved in the club \"for the foreseeable future\" while he receives cancer treatment.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nEsther Kidemba died after apparently jumping from a building during a drill at the private Strathmore University in the capital, Nairobi.\nGunshots were fired during the drill, causing panic on the campus.\nMilitant Islamist group al-Shabab killed some 150 people in an attack on Garissa University College in April.\nStrathmore University Vice-Chancellor John Odhiambo said he offered his deepest condolences to the family of Ms Kidemba, who was a staff member, and an \"unreserved apology to every student, parent, family, colleague and stakeholder for the unfortunate outcome of the security drill\".\nStrathmore would pay for the medical expenses of 30 students and staff injured during the drill, and would arrange post-traumatic counselling, he added.\nThe drill had been carried out by the university's security team in co-ordination with local police to assess how they would deal with any attack, a statement by the university said.\nOn Monday, Nairobi police chief Japheth Koome said all the proper procedures had been followed for the mock security exercise, Reuters news agency reports.\nBut in his reaction, police chief Joseph Boinett said: \"This must not happen again.\"\nKenya's security forces were on high alert to deal with threats, and drills should be conducted only with the authorisation of the \"highest security office in the country\", he added.\nApril's day-long assault on Garissa University College in north-eastern Kenya was the deadliest by al-Shabab in the East African state.\nIn 2013, at least 67 people were killed in an attack by the al-Qaeda-linked group, which is headquartered in neighbouring Somalia, on the upmarket Westgate shopping centre in Nairobi.\nAl-Shabab says it is opposed to the presence of Kenyan troops in Somalia.\nThe troops are part of an African Union (AU) force helping the weak Somali government fight the militants who want to establish Islamic rule in the country.",
"input": "",
"output": "The summary is: \nKenya's police chief has warned universities not to carry out security drills without his approval following the death of a woman on Monday.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nSources earlier told BBC home affairs correspondent Danny Shaw that one wing of HMP The Mount in Hertfordshire and half of another wing had been \"lost\".\nThe Ministry of Justice (MoJ) later said the incident was \"resolved\" and no staff or prisoners had been injured.\nA report into the jail published earlier highlighted staffing problems and said violence was an issue.\nThe Mount, in Bovingdon village near Hemel Hempstead, opened in 1987 and is classed as a category C male prison.\nA \"tornado team\" made up of riot-trained staff arrived at the jail at about 18:30, equipped with shields and batons while fire, police and ambulance crews were on standby outside.\nThe MoJ said officers had dealt with an \"incident involving a number of prisoners\".\nThe BBC understands the wings involved were H and L, which house 110 and 117 prisoners.\nAt about 23:45, a Prison Service spokesperson said: \"Specialist prison staff resolved an incident involving a number of prisoners. There were no injuries to staff or prisoners.\n\"The offenders responsible will be referred to the police and could spend longer behind bars.\"\nEarlier on Monday, the Independent Monitoring Board published its annual review into conditions at Mount Prison and said it had \"struggled\" with staff shortages.\nThere were 24 vacancies out of a total of 136 officers in February, it added.\nIt also claimed violence \"grew considerably\" throughout the year and that drugs were readily available, in particular the synthetic cannabis substitute spice.\nThe report says concerns raised last year had not been addressed by the MoJ.\nThe Prison Reform Trust calls this type of institution one where \"prison staff think [inmates] will not escape\", while acknowledging they \"cannot be trusted in an open prison\".\nPrison affairs academic and blogger Alex Cavendish had tweeted on Saturday: \"Staff shortages at HMP The Mount (Herts) are so severe that this is the 3rd weekend of total lockdown. Meals given at cell door. Trouble brewing.\"\nMark Fairhurst, of the Prison Officers Association, said staff shortages in UK jails were \"an epidemic\" and partly due to \"poor salaries\".\n\"We need to increase the starting salary to incentivise people to join and then we need to give them regular increments to incentivise them to stay,\" he said.\nMr Fairhurst added it was difficult to retain staff because of \"adverse working conditions, the violence they face and poor salary\".\nThe Mount is built on a former RAF station site and has more than 1,000 prisoners, according to the MoJ.\nIt is described as a \"hybrid training and resettlement prison\" for inmates in the final six months of their sentences.\nA 2015 inspection of the prison found The Mount was \"reasonably safe and felt calm and well ordered\", but chief inspector of prisons Nick Hardwick added that there was \"room for improvement\".\nIn March 2016 an inmate at The Mount stabbed a fellow prisoner with a shard of glass from a microwave.",
"input": "",
"output": "The summary is: \nRiot-trained prison staff were sent to a jail amid reports of violence on two wings.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nThe decision to call off the fixture was made following a morning pitch inspection and the game has been rescheduled for Tuesday, 15 December.\nCarrick's planned league game with Dungannon on that date will be moved.\nThe other semi-final between Larne and Ballymena United goes ahead after the pitch passed a lunchtime inspection.\nCarrick have had three home Premiership games postponed in recent weeks - their league fixture against Dungannon Swifts has been called off twice.\nGary Haveron's side were also forced to postpone their match against Cliftonville on Saturday because of a waterlogged pitch.\nLarne will be out to cause an upset against the Sky Blues at Inver Park.\n\"We know we are nowhere near winning the league so it's important to compete for other silverware,\" said Ballymena manager Glenn Ferguson ahead of the trip to their County Antrim rivals.\n\"We want to reach as many cup semi-finals and finals as we can.\"\n\"We played Larne in pre-season so that gives us some idea what to expect. It will be a tough match as all the teams near the top of the Championship are capable of giving the senior teams a game,\" he added.\nThe Sky Blues progressed to the last four by beating league champions and leaders Crusaders 2-0 at Seaview, courtesy of goals from David Cushley and Tony Kane.\nTheir opponents lie third in Championship One after a 4-4 draw with Armagh City on Saturday.\nDavid McAlinden's side saw off Ards 2-1 to reach the semi-finals and will take heart from their League Cup performance against Portadown earlier in the season, taking the Premiership outfit to extra-time before losing 4-1.\nThere will be coverage of Larne v Ballymena United on a Sportsound Special on BBC Radio Ulster medium wave and the BBC Sport website on Tuesday night from 19:30 GMT.",
"input": "",
"output": "The summary is: \nTuesday night's Co Antrim Shield semi-final between Carrick Rangers and Linfield has been postponed because of a waterlogged pitch.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nThe chancellor claimed Britain was \"walking tall again\" after five years of austerity.\nHe also cut 1p from beer duty, 2% from cider and whisky and froze fuel and wine duty. Cigarettes will go up by 16p a pack as earlier planned.\nLabour leader Ed Miliband said Mr Osborne had \"failed working families\".\n\"This a Budget that people won't believe from a government that is not on their side,\" Mr Miliband told MPs.\nThe Lib Dems - who will set out their own tax and spending plans on Thursday - claimed credit for coalition plans to raise the personal income tax allowance to £10,800 next year, more than previously expected.\nUsers of the BBC News app tap here for the Budget Calculator.\nLib Dem Business Secretary Vince Cable told BBC News: \"Where we differ from the Conservatives... is that we don't believe you can deal with this deficit problem simply by cutting public spending.\n\"Many of our public services are already under a lot of pressure so we will be arguing for a different mix.\"\nIn his final set-piece pitch to voters before May's election, Mr Osborne announced that if the Conservatives win power the first £1,000 of savings interest would be tax free - meaning 95% of savers would pay no tax.\nHe also said savings put aside for a deposit by first-time buyers would be topped up by the government - to the tune of £50 for every £200 saved - in a move that will come into force this Autumn. The new Help to Buy ISA accounts will be made available through banks and building societies.\nOther measures include:\nMr Osborne hailed slightly better than expected growth figures, which suggest the economy will expand by 2.5% this year, rather than 2.4% and described his economic package as a \"Budget for Britain - a comeback country\".\nHe said the government had met its 2010 target to end this Parliament with Britain's national debt falling as a share of GDP, meaning the \"the hard work and sacrifice of the British people has paid off\".\nMost computers will open PDF documents automatically, but you may need Adobe Reader\nDownload the reader here\nSetting out his plans in the Commons, Mr Osborne said: \"We took difficult decisions in the teeth of opposition and it worked. Britain is walking tall again.\n\"Five years ago, our economy had suffered a collapse greater than almost any country.\n\"Today, I can confirm: in the last year we have grown faster than any other major advanced economy in the world.\"\nHe said he would use a boost in the public finances caused by lower inflation and welfare payments to pay off some of the national debt and end the squeeze on public spending a year earlier than planned.\nIn 2019/20 spending will grow in line with the growth of the economy - bringing state spending as a share of national income to the same level as in 2000, the chancellor told MPs.\nThe BBC's Robert Peston said this was a move aimed at neutralising Labour's claim that the Conservatives would cut spending to 1930s levels.\nBut Shadow Chancellor Ed Balls said the Treasury's own figures showed spending at the \"lowest level since 1938\" in 2018/19 while the independent Office for Budget Responsibility, in its analysis, said Mr Osborne's plans implied a \"rollercoaster profile\" for expenditure over the next five years.\nMr Osborne insisted that deficit reduction remained his top priority but also unveiled measures aimed at increasing the amount people can earn before paying tax to £10,800 next year and an above inflation rise to £43,300 by 2017 for the amount people can earn before having to pay the 40p tax rate.\nSome of the plans in Mr Osborne's statement are likely to depend on a Conservative victory on 7 May. Whoever wins the election is likely to set out another Budget later this year.\nIf you think getting the debt down is the big priority, the last five years have seen a good deal of austerity for very delayed gain.\nIt is taking precisely twice as long as George Osborne hoped to get the debt down to 70% of GDP.\nAnd to achieve that deferred gain, the Office for Budget Responsibility says the acuteness of austerity will get worse, before it gets a lot better.\nRead Robert's full analysis here.\nLabour leader Ed Miliband claimed the Conservatives had a \"secret plan\" to cut the NHS because they would not be able to deliver their planned \"colossal cuts\" to other areas of public spending and they would also be forced to increase VAT.\nHe said Labour would reverse the tax cuts for millionaires and introduce a mansion tax to fund the NHS.\nHe also pledged to abolish the \"vindictive and unfair\" housing benefit changes he calls the \"bedroom tax\".\nThe SNP said Mr Osborne had \"blown his last chance\" to deliver for Scotland.\nSNP deputy leader and Treasury spokesman Stewart Hosie said: \"Today George Osborne could have delivered a Budget focused on delivering economic growth by tackling inequality.\n\"He has not - he has decided to continue with his utterly failed austerity agenda.\"\nThe chancellor sprayed largesse far and wide - on drinkers and drivers, orchestras and air ambulances, churches and charities.\nSo yes no massive giveaway, no massive rabbit, as some Tory MPs had wanted, but a glimpse of better things to come and the road - as Mr Osborne put it - from austerity to prosperity. But the essential political argument hasn't changed.\nRead James's full analysis here.\nUKIP Leader Nigel Farage said: \"Mr Osborne talks about a long-term economic plan, today he pushed all his targets back and created a long grass economic plan.\"\nGreen Party leader Natalie Bennett said the chancellor's \"triumphalist tone\" would \"leave a bad taste in the mouth\" of people on zero hours contracts or struggling to put food on the table.\nPlaid Cymru Treasury spokesman Jonathan Edwards said: \"This was a 'jam tomorrow' Budget from a chancellor who is busy sharpening the axe ready for the next Parliament.\"\nMr Osborne's sixth Budget statement came against a backdrop of a strengthening economic recovery, a fresh fall in unemployment and a rosier picture expected as a result of falling oil prices dragging down inflation.\nIn was, in parts, an openly electioneering Budget, with Mr Osborne saying: \"The critical choice facing the country now is this: do we return to the chaos of the past? Or do we say to the British people, let's work through the plan that is delivering for you?\"\nThe Budget was largely welcomed by business leaders with CBI Director General John Cridland saying it would provide the \"stability and consistency\" needed to boost growth.\nThe trade unions were less impressed, with the TUC General Secretary Frances O'Grady saying: \"He did not spell out where, if re-elected, he will make the huge spending cuts he plans for the next Parliament, nor did he tell Britain's low paid workers which of their benefits he will cut.\"",
"input": "",
"output": "The summary is: \nGeorge Osborne has announced tax cuts for first-time buyers, workers and savers in his final Budget before May's general election.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\n8 May 2017 Last updated at 15:13 BST\nEmmanuel Macron won to become the country's youngest president at 39-years-old.\nHe beat rival Marine Le Pen comfortably.\nJenny spoke to two kids in Paris to find out what they think of the result.",
"input": "",
"output": "The summary is: \nThe people of France had a big vote on Sunday to decide who they want to run their country for the next five years.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nThe legal team of the former pop star, real name Paul Gadd, argued media coverage had made a fair trial impossible.\nBut three judges said there was nothing \"unsafe\" about the conviction.\nThe 71-year-old was jailed for 16 years in February for offences at the height of his fame, between 1975 and 1980.\nHe had denied the allegations against him.\nA jury at Southwark Crown Court found him guilty of one count of attempted rape, one of unlawful sexual intercourse with a girl under 13, and four counts of indecent assault.\nAt his sentencing, Judge McCreath told him his victims were \"profoundly affected\".\nHe said the offence of attempted rape was \"so serious\" as to justify the maximum available sentence.\nGadd was jailed in Vietnam in 2006 for molesting two girls aged 11 and 12.\nHe later became the first person to be arrested under Operation Yewtree, the investigation launched by the Metropolitan Police following the Jimmy Savile scandal.",
"input": "",
"output": "The summary is: \nDisgraced singer Gary Glitter has lost a Court of Appeal challenge against his conviction for sexually abusing three young girls.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nThe database is reported to contain information on 100,000 US Department of Defense employees, among others.\nTroy Hunt, who published news of the leak, said the information had \"enormous\" potential for scammers.\nBusiness services firm Dun & Bradstreet confirmed to tech news site ZDNet that it owns the data.\nInformation on government departments and private sector employees is commonly collated by business services that sell the data to other companies, such as marketing firms.\nIn this case, the records - including names, job titles and contact details - were originally compiled by NetProspex, which was acquired by Dun & Bradstreet in 2015.\nOrganisations with employees mentioned in the data include the US Postal Service, telecoms giant AT&T and the retailer Walmart.\nMr Hunt pointed out that people might try to use the names and email addresses in the database to scam or retrieve sensitive information from recipients - a practice known as spear phishing.\n\"The value for very targeted spear phishing is enormous because you can carefully craft messages that refer to specific individuals of influence and their roles within the organisation,\" he wrote on his blog.\nDun & Bradstreet told ZDNet: \"Based on our analysis, it was not accessed or exposed through a Dun & Bradstreet system.\"\nThe leak is the latest in a long string of personal data caches dumped online.\nIn January, personal information of health workers in the US Army was found online by another security professional.",
"input": "",
"output": "The summary is: \nDetails of more than 33 million US employees - including military staff - have been released online, according to a security researcher.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nThe watercolour, attributed to Queen Victoria's favourite artist Sir Edwin Landseer, was sold by JP Humbert of Whittlebury, Northamptonshire.\nThe painting has attracted both firm supporters and those who doubt whether it does depict the Brontes.\nIt was sold to a collector who plans to do more research and resell it.\nBidding took off just 15 minutes before the end of the \"timed\" online auction with the painting sold for £40,550 hammer price (£50,038 including buyers premium) to an private investor believed to be based in the UK.\nAuctioneer Jonathan Humbert said: \"We are very pleased our theory has been accepted and endorsed by the establishment.\n\"The evidence was compelling that this is the Brontes as painted by Landseer and its successful sale has proved that research and factual evidence will overcome apathy and negativity.\"\nMr Humbert had decided to pull the picture, which he believes to be of \"national importance\", from an auction in 2012 so more research could be done.\nLandseer was a popular Victorian painter best known for his animal portraits and designing the bronze lions in London's Trafalgar Square.\nThe Bronte family moved to Haworth, West Yorkshire, in 1820 where the Reverend Patrick Bronte was appointed Curate of Haworth.\nThey lived at the Haworth Parsonage from 1820 to 1861, which is now the Bronte Parsonage Museum.",
"input": "",
"output": "The summary is: \nA painting claimed to be a previously unknown portrait of the three Bronte sisters has sold for more than £40,000 at auction.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nThe suspect, 28, handed himself in at an east London Police station on Friday, the Met said.\nHe arrested on suspicion of violent disorder and was bailed until mid-August.\nImages of three men still wanted over Tuesday's attack were released by the force.\nBottles and other objects were thrown at the coach as it got stuck in traffic en route to the stadium.\nThe disorder left four policemen injured and West Ham said it would ban for life any fan found responsible.\nTwo men, aged 18 and 47, who were arrested for pitch incursion have been bailed to return on a date in late May.\nA 20-year-old man who was arrested for throwing bottles at police officers has been bailed to return on a date in August.",
"input": "",
"output": "The summary is: \nA man has been arrested in connection with an attack on Manchester United's team bus outside West Ham's Boleyn Ground.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nMedia playback is not supported on this device\nSean Geddes' two goals helped the Conference North side through at the expense of the League One Sky Blues, Cup winners in 1987.\nLeague Two Portsmouth, winners in 2008, left it late to draw 2-2 at home with Conference club Aldershot.\nNorthern Premier Blyth Spartans beat Conference Altrincham 4-1 to take their place in the next round.\nMedia playback is not supported on this device\nWorcester took full advantage after Coventry goalkeeper Lee Burge was sent off in the first half for lashing out at visiting striker Daniel Nti.\nGeddes scored the resulting penalty and added a second goal 10 minutes into the second half.\nCoventry's Reda Johnson missed a penalty before the break and, though he scored late on, Worcester held on.\n\"We were the better side,\" Worcester manager Carl Heeley told BBC Sport. \"We're nine games unbeaten now, so we're a good footballing side. But to come to a League One club and to beat them on their own patch - it's a brilliant day.\"\nCoventry boss Steven Pressley said: \"This defeat ranks as one of the worst in the club's history.\"\nBlyth's Robbie Dale maintained his record of scoring in every round of this season's competition with two against Altrincham, while Danny Maguire also scored twice for team from the seventh tier of English football.\nAldershot were nine minutes away from a famous win over their Hampshire rivals, but Danny Hollands' header earned the former Premier League side a replay.\nBradford City came from behind against non-league opposition, overcoming an early FC Halifax Town goal to win 2-1 thanks to two goals in quick succession early in the second half.\nTwo deflected goals from Gary McSheffrey were enough to give Scunthorpe United victory at Forest Green Rovers, while League One pair Chesterfield and Colchester put six past Braintree and Gosport Borough respectively.\nMaidstone United, of the Ryman Premier Division, held Stevenage of League Two to a 0-0 draw, while a lacklustre encounter between Notts County and Accrington Stanley ended in the same scoreline.\nRob Ramshaw hit a hat-trick as Gateshead eased to a 4-0 win away at Norton United, while Wrexham are also into the second round after a 3-0 victory at home to fellow Conference side Woking.\nThe second-round draw takes place on Monday at 19:00 GMT. You can watch it live on BBC Two and the BBC Sport website.",
"input": "",
"output": "The summary is: \nWorcester City beat Coventry City 2-1 to produce the biggest shock of Sunday's FA Cup first-round ties.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nBridgewater Community Healthcare (BCH) said it had \"made a lot of progress\" since an inspection last summer.\nBut Labour MP Rosie Cooper said it was \"staggering\" that BCH was due to take over most of Liverpool's community health services from July.\nThe Department of Health is yet to respond.\nThe Care Quality Commission (CQC) conducted its first full inspection at BCH - which is used by about 1.5m people in north-west England annually - in May and June 2016.\nIt said it measured 40 domains across the services with one rated as outstanding, 27 as good and 12 as requiring improvement.\nOverall, the trust received a rating of \"requires improvement\".\nMs Cooper, MP for West Lancashire, said she called on the CQC in July to publish its inspection report ahead of any decision on \"awarding the multimillion-pound contract for Liverpool community health services\".\nShe said: \"What this report tells us is Bridgewater Community Healthcare needs to improve the services they have currently got.\"\nLast November, BCH was chosen to run most of the city's community health services by NHS England and Liverpool Clinical Commissioning Group (CCG).\nMs Cooper said that the inspection rating \"raises some very serious questions about the entire transaction process in Liverpool\".\n\"I have called on the Secretary of State to review this sorry state of affairs and intervene to uphold NHS rules,\" she added.\nColin Scales, chief executive at BCH, said: \"All the essential actions the CQC has asked us look at have already been addressed since the inspectors were on site, so we've made a lot of progress and are in a stronger position now as we move forward.\"\nKatherine Sheerin, chief officer for NHS Liverpool CCG, said: \"Bridgewater NHS Foundation Trust was identified as the preferred provider of community services in Liverpool because we believe it is the best organisation to help accelerate our Healthy Liverpool plans for making more care available in the community so that people do not end up in hospital.\"\nShe added the CCG was \"confident the Trust is already taking action to address the issues which have been identified\".",
"input": "",
"output": "The summary is: \nAn MP has called on the government to review a decision allowing an NHS trust that \"requires improvement\" to run community health services in Liverpool.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nThe endorsement came after Mr Rajoy's Popular Party (PP) gained the backing of the Ciudadanos (\"Citizens\") party and tacit support from the Socialists.\nSocialist lawmakers are said to have been among the 68 abstentions.\nThe country had faced the prospect of a third general election inside a year.\nBut the Socialists forced out their leader, Pedro Sanchez, earlier this month after he rejected abstention.\nMr Rajoy has led a caretaker administration since losing his overall majority in an election last December. A repeat election in June failed to end the impasse but strengthened his hand.\nThe Socialists (commonly known by their Spanish abbreviation, the PSOE) came second on both occasions, their support eroded by radical leftist newcomers Podemos.\nFor decades, the PSOE and PP took turns in governing the country on their own but last year the popular vote split four ways - the new centrist Ciudadanos party came fourth.\nSpain country profile\nThe PSOE has 85 seats to the 137 won by the PP in June.\nPodemos's Ikea-style appeal to young voters\nResisting change in a dying Spanish village\nTaking back Barcelona's apartments",
"input": "",
"output": "The summary is: \nSpain's parliament has voted to allow conservative leader Mariano Rajoy to lead a minority government after a 10-month political deadlock following inconclusive elections.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nGavin Egan, 34, was found in Peasholm Park, Scarborough, on 24 February 2016.\nThe Independent Police Complaints Commission (IPCC) said PC Helen Hardie \"had a case to answer for gross misconduct\".\nThe force said it \"disagreed with the content of their [IPCC] report\".\nMore on this and other North Yorkshire stories\nIn its report, the IPCC said an ambulance had gone to the park after a member of the public had contacted them to say he had pulled a man out out of the lake.\nA paramedic searched the park for about 38 minutes but could not find the missing man, so he called the police.\nPC Hardie attended the scene at about 04:00 GMT.\nThe IPCC report said: \"A four-minute search was carried out before PC Hardie left the area, she did not seek assistance and the incident log was closed soon after.\"\nIt added that the officer concluded the missing man had fled the scene \"despite a paramedic's view that he would be incapable of such action because of freezing temperatures\".\nShe later told an inspector \"there was no evidence a man had been pulled from the lake\".\nMr Egan's body was found at about 11:30 GMT.\nThe IPCC investigator said that in his opinion \"PC Hardie had a case to answer for gross misconduct\".\nIn a statement, North Yorkshire Police said: \"We disagreed with the content of their report and their finding that it amounted to gross misconduct.\n\"We appealed their report and it was subsequently agreed with the IPCC that a misconduct meeting would be held.\n\"This has been carried out and the officer has been issued with a written warning.\"",
"input": "",
"output": "The summary is: \nA North Yorkshire Police officer made \"errors\" in the search for a missing man who was later found dead in the lake of a public park, the police watchdog has found.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nAt a UN oceans summit, delegates from China, Thailand, Indonesia and the Philippines said they would work to keep plastics out of the seas.\nSome of the promises are not yet formalised and environmentalists say the measures proposed are not nearly urgent enough.\nBut UN officials praised the statement.\nMeeting in New York, they said it was part of a clear international shift against ocean pollution.\nEric Solheim, the UN's environment director, told BBC News: \"There are quite encouraging signs, with nations taking the ocean much more seriously. Of course, there is a very long way to go because the problems are huge.\"\nIt is estimated that 5-13 million tonnes of plastics flow into the world's oceans annually. Much of it is ingested by birds and fish – and fragments of plastic have even been found in organisms at the bottom of the ocean.\nA recent paper said much of the marine plastic often originates far from the sea – especially in countries which have developed consumer economies faster than their ability to manage waste.\nThe Helmholtz Centre in Leipzig, Germany, estimated that 75% of land-borne marine pollution comes from just 10 rivers, predominantly in Asia.\nReducing the plastic loads in these rivers by 50% would reduce global plastic inputs by 37%, it said.\nTom Dillon from the Pew Charitable Trusts, which campaign on oceans, urged China to move quickly.\nHe told BBC News: \"For thousands of years the Maritime Silk Road was a pathway for export of Chinese culture and influence. Will the ocean be a vehicle for export of Chinese pollution, or a new culture of conservation and sustainability?\"\nA report to the UN conference from the Thailand government says most marine plastic debris is land-based, caused by inefficient waste management and poor handling of plastic wastes.\nIn Thailand, the total amount of garbage finding its way into the sea was estimated at 2.83 million tonnes in 2016 - of which 12% was plastic.\nThe Thai government says the nation has established a 20-year strategy to tackle the problem, including developing financial incentives for keeping plastic out of the sea and encouraging eco-packaging design and eco-friendly substitutes for plastics.\nIn Indonesia, the government is starting a mass education programme for schoolchildren, and in the Philippines new laws are being developed.\nPart of the challenge is finding substitutes for plastics. An international prize for smarter materials and design for packaging was launched recently by the Ellen MacArthur Foundation.\nFollow Roger on Twitter @rharrabin",
"input": "",
"output": "The summary is: \nNations responsible for much of the world's ocean plastic pollution have promised to start cleaning up their act.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nMedia playback is not supported on this device\nGoals from Liam Polworth and Ross Draper in the space of five minutes put Caley 2-0 ahead at half-time.\nA fabulous Miles Storey volley early in the second half sealed a first win in eight games and lifted Thistle to within four points of County in sixth.\nCounty, who were due to parade the League Cup around Dingwall afterwards, spurned a handful of chances.\nIt was a horrible end to a fabulous week for the hosts, who had recovered from their Hampden heroics to salvage a point at St Johnstone on Wednesday.\nMedia playback is not supported on this device\nBut this proved a match too far after a hectic recent schedule in front of a crowd of just under 6,000.\nAlex Schalk, who scored the winner against Hibs at Hampden, nearly gave Ross the perfect start, but his diving header from Richard Foster's cross was well saved by Owain Fon Williams.\nAndrew Davies, who directed a header wide, Jackson Irvine - flicking the ball past Fon Williams but clipping the outside of the post - and Liam Boyce with a header all missed further chances for County.\nStorey and Carl Tremarco, with a thunderous volley that cracked off the bar, might have scored for Caley before they broke the deadlock in the 32nd minute.\nMarcus Fraser lost possession with County lacking numbers at the back and Storey broke away, found Polworth on the edge of the box and he confidently picked his spot across Woods.\nThey swiftly doubled their lead when Draper capitalised on space in the area and coolly slotted past the onrushing Woods.\nThistle have struggled for goals of late, but three minutes after the resumption, Storey fired a stunning dipping volley past Woods from the edge of the area to send the visiting fans wild.\nCaley were then able to try and pick off their hosts and Josh Meekings came close to connecting at the back post.\nPolworth also clipped the bar from distance as a well-executed tactical plan brought a comfortable win.",
"input": "",
"output": "The summary is: \nA clinical Inverness Caley Thistle spoiled Ross County's celebrations with a fifth straight derby win at Dingwall.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nThe think tank said the city's 1,536 schools needed to save £360m in the first year if the government's National Funding Formula (NFF) plan goes ahead.\nThe amount is the equivalent of 12,857 qualified teachers, on an average salary of £28,000.\nThe government said London was the highest funded part of the country.\nIt added that under the plans, which are under consultation, inner-city schools would be allocated 30% more money per pupil than the national average.\nBut London Councils, which represents the city's 32 boroughs and the City, said no school would gain enough funding from the NFF to compensate for increased cost pressures from inflation, higher pension contributions and national insurance.\nMinisters said the new formula was needed to tackle uneven levels of funding across England, with the best funded areas getting more than £6,300 per pupil per year, while the worst-funded averaging £4,200.\nIt said the funding cut was on top of National Audit Office figures which showed England schools faced an eight per cent real-terms cut per pupil by 2019-20 because it wider cost pressures.\nIn a statement, London Councils said: \"At a time when UK schools are seen as underperforming by international standards, and when businesses based in London are facing massive uncertainty about recruiting skilled staff, there is an urgent need to invest in schools in London and across the rest of the country.\"\nIt added: \"Without the right qualifications and skills, London's children will be unable to access jobs and contribute to the national economy. Over 60% of jobs in inner London require a degree and around 45% of jobs in the rest of the capital require a degree.\"",
"input": "",
"output": "The summary is: \nAbout 70% of London schools could face budget cuts under government plans to change how they are funded, according to London Councils.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nIt's all about #TheDress and whether it's blue and black (#blueandblack) or white and gold (#whiteandgold).\nThe whole debate started when Scottish singer Caitlin McNeill posted a picture of a dress on her Tumblr blog.\nShe asked her followers: \"Guys please help me - is this dress white and gold, or blue and black? Me and my friends can't agree and we are freaking... out.\"\nCaitlin told Newsbeat that it all started when her friend's mum wore the dress at a wedding.\n\"Two of my very good friends were getting married and they has asked me to put together a band to come and play at the wedding,\" she says.\n\"This was a wedding on the tiny island that we come from on the west coast of Scotland called Colonsay and about 100 people were there.\n\"A week beforehand the bride had been sent by her mother a picture of the dress she was going to wear and when the bride showed her fiance, they disagreed about what colour it was.\n\"She was like, 'It's white and gold' and he said, 'It's blue and black'.\n\"So they posted it on Facebook to try and see what their friends were saying but that caused carnage on Facebook.\n\"We forgot about it until we saw it at the wedding which the mother of the bride was wearing and it was obviously blue and black.\"\nRead Newsbeat's interview with Caitlin McNeill\nYouTube talent manager Sarah Weichel then spotted it on Tumblr and the rest is Twitter history...\nTurns out a lot of people cared and thousands are still debating the colour of that badly-taken snapshot.\nVarious US news outlets have written stories about how the human eyes see different colours and why some people see blue and black while others see gold and white.\nBuzzFeed's original article has been shared more than 20 million times and tech site Wired explains the science of colour.\nThe prime minster of Singapore liked the bit about science so much, he posted about it on his Facebook page.\nAnd photo experts Adobe got involved as well, sending out this tweet.\nIt got celebrities talking on Twitter.\nAnd then the memes started...\nIt's all great news for the makers of the £50 dress.\nA quick check online shows Roman Women's Lace Detail Bodycon Dress is available in Royal Blue - so blue then...\nAnd the company says it's looking into doing a gold and white version of the dress.\nA spokesman told Newsbeat: \"We're looking into getting it through the approval stages.\n\"We want to do it but it depends on the speed. We're trying to get it done as soon as possible.\n\"We are in contact with the suppliers to establish if we can get it manufactured in white and gold.\"\nFollow @BBCNewsbeat on Twitter, BBCNewsbeat on Instagram and Radio1Newsbeat on YouTube",
"input": "",
"output": "The summary is: \nIt's the debate of the year so far - well - on Twitter at least and has been the top trending topic worldwide.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nIt had submitted plans for a new short-term holding facility near Glasgow Airport, which would have replaced the Lanarkshire detention centre.\nBut Renfrewshire Council rejected the planning application for the new facility.\nAs a result, the Home Office said it will retain Dungavel House for people who are facing removal.\nThe Home Office said it had been \"disappointed\" by the council's decision to block a new holding centre.\nIt said the Glasgow Airport plan would have created a \"modern and secure facility\" for \"those with no right to be in the UK\".\nA spokesman said: \"We always made clear that the closure of Dungavel immigration removal centre was dependent on the opening of a new short-term holding facility in Scotland.\n\"As the application for a new facility at Paisley was rejected, Dungavel will remain open.\"\nThe replacement would have used to detain people under immigration powers for up to seven days before they were moved on to an airport for deportation or to an immigration removal centre.\nThe Home Office has said it believes detention and removal are essential parts of effective immigration controls but insists they are carried out with dignity and respect.\nOfficials say that when people are detained, it is for the minimum time possible.\nThey pointed out the most recent inspection of Dungavel by Her Majesty's Inspector of Prisons found that the centre was a safe place where detainees are given the support and help they need.\nThe Lanarkshire detention centre has attracted protests from opponents who described it as \"racist and inhumane\".",
"input": "",
"output": "The summary is: \nThe Home Office has abandoned plans to replace the immigration removal centre at Dungavel House.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nPeter Barnett, 44, travelled from Haddenham and Thame Parkway to London Marylebone, but dodged the full fare by claiming his journey began at Wembley in north-west London.\nChiltern Railways had argued he should pay back nearly £20,000 but the defence said the true value was £6,000.\nBarnett, from Oxford, admitted fraud by false representation.\nDeputy District Judge Olalekan Omotosho said: \"There is a need not just to punish you for the offences but also deter others from committing offences.\"\nShe added: \"It remains unclear why you acted so badly.\n\"You let yourself down and your family down, particularly in light of your profession as a lawyer.\"\nBarnett admitted six counts of fraud by false representation between April 2012 and November 2014 and was ordered to pay back nearly £6,000.\nCity of London Magistrates' Court heard that Barnett - an Oxford graduate and former Rhodes scholar who also worked in the financial services sector - failed to pay for journeys on Chiltern Railways on 655 days between April 2012 and November 2014.\nHe was thought to have simply \"tapped out\" with an Oyster card, automatically being charged the maximum Transport for London fare.\nProsecutors had argued he should pay back £19,689, the full value of the cost of daily returns for the trips he made.\nHowever, the defence claimed the value was a penalty imposed by the railway company rather than the true value, because if Barnett had bought a ticket it would have been a weekly one - rather than paying a daily fare.\nThe court heard that Barnett ran off when a member of station staff became suspicious about his story and called a supervisor, but had a change of heart and later handed himself in.\nDuring an interview with British Transport Police, he confessed that he had been carrying out the scam since April 2012.\nBarnett was also ordered to carry out 200 hours of unpaid work and be supervised for 12 months.",
"input": "",
"output": "The summary is: \nA barrister who commuted by train for two years without paying has been given a suspended 16-week prison sentence.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nHere we highlight some of the stories that readers enjoyed the most during the course of the year.\nNo year could pass without a major product launch and this year's most read was the announcement in September that Apple was entering the crowded wearable market with a smart watch of its own.\nEvery year could be described as the year of the hack but 2014 saw some pretty significant ones, not least the Heartbleed bug which put security experts into a tailspin in April.\nMeanwhile one of the most read stories of the year was also one of our last (of the year, not ever I hope). Prof Stephen Hawking's assessment of artificial intelligence offers a tantalising glimpse into what we might be reading more about next year - the endless rise of the intelligent machine.\nJanuary: 'Super-rare' Nintendo games hits eBay\nFans of old video games were queuing up to get their hands on a copy of Nintendo World Championships, one of only 116 copies made as part of a special event in 1990.\nA first bid of $4,999 (£3,000) set the tone for the eBay auction.\nEven though the cartridge was in poor condition, its rarity - designed for a competition and never put on general sale - meant it would be something of a holy grail for keen collectors, according to gaming experts.\nThe game eventually sold to a bidder for over $99,0000 - although the winning bidder later backed out.\nFebruary: Flappy Bird creator removes game from app stores\nThe game theme continued into February when everyone was talking about the new gaming sensation - Flappy Bird. Seen as a natural successor to Angry Birds, it proved short-lived when its Vietnamese creator Dong Nguyen announced that he was removing the game from online stores.\nMany questioned whether the real reason was because he may have faced legal action from Nintendo as the main characters resembled those in Super Mario Bros.\nBy the time it was withdrawn the game had been downloaded more than 50 million times, making it the most popular game of the year.\nFor those who had it installed on their phones, it put an immediate premium on the device with people selling handsets on eBay for $1,000 until the online auction site put an end to such trades.\nMarch: Thousands make #nomakeupselfie donation error\nBefore the ice bucket challenge took hold this summer, another charity-raising craze hit social media in the spring - taking a self portrait wearing no make-up.\nOnce the picture was posted on social media, users were asked to donate to Cancer Research UK.\nBut unfortunately not all of the funds raised by the stunt went to the intended cause.\nThe BBC discovered that thousands of pounds were accidentally donated to Unicef instead of Cancer Research UK, as people sent money by texting DONATE rather than BEAT.\nUnicef told the BBC that over £18,000 had been identified as being accidentally pledged to it and that it was working with Cancer Research to transfer the money.\nApril: Heartbleed Bug: Tech firms urge password reset\nIt took a quarter of a year for the first big security story to hit but when it did it was a big one.\nThe Heartbleed bug affected software used by millions of web servers with some advising people to stay away from the internet entirely until it was fixed. One security expert said that on a scale of one to 10, Heartbleed was an 11.\nThe biggest story of the month was not the one revealing the bug but a follow-up in which several tech firms advised people to change all their passwords.\nThat suggestion proved controversial, with other experts later pointing out that even if users created brand new logins they would remain at risk until all the online services they used updated their servers.\nMay: eBay makes users change their passwords after hack\nThe security theme continued through the spring with May's biggest story focusing on eBay's security woes.\nThe US online marketplace admitted that a database had been hacked between late February and early March, which had contained encrypted passwords and other non-financial data.\nBut there was confusion about how eBay would communicate the problem to its 128 million active users.\nInitially it said it would oblige users to choose new passwords but later said that it would be optional.\nJune: Android and Windows to get 'kill switch'\nThe middle of the year saw efforts by the tech industry to combat the problem of mobile phone theft which police said had risen dramatically.\nAccording to a report by the US authorities, some 3.1 million mobile device were stolen in the US in 2013, double the number stolen in 2012.\nBy adding a kill switch, a function that would render any stolen device useless, they hoped to cut the crime.\nIn June Google and Microsoft announcing that they will add a kill switch feature to their phone operating systems.\nApple already offered such a switch and according to US reports the theft of iPhones had fallen significantly in the months following the launch.\nJuly: Naked selfies extracted from 'factory reset' phones\nIn July it was reported that a Czech-based security firm had managed to extract thousands of pictures, including naked selfies from mobile phones that users thought had been wiped.\nAvast used publicly-available forensic security tools to find the images from second-hand phones bought on eBay.\nIt warned that the only way to completely delete data is to destroy your phone.\nAugust: Man jailed for filming Fast and Furious in cinema\nPiracy is always a hot topic and there was proof in August that it is not going away any time soon with news that a man had been jailed after recording Fast And Furious 6 from the back of a cinema in Walsall.\nThe Federation Against Copyright Theft (Fact) claimed it meant millions of pounds lost for the film's distributor, Universal Pictures.\nPhilip Danks, 25, was jailed for 33 months after the movie he uploaded was downloaded 700,000 times.\nThe judge said his behaviour was \"bold, arrogant and cocksure.\"\nSeptember: Apple Watch unveiled alongside larger iPhones.\nIt wouldn't be possible to get through a year in tech without a new product launch and this time it was details about Apple's smartwatch that grabbed attention.\nThe watch will offer 11 different watch faces and will run Siri - Apple's voice-activated digital assistant.\nIt will also offer maps and act as a heart rate monitor.\nThe watch goes on sale in 2015 and will compete with a crowded market but Apple chief executive Tim Cook was hopeful that its history of entering sectors relatively late and then changing the landscape would prove true for watches as well as phones and music players.\nOctober: Nude 'Snapchat images' put online by hackers\nThe readership of the BBC technology site seem to like a headline if it includes the phrase 'naked images' - and in October a second such tale took their fancy.\nThis time it was news that hackers had put explicit images sent through messaging service Snapchat online with threats to upload more.\nHalf of Snapchat's users are aged between 13 and 17, raising concern that many of the images may be of children.\nSnapchat blamed third-party apps but security experts said that it too had to take more responsibility over user data.\nNovember: Breached webcam and baby monitor site flagged by watchdogs\nNovember saw the extraordinary tale of a website containing thousands of live feeds to baby monitors and CCTV systems around the world.\nIt included a child's bedroom in Birmingham, an office in Warwickshire and a shop in London.\nThe site, based in Russia, broadcast footage from all the systems that used either default passwords or no log-in codes at all.\nIt claimed that it was simply highlighting the dangers of weakly protected cameras but others felt it was a gross violation of people's privacy.\nThe UK's information commissioner Christopher Graham described the feeds on show as \"spooky\" and said he was working with the Russian authorities to have the site shut down.\nDecember: Stephen Hawking warns artificial intelligence could end mankind\nA cheery note to end the year when world-renowned scientist Prof Stephen Hawking revealed his fears about the development of artificial intelligence.\nHis view was that the rise of the intelligent machine could signal the end of the human race and his thoughts hit a note with the public - the story was one of the most read of the entire year.\nMost industry watchers are marking out artificial intelligence as one of their 'technologies to watch' next year although it may be a little longer until it poses any real threat to its human overlords.",
"input": "",
"output": "The summary is: \nPrivacy, bugs and naked selfies - just part of a day's work for the BBC technology team in 2014.",
"task_type": 0
},
{
"instruction": "Read the following document and then please summarize this document. \n Document:\nAfter the bus explosion, huge blasts were heard in the Gaza Strip as the Israeli bombardment of the Palestinian territory continued.\nEleven people were killed in Gaza on Wednesday, the health ministry said.\nUnnamed Palestinian officials told news agencies a ceasefire between Hamas and Israel would be announced within hours.\nAfter eight days of exchanges of fire between Israel and Palestinian militants in Gaza, US Secretary of State Hillary Clinton and UN Secretary General Ban Ki-moon are now in Cairo for talks with the Egyptian president.\nThere were \"many details to work out\" before a ceasefire could be reached, Mr Ban warned.\nBy Kevin ConnollyBBC News, Tel Aviv\nIn the immediate aftermath of the bus bombing, there was a palpable sense of shock hanging in the air around the scene.\nIsrael's largest city has seen nothing like this for six-and-a-half years.\nOne resident - when told that the news of the explosion broadcast from mosques in Gaza has been greeted with a sound of celebratory gunfire there - said that they need to celebrate this as some kind of victory because they have nothing else to offer but violence.\nA police helicopter still circled overhead and there were roadblocks at many main junctions around the scene as police hunted for the bomber or bombers seen running away from the scene.\nParadoxically, the explosion and the waves of Israeli air raids on Gaza this morning do not necessarily mean that the search for a ceasefire is dead.\nIt may mean that both sides are sending a signal that if a deal is agreed, they will be reaching it from what they regard as a position of strength.\nThe search for a diplomatic solution reaches a critical phase this afternoon when US Secretary of St
gitextract_holo9v9h/
├── .gitignore
├── README.md
├── __init__.py
├── build_dataset.py
├── data/
│ └── tiny_data/
│ ├── test.json
│ └── train/
│ └── train.json
├── ds_zero2_no_offload.json
├── ds_zero3_nvme_offload.json
├── ds_zero3_offload.json
├── environment.yml
├── flash_attn_patch.py
├── peft/
│ ├── __init__.py
│ ├── mapping.py
│ ├── peft_model.py
│ ├── tuners/
│ │ ├── __init__.py
│ │ ├── lora.py
│ │ ├── p_tuning.py
│ │ ├── prefix_tuning.py
│ │ └── prompt_tuning.py
│ └── utils/
│ ├── __init__.py
│ ├── adapters_utils.py
│ ├── config.py
│ ├── other.py
│ └── save_and_load.py
├── requirements.txt
├── run_loramoe.py
├── run_loramoe.sh
└── transformers/
├── __init__.py
├── activations.py
├── activations_tf.py
├── audio_utils.py
├── benchmark/
│ ├── __init__.py
│ ├── benchmark.py
│ ├── benchmark_args.py
│ ├── benchmark_args_tf.py
│ ├── benchmark_args_utils.py
│ ├── benchmark_tf.py
│ └── benchmark_utils.py
├── commands/
│ ├── __init__.py
│ ├── add_new_model.py
│ ├── add_new_model_like.py
│ ├── convert.py
│ ├── download.py
│ ├── env.py
│ ├── lfs.py
│ ├── pt_to_tf.py
│ ├── run.py
│ ├── serving.py
│ ├── train.py
│ ├── transformers_cli.py
│ └── user.py
├── configuration_utils.py
├── convert_graph_to_onnx.py
├── convert_pytorch_checkpoint_to_tf2.py
├── convert_slow_tokenizer.py
├── convert_slow_tokenizers_checkpoints_to_fast.py
├── convert_tf_hub_seq_to_seq_bert_to_pytorch.py
├── data/
│ ├── __init__.py
│ ├── data_collator.py
│ ├── datasets/
│ │ ├── __init__.py
│ │ ├── glue.py
│ │ ├── language_modeling.py
│ │ └── squad.py
│ ├── metrics/
│ │ ├── __init__.py
│ │ └── squad_metrics.py
│ └── processors/
│ ├── __init__.py
│ ├── glue.py
│ ├── squad.py
│ ├── utils.py
│ └── xnli.py
├── debug_utils.py
├── deepspeed.py
├── dependency_versions_check.py
├── dependency_versions_table.py
├── dynamic_module_utils.py
├── feature_extraction_sequence_utils.py
├── feature_extraction_utils.py
├── file_utils.py
├── generation/
│ ├── __init__.py
│ ├── beam_constraints.py
│ ├── beam_search.py
│ ├── configuration_utils.py
│ ├── flax_logits_process.py
│ ├── flax_utils.py
│ ├── logits_process.py
│ ├── stopping_criteria.py
│ ├── streamers.py
│ ├── tf_logits_process.py
│ ├── tf_utils.py
│ └── utils.py
├── generation_flax_utils.py
├── generation_tf_utils.py
├── generation_utils.py
├── hf_argparser.py
├── image_processing_utils.py
├── image_transforms.py
├── image_utils.py
├── integrations.py
├── keras_callbacks.py
├── kernels/
│ ├── deformable_detr/
│ │ ├── cpu/
│ │ │ ├── ms_deform_attn_cpu.cpp
│ │ │ └── ms_deform_attn_cpu.h
│ │ ├── cuda/
│ │ │ ├── ms_deform_attn_cuda.cu
│ │ │ ├── ms_deform_attn_cuda.cuh
│ │ │ ├── ms_deform_attn_cuda.h
│ │ │ └── ms_deform_im2col_cuda.cuh
│ │ ├── ms_deform_attn.h
│ │ └── vision.cpp
│ ├── rwkv/
│ │ ├── wkv_cuda.cu
│ │ ├── wkv_cuda_bf16.cu
│ │ └── wkv_op.cpp
│ └── yoso/
│ ├── common.h
│ ├── common_cuda.h
│ ├── common_cuda_device.h
│ ├── fast_lsh_cumulation.cu
│ ├── fast_lsh_cumulation.h
│ ├── fast_lsh_cumulation_cuda.cu
│ ├── fast_lsh_cumulation_cuda.h
│ └── fast_lsh_cumulation_torch.cpp
├── modelcard.py
├── modeling_flax_outputs.py
├── modeling_flax_pytorch_utils.py
├── modeling_flax_utils.py
├── modeling_outputs.py
├── modeling_tf_outputs.py
├── modeling_tf_pytorch_utils.py
├── modeling_tf_utils.py
├── modeling_utils.py
├── models/
│ ├── __init__.py
│ ├── albert/
│ │ ├── __init__.py
│ │ ├── configuration_albert.py
│ │ ├── convert_albert_original_tf_checkpoint_to_pytorch.py
│ │ ├── modeling_albert.py
│ │ ├── modeling_flax_albert.py
│ │ ├── modeling_tf_albert.py
│ │ ├── tokenization_albert.py
│ │ └── tokenization_albert_fast.py
│ ├── align/
│ │ ├── __init__.py
│ │ ├── configuration_align.py
│ │ ├── convert_align_tf_to_hf.py
│ │ ├── modeling_align.py
│ │ └── processing_align.py
│ ├── altclip/
│ │ ├── __init__.py
│ │ ├── configuration_altclip.py
│ │ ├── modeling_altclip.py
│ │ └── processing_altclip.py
│ ├── audio_spectrogram_transformer/
│ │ ├── __init__.py
│ │ ├── configuration_audio_spectrogram_transformer.py
│ │ ├── convert_audio_spectrogram_transformer_original_to_pytorch.py
│ │ ├── feature_extraction_audio_spectrogram_transformer.py
│ │ └── modeling_audio_spectrogram_transformer.py
│ ├── auto/
│ │ ├── __init__.py
│ │ ├── auto_factory.py
│ │ ├── configuration_auto.py
│ │ ├── feature_extraction_auto.py
│ │ ├── image_processing_auto.py
│ │ ├── modeling_auto.py
│ │ ├── modeling_flax_auto.py
│ │ ├── modeling_tf_auto.py
│ │ ├── processing_auto.py
│ │ └── tokenization_auto.py
│ ├── autoformer/
│ │ ├── __init__.py
│ │ ├── configuration_autoformer.py
│ │ └── modeling_autoformer.py
│ ├── bart/
│ │ ├── __init__.py
│ │ ├── configuration_bart.py
│ │ ├── convert_bart_original_pytorch_checkpoint_to_pytorch.py
│ │ ├── modeling_bart.py
│ │ ├── modeling_flax_bart.py
│ │ ├── modeling_tf_bart.py
│ │ ├── tokenization_bart.py
│ │ └── tokenization_bart_fast.py
│ ├── barthez/
│ │ ├── __init__.py
│ │ ├── tokenization_barthez.py
│ │ └── tokenization_barthez_fast.py
│ ├── bartpho/
│ │ ├── __init__.py
│ │ └── tokenization_bartpho.py
│ ├── beit/
│ │ ├── __init__.py
│ │ ├── configuration_beit.py
│ │ ├── convert_beit_unilm_to_pytorch.py
│ │ ├── feature_extraction_beit.py
│ │ ├── image_processing_beit.py
│ │ ├── modeling_beit.py
│ │ └── modeling_flax_beit.py
│ ├── bert/
│ │ ├── __init__.py
│ │ ├── configuration_bert.py
│ │ ├── convert_bert_original_tf2_checkpoint_to_pytorch.py
│ │ ├── convert_bert_original_tf_checkpoint_to_pytorch.py
│ │ ├── convert_bert_pytorch_checkpoint_to_original_tf.py
│ │ ├── convert_bert_token_dropping_original_tf2_checkpoint_to_pytorch.py
│ │ ├── modeling_bert.py
│ │ ├── modeling_flax_bert.py
│ │ ├── modeling_tf_bert.py
│ │ ├── tokenization_bert.py
│ │ ├── tokenization_bert_fast.py
│ │ └── tokenization_bert_tf.py
│ ├── bert_generation/
│ │ ├── __init__.py
│ │ ├── configuration_bert_generation.py
│ │ ├── modeling_bert_generation.py
│ │ └── tokenization_bert_generation.py
│ ├── bert_japanese/
│ │ ├── __init__.py
│ │ └── tokenization_bert_japanese.py
│ ├── bertweet/
│ │ ├── __init__.py
│ │ └── tokenization_bertweet.py
│ ├── big_bird/
│ │ ├── __init__.py
│ │ ├── configuration_big_bird.py
│ │ ├── convert_bigbird_original_tf_checkpoint_to_pytorch.py
│ │ ├── modeling_big_bird.py
│ │ ├── modeling_flax_big_bird.py
│ │ ├── tokenization_big_bird.py
│ │ └── tokenization_big_bird_fast.py
│ ├── bigbird_pegasus/
│ │ ├── __init__.py
│ │ ├── configuration_bigbird_pegasus.py
│ │ ├── convert_bigbird_pegasus_tf_to_pytorch.py
│ │ └── modeling_bigbird_pegasus.py
│ ├── biogpt/
│ │ ├── __init__.py
│ │ ├── configuration_biogpt.py
│ │ ├── convert_biogpt_original_pytorch_checkpoint_to_pytorch.py
│ │ ├── modeling_biogpt.py
│ │ └── tokenization_biogpt.py
│ ├── bit/
│ │ ├── __init__.py
│ │ ├── configuration_bit.py
│ │ ├── convert_bit_to_pytorch.py
│ │ ├── image_processing_bit.py
│ │ └── modeling_bit.py
│ ├── blenderbot/
│ │ ├── __init__.py
│ │ ├── configuration_blenderbot.py
│ │ ├── convert_blenderbot_original_pytorch_checkpoint_to_pytorch.py
│ │ ├── modeling_blenderbot.py
│ │ ├── modeling_flax_blenderbot.py
│ │ ├── modeling_tf_blenderbot.py
│ │ ├── tokenization_blenderbot.py
│ │ └── tokenization_blenderbot_fast.py
│ ├── blenderbot_small/
│ │ ├── __init__.py
│ │ ├── configuration_blenderbot_small.py
│ │ ├── modeling_blenderbot_small.py
│ │ ├── modeling_flax_blenderbot_small.py
│ │ ├── modeling_tf_blenderbot_small.py
│ │ ├── tokenization_blenderbot_small.py
│ │ └── tokenization_blenderbot_small_fast.py
│ ├── blip/
│ │ ├── __init__.py
│ │ ├── configuration_blip.py
│ │ ├── convert_blip_original_pytorch_to_hf.py
│ │ ├── image_processing_blip.py
│ │ ├── modeling_blip.py
│ │ ├── modeling_blip_text.py
│ │ ├── modeling_tf_blip.py
│ │ ├── modeling_tf_blip_text.py
│ │ └── processing_blip.py
│ ├── blip_2/
│ │ ├── __init__.py
│ │ ├── configuration_blip_2.py
│ │ ├── convert_blip_2_original_to_pytorch.py
│ │ ├── modeling_blip_2.py
│ │ └── processing_blip_2.py
│ ├── bloom/
│ │ ├── __init__.py
│ │ ├── configuration_bloom.py
│ │ ├── convert_bloom_original_checkpoint_to_pytorch.py
│ │ ├── modeling_bloom.py
│ │ └── tokenization_bloom_fast.py
│ ├── bort/
│ │ ├── __init__.py
│ │ └── convert_bort_original_gluonnlp_checkpoint_to_pytorch.py
│ ├── bridgetower/
│ │ ├── __init__.py
│ │ ├── configuration_bridgetower.py
│ │ ├── image_processing_bridgetower.py
│ │ ├── modeling_bridgetower.py
│ │ └── processing_bridgetower.py
│ ├── byt5/
│ │ ├── __init__.py
│ │ ├── convert_byt5_original_tf_checkpoint_to_pytorch.py
│ │ └── tokenization_byt5.py
│ ├── camembert/
│ │ ├── __init__.py
│ │ ├── configuration_camembert.py
│ │ ├── modeling_camembert.py
│ │ ├── modeling_tf_camembert.py
│ │ ├── tokenization_camembert.py
│ │ └── tokenization_camembert_fast.py
│ ├── canine/
│ │ ├── __init__.py
│ │ ├── configuration_canine.py
│ │ ├── convert_canine_original_tf_checkpoint_to_pytorch.py
│ │ ├── modeling_canine.py
│ │ └── tokenization_canine.py
│ ├── chinese_clip/
│ │ ├── __init__.py
│ │ ├── configuration_chinese_clip.py
│ │ ├── convert_chinese_clip_original_pytorch_to_hf.py
│ │ ├── feature_extraction_chinese_clip.py
│ │ ├── image_processing_chinese_clip.py
│ │ ├── modeling_chinese_clip.py
│ │ └── processing_chinese_clip.py
│ ├── clap/
│ │ ├── __init__.py
│ │ ├── configuration_clap.py
│ │ ├── convert_clap_original_pytorch_to_hf.py
│ │ ├── feature_extraction_clap.py
│ │ ├── modeling_clap.py
│ │ └── processing_clap.py
│ ├── clip/
│ │ ├── __init__.py
│ │ ├── configuration_clip.py
│ │ ├── convert_clip_original_pytorch_to_hf.py
│ │ ├── feature_extraction_clip.py
│ │ ├── image_processing_clip.py
│ │ ├── modeling_clip.py
│ │ ├── modeling_flax_clip.py
│ │ ├── modeling_tf_clip.py
│ │ ├── processing_clip.py
│ │ ├── tokenization_clip.py
│ │ └── tokenization_clip_fast.py
│ ├── clipseg/
│ │ ├── __init__.py
│ │ ├── configuration_clipseg.py
│ │ ├── convert_clipseg_original_pytorch_to_hf.py
│ │ ├── modeling_clipseg.py
│ │ └── processing_clipseg.py
│ ├── codegen/
│ │ ├── __init__.py
│ │ ├── configuration_codegen.py
│ │ ├── modeling_codegen.py
│ │ ├── tokenization_codegen.py
│ │ └── tokenization_codegen_fast.py
│ ├── conditional_detr/
│ │ ├── __init__.py
│ │ ├── configuration_conditional_detr.py
│ │ ├── convert_conditional_detr_original_pytorch_checkpoint_to_pytorch.py
│ │ ├── feature_extraction_conditional_detr.py
│ │ ├── image_processing_conditional_detr.py
│ │ └── modeling_conditional_detr.py
│ ├── convbert/
│ │ ├── __init__.py
│ │ ├── configuration_convbert.py
│ │ ├── convert_convbert_original_tf1_checkpoint_to_pytorch_and_tf2.py
│ │ ├── modeling_convbert.py
│ │ ├── modeling_tf_convbert.py
│ │ ├── tokenization_convbert.py
│ │ └── tokenization_convbert_fast.py
│ ├── convnext/
│ │ ├── __init__.py
│ │ ├── configuration_convnext.py
│ │ ├── convert_convnext_to_pytorch.py
│ │ ├── feature_extraction_convnext.py
│ │ ├── image_processing_convnext.py
│ │ ├── modeling_convnext.py
│ │ └── modeling_tf_convnext.py
│ ├── convnextv2/
│ │ ├── __init__.py
│ │ ├── configuration_convnextv2.py
│ │ ├── convert_convnextv2_to_pytorch.py
│ │ └── modeling_convnextv2.py
│ ├── cpm/
│ │ ├── __init__.py
│ │ ├── tokenization_cpm.py
│ │ └── tokenization_cpm_fast.py
│ ├── cpmant/
│ │ ├── __init__.py
│ │ ├── configuration_cpmant.py
│ │ ├── modeling_cpmant.py
│ │ └── tokenization_cpmant.py
│ ├── ctrl/
│ │ ├── __init__.py
│ │ ├── configuration_ctrl.py
│ │ ├── modeling_ctrl.py
│ │ ├── modeling_tf_ctrl.py
│ │ └── tokenization_ctrl.py
│ ├── cvt/
│ │ ├── __init__.py
│ │ ├── configuration_cvt.py
│ │ ├── convert_cvt_original_pytorch_checkpoint_to_pytorch.py
│ │ ├── modeling_cvt.py
│ │ └── modeling_tf_cvt.py
│ ├── data2vec/
│ │ ├── __init__.py
│ │ ├── configuration_data2vec_audio.py
│ │ ├── configuration_data2vec_text.py
│ │ ├── configuration_data2vec_vision.py
│ │ ├── convert_data2vec_audio_original_pytorch_checkpoint_to_pytorch.py
│ │ ├── convert_data2vec_text_original_pytorch_checkpoint_to_pytorch.py
│ │ ├── convert_data2vec_vision_original_pytorch_checkpoint_to_pytorch.py
│ │ ├── modeling_data2vec_audio.py
│ │ ├── modeling_data2vec_text.py
│ │ ├── modeling_data2vec_vision.py
│ │ └── modeling_tf_data2vec_vision.py
│ ├── deberta/
│ │ ├── __init__.py
│ │ ├── configuration_deberta.py
│ │ ├── modeling_deberta.py
│ │ ├── modeling_tf_deberta.py
│ │ ├── tokenization_deberta.py
│ │ └── tokenization_deberta_fast.py
│ ├── deberta_v2/
│ │ ├── __init__.py
│ │ ├── configuration_deberta_v2.py
│ │ ├── modeling_deberta_v2.py
│ │ ├── modeling_tf_deberta_v2.py
│ │ ├── tokenization_deberta_v2.py
│ │ └── tokenization_deberta_v2_fast.py
│ ├── decision_transformer/
│ │ ├── __init__.py
│ │ ├── configuration_decision_transformer.py
│ │ └── modeling_decision_transformer.py
│ ├── deformable_detr/
│ │ ├── __init__.py
│ │ ├── configuration_deformable_detr.py
│ │ ├── convert_deformable_detr_to_pytorch.py
│ │ ├── feature_extraction_deformable_detr.py
│ │ ├── image_processing_deformable_detr.py
│ │ ├── load_custom.py
│ │ └── modeling_deformable_detr.py
│ ├── deit/
│ │ ├── __init__.py
│ │ ├── configuration_deit.py
│ │ ├── convert_deit_timm_to_pytorch.py
│ │ ├── feature_extraction_deit.py
│ │ ├── image_processing_deit.py
│ │ ├── modeling_deit.py
│ │ └── modeling_tf_deit.py
│ ├── deta/
│ │ ├── __init__.py
│ │ ├── configuration_deta.py
│ │ ├── convert_deta_resnet_to_pytorch.py
│ │ ├── convert_deta_swin_to_pytorch.py
│ │ ├── image_processing_deta.py
│ │ └── modeling_deta.py
│ ├── detr/
│ │ ├── __init__.py
│ │ ├── configuration_detr.py
│ │ ├── convert_detr_original_pytorch_checkpoint_to_pytorch.py
│ │ ├── convert_detr_to_pytorch.py
│ │ ├── feature_extraction_detr.py
│ │ ├── image_processing_detr.py
│ │ └── modeling_detr.py
│ ├── dialogpt/
│ │ ├── __init__.py
│ │ └── convert_dialogpt_original_pytorch_checkpoint_to_pytorch.py
│ ├── dinat/
│ │ ├── __init__.py
│ │ ├── configuration_dinat.py
│ │ └── modeling_dinat.py
│ ├── distilbert/
│ │ ├── __init__.py
│ │ ├── configuration_distilbert.py
│ │ ├── modeling_distilbert.py
│ │ ├── modeling_flax_distilbert.py
│ │ ├── modeling_tf_distilbert.py
│ │ ├── tokenization_distilbert.py
│ │ └── tokenization_distilbert_fast.py
│ ├── dit/
│ │ ├── __init__.py
│ │ └── convert_dit_unilm_to_pytorch.py
│ ├── donut/
│ │ ├── __init__.py
│ │ ├── configuration_donut_swin.py
│ │ ├── convert_donut_to_pytorch.py
│ │ ├── feature_extraction_donut.py
│ │ ├── image_processing_donut.py
│ │ ├── modeling_donut_swin.py
│ │ └── processing_donut.py
│ ├── dpr/
│ │ ├── __init__.py
│ │ ├── configuration_dpr.py
│ │ ├── convert_dpr_original_checkpoint_to_pytorch.py
│ │ ├── modeling_dpr.py
│ │ ├── modeling_tf_dpr.py
│ │ ├── tokenization_dpr.py
│ │ └── tokenization_dpr_fast.py
│ ├── dpt/
│ │ ├── __init__.py
│ │ ├── configuration_dpt.py
│ │ ├── convert_dpt_hybrid_to_pytorch.py
│ │ ├── convert_dpt_to_pytorch.py
│ │ ├── feature_extraction_dpt.py
│ │ ├── image_processing_dpt.py
│ │ └── modeling_dpt.py
│ ├── efficientformer/
│ │ ├── __init__.py
│ │ ├── configuration_efficientformer.py
│ │ ├── convert_efficientformer_original_pytorch_checkpoint_to_pytorch.py
│ │ ├── image_processing_efficientformer.py
│ │ ├── modeling_efficientformer.py
│ │ └── modeling_tf_efficientformer.py
│ ├── efficientnet/
│ │ ├── __init__.py
│ │ ├── configuration_efficientnet.py
│ │ ├── convert_efficientnet_to_pytorch.py
│ │ ├── image_processing_efficientnet.py
│ │ └── modeling_efficientnet.py
│ ├── electra/
│ │ ├── __init__.py
│ │ ├── configuration_electra.py
│ │ ├── convert_electra_original_tf_checkpoint_to_pytorch.py
│ │ ├── modeling_electra.py
│ │ ├── modeling_flax_electra.py
│ │ ├── modeling_tf_electra.py
│ │ ├── tokenization_electra.py
│ │ └── tokenization_electra_fast.py
│ ├── encoder_decoder/
│ │ ├── __init__.py
│ │ ├── configuration_encoder_decoder.py
│ │ ├── modeling_encoder_decoder.py
│ │ ├── modeling_flax_encoder_decoder.py
│ │ └── modeling_tf_encoder_decoder.py
│ ├── ernie/
│ │ ├── __init__.py
│ │ ├── configuration_ernie.py
│ │ └── modeling_ernie.py
│ ├── ernie_m/
│ │ ├── __init__.py
│ │ ├── configuration_ernie_m.py
│ │ ├── modeling_ernie_m.py
│ │ └── tokenization_ernie_m.py
│ ├── esm/
│ │ ├── __init__.py
│ │ ├── configuration_esm.py
│ │ ├── convert_esm.py
│ │ ├── modeling_esm.py
│ │ ├── modeling_esmfold.py
│ │ ├── modeling_tf_esm.py
│ │ ├── openfold_utils/
│ │ │ ├── __init__.py
│ │ │ ├── chunk_utils.py
│ │ │ ├── data_transforms.py
│ │ │ ├── feats.py
│ │ │ ├── loss.py
│ │ │ ├── protein.py
│ │ │ ├── residue_constants.py
│ │ │ ├── rigid_utils.py
│ │ │ └── tensor_utils.py
│ │ └── tokenization_esm.py
│ ├── flaubert/
│ │ ├── __init__.py
│ │ ├── configuration_flaubert.py
│ │ ├── modeling_flaubert.py
│ │ ├── modeling_tf_flaubert.py
│ │ └── tokenization_flaubert.py
│ ├── flava/
│ │ ├── __init__.py
│ │ ├── configuration_flava.py
│ │ ├── convert_dalle_to_flava_codebook.py
│ │ ├── convert_flava_original_pytorch_to_hf.py
│ │ ├── feature_extraction_flava.py
│ │ ├── image_processing_flava.py
│ │ ├── modeling_flava.py
│ │ └── processing_flava.py
│ ├── fnet/
│ │ ├── __init__.py
│ │ ├── configuration_fnet.py
│ │ ├── convert_fnet_original_flax_checkpoint_to_pytorch.py
│ │ ├── modeling_fnet.py
│ │ ├── tokenization_fnet.py
│ │ └── tokenization_fnet_fast.py
│ ├── focalnet/
│ │ ├── __init__.py
│ │ ├── configuration_focalnet.py
│ │ ├── convert_focalnet_to_hf_format.py
│ │ └── modeling_focalnet.py
│ ├── fsmt/
│ │ ├── __init__.py
│ │ ├── configuration_fsmt.py
│ │ ├── convert_fsmt_original_pytorch_checkpoint_to_pytorch.py
│ │ ├── modeling_fsmt.py
│ │ └── tokenization_fsmt.py
│ ├── funnel/
│ │ ├── __init__.py
│ │ ├── configuration_funnel.py
│ │ ├── convert_funnel_original_tf_checkpoint_to_pytorch.py
│ │ ├── modeling_funnel.py
│ │ ├── modeling_tf_funnel.py
│ │ ├── tokenization_funnel.py
│ │ └── tokenization_funnel_fast.py
│ ├── git/
│ │ ├── __init__.py
│ │ ├── configuration_git.py
│ │ ├── convert_git_to_pytorch.py
│ │ ├── modeling_git.py
│ │ └── processing_git.py
│ ├── glpn/
│ │ ├── __init__.py
│ │ ├── configuration_glpn.py
│ │ ├── convert_glpn_to_pytorch.py
│ │ ├── feature_extraction_glpn.py
│ │ ├── image_processing_glpn.py
│ │ └── modeling_glpn.py
│ ├── gpt2/
│ │ ├── __init__.py
│ │ ├── configuration_gpt2.py
│ │ ├── convert_gpt2_original_tf_checkpoint_to_pytorch.py
│ │ ├── modeling_flax_gpt2.py
│ │ ├── modeling_gpt2.py
│ │ ├── modeling_tf_gpt2.py
│ │ ├── tokenization_gpt2.py
│ │ ├── tokenization_gpt2_fast.py
│ │ └── tokenization_gpt2_tf.py
│ ├── gpt_bigcode/
│ │ ├── __init__.py
│ │ ├── configuration_gpt_bigcode.py
│ │ └── modeling_gpt_bigcode.py
│ ├── gpt_neo/
│ │ ├── __init__.py
│ │ ├── configuration_gpt_neo.py
│ │ ├── convert_gpt_neo_mesh_tf_to_pytorch.py
│ │ ├── modeling_flax_gpt_neo.py
│ │ └── modeling_gpt_neo.py
│ ├── gpt_neox/
│ │ ├── __init__.py
│ │ ├── configuration_gpt_neox.py
│ │ ├── modeling_gpt_neox.py
│ │ └── tokenization_gpt_neox_fast.py
│ ├── gpt_neox_japanese/
│ │ ├── __init__.py
│ │ ├── configuration_gpt_neox_japanese.py
│ │ ├── modeling_gpt_neox_japanese.py
│ │ └── tokenization_gpt_neox_japanese.py
│ ├── gpt_sw3/
│ │ ├── __init__.py
│ │ ├── convert_megatron_to_pytorch.py
│ │ └── tokenization_gpt_sw3.py
│ ├── gptj/
│ │ ├── __init__.py
│ │ ├── configuration_gptj.py
│ │ ├── modeling_flax_gptj.py
│ │ ├── modeling_gptj.py
│ │ └── modeling_tf_gptj.py
│ ├── gptsan_japanese/
│ │ ├── __init__.py
│ │ ├── configuration_gptsan_japanese.py
│ │ ├── convert_gptsan_tf_checkpoint_to_pytorch.py
│ │ ├── modeling_gptsan_japanese.py
│ │ └── tokenization_gptsan_japanese.py
│ ├── graphormer/
│ │ ├── __init__.py
│ │ ├── algos_graphormer.pyx
│ │ ├── collating_graphormer.py
│ │ ├── configuration_graphormer.py
│ │ └── modeling_graphormer.py
│ ├── groupvit/
│ │ ├── __init__.py
│ │ ├── configuration_groupvit.py
│ │ ├── convert_groupvit_nvlab_to_hf.py
│ │ ├── modeling_groupvit.py
│ │ └── modeling_tf_groupvit.py
│ ├── herbert/
│ │ ├── __init__.py
│ │ ├── tokenization_herbert.py
│ │ └── tokenization_herbert_fast.py
│ ├── hubert/
│ │ ├── __init__.py
│ │ ├── configuration_hubert.py
│ │ ├── convert_distilhubert_original_s3prl_checkpoint_to_pytorch.py
│ │ ├── convert_hubert_original_pytorch_checkpoint_to_pytorch.py
│ │ ├── convert_hubert_original_s3prl_checkpoint_to_pytorch.py
│ │ ├── modeling_hubert.py
│ │ └── modeling_tf_hubert.py
│ ├── ibert/
│ │ ├── __init__.py
│ │ ├── configuration_ibert.py
│ │ ├── modeling_ibert.py
│ │ └── quant_modules.py
│ ├── imagegpt/
│ │ ├── __init__.py
│ │ ├── configuration_imagegpt.py
│ │ ├── convert_imagegpt_original_tf2_to_pytorch.py
│ │ ├── feature_extraction_imagegpt.py
│ │ ├── image_processing_imagegpt.py
│ │ └── modeling_imagegpt.py
│ ├── informer/
│ │ ├── __init__.py
│ │ ├── configuration_informer.py
│ │ └── modeling_informer.py
│ ├── jukebox/
│ │ ├── __init__.py
│ │ ├── configuration_jukebox.py
│ │ ├── convert_jukebox.py
│ │ ├── modeling_jukebox.py
│ │ └── tokenization_jukebox.py
│ ├── layoutlm/
│ │ ├── __init__.py
│ │ ├── configuration_layoutlm.py
│ │ ├── modeling_layoutlm.py
│ │ ├── modeling_tf_layoutlm.py
│ │ ├── tokenization_layoutlm.py
│ │ └── tokenization_layoutlm_fast.py
│ ├── layoutlmv2/
│ │ ├── __init__.py
│ │ ├── configuration_layoutlmv2.py
│ │ ├── feature_extraction_layoutlmv2.py
│ │ ├── image_processing_layoutlmv2.py
│ │ ├── modeling_layoutlmv2.py
│ │ ├── processing_layoutlmv2.py
│ │ ├── tokenization_layoutlmv2.py
│ │ └── tokenization_layoutlmv2_fast.py
│ ├── layoutlmv3/
│ │ ├── __init__.py
│ │ ├── configuration_layoutlmv3.py
│ │ ├── feature_extraction_layoutlmv3.py
│ │ ├── image_processing_layoutlmv3.py
│ │ ├── modeling_layoutlmv3.py
│ │ ├── modeling_tf_layoutlmv3.py
│ │ ├── processing_layoutlmv3.py
│ │ ├── tokenization_layoutlmv3.py
│ │ └── tokenization_layoutlmv3_fast.py
│ ├── layoutxlm/
│ │ ├── __init__.py
│ │ ├── processing_layoutxlm.py
│ │ ├── tokenization_layoutxlm.py
│ │ └── tokenization_layoutxlm_fast.py
│ ├── led/
│ │ ├── __init__.py
│ │ ├── configuration_led.py
│ │ ├── modeling_led.py
│ │ ├── modeling_tf_led.py
│ │ ├── tokenization_led.py
│ │ └── tokenization_led_fast.py
│ ├── levit/
│ │ ├── __init__.py
│ │ ├── configuration_levit.py
│ │ ├── convert_levit_timm_to_pytorch.py
│ │ ├── feature_extraction_levit.py
│ │ ├── image_processing_levit.py
│ │ └── modeling_levit.py
│ ├── lilt/
│ │ ├── __init__.py
│ │ ├── configuration_lilt.py
│ │ └── modeling_lilt.py
│ ├── llama/
│ │ ├── __init__.py
│ │ ├── configuration_llama.py
│ │ ├── convert_llama_weights_to_hf.py
│ │ ├── modeling_llama.py
│ │ ├── tokenization_llama.py
│ │ └── tokenization_llama_fast.py
│ ├── longformer/
│ │ ├── __init__.py
│ │ ├── configuration_longformer.py
│ │ ├── convert_longformer_original_pytorch_lightning_to_pytorch.py
│ │ ├── modeling_longformer.py
│ │ ├── modeling_tf_longformer.py
│ │ ├── tokenization_longformer.py
│ │ └── tokenization_longformer_fast.py
│ ├── longt5/
│ │ ├── __init__.py
│ │ ├── configuration_longt5.py
│ │ ├── convert_longt5x_checkpoint_to_flax.py
│ │ ├── modeling_flax_longt5.py
│ │ └── modeling_longt5.py
│ ├── luke/
│ │ ├── __init__.py
│ │ ├── configuration_luke.py
│ │ ├── convert_luke_original_pytorch_checkpoint_to_pytorch.py
│ │ ├── modeling_luke.py
│ │ └── tokenization_luke.py
│ ├── lxmert/
│ │ ├── __init__.py
│ │ ├── configuration_lxmert.py
│ │ ├── convert_lxmert_original_tf_checkpoint_to_pytorch.py
│ │ ├── modeling_lxmert.py
│ │ ├── modeling_tf_lxmert.py
│ │ ├── tokenization_lxmert.py
│ │ └── tokenization_lxmert_fast.py
│ ├── m2m_100/
│ │ ├── __init__.py
│ │ ├── configuration_m2m_100.py
│ │ ├── convert_m2m100_original_checkpoint_to_pytorch.py
│ │ ├── modeling_m2m_100.py
│ │ └── tokenization_m2m_100.py
│ ├── marian/
│ │ ├── __init__.py
│ │ ├── configuration_marian.py
│ │ ├── convert_marian_tatoeba_to_pytorch.py
│ │ ├── convert_marian_to_pytorch.py
│ │ ├── modeling_flax_marian.py
│ │ ├── modeling_marian.py
│ │ ├── modeling_tf_marian.py
│ │ └── tokenization_marian.py
│ ├── markuplm/
│ │ ├── __init__.py
│ │ ├── configuration_markuplm.py
│ │ ├── feature_extraction_markuplm.py
│ │ ├── modeling_markuplm.py
│ │ ├── processing_markuplm.py
│ │ ├── tokenization_markuplm.py
│ │ └── tokenization_markuplm_fast.py
│ ├── mask2former/
│ │ ├── __init__.py
│ │ ├── configuration_mask2former.py
│ │ ├── convert_mask2former_original_pytorch_checkpoint_to_pytorch.py
│ │ ├── image_processing_mask2former.py
│ │ └── modeling_mask2former.py
│ ├── maskformer/
│ │ ├── __init__.py
│ │ ├── configuration_maskformer.py
│ │ ├── configuration_maskformer_swin.py
│ │ ├── convert_maskformer_original_pytorch_checkpoint_to_pytorch.py
│ │ ├── convert_maskformer_resnet_to_pytorch.py
│ │ ├── convert_maskformer_swin_to_pytorch.py
│ │ ├── feature_extraction_maskformer.py
│ │ ├── image_processing_maskformer.py
│ │ ├── modeling_maskformer.py
│ │ └── modeling_maskformer_swin.py
│ ├── mbart/
│ │ ├── __init__.py
│ │ ├── configuration_mbart.py
│ │ ├── convert_mbart_original_checkpoint_to_pytorch.py
│ │ ├── modeling_flax_mbart.py
│ │ ├── modeling_mbart.py
│ │ ├── modeling_tf_mbart.py
│ │ ├── tokenization_mbart.py
│ │ └── tokenization_mbart_fast.py
│ ├── mbart50/
│ │ ├── __init__.py
│ │ ├── tokenization_mbart50.py
│ │ └── tokenization_mbart50_fast.py
│ ├── mctct/
│ │ ├── __init__.py
│ │ ├── configuration_mctct.py
│ │ ├── feature_extraction_mctct.py
│ │ ├── modeling_mctct.py
│ │ └── processing_mctct.py
│ ├── mega/
│ │ ├── __init__.py
│ │ ├── configuration_mega.py
│ │ ├── convert_mega_original_pytorch_checkpoint_to_pytorch.py
│ │ └── modeling_mega.py
│ ├── megatron_bert/
│ │ ├── __init__.py
│ │ ├── configuration_megatron_bert.py
│ │ ├── convert_megatron_bert_checkpoint.py
│ │ └── modeling_megatron_bert.py
│ ├── megatron_gpt2/
│ │ ├── __init__.py
│ │ ├── checkpoint_reshaping_and_interoperability.py
│ │ └── convert_megatron_gpt2_checkpoint.py
│ ├── mgp_str/
│ │ ├── __init__.py
│ │ ├── configuration_mgp_str.py
│ │ ├── modeling_mgp_str.py
│ │ ├── processing_mgp_str.py
│ │ └── tokenization_mgp_str.py
│ ├── mluke/
│ │ ├── __init__.py
│ │ ├── convert_mluke_original_pytorch_checkpoint_to_pytorch.py
│ │ └── tokenization_mluke.py
│ ├── mmbt/
│ │ ├── __init__.py
│ │ ├── configuration_mmbt.py
│ │ └── modeling_mmbt.py
│ ├── mobilebert/
│ │ ├── __init__.py
│ │ ├── configuration_mobilebert.py
│ │ ├── convert_mobilebert_original_tf_checkpoint_to_pytorch.py
│ │ ├── modeling_mobilebert.py
│ │ ├── modeling_tf_mobilebert.py
│ │ ├── tokenization_mobilebert.py
│ │ └── tokenization_mobilebert_fast.py
│ ├── mobilenet_v1/
│ │ ├── __init__.py
│ │ ├── configuration_mobilenet_v1.py
│ │ ├── convert_original_tf_checkpoint_to_pytorch.py
│ │ ├── feature_extraction_mobilenet_v1.py
│ │ ├── image_processing_mobilenet_v1.py
│ │ └── modeling_mobilenet_v1.py
│ ├── mobilenet_v2/
│ │ ├── __init__.py
│ │ ├── configuration_mobilenet_v2.py
│ │ ├── convert_original_tf_checkpoint_to_pytorch.py
│ │ ├── feature_extraction_mobilenet_v2.py
│ │ ├── image_processing_mobilenet_v2.py
│ │ └── modeling_mobilenet_v2.py
│ ├── mobilevit/
│ │ ├── __init__.py
│ │ ├── configuration_mobilevit.py
│ │ ├── convert_mlcvnets_to_pytorch.py
│ │ ├── feature_extraction_mobilevit.py
│ │ ├── image_processing_mobilevit.py
│ │ ├── modeling_mobilevit.py
│ │ └── modeling_tf_mobilevit.py
│ ├── mobilevitv2/
│ │ ├── __init__.py
│ │ ├── configuration_mobilevitv2.py
│ │ ├── convert_mlcvnets_to_pytorch.py
│ │ └── modeling_mobilevitv2.py
│ ├── mpnet/
│ │ ├── __init__.py
│ │ ├── configuration_mpnet.py
│ │ ├── modeling_mpnet.py
│ │ ├── modeling_tf_mpnet.py
│ │ ├── tokenization_mpnet.py
│ │ └── tokenization_mpnet_fast.py
│ ├── mt5/
│ │ ├── __init__.py
│ │ ├── configuration_mt5.py
│ │ ├── modeling_flax_mt5.py
│ │ ├── modeling_mt5.py
│ │ └── modeling_tf_mt5.py
│ ├── mvp/
│ │ ├── __init__.py
│ │ ├── configuration_mvp.py
│ │ ├── modeling_mvp.py
│ │ ├── tokenization_mvp.py
│ │ └── tokenization_mvp_fast.py
│ ├── nat/
│ │ ├── __init__.py
│ │ ├── configuration_nat.py
│ │ └── modeling_nat.py
│ ├── nezha/
│ │ ├── __init__.py
│ │ ├── configuration_nezha.py
│ │ └── modeling_nezha.py
│ ├── nllb/
│ │ ├── __init__.py
│ │ ├── tokenization_nllb.py
│ │ └── tokenization_nllb_fast.py
│ ├── nllb_moe/
│ │ ├── __init__.py
│ │ ├── configuration_nllb_moe.py
│ │ ├── convert_nllb_moe_sharded_original_checkpoint_to_pytorch.py
│ │ └── modeling_nllb_moe.py
│ ├── nystromformer/
│ │ ├── __init__.py
│ │ ├── configuration_nystromformer.py
│ │ ├── convert_nystromformer_original_pytorch_checkpoint_to_pytorch.py
│ │ └── modeling_nystromformer.py
│ ├── oneformer/
│ │ ├── __init__.py
│ │ ├── configuration_oneformer.py
│ │ ├── convert_to_hf_oneformer.py
│ │ ├── image_processing_oneformer.py
│ │ ├── modeling_oneformer.py
│ │ └── processing_oneformer.py
│ ├── open_llama/
│ │ ├── __init__.py
│ │ ├── configuration_open_llama.py
│ │ └── modeling_open_llama.py
│ ├── openai/
│ │ ├── __init__.py
│ │ ├── configuration_openai.py
│ │ ├── convert_openai_original_tf_checkpoint_to_pytorch.py
│ │ ├── modeling_openai.py
│ │ ├── modeling_tf_openai.py
│ │ ├── tokenization_openai.py
│ │ └── tokenization_openai_fast.py
│ ├── opt/
│ │ ├── __init__.py
│ │ ├── configuration_opt.py
│ │ ├── convert_opt_original_pytorch_checkpoint_to_pytorch.py
│ │ ├── modeling_flax_opt.py
│ │ ├── modeling_opt.py
│ │ └── modeling_tf_opt.py
│ ├── owlvit/
│ │ ├── __init__.py
│ │ ├── configuration_owlvit.py
│ │ ├── convert_owlvit_original_flax_to_hf.py
│ │ ├── feature_extraction_owlvit.py
│ │ ├── image_processing_owlvit.py
│ │ ├── modeling_owlvit.py
│ │ └── processing_owlvit.py
│ ├── pegasus/
│ │ ├── __init__.py
│ │ ├── configuration_pegasus.py
│ │ ├── convert_pegasus_tf_to_pytorch.py
│ │ ├── modeling_flax_pegasus.py
│ │ ├── modeling_pegasus.py
│ │ ├── modeling_tf_pegasus.py
│ │ ├── tokenization_pegasus.py
│ │ └── tokenization_pegasus_fast.py
│ ├── pegasus_x/
│ │ ├── __init__.py
│ │ ├── configuration_pegasus_x.py
│ │ └── modeling_pegasus_x.py
│ ├── perceiver/
│ │ ├── __init__.py
│ │ ├── configuration_perceiver.py
│ │ ├── convert_perceiver_haiku_to_pytorch.py
│ │ ├── feature_extraction_perceiver.py
│ │ ├── image_processing_perceiver.py
│ │ ├── modeling_perceiver.py
│ │ └── tokenization_perceiver.py
│ ├── phobert/
│ │ ├── __init__.py
│ │ └── tokenization_phobert.py
│ ├── pix2struct/
│ │ ├── __init__.py
│ │ ├── configuration_pix2struct.py
│ │ ├── convert_pix2struct_original_pytorch_to_hf.py
│ │ ├── image_processing_pix2struct.py
│ │ ├── modeling_pix2struct.py
│ │ └── processing_pix2struct.py
│ ├── plbart/
│ │ ├── __init__.py
│ │ ├── configuration_plbart.py
│ │ ├── convert_plbart_original_checkpoint_to_torch.py
│ │ ├── modeling_plbart.py
│ │ └── tokenization_plbart.py
│ ├── poolformer/
│ │ ├── __init__.py
│ │ ├── configuration_poolformer.py
│ │ ├── convert_poolformer_original_to_pytorch.py
│ │ ├── feature_extraction_poolformer.py
│ │ ├── image_processing_poolformer.py
│ │ └── modeling_poolformer.py
│ ├── prophetnet/
│ │ ├── __init__.py
│ │ ├── configuration_prophetnet.py
│ │ ├── convert_prophetnet_original_pytorch_checkpoint_to_pytorch.py
│ │ ├── modeling_prophetnet.py
│ │ └── tokenization_prophetnet.py
│ ├── qdqbert/
│ │ ├── __init__.py
│ │ ├── configuration_qdqbert.py
│ │ └── modeling_qdqbert.py
│ ├── rag/
│ │ ├── __init__.py
│ │ ├── configuration_rag.py
│ │ ├── modeling_rag.py
│ │ ├── modeling_tf_rag.py
│ │ ├── retrieval_rag.py
│ │ └── tokenization_rag.py
│ ├── realm/
│ │ ├── __init__.py
│ │ ├── configuration_realm.py
│ │ ├── modeling_realm.py
│ │ ├── retrieval_realm.py
│ │ ├── tokenization_realm.py
│ │ └── tokenization_realm_fast.py
│ ├── reformer/
│ │ ├── __init__.py
│ │ ├── configuration_reformer.py
│ │ ├── convert_reformer_trax_checkpoint_to_pytorch.py
│ │ ├── modeling_reformer.py
│ │ ├── tokenization_reformer.py
│ │ └── tokenization_reformer_fast.py
│ ├── regnet/
│ │ ├── __init__.py
│ │ ├── configuration_regnet.py
│ │ ├── convert_regnet_seer_10b_to_pytorch.py
│ │ ├── convert_regnet_to_pytorch.py
│ │ ├── modeling_flax_regnet.py
│ │ ├── modeling_regnet.py
│ │ └── modeling_tf_regnet.py
│ ├── rembert/
│ │ ├── __init__.py
│ │ ├── configuration_rembert.py
│ │ ├── convert_rembert_tf_checkpoint_to_pytorch.py
│ │ ├── modeling_rembert.py
│ │ ├── modeling_tf_rembert.py
│ │ ├── tokenization_rembert.py
│ │ └── tokenization_rembert_fast.py
│ ├── resnet/
│ │ ├── __init__.py
│ │ ├── configuration_resnet.py
│ │ ├── convert_resnet_to_pytorch.py
│ │ ├── modeling_flax_resnet.py
│ │ ├── modeling_resnet.py
│ │ └── modeling_tf_resnet.py
│ ├── retribert/
│ │ ├── __init__.py
│ │ ├── configuration_retribert.py
│ │ ├── modeling_retribert.py
│ │ ├── tokenization_retribert.py
│ │ └── tokenization_retribert_fast.py
│ ├── roberta/
│ │ ├── __init__.py
│ │ ├── configuration_roberta.py
│ │ ├── convert_roberta_original_pytorch_checkpoint_to_pytorch.py
│ │ ├── modeling_flax_roberta.py
│ │ ├── modeling_roberta.py
│ │ ├── modeling_tf_roberta.py
│ │ ├── tokenization_roberta.py
│ │ └── tokenization_roberta_fast.py
│ ├── roberta_prelayernorm/
│ │ ├── __init__.py
│ │ ├── configuration_roberta_prelayernorm.py
│ │ ├── convert_roberta_prelayernorm_original_pytorch_checkpoint_to_pytorch.py
│ │ ├── modeling_flax_roberta_prelayernorm.py
│ │ ├── modeling_roberta_prelayernorm.py
│ │ └── modeling_tf_roberta_prelayernorm.py
│ ├── roc_bert/
│ │ ├── __init__.py
│ │ ├── configuration_roc_bert.py
│ │ ├── modeling_roc_bert.py
│ │ └── tokenization_roc_bert.py
│ ├── roformer/
│ │ ├── __init__.py
│ │ ├── configuration_roformer.py
│ │ ├── convert_roformer_original_tf_checkpoint_to_pytorch.py
│ │ ├── modeling_flax_roformer.py
│ │ ├── modeling_roformer.py
│ │ ├── modeling_tf_roformer.py
│ │ ├── tokenization_roformer.py
│ │ ├── tokenization_roformer_fast.py
│ │ └── tokenization_utils.py
│ ├── rwkv/
│ │ ├── __init__.py
│ │ ├── configuration_rwkv.py
│ │ ├── convert_rwkv_checkpoint_to_hf.py
│ │ └── modeling_rwkv.py
│ ├── sam/
│ │ ├── __init__.py
│ │ ├── configuration_sam.py
│ │ ├── convert_sam_original_to_hf_format.py
│ │ ├── image_processing_sam.py
│ │ ├── modeling_sam.py
│ │ ├── modeling_tf_sam.py
│ │ └── processing_sam.py
│ ├── segformer/
│ │ ├── __init__.py
│ │ ├── configuration_segformer.py
│ │ ├── convert_segformer_original_to_pytorch.py
│ │ ├── feature_extraction_segformer.py
│ │ ├── image_processing_segformer.py
│ │ ├── modeling_segformer.py
│ │ └── modeling_tf_segformer.py
│ ├── sew/
│ │ ├── __init__.py
│ │ ├── configuration_sew.py
│ │ ├── convert_sew_original_pytorch_checkpoint_to_pytorch.py
│ │ └── modeling_sew.py
│ ├── sew_d/
│ │ ├── __init__.py
│ │ ├── configuration_sew_d.py
│ │ ├── convert_sew_d_original_pytorch_checkpoint_to_pytorch.py
│ │ └── modeling_sew_d.py
│ ├── speech_encoder_decoder/
│ │ ├── __init__.py
│ │ ├── configuration_speech_encoder_decoder.py
│ │ ├── convert_mbart_wav2vec2_seq2seq_original_to_pytorch.py
│ │ ├── convert_speech_to_text_wav2vec2_seq2seq_original_to_pytorch.py
│ │ ├── modeling_flax_speech_encoder_decoder.py
│ │ └── modeling_speech_encoder_decoder.py
│ ├── speech_to_text/
│ │ ├── __init__.py
│ │ ├── configuration_speech_to_text.py
│ │ ├── convert_s2t_fairseq_to_tfms.py
│ │ ├── feature_extraction_speech_to_text.py
│ │ ├── modeling_speech_to_text.py
│ │ ├── modeling_tf_speech_to_text.py
│ │ ├── processing_speech_to_text.py
│ │ └── tokenization_speech_to_text.py
│ ├── speech_to_text_2/
│ │ ├── __init__.py
│ │ ├── configuration_speech_to_text_2.py
│ │ ├── modeling_speech_to_text_2.py
│ │ ├── processing_speech_to_text_2.py
│ │ └── tokenization_speech_to_text_2.py
│ ├── speecht5/
│ │ ├── __init__.py
│ │ ├── configuration_speecht5.py
│ │ ├── convert_hifigan.py
│ │ ├── convert_speecht5_original_pytorch_checkpoint_to_pytorch.py
│ │ ├── feature_extraction_speecht5.py
│ │ ├── modeling_speecht5.py
│ │ ├── processing_speecht5.py
│ │ └── tokenization_speecht5.py
│ ├── splinter/
│ │ ├── __init__.py
│ │ ├── configuration_splinter.py
│ │ ├── modeling_splinter.py
│ │ ├── tokenization_splinter.py
│ │ └── tokenization_splinter_fast.py
│ ├── squeezebert/
│ │ ├── __init__.py
│ │ ├── configuration_squeezebert.py
│ │ ├── modeling_squeezebert.py
│ │ ├── tokenization_squeezebert.py
│ │ └── tokenization_squeezebert_fast.py
│ ├── swiftformer/
│ │ ├── __init__.py
│ │ ├── configuration_swiftformer.py
│ │ ├── convert_swiftformer_original_to_hf.py
│ │ └── modeling_swiftformer.py
│ ├── swin/
│ │ ├── __init__.py
│ │ ├── configuration_swin.py
│ │ ├── convert_swin_simmim_to_pytorch.py
│ │ ├── convert_swin_timm_to_pytorch.py
│ │ ├── modeling_swin.py
│ │ └── modeling_tf_swin.py
│ ├── swin2sr/
│ │ ├── __init__.py
│ │ ├── configuration_swin2sr.py
│ │ ├── convert_swin2sr_original_to_pytorch.py
│ │ ├── image_processing_swin2sr.py
│ │ └── modeling_swin2sr.py
│ ├── swinv2/
│ │ ├── __init__.py
│ │ ├── configuration_swinv2.py
│ │ ├── convert_swinv2_timm_to_pytorch.py
│ │ └── modeling_swinv2.py
│ ├── switch_transformers/
│ │ ├── __init__.py
│ │ ├── configuration_switch_transformers.py
│ │ ├── convert_big_switch.py
│ │ ├── convert_switch_transformers_original_flax_checkpoint_to_pytorch.py
│ │ └── modeling_switch_transformers.py
│ ├── t5/
│ │ ├── __init__.py
│ │ ├── configuration_t5.py
│ │ ├── convert_t5_original_tf_checkpoint_to_pytorch.py
│ │ ├── convert_t5x_checkpoint_to_flax.py
│ │ ├── convert_t5x_checkpoint_to_pytorch.py
│ │ ├── modeling_flax_t5.py
│ │ ├── modeling_t5.py
│ │ ├── modeling_tf_t5.py
│ │ ├── tokenization_t5.py
│ │ └── tokenization_t5_fast.py
│ ├── table_transformer/
│ │ ├── __init__.py
│ │ ├── configuration_table_transformer.py
│ │ ├── convert_table_transformer_original_pytorch_checkpoint_to_pytorch.py
│ │ └── modeling_table_transformer.py
│ ├── tapas/
│ │ ├── __init__.py
│ │ ├── configuration_tapas.py
│ │ ├── convert_tapas_original_tf_checkpoint_to_pytorch.py
│ │ ├── modeling_tapas.py
│ │ ├── modeling_tf_tapas.py
│ │ └── tokenization_tapas.py
│ ├── tapex/
│ │ ├── __init__.py
│ │ └── tokenization_tapex.py
│ ├── time_series_transformer/
│ │ ├── __init__.py
│ │ ├── configuration_time_series_transformer.py
│ │ └── modeling_time_series_transformer.py
│ ├── timesformer/
│ │ ├── __init__.py
│ │ ├── configuration_timesformer.py
│ │ ├── convert_timesformer_to_pytorch.py
│ │ └── modeling_timesformer.py
│ ├── timm_backbone/
│ │ ├── __init__.py
│ │ ├── configuration_timm_backbone.py
│ │ └── modeling_timm_backbone.py
│ ├── trajectory_transformer/
│ │ ├── __init__.py
│ │ ├── configuration_trajectory_transformer.py
│ │ ├── convert_trajectory_transformer_original_pytorch_checkpoint_to_pytorch.py
│ │ └── modeling_trajectory_transformer.py
│ ├── transfo_xl/
│ │ ├── __init__.py
│ │ ├── configuration_transfo_xl.py
│ │ ├── convert_transfo_xl_original_tf_checkpoint_to_pytorch.py
│ │ ├── modeling_tf_transfo_xl.py
│ │ ├── modeling_tf_transfo_xl_utilities.py
│ │ ├── modeling_transfo_xl.py
│ │ ├── modeling_transfo_xl_utilities.py
│ │ └── tokenization_transfo_xl.py
│ ├── trocr/
│ │ ├── __init__.py
│ │ ├── configuration_trocr.py
│ │ ├── convert_trocr_unilm_to_pytorch.py
│ │ ├── modeling_trocr.py
│ │ └── processing_trocr.py
│ ├── tvlt/
│ │ ├── __init__.py
│ │ ├── configuration_tvlt.py
│ │ ├── feature_extraction_tvlt.py
│ │ ├── image_processing_tvlt.py
│ │ ├── modeling_tvlt.py
│ │ └── processing_tvlt.py
│ ├── unispeech/
│ │ ├── __init__.py
│ │ ├── configuration_unispeech.py
│ │ ├── convert_unispeech_original_pytorch_checkpoint_to_pytorch.py
│ │ └── modeling_unispeech.py
│ ├── unispeech_sat/
│ │ ├── __init__.py
│ │ ├── configuration_unispeech_sat.py
│ │ ├── convert_unispeech_original_s3prl_checkpoint_to_pytorch.py
│ │ ├── convert_unispeech_sat_original_pytorch_checkpoint_to_pytorch.py
│ │ └── modeling_unispeech_sat.py
│ ├── upernet/
│ │ ├── __init__.py
│ │ ├── configuration_upernet.py
│ │ ├── convert_convnext_upernet_to_pytorch.py
│ │ ├── convert_swin_upernet_to_pytorch.py
│ │ └── modeling_upernet.py
│ ├── van/
│ │ ├── __init__.py
│ │ ├── configuration_van.py
│ │ ├── convert_van_to_pytorch.py
│ │ └── modeling_van.py
│ ├── videomae/
│ │ ├── __init__.py
│ │ ├── configuration_videomae.py
│ │ ├── convert_videomae_to_pytorch.py
│ │ ├── feature_extraction_videomae.py
│ │ ├── image_processing_videomae.py
│ │ └── modeling_videomae.py
│ ├── vilt/
│ │ ├── __init__.py
│ │ ├── configuration_vilt.py
│ │ ├── convert_vilt_original_to_pytorch.py
│ │ ├── feature_extraction_vilt.py
│ │ ├── image_processing_vilt.py
│ │ ├── modeling_vilt.py
│ │ └── processing_vilt.py
│ ├── vision_encoder_decoder/
│ │ ├── __init__.py
│ │ ├── configuration_vision_encoder_decoder.py
│ │ ├── modeling_flax_vision_encoder_decoder.py
│ │ ├── modeling_tf_vision_encoder_decoder.py
│ │ └── modeling_vision_encoder_decoder.py
│ ├── vision_text_dual_encoder/
│ │ ├── __init__.py
│ │ ├── configuration_vision_text_dual_encoder.py
│ │ ├── modeling_flax_vision_text_dual_encoder.py
│ │ ├── modeling_tf_vision_text_dual_encoder.py
│ │ ├── modeling_vision_text_dual_encoder.py
│ │ └── processing_vision_text_dual_encoder.py
│ ├── visual_bert/
│ │ ├── __init__.py
│ │ ├── configuration_visual_bert.py
│ │ ├── convert_visual_bert_original_pytorch_checkpoint_to_pytorch.py
│ │ └── modeling_visual_bert.py
│ ├── vit/
│ │ ├── __init__.py
│ │ ├── configuration_vit.py
│ │ ├── convert_dino_to_pytorch.py
│ │ ├── convert_vit_timm_to_pytorch.py
│ │ ├── feature_extraction_vit.py
│ │ ├── image_processing_vit.py
│ │ ├── modeling_flax_vit.py
│ │ ├── modeling_tf_vit.py
│ │ └── modeling_vit.py
│ ├── vit_hybrid/
│ │ ├── __init__.py
│ │ ├── configuration_vit_hybrid.py
│ │ ├── convert_vit_hybrid_timm_to_pytorch.py
│ │ ├── image_processing_vit_hybrid.py
│ │ └── modeling_vit_hybrid.py
│ ├── vit_mae/
│ │ ├── __init__.py
│ │ ├── configuration_vit_mae.py
│ │ ├── convert_vit_mae_to_pytorch.py
│ │ ├── modeling_tf_vit_mae.py
│ │ └── modeling_vit_mae.py
│ ├── vit_msn/
│ │ ├── __init__.py
│ │ ├── configuration_vit_msn.py
│ │ ├── convert_msn_to_pytorch.py
│ │ └── modeling_vit_msn.py
│ ├── wav2vec2/
│ │ ├── __init__.py
│ │ ├── configuration_wav2vec2.py
│ │ ├── convert_wav2vec2_original_pytorch_checkpoint_to_pytorch.py
│ │ ├── convert_wav2vec2_original_s3prl_checkpoint_to_pytorch.py
│ │ ├── feature_extraction_wav2vec2.py
│ │ ├── modeling_flax_wav2vec2.py
│ │ ├── modeling_tf_wav2vec2.py
│ │ ├── modeling_wav2vec2.py
│ │ ├── processing_wav2vec2.py
│ │ └── tokenization_wav2vec2.py
│ ├── wav2vec2_conformer/
│ │ ├── __init__.py
│ │ ├── configuration_wav2vec2_conformer.py
│ │ ├── convert_wav2vec2_conformer_original_pytorch_checkpoint_to_pytorch.py
│ │ └── modeling_wav2vec2_conformer.py
│ ├── wav2vec2_phoneme/
│ │ ├── __init__.py
│ │ └── tokenization_wav2vec2_phoneme.py
│ ├── wav2vec2_with_lm/
│ │ ├── __init__.py
│ │ └── processing_wav2vec2_with_lm.py
│ ├── wavlm/
│ │ ├── __init__.py
│ │ ├── configuration_wavlm.py
│ │ ├── convert_wavlm_original_pytorch_checkpoint_to_pytorch.py
│ │ ├── convert_wavlm_original_s3prl_checkpoint_to_pytorch.py
│ │ └── modeling_wavlm.py
│ ├── whisper/
│ │ ├── __init__.py
│ │ ├── configuration_whisper.py
│ │ ├── convert_openai_to_hf.py
│ │ ├── english_normalizer.py
│ │ ├── feature_extraction_whisper.py
│ │ ├── modeling_flax_whisper.py
│ │ ├── modeling_tf_whisper.py
│ │ ├── modeling_whisper.py
│ │ ├── processing_whisper.py
│ │ ├── tokenization_whisper.py
│ │ └── tokenization_whisper_fast.py
│ ├── x_clip/
│ │ ├── __init__.py
│ │ ├── configuration_x_clip.py
│ │ ├── convert_x_clip_original_pytorch_to_hf.py
│ │ ├── modeling_x_clip.py
│ │ └── processing_x_clip.py
│ ├── xglm/
│ │ ├── __init__.py
│ │ ├── configuration_xglm.py
│ │ ├── convert_xglm_original_ckpt_to_trfms.py
│ │ ├── modeling_flax_xglm.py
│ │ ├── modeling_tf_xglm.py
│ │ ├── modeling_xglm.py
│ │ ├── tokenization_xglm.py
│ │ └── tokenization_xglm_fast.py
│ ├── xlm/
│ │ ├── __init__.py
│ │ ├── configuration_xlm.py
│ │ ├── convert_xlm_original_pytorch_checkpoint_to_pytorch.py
│ │ ├── modeling_tf_xlm.py
│ │ ├── modeling_xlm.py
│ │ └── tokenization_xlm.py
│ ├── xlm_prophetnet/
│ │ ├── __init__.py
│ │ ├── configuration_xlm_prophetnet.py
│ │ ├── modeling_xlm_prophetnet.py
│ │ └── tokenization_xlm_prophetnet.py
│ ├── xlm_roberta/
│ │ ├── __init__.py
│ │ ├── configuration_xlm_roberta.py
│ │ ├── modeling_flax_xlm_roberta.py
│ │ ├── modeling_tf_xlm_roberta.py
│ │ ├── modeling_xlm_roberta.py
│ │ ├── tokenization_xlm_roberta.py
│ │ └── tokenization_xlm_roberta_fast.py
│ ├── xlm_roberta_xl/
│ │ ├── __init__.py
│ │ ├── configuration_xlm_roberta_xl.py
│ │ ├── convert_xlm_roberta_xl_original_pytorch_checkpoint_to_pytorch.py
│ │ └── modeling_xlm_roberta_xl.py
│ ├── xlnet/
│ │ ├── __init__.py
│ │ ├── configuration_xlnet.py
│ │ ├── convert_xlnet_original_tf_checkpoint_to_pytorch.py
│ │ ├── modeling_tf_xlnet.py
│ │ ├── modeling_xlnet.py
│ │ ├── tokenization_xlnet.py
│ │ └── tokenization_xlnet_fast.py
│ ├── xmod/
│ │ ├── __init__.py
│ │ ├── configuration_xmod.py
│ │ ├── convert_xmod_original_pytorch_checkpoint_to_pytorch.py
│ │ └── modeling_xmod.py
│ ├── yolos/
│ │ ├── __init__.py
│ │ ├── configuration_yolos.py
│ │ ├── convert_yolos_to_pytorch.py
│ │ ├── feature_extraction_yolos.py
│ │ ├── image_processing_yolos.py
│ │ └── modeling_yolos.py
│ └── yoso/
│ ├── __init__.py
│ ├── configuration_yoso.py
│ ├── convert_yoso_pytorch_to_pytorch.py
│ └── modeling_yoso.py
├── onnx/
│ ├── __init__.py
│ ├── __main__.py
│ ├── config.py
│ ├── convert.py
│ ├── features.py
│ └── utils.py
├── optimization.py
├── optimization_tf.py
├── pipelines/
│ ├── __init__.py
│ ├── audio_classification.py
│ ├── audio_utils.py
│ ├── automatic_speech_recognition.py
│ ├── base.py
│ ├── conversational.py
│ ├── depth_estimation.py
│ ├── document_question_answering.py
│ ├── feature_extraction.py
│ ├── fill_mask.py
│ ├── image_classification.py
│ ├── image_segmentation.py
│ ├── image_to_text.py
│ ├── mask_generation.py
│ ├── object_detection.py
│ ├── pt_utils.py
│ ├── question_answering.py
│ ├── table_question_answering.py
│ ├── text2text_generation.py
│ ├── text_classification.py
│ ├── text_generation.py
│ ├── token_classification.py
│ ├── video_classification.py
│ ├── visual_question_answering.py
│ ├── zero_shot_audio_classification.py
│ ├── zero_shot_classification.py
│ ├── zero_shot_image_classification.py
│ └── zero_shot_object_detection.py
├── processing_utils.py
├── pytorch_utils.py
├── sagemaker/
│ ├── __init__.py
│ ├── trainer_sm.py
│ └── training_args_sm.py
├── testing_utils.py
├── tf_utils.py
├── time_series_utils.py
├── tokenization_utils.py
├── tokenization_utils_base.py
├── tokenization_utils_fast.py
├── tools/
│ ├── __init__.py
│ ├── agents.py
│ ├── base.py
│ ├── document_question_answering.py
│ ├── evaluate_agent.py
│ ├── image_captioning.py
│ ├── image_question_answering.py
│ ├── image_segmentation.py
│ ├── prompts.py
│ ├── python_interpreter.py
│ ├── speech_to_text.py
│ ├── text_classification.py
│ ├── text_question_answering.py
│ ├── text_summarization.py
│ ├── text_to_speech.py
│ └── translation.py
├── trainer.py
├── trainer_callback.py
├── trainer_pt_utils.py
├── trainer_seq2seq.py
├── trainer_tf.py
├── trainer_utils.py
├── training_args.py
├── training_args_seq2seq.py
├── training_args_tf.py
└── utils/
├── __init__.py
├── backbone_utils.py
├── bitsandbytes.py
├── constants.py
├── doc.py
├── dummy_detectron2_objects.py
├── dummy_flax_objects.py
├── dummy_keras_nlp_objects.py
├── dummy_pt_objects.py
├── dummy_sentencepiece_and_tokenizers_objects.py
├── dummy_sentencepiece_objects.py
├── dummy_speech_objects.py
├── dummy_tensorflow_text_objects.py
├── dummy_tf_objects.py
├── dummy_tokenizers_objects.py
├── dummy_vision_objects.py
├── fx.py
├── generic.py
├── hp_naming.py
├── hub.py
├── import_utils.py
├── logging.py
├── model_parallel_utils.py
├── notebook.py
├── quantization_config.py
├── sentencepiece_model_pb2.py
└── versions.py
Showing preview only (2,527K chars total). Download the full file or copy to clipboard to get everything.
SYMBOL INDEX (30522 symbols across 1110 files)
FILE: build_dataset.py
function build_instruction_dataset (line 19) | def build_instruction_dataset(data_path: Union[List[str],str],
class DataCollatorForSupervisedDataset (line 89) | class DataCollatorForSupervisedDataset(object):
method __call__ (line 94) | def __call__(self, instances: Sequence[Dict]) -> Dict[str, torch.Tensor]:
FILE: flash_attn_patch.py
function forward (line 17) | def forward(
function _prepare_decoder_attention_mask (line 105) | def _prepare_decoder_attention_mask(
function replace_llama_attn_with_flash_attn (line 112) | def replace_llama_attn_with_flash_attn():
FILE: peft/mapping.py
function get_peft_config (line 63) | def get_peft_config(config_dict):
function _prepare_prompt_learning_config (line 74) | def _prepare_prompt_learning_config(peft_config, model_config):
function _prepare_lora_config (line 116) | def _prepare_lora_config(peft_config, model_config):
function get_peft_model (line 129) | def get_peft_model(model, peft_config):
FILE: peft/peft_model.py
class PeftModel (line 48) | class PeftModel(PushToHubMixin, torch.nn.Module):
method __init__ (line 72) | def __init__(self, model, peft_config: PeftConfig): # casualLM, LoraCo...
method save_pretrained (line 87) | def save_pretrained(self, save_directory, **kwargs):
method from_pretrained (line 120) | def from_pretrained(cls, model, model_id, **kwargs):
method _setup_prompt_encoder (line 189) | def _setup_prompt_encoder(self):
method get_prompt_embedding_to_save (line 223) | def get_prompt_embedding_to_save(self):
method get_prompt (line 234) | def get_prompt(self, batch_size):
method print_trainable_parameters (line 268) | def print_trainable_parameters(self):
method __getattr__ (line 290) | def __getattr__(self, name: str):
method forward (line 297) | def forward(self, *args, **kwargs): # pylint: disable=E0202
method disable_adapter (line 304) | def disable_adapter(self):
method get_base_model (line 319) | def get_base_model(self):
class PeftModelForSequenceClassification (line 326) | class PeftModelForSequenceClassification(PeftModel):
method __init__ (line 352) | def __init__(self, model, peft_config: PeftConfig):
method forward (line 364) | def forward( # pylint: disable=W0221
method _prefix_tuning_forward (line 425) | def _prefix_tuning_forward(
class PeftModelForCausalLM (line 497) | class PeftModelForCausalLM(PeftModel):
method __init__ (line 518) | def __init__(self, model, peft_config: PeftConfig): # casualLM, LoraCo...
method forward (line 522) | def forward(# pylint: disable=W0221
method generate (line 584) | def generate(self, **kwargs):
method prepare_inputs_for_generation (line 595) | def prepare_inputs_for_generation(self, *args, **kwargs):
class PeftModelForSeq2SeqLM (line 613) | class PeftModelForSeq2SeqLM(PeftModel):
method __init__ (line 635) | def __init__(self, model, peft_config: PeftConfig):
method forward (line 642) | def forward(
method generate (line 732) | def generate(self, **kwargs):
method prepare_inputs_for_generation (line 771) | def prepare_inputs_for_generation(self, *args, **kwargs):
class PeftModelForTokenClassification (line 780) | class PeftModelForTokenClassification(PeftModel):
method __init__ (line 806) | def __init__(self, model, peft_config: PeftConfig):
method forward (line 818) | def forward(
method _prefix_tuning_forward (line 879) | def _prefix_tuning_forward(
FILE: peft/tuners/lora.py
class LoraConfig (line 32) | class LoraConfig(PeftConfig):
method __post_init__ (line 82) | def __post_init__(self):
class LoraModel (line 86) | class LoraModel(torch.nn.Module):
method __init__ (line 110) | def __init__(self, config, model): # LoraConfig, CasualLM
method _find_and_replace (line 118) | def _find_and_replace(self):
method _get_submodules (line 161) | def _get_submodules(self, key):
method _replace_module (line 167) | def _replace_module(self, parent_module, child_name, new_module, old_m...
method __getattr__ (line 181) | def __getattr__(self, name: str):
method modules_to_save (line 189) | def modules_to_save(self):
method get_peft_config_as_dict (line 192) | def get_peft_config_as_dict(self, inference: bool = False):
method _set_adapter_layers (line 198) | def _set_adapter_layers(self, enabled=True):
method enable_adapter_layers (line 203) | def enable_adapter_layers(self):
method disable_adapter_layers (line 206) | def disable_adapter_layers(self):
function mark_only_lora_as_trainable (line 221) | def mark_only_lora_as_trainable(model: nn.Module, bias: str = "none") ->...
class LoraLayer (line 239) | class LoraLayer:
method __init__ (line 240) | def __init__(
class Linear (line 260) | class Linear(nn.Linear, LoraLayer):
method __init__ (line 262) | def __init__(
method reset_parameters (line 299) | def reset_parameters(self):
method train (line 309) | def train(self, mode: bool = True):
method eval (line 316) | def eval(self):
method cv_squared (line 323) | def cv_squared(self, x):
method forward (line 338) | def forward(self, x: torch.Tensor, task_types=None):
FILE: peft/tuners/p_tuning.py
class PromptEncoderReparameterizationType (line 26) | class PromptEncoderReparameterizationType(str, enum.Enum):
class PromptEncoderConfig (line 32) | class PromptEncoderConfig(PromptLearningConfig):
method __post_init__ (line 61) | def __post_init__(self):
class PromptEncoder (line 67) | class PromptEncoder(torch.nn.Module):
method __init__ (line 103) | def __init__(self, config):
method forward (line 150) | def forward(self, indices):
FILE: peft/tuners/prefix_tuning.py
class PrefixTuningConfig (line 25) | class PrefixTuningConfig(PromptLearningConfig):
method __post_init__ (line 43) | def __post_init__(self):
class PrefixEncoder (line 49) | class PrefixEncoder(torch.nn.Module):
method __init__ (line 77) | def __init__(self, config):
method forward (line 95) | def forward(self, prefix: torch.Tensor):
FILE: peft/tuners/prompt_tuning.py
class PromptTuningInit (line 26) | class PromptTuningInit(str, enum.Enum):
class PromptTuningConfig (line 32) | class PromptTuningConfig(PromptLearningConfig):
method __post_init__ (line 61) | def __post_init__(self):
class PromptEmbedding (line 65) | class PromptEmbedding(torch.nn.Module):
method __init__ (line 93) | def __init__(self, config, word_embeddings):
method forward (line 117) | def forward(self, indices):
FILE: peft/utils/config.py
class PeftType (line 27) | class PeftType(str, enum.Enum):
class TaskType (line 34) | class TaskType(str, enum.Enum):
class PeftConfigMixin (line 41) | class PeftConfigMixin(PushToHubMixin):
method __dict__ (line 54) | def __dict__(self):
method to_dict (line 57) | def to_dict(self):
method save_pretrained (line 60) | def save_pretrained(self, save_directory, **kwargs):
method from_pretrained (line 84) | def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):
method from_json_file (line 113) | def from_json_file(cls, path_json_file, **kwargs):
class PeftConfig (line 128) | class PeftConfig(PeftConfigMixin):
class PromptLearningConfig (line 145) | class PromptLearningConfig(PeftConfig):
FILE: peft/utils/other.py
function bloom_model_postprocess_past_key_value (line 20) | def bloom_model_postprocess_past_key_value(past_key_values):
function shift_tokens_right (line 39) | def shift_tokens_right(input_ids: torch.Tensor, pad_token_id: int, decod...
function _set_trainable (line 60) | def _set_trainable(model):
function fsdp_auto_wrap_policy (line 67) | def fsdp_auto_wrap_policy(model):
function transpose (line 102) | def transpose(weight, fan_in_fan_out):
FILE: peft/utils/save_and_load.py
function get_peft_model_state_dict (line 19) | def get_peft_model_state_dict(model, state_dict=None):
function set_peft_model_state_dict (line 65) | def set_peft_model_state_dict(model, peft_model_state_dict):
FILE: run_loramoe.py
class SavePeftModelCallback (line 60) | class SavePeftModelCallback(transformers.TrainerCallback):
method save_model (line 61) | def save_model(self, args, state, kwargs):
method on_save (line 71) | def on_save(self, args, state, control, **kwargs):
method on_train_end (line 75) | def on_train_end(self, args, state, control, **kwargs):
function prepare_model_for_kbit_training (line 81) | def prepare_model_for_kbit_training(model, use_gradient_checkpointing=Tr...
class ModelArguments (line 122) | class ModelArguments:
method __post_init__ (line 191) | def __post_init__(self):
class DataTrainingArguments (line 199) | class DataTrainingArguments:
class MyTrainingArguments (line 236) | class MyTrainingArguments(TrainingArguments):
function main (line 256) | def main():
FILE: transformers/activations.py
class PytorchGELUTanh (line 28) | class PytorchGELUTanh(nn.Module):
method __init__ (line 37) | def __init__(self):
method forward (line 45) | def forward(self, input: Tensor) -> Tensor:
class NewGELUActivation (line 49) | class NewGELUActivation(nn.Module):
method forward (line 55) | def forward(self, input: Tensor) -> Tensor:
class GELUActivation (line 59) | class GELUActivation(nn.Module):
method __init__ (line 67) | def __init__(self, use_gelu_python: bool = False):
method _gelu_python (line 74) | def _gelu_python(self, input: Tensor) -> Tensor:
method forward (line 77) | def forward(self, input: Tensor) -> Tensor:
class FastGELUActivation (line 81) | class FastGELUActivation(nn.Module):
method forward (line 86) | def forward(self, input: Tensor) -> Tensor:
class QuickGELUActivation (line 90) | class QuickGELUActivation(nn.Module):
method forward (line 95) | def forward(self, input: Tensor) -> Tensor:
class ClippedGELUActivation (line 99) | class ClippedGELUActivation(nn.Module):
method __init__ (line 112) | def __init__(self, min: float, max: float):
method forward (line 120) | def forward(self, x: Tensor) -> Tensor:
class AccurateGELUActivation (line 124) | class AccurateGELUActivation(nn.Module):
method __init__ (line 132) | def __init__(self):
method forward (line 136) | def forward(self, input: Tensor) -> Tensor:
class SiLUActivation (line 140) | class SiLUActivation(nn.Module):
method forward (line 149) | def forward(self, input: Tensor) -> Tensor:
class MishActivation (line 153) | class MishActivation(nn.Module):
method __init__ (line 159) | def __init__(self):
method _mish_python (line 166) | def _mish_python(self, input: Tensor) -> Tensor:
method forward (line 169) | def forward(self, input: Tensor) -> Tensor:
class LinearActivation (line 173) | class LinearActivation(nn.Module):
method forward (line 178) | def forward(self, input: Tensor) -> Tensor:
class LaplaceActivation (line 182) | class LaplaceActivation(nn.Module):
method forward (line 190) | def forward(self, input, mu=0.707107, sigma=0.282095):
class ReLUSquaredActivation (line 195) | class ReLUSquaredActivation(nn.Module):
method forward (line 200) | def forward(self, input):
class ClassInstantier (line 206) | class ClassInstantier(OrderedDict):
method __getitem__ (line 207) | def __getitem__(self, key):
function get_activation (line 236) | def get_activation(activation_string):
FILE: transformers/activations_tf.py
function _gelu (line 21) | def _gelu(x):
function _gelu_new (line 34) | def _gelu_new(x):
function mish (line 52) | def mish(x):
function gelu_fast (line 58) | def gelu_fast(x):
function quick_gelu (line 66) | def quick_gelu(x):
function gelu_10 (line 72) | def gelu_10(x):
function glu (line 86) | def glu(x, axis=-1):
function approximate_gelu_wrap (line 104) | def approximate_gelu_wrap(x):
function get_tf_activation (line 130) | def get_tf_activation(activation_string):
FILE: transformers/audio_utils.py
function hertz_to_mel (line 25) | def hertz_to_mel(freq: Union[float, np.ndarray], mel_scale: str = "htk")...
function mel_to_hertz (line 59) | def mel_to_hertz(mels: Union[float, np.ndarray], mel_scale: str = "htk")...
function _create_triangular_filter_bank (line 93) | def _create_triangular_filter_bank(fft_freqs: np.ndarray, filter_freqs: ...
function mel_filter_bank (line 115) | def mel_filter_bank(
function optimal_fft_length (line 193) | def optimal_fft_length(window_length: int) -> int:
function window_function (line 206) | def window_function(
function spectrogram (line 267) | def spectrogram(
function power_to_db (line 455) | def power_to_db(
function amplitude_to_db (line 506) | def amplitude_to_db(
function get_mel_filter_banks (line 558) | def get_mel_filter_banks(
function fram_wave (line 582) | def fram_wave(waveform: np.array, hop_length: int = 160, fft_window_size...
function stft (line 638) | def stft(frames: np.array, windowing_function: np.array, fft_window_size...
FILE: transformers/benchmark/benchmark.py
class PyTorchBenchmark (line 50) | class PyTorchBenchmark(Benchmark):
method framework_version (line 56) | def framework_version(self):
method _inference_speed (line 59) | def _inference_speed(self, model_name: str, batch_size: int, sequence_...
method _inference_memory (line 63) | def _inference_memory(
method _train_speed (line 69) | def _train_speed(self, model_name: str, batch_size: int, sequence_leng...
method _train_memory (line 73) | def _train_memory(
method _prepare_inference_func (line 79) | def _prepare_inference_func(self, model_name: str, batch_size: int, se...
method _prepare_train_func (line 138) | def _prepare_train_func(self, model_name: str, batch_size: int, sequen...
method _measure_speed (line 198) | def _measure_speed(self, func) -> float:
method _measure_memory (line 226) | def _measure_memory(self, func: Callable[[], None]) -> [Memory, Memory...
FILE: transformers/benchmark/benchmark_args.py
class PyTorchBenchmarkArguments (line 35) | class PyTorchBenchmarkArguments(BenchmarkArguments):
method __init__ (line 46) | def __init__(self, **kwargs):
method _setup_devices (line 78) | def _setup_devices(self) -> Tuple["torch.device", int]:
method is_tpu (line 93) | def is_tpu(self):
method device_idx (line 97) | def device_idx(self) -> int:
method device (line 103) | def device(self) -> "torch.device":
method n_gpu (line 108) | def n_gpu(self):
method is_gpu (line 113) | def is_gpu(self):
FILE: transformers/benchmark/benchmark_args_tf.py
class TensorFlowBenchmarkArguments (line 32) | class TensorFlowBenchmarkArguments(BenchmarkArguments):
method __init__ (line 43) | def __init__(self, **kwargs):
method _setup_tpu (line 79) | def _setup_tpu(self) -> Tuple["tf.distribute.cluster_resolver.TPUClust...
method _setup_strategy (line 93) | def _setup_strategy(self) -> Tuple["tf.distribute.Strategy", "tf.distr...
method is_tpu (line 113) | def is_tpu(self) -> bool:
method strategy (line 118) | def strategy(self) -> "tf.distribute.Strategy":
method gpu_list (line 123) | def gpu_list(self):
method n_gpu (line 128) | def n_gpu(self) -> int:
method is_gpu (line 135) | def is_gpu(self) -> bool:
FILE: transformers/benchmark/benchmark_args_utils.py
function list_field (line 30) | def list_field(default=None, metadata=None):
class BenchmarkArguments (line 35) | class BenchmarkArguments:
method __post_init__ (line 135) | def __post_init__(self):
method to_json_string (line 143) | def to_json_string(self):
method model_names (line 150) | def model_names(self):
method do_multi_processing (line 158) | def do_multi_processing(self):
FILE: transformers/benchmark/benchmark_tf.py
function run_with_tf_optimizations (line 51) | def run_with_tf_optimizations(do_eager_mode: bool, use_xla: bool):
function random_input_ids (line 73) | def random_input_ids(batch_size: int, sequence_length: int, vocab_size: ...
class TensorFlowBenchmark (line 79) | class TensorFlowBenchmark(Benchmark):
method framework_version (line 85) | def framework_version(self):
method _inference_speed (line 88) | def _inference_speed(self, model_name: str, batch_size: int, sequence_...
method _train_speed (line 95) | def _train_speed(self, model_name: str, batch_size: int, sequence_leng...
method _inference_memory (line 101) | def _inference_memory(
method _train_memory (line 112) | def _train_memory(
method _prepare_inference_func (line 123) | def _prepare_inference_func(self, model_name: str, batch_size: int, se...
method _prepare_train_func (line 164) | def _prepare_train_func(self, model_name: str, batch_size: int, sequen...
method _measure_speed (line 213) | def _measure_speed(self, func) -> float:
method _measure_memory (line 232) | def _measure_memory(self, func: Callable[[], None]) -> [Memory, Memory...
FILE: transformers/benchmark/benchmark_utils.py
function separate_process_wrapper_fn (line 76) | def separate_process_wrapper_fn(func: Callable[[], None], do_multi_proce...
function is_memory_tracing_enabled (line 112) | def is_memory_tracing_enabled():
class Frame (line 117) | class Frame(NamedTuple):
class UsedMemoryState (line 135) | class UsedMemoryState(NamedTuple):
class Memory (line 151) | class Memory(NamedTuple):
method __repr__ (line 161) | def __repr__(self) -> str:
class MemoryState (line 165) | class MemoryState(NamedTuple):
class MemorySummary (line 181) | class MemorySummary(NamedTuple):
function measure_peak_memory_cpu (line 204) | def measure_peak_memory_cpu(function: Callable[[], None], interval=0.5, ...
function start_memory_tracing (line 323) | def start_memory_tracing(
function stop_memory_tracing (line 475) | def stop_memory_tracing(
function bytes_to_mega_bytes (line 591) | def bytes_to_mega_bytes(memory_amount: int) -> int:
class Benchmark (line 596) | class Benchmark(ABC):
method __init__ (line 606) | def __init__(self, args: BenchmarkArguments = None, configs: Pretraine...
method print_fn (line 633) | def print_fn(self):
method framework_version (line 649) | def framework_version(self):
method _inference_speed (line 653) | def _inference_speed(self, model_name: str, batch_size: int, sequence_...
method _train_speed (line 657) | def _train_speed(self, model_name: str, batch_size: int, sequence_leng...
method _inference_memory (line 661) | def _inference_memory(
method _train_memory (line 667) | def _train_memory(
method inference_speed (line 672) | def inference_speed(self, *args, **kwargs) -> float:
method train_speed (line 675) | def train_speed(self, *args, **kwargs) -> float:
method inference_memory (line 678) | def inference_memory(self, *args, **kwargs) -> [Memory, Optional[Memor...
method train_memory (line 681) | def train_memory(self, *args, **kwargs) -> [Memory, Optional[MemorySum...
method run (line 684) | def run(self):
method environment_info (line 784) | def environment_info(self):
method print_results (line 842) | def print_results(self, result_dict, type_label):
method print_memory_trace_statistics (line 864) | def print_memory_trace_statistics(self, summary: MemorySummary):
method save_to_csv (line 888) | def save_to_csv(self, result_dict, filename):
FILE: transformers/commands/__init__.py
class BaseTransformersCLICommand (line 19) | class BaseTransformersCLICommand(ABC):
method register_subcommand (line 22) | def register_subcommand(parser: ArgumentParser):
method run (line 26) | def run(self):
FILE: transformers/commands/add_new_model.py
function add_new_model_command_factory (line 37) | def add_new_model_command_factory(args: Namespace):
class AddNewModelCommand (line 41) | class AddNewModelCommand(BaseTransformersCLICommand):
method register_subcommand (line 43) | def register_subcommand(parser: ArgumentParser):
method __init__ (line 52) | def __init__(self, testing: bool, testing_file: str, path=None, *args):
method run (line 57) | def run(self):
FILE: transformers/commands/add_new_model_like.py
class ModelPatterns (line 41) | class ModelPatterns:
method __post_init__ (line 86) | def __post_init__(self):
function is_empty_line (line 119) | def is_empty_line(line: str) -> bool:
function find_indent (line 126) | def find_indent(line: str) -> int:
function parse_module_content (line 136) | def parse_module_content(content: str) -> List[str]:
function extract_block (line 176) | def extract_block(content: str, indent_level: int = 0) -> str:
function add_content_to_text (line 226) | def add_content_to_text(
function add_content_to_file (line 284) | def add_content_to_file(
function replace_model_patterns (line 322) | def replace_model_patterns(
function simplify_replacements (line 388) | def simplify_replacements(replacements):
function get_module_from_file (line 425) | def get_module_from_file(module_file: Union[str, os.PathLike]) -> str:
function remove_attributes (line 455) | def remove_attributes(obj, target_attr):
function duplicate_module (line 495) | def duplicate_module(
function filter_framework_files (line 595) | def filter_framework_files(
function get_model_files (line 628) | def get_model_files(model_type: str, frameworks: Optional[List[str]] = N...
function find_base_model_checkpoint (line 673) | def find_base_model_checkpoint(
function get_default_frameworks (line 707) | def get_default_frameworks():
function retrieve_model_classes (line 724) | def retrieve_model_classes(model_type: str, frameworks: Optional[List[st...
function retrieve_info_for_model (line 765) | def retrieve_info_for_model(model_type, frameworks: Optional[List[str]] ...
function clean_frameworks_in_init (line 850) | def clean_frameworks_in_init(
function add_model_to_main_init (line 929) | def add_model_to_main_init(
function insert_tokenizer_in_auto_module (line 1018) | def insert_tokenizer_in_auto_module(old_model_patterns: ModelPatterns, n...
function add_model_to_auto_classes (line 1080) | def add_model_to_auto_classes(
function duplicate_doc_file (line 1173) | def duplicate_doc_file(
function create_new_model_like (line 1271) | def create_new_model_like(
function add_new_model_like_command_factory (line 1447) | def add_new_model_like_command_factory(args: Namespace):
class AddNewModelLikeCommand (line 1451) | class AddNewModelLikeCommand(BaseTransformersCLICommand):
method register_subcommand (line 1453) | def register_subcommand(parser: ArgumentParser):
method __init__ (line 1463) | def __init__(self, config_file=None, path_to_repo=None, *args):
method run (line 1483) | def run(self):
function get_user_field (line 1501) | def get_user_field(
function convert_to_bool (line 1553) | def convert_to_bool(x: str) -> bool:
function get_user_input (line 1564) | def get_user_input():
FILE: transformers/commands/convert.py
function convert_command_factory (line 21) | def convert_command_factory(args: Namespace):
class ConvertCommand (line 38) | class ConvertCommand(BaseTransformersCLICommand):
method register_subcommand (line 40) | def register_subcommand(parser: ArgumentParser):
method __init__ (line 67) | def __init__(
method run (line 85) | def run(self):
FILE: transformers/commands/download.py
function download_command_factory (line 20) | def download_command_factory(args):
class DownloadCommand (line 24) | class DownloadCommand(BaseTransformersCLICommand):
method register_subcommand (line 26) | def register_subcommand(parser: ArgumentParser):
method __init__ (line 37) | def __init__(self, model: str, cache: str, force: bool):
method run (line 42) | def run(self):
FILE: transformers/commands/env.py
function info_command_factory (line 26) | def info_command_factory(_):
class EnvironmentCommand (line 30) | class EnvironmentCommand(BaseTransformersCLICommand):
method register_subcommand (line 32) | def register_subcommand(parser: ArgumentParser):
method run (line 36) | def run(self):
method format_dict (line 103) | def format_dict(d):
FILE: transformers/commands/lfs.py
class LfsCommands (line 36) | class LfsCommands(BaseTransformersCLICommand):
method register_subcommand (line 55) | def register_subcommand(parser: ArgumentParser):
class LfsEnableCommand (line 75) | class LfsEnableCommand:
method __init__ (line 76) | def __init__(self, args):
method run (line 79) | def run(self):
function write_msg (line 98) | def write_msg(msg: Dict):
function read_msg (line 105) | def read_msg() -> Optional[Dict]:
class FileSlice (line 120) | class FileSlice(AbstractContextManager):
method __init__ (line 127) | def __init__(self, filepath: str, seek_from: int, read_limit: int):
method __enter__ (line 133) | def __enter__(self):
method __len__ (line 138) | def __len__(self):
method read (line 142) | def read(self, n=-1):
method __iter__ (line 150) | def __iter__(self):
method __exit__ (line 153) | def __exit__(self, *args):
class LfsUploadCommand (line 157) | class LfsUploadCommand:
method __init__ (line 158) | def __init__(self, args):
method run (line 161) | def run(self):
FILE: transformers/commands/pt_to_tf.py
function convert_command_factory (line 57) | def convert_command_factory(args: Namespace):
class PTtoTFCommand (line 75) | class PTtoTFCommand(BaseTransformersCLICommand):
method register_subcommand (line 77) | def register_subcommand(parser: ArgumentParser):
method find_pt_tf_differences (line 140) | def find_pt_tf_differences(pt_outputs, tf_outputs):
method __init__ (line 177) | def __init__(
method get_inputs (line 199) | def get_inputs(self, pt_model, tf_dummy_inputs, config):
method run (line 269) | def run(self):
FILE: transformers/commands/run.py
function try_infer_format_from_ext (line 25) | def try_infer_format_from_ext(path: str):
function run_command_factory (line 39) | def run_command_factory(args):
class RunCommand (line 58) | class RunCommand(BaseTransformersCLICommand):
method __init__ (line 59) | def __init__(self, nlp: Pipeline, reader: PipelineDataFormat):
method register_subcommand (line 64) | def register_subcommand(parser: ArgumentParser):
method run (line 95) | def run(self):
FILE: transformers/commands/serving.py
function Body (line 34) | def Body(*x, **y):
function serve_command_factory (line 43) | def serve_command_factory(args: Namespace):
class ServeModelInfoResult (line 59) | class ServeModelInfoResult(BaseModel):
class ServeTokenizeResult (line 67) | class ServeTokenizeResult(BaseModel):
class ServeDeTokenizeResult (line 76) | class ServeDeTokenizeResult(BaseModel):
class ServeForwardResult (line 84) | class ServeForwardResult(BaseModel):
class ServeCommand (line 92) | class ServeCommand(BaseTransformersCLICommand):
method register_subcommand (line 94) | def register_subcommand(parser: ArgumentParser):
method __init__ (line 124) | def __init__(self, pipeline: Pipeline, host: str, port: int, workers: ...
method run (line 173) | def run(self):
method model_info (line 176) | def model_info(self):
method tokenize (line 179) | def tokenize(self, text_input: str = Body(None, embed=True), return_id...
method detokenize (line 197) | def detokenize(
method forward (line 214) | async def forward(self, inputs=Body(None, embed=True)):
FILE: transformers/commands/train.py
function train_command_factory (line 32) | def train_command_factory(args: Namespace):
class TrainCommand (line 41) | class TrainCommand(BaseTransformersCLICommand):
method register_subcommand (line 43) | def register_subcommand(parser: ArgumentParser):
method __init__ (line 93) | def __init__(self, args: Namespace):
method run (line 138) | def run(self):
method run_torch (line 143) | def run_torch(self):
method run_tf (line 146) | def run_tf(self):
FILE: transformers/commands/transformers_cli.py
function main (line 30) | def main():
FILE: transformers/commands/user.py
class UserCommands (line 25) | class UserCommands(BaseTransformersCLICommand):
method register_subcommand (line 27) | def register_subcommand(parser: ArgumentParser):
class ANSI (line 56) | class ANSI:
method bold (line 67) | def bold(cls, s):
method red (line 71) | def red(cls, s):
method gray (line 75) | def gray(cls, s):
function tabulate (line 79) | def tabulate(rows: List[List[Union[str, int]]], headers: List[str]) -> str:
class BaseUserCommand (line 96) | class BaseUserCommand:
method __init__ (line 97) | def __init__(self, args):
class LoginCommand (line 101) | class LoginCommand(BaseUserCommand):
method run (line 102) | def run(self):
class WhoamiCommand (line 112) | class WhoamiCommand(BaseUserCommand):
method run (line 113) | def run(self):
class LogoutCommand (line 135) | class LogoutCommand(BaseUserCommand):
method run (line 136) | def run(self):
class RepoCreateCommand (line 146) | class RepoCreateCommand(BaseUserCommand):
method run (line 147) | def run(self):
FILE: transformers/configuration_utils.py
class PretrainedConfig (line 49) | class PretrainedConfig(PushToHubMixin):
method __setattr__ (line 253) | def __setattr__(self, key, value):
method __getattribute__ (line 258) | def __getattribute__(self, key):
method __init__ (line 263) | def __init__(self, **kwargs):
method name_or_path (line 395) | def name_or_path(self) -> str:
method name_or_path (line 399) | def name_or_path(self, value):
method use_return_dict (line 403) | def use_return_dict(self) -> bool:
method num_labels (line 411) | def num_labels(self) -> int:
method num_labels (line 418) | def num_labels(self, num_labels: int):
method save_pretrained (line 423) | def save_pretrained(self, save_directory: Union[str, os.PathLike], pus...
method from_pretrained (line 470) | def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os....
method get_config_dict (line 557) | def get_config_dict(
method _get_config_dict (line 588) | def _get_config_dict(
method from_dict (line 678) | def from_dict(cls, config_dict: Dict[str, Any], **kwargs) -> "Pretrain...
method from_json_file (line 732) | def from_json_file(cls, json_file: Union[str, os.PathLike]) -> "Pretra...
method _dict_from_json_file (line 748) | def _dict_from_json_file(cls, json_file: Union[str, os.PathLike]):
method __eq__ (line 753) | def __eq__(self, other):
method __repr__ (line 756) | def __repr__(self):
method to_diff_dict (line 759) | def to_diff_dict(self) -> Dict[str, Any]:
method to_dict (line 798) | def to_dict(self) -> Dict[str, Any]:
method to_json_string (line 827) | def to_json_string(self, use_diff: bool = True) -> str:
method to_json_file (line 845) | def to_json_file(self, json_file_path: Union[str, os.PathLike], use_di...
method update (line 859) | def update(self, config_dict: Dict[str, Any]):
method update_from_string (line 869) | def update_from_string(self, update_str: str):
method dict_torch_dtype_to_str (line 907) | def dict_torch_dtype_to_str(self, d: Dict[str, Any]) -> None:
method register_for_auto_class (line 920) | def register_for_auto_class(cls, auto_class="AutoConfig"):
function get_configuration_file (line 946) | def get_configuration_file(configuration_files: List[str]) -> str:
FILE: transformers/convert_graph_to_onnx.py
class OnnxConverterArgumentParser (line 46) | class OnnxConverterArgumentParser(ArgumentParser):
method __init__ (line 51) | def __init__(self):
function generate_identified_filename (line 92) | def generate_identified_filename(filename: Path, identifier: str) -> Path:
function check_onnxruntime_requirements (line 105) | def check_onnxruntime_requirements(minimum_version: Version):
function ensure_valid_input (line 134) | def ensure_valid_input(model, tokens, input_names):
function infer_shapes (line 162) | def infer_shapes(nlp: Pipeline, framework: str) -> Tuple[List[str], List...
function load_graph_from_args (line 226) | def load_graph_from_args(
function convert_pytorch (line 257) | def convert_pytorch(nlp: Pipeline, opset: int, output: Path, use_externa...
function convert_tensorflow (line 312) | def convert_tensorflow(nlp: Pipeline, opset: int, output: Path):
function convert (line 352) | def convert(
function optimize (line 402) | def optimize(onnx_model_path: Path) -> Path:
function quantize (line 427) | def quantize(onnx_model_path: Path) -> Path:
function verify (line 502) | def verify(path: Path):
FILE: transformers/convert_pytorch_checkpoint_to_tf2.py
function convert_pt_checkpoint_to_tf (line 304) | def convert_pt_checkpoint_to_tf(
function convert_all_pt_checkpoints_to_tf (line 351) | def convert_all_pt_checkpoints_to_tf(
FILE: transformers/convert_slow_tokenizer.py
class SentencePieceExtractor (line 31) | class SentencePieceExtractor:
method __init__ (line 36) | def __init__(self, model: str):
method extract (line 43) | def extract(self, vocab_scores=None) -> Tuple[Dict[str, int], List[Tup...
function check_number_comma (line 68) | def check_number_comma(piece: str) -> bool:
class Converter (line 72) | class Converter:
method __init__ (line 73) | def __init__(self, original_tokenizer):
method converted (line 76) | def converted(self) -> Tokenizer:
class BertConverter (line 80) | class BertConverter(Converter):
method converted (line 81) | def converted(self) -> Tokenizer:
class SplinterConverter (line 119) | class SplinterConverter(Converter):
method converted (line 120) | def converted(self) -> Tokenizer:
class FunnelConverter (line 169) | class FunnelConverter(Converter):
method converted (line 170) | def converted(self) -> Tokenizer:
class MPNetConverter (line 208) | class MPNetConverter(Converter):
method converted (line 209) | def converted(self) -> Tokenizer:
class OpenAIGPTConverter (line 247) | class OpenAIGPTConverter(Converter):
method converted (line 248) | def converted(self) -> Tokenizer:
class GPT2Converter (line 274) | class GPT2Converter(Converter):
method converted (line 275) | def converted(self) -> Tokenizer:
class HerbertConverter (line 309) | class HerbertConverter(Converter):
method converted (line 310) | def converted(self) -> Tokenizer:
class RobertaConverter (line 340) | class RobertaConverter(Converter):
method converted (line 341) | def converted(self) -> Tokenizer:
class RoFormerConverter (line 369) | class RoFormerConverter(Converter):
method converted (line 370) | def converted(self) -> Tokenizer:
class DebertaConverter (line 408) | class DebertaConverter(Converter):
method converted (line 409) | def converted(self) -> Tokenizer:
class SpmConverter (line 439) | class SpmConverter(Converter):
method __init__ (line 440) | def __init__(self, *args):
method vocab (line 461) | def vocab(self, proto):
method unk_id (line 464) | def unk_id(self, proto):
method tokenizer (line 467) | def tokenizer(self, proto):
method normalizer (line 492) | def normalizer(self, proto):
method pre_tokenizer (line 501) | def pre_tokenizer(self, replacement, add_prefix_space):
method post_processor (line 504) | def post_processor(self):
method decoder (line 507) | def decoder(self, replacement, add_prefix_space):
method converted (line 510) | def converted(self) -> Tokenizer:
class AlbertConverter (line 532) | class AlbertConverter(SpmConverter):
method vocab (line 533) | def vocab(self, proto):
method normalizer (line 539) | def normalizer(self, proto):
method post_processor (line 555) | def post_processor(self):
class BarthezConverter (line 566) | class BarthezConverter(SpmConverter):
method unk_id (line 567) | def unk_id(self, proto):
method post_processor (line 571) | def post_processor(self):
class CamembertConverter (line 582) | class CamembertConverter(SpmConverter):
method vocab (line 583) | def vocab(self, proto):
method unk_id (line 596) | def unk_id(self, proto):
method post_processor (line 600) | def post_processor(self):
class DebertaV2Converter (line 611) | class DebertaV2Converter(SpmConverter):
method pre_tokenizer (line 612) | def pre_tokenizer(self, replacement, add_prefix_space):
method normalizer (line 619) | def normalizer(self, proto):
method post_processor (line 632) | def post_processor(self):
class MBartConverter (line 643) | class MBartConverter(SpmConverter):
method vocab (line 644) | def vocab(self, proto):
method unk_id (line 682) | def unk_id(self, proto):
method post_processor (line 685) | def post_processor(self):
class MBart50Converter (line 696) | class MBart50Converter(SpmConverter):
method vocab (line 697) | def vocab(self, proto):
method unk_id (line 711) | def unk_id(self, proto):
method post_processor (line 714) | def post_processor(self):
class NllbConverter (line 725) | class NllbConverter(SpmConverter):
method vocab (line 726) | def vocab(self, proto):
method unk_id (line 742) | def unk_id(self, proto):
method post_processor (line 745) | def post_processor(self):
class XLMRobertaConverter (line 756) | class XLMRobertaConverter(SpmConverter):
method vocab (line 757) | def vocab(self, proto):
method unk_id (line 768) | def unk_id(self, proto):
method post_processor (line 772) | def post_processor(self):
class XLNetConverter (line 783) | class XLNetConverter(SpmConverter):
method vocab (line 784) | def vocab(self, proto):
method normalizer (line 790) | def normalizer(self, proto):
method post_processor (line 806) | def post_processor(self):
class ReformerConverter (line 817) | class ReformerConverter(SpmConverter):
class RemBertConverter (line 821) | class RemBertConverter(SpmConverter):
method normalizer (line 823) | def normalizer(self, proto):
method post_processor (line 839) | def post_processor(self):
class BertGenerationConverter (line 850) | class BertGenerationConverter(SpmConverter):
class PegasusConverter (line 854) | class PegasusConverter(SpmConverter):
method vocab (line 855) | def vocab(self, proto):
method unk_id (line 874) | def unk_id(self, proto):
method pre_tokenizer (line 877) | def pre_tokenizer(self, replacement, add_prefix_space):
method post_processor (line 885) | def post_processor(self):
class T5Converter (line 893) | class T5Converter(SpmConverter):
method vocab (line 894) | def vocab(self, proto):
method post_processor (line 900) | def post_processor(self):
class WhisperConverter (line 910) | class WhisperConverter(Converter):
method converted (line 911) | def converted(self) -> Tokenizer:
class BigBirdConverter (line 946) | class BigBirdConverter(SpmConverter):
method post_processor (line 947) | def post_processor(self):
class CLIPConverter (line 958) | class CLIPConverter(Converter):
method converted (line 959) | def converted(self) -> Tokenizer:
class LayoutLMv2Converter (line 1001) | class LayoutLMv2Converter(Converter):
method converted (line 1002) | def converted(self) -> Tokenizer:
class BlenderbotConverter (line 1040) | class BlenderbotConverter(Converter):
method converted (line 1041) | def converted(self) -> Tokenizer:
class XGLMConverter (line 1069) | class XGLMConverter(SpmConverter):
method vocab (line 1070) | def vocab(self, proto):
method unk_id (line 1083) | def unk_id(self, proto):
method post_processor (line 1087) | def post_processor(self):
class LlamaConverter (line 1098) | class LlamaConverter(SpmConverter):
method vocab (line 1101) | def vocab(self, proto):
method unk_id (line 1110) | def unk_id(self, proto):
method decoder (line 1114) | def decoder(self, replacement, add_prefix_space):
method tokenizer (line 1124) | def tokenizer(self, proto):
method normalizer (line 1149) | def normalizer(self, proto):
method pre_tokenizer (line 1157) | def pre_tokenizer(self, replacement, add_prefix_space):
method post_processor (line 1160) | def post_processor(self):
class MarkupLMConverter (line 1189) | class MarkupLMConverter(Converter):
method converted (line 1190) | def converted(self) -> Tokenizer:
function convert_slow_tokenizer (line 1282) | def convert_slow_tokenizer(transformer_tokenizer) -> Tokenizer:
FILE: transformers/convert_slow_tokenizers_checkpoints_to_fast.py
function convert_slow_checkpoint_to_fast (line 34) | def convert_slow_checkpoint_to_fast(tokenizer_name, checkpoint_name, dum...
FILE: transformers/convert_tf_hub_seq_to_seq_bert_to_pytorch.py
function convert_tf_checkpoint_to_pytorch (line 33) | def convert_tf_checkpoint_to_pytorch(tf_hub_path, pytorch_dump_path, is_...
FILE: transformers/data/data_collator.py
class DataCollatorMixin (line 38) | class DataCollatorMixin:
method __call__ (line 39) | def __call__(self, features, return_tensors=None):
function default_data_collator (line 52) | def default_data_collator(features: List[InputDataClass], return_tensors...
class DefaultDataCollator (line 78) | class DefaultDataCollator(DataCollatorMixin):
method __call__ (line 99) | def __call__(self, features: List[Dict[str, Any]], return_tensors=None...
function torch_default_data_collator (line 105) | def torch_default_data_collator(features: List[InputDataClass]) -> Dict[...
function tf_default_data_collator (line 141) | def tf_default_data_collator(features: List[InputDataClass]) -> Dict[str...
function numpy_default_data_collator (line 182) | def numpy_default_data_collator(features: List[InputDataClass]) -> Dict[...
class DataCollatorWithPadding (line 215) | class DataCollatorWithPadding:
method __call__ (line 248) | def __call__(self, features: List[Dict[str, Any]]) -> Dict[str, Any]:
class DataCollatorForTokenClassification (line 266) | class DataCollatorForTokenClassification(DataCollatorMixin):
method torch_call (line 302) | def torch_call(self, features):
method tf_call (line 341) | def tf_call(self, features):
method numpy_call (line 372) | def numpy_call(self, features):
function _torch_collate_batch (line 402) | def _torch_collate_batch(examples, tokenizer, pad_to_multiple_of: Option...
function _tf_collate_batch (line 438) | def _tf_collate_batch(examples, tokenizer, pad_to_multiple_of: Optional[...
function _numpy_collate_batch (line 476) | def _numpy_collate_batch(examples, tokenizer, pad_to_multiple_of: Option...
function tolist (line 508) | def tolist(x):
class DataCollatorForSeq2Seq (line 517) | class DataCollatorForSeq2Seq:
method __call__ (line 559) | def __call__(self, features, return_tensors=None):
class DataCollatorForLanguageModeling (line 607) | class DataCollatorForLanguageModeling(DataCollatorMixin):
method __post_init__ (line 641) | def __post_init__(self):
method tf_bernoulli (line 653) | def tf_bernoulli(shape, probability):
method tf_mask_tokens (line 659) | def tf_mask_tokens(
method tf_call (line 690) | def tf_call(self, examples: List[Union[List[int], Any, Dict[str, Any]]...
method torch_call (line 729) | def torch_call(self, examples: List[Union[List[int], Any, Dict[str, An...
method torch_mask_tokens (line 751) | def torch_mask_tokens(self, inputs: Any, special_tokens_mask: Optional...
method numpy_call (line 784) | def numpy_call(self, examples: List[Union[List[int], Any, Dict[str, An...
method numpy_mask_tokens (line 806) | def numpy_mask_tokens(self, inputs: Any, special_tokens_mask: Optional...
class DataCollatorForWholeWordMask (line 845) | class DataCollatorForWholeWordMask(DataCollatorForLanguageModeling):
method torch_call (line 860) | def torch_call(self, examples: List[Union[List[int], Any, Dict[str, An...
method tf_call (line 888) | def tf_call(self, examples: List[Union[List[int], Any, Dict[str, Any]]...
method numpy_call (line 918) | def numpy_call(self, examples: List[Union[List[int], Any, Dict[str, An...
method _whole_word_mask (line 946) | def _whole_word_mask(self, input_tokens: List[str], max_predictions=512):
method torch_mask_tokens (line 993) | def torch_mask_tokens(self, inputs: Any, mask_labels: Any) -> Tuple[An...
method tf_mask_tokens (line 1033) | def tf_mask_tokens(self, inputs: Any, mask_labels: Any) -> Tuple[Any, ...
method numpy_mask_tokens (line 1075) | def numpy_mask_tokens(self, inputs: Any, mask_labels: Any) -> Tuple[An...
class DataCollatorForSOP (line 1117) | class DataCollatorForSOP(DataCollatorForLanguageModeling):
method __init__ (line 1125) | def __init__(self, *args, **kwargs):
method __call__ (line 1132) | def __call__(self, examples: List[Dict[str, Any]]) -> Dict[str, Any]:
method mask_tokens (line 1155) | def mask_tokens(self, inputs: Any) -> Tuple[Any, Any, Any]:
class DataCollatorForPermutationLanguageModeling (line 1200) | class DataCollatorForPermutationLanguageModeling(DataCollatorMixin):
method torch_call (line 1213) | def torch_call(self, examples: List[Union[List[int], Any, Dict[str, An...
method tf_call (line 1220) | def tf_call(self, examples: List[Union[List[int], Any, Dict[str, Any]]...
method numpy_call (line 1227) | def numpy_call(self, examples: List[Union[List[int], Any, Dict[str, An...
method torch_mask_tokens (line 1234) | def torch_mask_tokens(self, inputs: Any) -> Tuple[Any, Any, Any, Any]:
method tf_mask_tokens (line 1333) | def tf_mask_tokens(self, inputs: Any) -> Tuple[Any, Any, Any, Any]:
method numpy_mask_tokens (line 1440) | def numpy_mask_tokens(self, inputs: Any) -> Tuple[Any, Any, Any, Any]:
FILE: transformers/data/datasets/glue.py
class GlueDataTrainingArguments (line 36) | class GlueDataTrainingArguments:
method __post_init__ (line 61) | def __post_init__(self):
class Split (line 65) | class Split(Enum):
class GlueDataset (line 71) | class GlueDataset(Dataset):
method __init__ (line 80) | def __init__(
method __len__ (line 154) | def __len__(self):
method __getitem__ (line 157) | def __getitem__(self, i) -> InputFeatures:
method get_labels (line 160) | def get_labels(self):
FILE: transformers/data/datasets/language_modeling.py
class TextDataset (line 40) | class TextDataset(Dataset):
method __init__ (line 45) | def __init__(
method __len__ (line 106) | def __len__(self):
method __getitem__ (line 109) | def __getitem__(self, i) -> torch.Tensor:
class LineByLineTextDataset (line 113) | class LineByLineTextDataset(Dataset):
method __init__ (line 118) | def __init__(self, tokenizer: PreTrainedTokenizer, file_path: str, blo...
method __len__ (line 139) | def __len__(self):
method __getitem__ (line 142) | def __getitem__(self, i) -> Dict[str, torch.tensor]:
class LineByLineWithRefDataset (line 146) | class LineByLineWithRefDataset(Dataset):
method __init__ (line 151) | def __init__(self, tokenizer: PreTrainedTokenizer, file_path: str, blo...
method __len__ (line 187) | def __len__(self):
method __getitem__ (line 190) | def __getitem__(self, i) -> Dict[str, torch.tensor]:
class LineByLineWithSOPTextDataset (line 194) | class LineByLineWithSOPTextDataset(Dataset):
method __init__ (line 199) | def __init__(self, tokenizer: PreTrainedTokenizer, file_dir: str, bloc...
method create_examples_from_document (line 240) | def create_examples_from_document(self, document, block_size, tokenize...
method __len__ (line 339) | def __len__(self):
method __getitem__ (line 342) | def __getitem__(self, i) -> Dict[str, torch.tensor]:
class TextDatasetForNextSentencePrediction (line 346) | class TextDatasetForNextSentencePrediction(Dataset):
method __init__ (line 351) | def __init__(
method create_examples_from_document (line 436) | def create_examples_from_document(self, document: List[List[int]], doc...
method __len__ (line 526) | def __len__(self):
method __getitem__ (line 529) | def __getitem__(self, i):
FILE: transformers/data/datasets/squad.py
class SquadDataTrainingArguments (line 38) | class SquadDataTrainingArguments:
class Split (line 104) | class Split(Enum):
class SquadDataset (line 109) | class SquadDataset(Dataset):
method __init__ (line 119) | def __init__(
method __len__ (line 194) | def __len__(self):
method __getitem__ (line 197) | def __getitem__(self, i) -> Dict[str, torch.Tensor]:
FILE: transformers/data/metrics/__init__.py
function simple_accuracy (line 30) | def simple_accuracy(preds, labels):
function acc_and_f1 (line 36) | def acc_and_f1(preds, labels):
function pearson_and_spearman (line 48) | def pearson_and_spearman(preds, labels):
function glue_compute_metrics (line 60) | def glue_compute_metrics(task_name, preds, labels):
function xnli_compute_metrics (line 90) | def xnli_compute_metrics(task_name, preds, labels):
FILE: transformers/data/metrics/squad_metrics.py
function normalize_answer (line 37) | def normalize_answer(s):
function get_tokens (line 57) | def get_tokens(s):
function compute_exact (line 63) | def compute_exact(a_gold, a_pred):
function compute_f1 (line 67) | def compute_f1(a_gold, a_pred):
function get_raw_scores (line 83) | def get_raw_scores(examples, preds):
function apply_no_ans_threshold (line 109) | def apply_no_ans_threshold(scores, na_probs, qid_to_has_ans, na_prob_thr...
function make_eval_dict (line 120) | def make_eval_dict(exact_scores, f1_scores, qid_list=None):
function merge_eval (line 141) | def merge_eval(main_eval, new_eval, prefix):
function find_best_thresh_v2 (line 146) | def find_best_thresh_v2(preds, scores, na_probs, qid_to_has_ans):
function find_all_best_thresh_v2 (line 180) | def find_all_best_thresh_v2(main_eval, preds, exact_raw, f1_raw, na_prob...
function find_best_thresh (line 191) | def find_best_thresh(preds, scores, na_probs, qid_to_has_ans):
function find_all_best_thresh (line 214) | def find_all_best_thresh(main_eval, preds, exact_raw, f1_raw, na_probs, ...
function squad_evaluate (line 224) | def squad_evaluate(examples, preds, no_answer_probs=None, no_answer_prob...
function get_final_text (line 255) | def get_final_text(pred_text, orig_text, do_lower_case, verbose_logging=...
function _get_best_indexes (line 349) | def _get_best_indexes(logits, n_best_size):
function _compute_softmax (line 361) | def _compute_softmax(scores):
function compute_predictions_logits (line 384) | def compute_predictions_logits(
function compute_predictions_log_probs (line 591) | def compute_predictions_log_probs(
FILE: transformers/data/processors/glue.py
function glue_convert_examples_to_features (line 41) | def glue_convert_examples_to_features(
function _tf_glue_convert_examples_to_features (line 78) | def _tf_glue_convert_examples_to_features(
function _glue_convert_examples_to_features (line 109) | def _glue_convert_examples_to_features(
class OutputMode (line 164) | class OutputMode(Enum):
class MrpcProcessor (line 169) | class MrpcProcessor(DataProcessor):
method __init__ (line 172) | def __init__(self, *args, **kwargs):
method get_example_from_tensor_dict (line 176) | def get_example_from_tensor_dict(self, tensor_dict):
method get_train_examples (line 185) | def get_train_examples(self, data_dir):
method get_dev_examples (line 190) | def get_dev_examples(self, data_dir):
method get_test_examples (line 194) | def get_test_examples(self, data_dir):
method get_labels (line 198) | def get_labels(self):
method _create_examples (line 202) | def _create_examples(self, lines, set_type):
class MnliProcessor (line 216) | class MnliProcessor(DataProcessor):
method __init__ (line 219) | def __init__(self, *args, **kwargs):
method get_example_from_tensor_dict (line 223) | def get_example_from_tensor_dict(self, tensor_dict):
method get_train_examples (line 232) | def get_train_examples(self, data_dir):
method get_dev_examples (line 236) | def get_dev_examples(self, data_dir):
method get_test_examples (line 240) | def get_test_examples(self, data_dir):
method get_labels (line 244) | def get_labels(self):
method _create_examples (line 248) | def _create_examples(self, lines, set_type):
class MnliMismatchedProcessor (line 262) | class MnliMismatchedProcessor(MnliProcessor):
method __init__ (line 265) | def __init__(self, *args, **kwargs):
method get_dev_examples (line 269) | def get_dev_examples(self, data_dir):
method get_test_examples (line 273) | def get_test_examples(self, data_dir):
class ColaProcessor (line 278) | class ColaProcessor(DataProcessor):
method __init__ (line 281) | def __init__(self, *args, **kwargs):
method get_example_from_tensor_dict (line 285) | def get_example_from_tensor_dict(self, tensor_dict):
method get_train_examples (line 294) | def get_train_examples(self, data_dir):
method get_dev_examples (line 298) | def get_dev_examples(self, data_dir):
method get_test_examples (line 302) | def get_test_examples(self, data_dir):
method get_labels (line 306) | def get_labels(self):
method _create_examples (line 310) | def _create_examples(self, lines, set_type):
class Sst2Processor (line 325) | class Sst2Processor(DataProcessor):
method __init__ (line 328) | def __init__(self, *args, **kwargs):
method get_example_from_tensor_dict (line 332) | def get_example_from_tensor_dict(self, tensor_dict):
method get_train_examples (line 341) | def get_train_examples(self, data_dir):
method get_dev_examples (line 345) | def get_dev_examples(self, data_dir):
method get_test_examples (line 349) | def get_test_examples(self, data_dir):
method get_labels (line 353) | def get_labels(self):
method _create_examples (line 357) | def _create_examples(self, lines, set_type):
class StsbProcessor (line 371) | class StsbProcessor(DataProcessor):
method __init__ (line 374) | def __init__(self, *args, **kwargs):
method get_example_from_tensor_dict (line 378) | def get_example_from_tensor_dict(self, tensor_dict):
method get_train_examples (line 387) | def get_train_examples(self, data_dir):
method get_dev_examples (line 391) | def get_dev_examples(self, data_dir):
method get_test_examples (line 395) | def get_test_examples(self, data_dir):
method get_labels (line 399) | def get_labels(self):
method _create_examples (line 403) | def _create_examples(self, lines, set_type):
class QqpProcessor (line 417) | class QqpProcessor(DataProcessor):
method __init__ (line 420) | def __init__(self, *args, **kwargs):
method get_example_from_tensor_dict (line 424) | def get_example_from_tensor_dict(self, tensor_dict):
method get_train_examples (line 433) | def get_train_examples(self, data_dir):
method get_dev_examples (line 437) | def get_dev_examples(self, data_dir):
method get_test_examples (line 441) | def get_test_examples(self, data_dir):
method get_labels (line 445) | def get_labels(self):
method _create_examples (line 449) | def _create_examples(self, lines, set_type):
class QnliProcessor (line 469) | class QnliProcessor(DataProcessor):
method __init__ (line 472) | def __init__(self, *args, **kwargs):
method get_example_from_tensor_dict (line 476) | def get_example_from_tensor_dict(self, tensor_dict):
method get_train_examples (line 485) | def get_train_examples(self, data_dir):
method get_dev_examples (line 489) | def get_dev_examples(self, data_dir):
method get_test_examples (line 493) | def get_test_examples(self, data_dir):
method get_labels (line 497) | def get_labels(self):
method _create_examples (line 501) | def _create_examples(self, lines, set_type):
class RteProcessor (line 515) | class RteProcessor(DataProcessor):
method __init__ (line 518) | def __init__(self, *args, **kwargs):
method get_example_from_tensor_dict (line 522) | def get_example_from_tensor_dict(self, tensor_dict):
method get_train_examples (line 531) | def get_train_examples(self, data_dir):
method get_dev_examples (line 535) | def get_dev_examples(self, data_dir):
method get_test_examples (line 539) | def get_test_examples(self, data_dir):
method get_labels (line 543) | def get_labels(self):
method _create_examples (line 547) | def _create_examples(self, lines, set_type):
class WnliProcessor (line 561) | class WnliProcessor(DataProcessor):
method __init__ (line 564) | def __init__(self, *args, **kwargs):
method get_example_from_tensor_dict (line 568) | def get_example_from_tensor_dict(self, tensor_dict):
method get_train_examples (line 577) | def get_train_examples(self, data_dir):
method get_dev_examples (line 581) | def get_dev_examples(self, data_dir):
method get_test_examples (line 585) | def get_test_examples(self, data_dir):
method get_labels (line 589) | def get_labels(self):
method _create_examples (line 593) | def _create_examples(self, lines, set_type):
FILE: transformers/data/processors/squad.py
function _improve_answer_span (line 43) | def _improve_answer_span(doc_tokens, input_start, input_end, tokenizer, ...
function _check_is_max_context (line 56) | def _check_is_max_context(doc_spans, cur_span_index, position):
function _new_check_is_max_context (line 76) | def _new_check_is_max_context(doc_spans, cur_span_index, position):
function _is_whitespace (line 98) | def _is_whitespace(c):
function squad_convert_example_to_features (line 104) | def squad_convert_example_to_features(
function squad_convert_example_to_features_init (line 311) | def squad_convert_example_to_features_init(tokenizer_for_convert: PreTra...
function squad_convert_examples_to_features (line 316) | def squad_convert_examples_to_features(
class SquadProcessor (line 541) | class SquadProcessor(DataProcessor):
method _get_example_from_tensor_dict (line 550) | def _get_example_from_tensor_dict(self, tensor_dict, evaluate=False):
method get_examples_from_dataset (line 574) | def get_examples_from_dataset(self, dataset, evaluate=False):
method get_train_examples (line 607) | def get_train_examples(self, data_dir, filename=None):
method get_dev_examples (line 629) | def get_dev_examples(self, data_dir, filename=None):
method _create_examples (line 650) | def _create_examples(self, input_data, set_type):
class SquadV1Processor (line 687) | class SquadV1Processor(SquadProcessor):
class SquadV2Processor (line 692) | class SquadV2Processor(SquadProcessor):
class SquadExample (line 697) | class SquadExample:
method __init__ (line 712) | def __init__(
class SquadFeatures (line 760) | class SquadFeatures:
method __init__ (line 787) | def __init__(
class SquadResult (line 827) | class SquadResult:
method __init__ (line 837) | def __init__(self, unique_id, start_logits, end_logits, start_top_inde...
FILE: transformers/data/processors/utils.py
class InputExample (line 30) | class InputExample:
method to_json_string (line 49) | def to_json_string(self):
class InputFeatures (line 55) | class InputFeatures:
method to_json_string (line 75) | def to_json_string(self):
class DataProcessor (line 80) | class DataProcessor:
method get_example_from_tensor_dict (line 83) | def get_example_from_tensor_dict(self, tensor_dict):
method get_train_examples (line 93) | def get_train_examples(self, data_dir):
method get_dev_examples (line 97) | def get_dev_examples(self, data_dir):
method get_test_examples (line 101) | def get_test_examples(self, data_dir):
method get_labels (line 105) | def get_labels(self):
method tfds_map (line 109) | def tfds_map(self, example):
method _read_tsv (line 119) | def _read_tsv(cls, input_file, quotechar=None):
class SingleSentenceClassificationProcessor (line 125) | class SingleSentenceClassificationProcessor(DataProcessor):
method __init__ (line 128) | def __init__(self, labels=None, examples=None, mode="classification", ...
method __len__ (line 134) | def __len__(self):
method __getitem__ (line 137) | def __getitem__(self, idx):
method create_from_csv (line 143) | def create_from_csv(
method create_from_examples (line 160) | def create_from_examples(cls, texts_or_text_and_labels, labels=None, *...
method add_examples_from_csv (line 165) | def add_examples_from_csv(
method add_examples (line 195) | def add_examples(
method get_features (line 232) | def get_features(
FILE: transformers/data/processors/xnli.py
class XnliProcessor (line 28) | class XnliProcessor(DataProcessor):
method __init__ (line 34) | def __init__(self, language, train_language=None):
method get_train_examples (line 38) | def get_train_examples(self, data_dir):
method get_test_examples (line 59) | def get_test_examples(self, data_dir):
method get_labels (line 82) | def get_labels(self):
FILE: transformers/debug_utils.py
class DebugUnderflowOverflow (line 27) | class DebugUnderflowOverflow:
method __init__ (line 145) | def __init__(self, model, max_frames_to_save=21, trace_batch_nums=[], ...
method save_frame (line 162) | def save_frame(self, frame=None):
method expand_frame (line 168) | def expand_frame(self, line):
method trace_frames (line 171) | def trace_frames(self):
method reset_saved_frames (line 175) | def reset_saved_frames(self):
method dump_saved_frames (line 178) | def dump_saved_frames(self):
method analyse_model (line 186) | def analyse_model(self):
method analyse_variable (line 194) | def analyse_variable(self, var, ctx):
method batch_start_frame (line 204) | def batch_start_frame(self):
method batch_end_frame (line 208) | def batch_end_frame(self):
method create_frame (line 211) | def create_frame(self, module, input, output):
method register_forward_hook (line 239) | def register_forward_hook(self):
method _register_forward_hook (line 242) | def _register_forward_hook(self, module):
method forward_hook (line 245) | def forward_hook(self, module, input, output):
function get_abs_min_max (line 293) | def get_abs_min_max(var, ctx):
function detect_overflow (line 298) | def detect_overflow(var, ctx):
class DebugOption (line 344) | class DebugOption(ExplicitEnum):
FILE: transformers/deepspeed.py
function is_deepspeed_available (line 32) | def is_deepspeed_available():
class HfDeepSpeedConfig (line 44) | class HfDeepSpeedConfig(DeepSpeedConfig):
method __init__ (line 61) | def __init__(self, config_file_or_dict):
class HfTrainerDeepSpeedConfig (line 69) | class HfTrainerDeepSpeedConfig(HfDeepSpeedConfig):
method __init__ (line 75) | def __init__(self, config_file_or_dict):
method dtype (line 80) | def dtype(self):
method is_auto (line 85) | def is_auto(self, ds_key_long):
method fill_match (line 92) | def fill_match(self, ds_key_long, hf_val, hf_key=None, must_match=True):
method trainer_config_process (line 120) | def trainer_config_process(self, args):
method trainer_config_finalize (line 178) | def trainer_config_finalize(self, args, model, num_training_steps):
function set_hf_deepspeed_config (line 230) | def set_hf_deepspeed_config(hf_deepspeed_config_obj):
function unset_hf_deepspeed_config (line 238) | def unset_hf_deepspeed_config():
function is_deepspeed_zero3_enabled (line 244) | def is_deepspeed_zero3_enabled():
function deepspeed_config (line 251) | def deepspeed_config():
function deepspeed_optim_sched (line 258) | def deepspeed_optim_sched(trainer, hf_deepspeed_config, args, num_traini...
function deepspeed_init (line 316) | def deepspeed_init(trainer, num_training_steps, inference=False):
function deepspeed_load_checkpoint (line 371) | def deepspeed_load_checkpoint(deepspeed_engine, checkpoint_path):
FILE: transformers/dependency_versions_check.py
function dep_version_check (line 46) | def dep_version_check(pkg, hint=None):
FILE: transformers/dynamic_module_utils.py
function init_hf_modules (line 40) | def init_hf_modules():
function create_dynamic_module (line 56) | def create_dynamic_module(name: Union[str, os.PathLike]):
function get_relative_imports (line 72) | def get_relative_imports(module_file):
function get_relative_import_files (line 90) | def get_relative_import_files(module_file):
function get_imports (line 119) | def get_imports(filename):
function check_imports (line 138) | def check_imports(filename):
function get_class_in_module (line 159) | def get_class_in_module(class_name, module_path):
function get_cached_module_file (line 168) | def get_cached_module_file(
function get_class_from_dynamic_module (line 333) | def get_class_from_dynamic_module(
function custom_object_save (line 446) | def custom_object_save(obj, folder, config=None):
function _raise_timeout_error (line 519) | def _raise_timeout_error(signum, frame):
function resolve_trust_remote_code (line 530) | def resolve_trust_remote_code(trust_remote_code, model_name, has_local_c...
FILE: transformers/feature_extraction_sequence_utils.py
class SequenceFeatureExtractor (line 29) | class SequenceFeatureExtractor(FeatureExtractionMixin):
method __init__ (line 42) | def __init__(self, feature_size: int, sampling_rate: int, padding_valu...
method pad (line 52) | def pad(
method _pad (line 226) | def _pad(
method _truncate (line 298) | def _truncate(
method _get_padding_strategies (line 341) | def _get_padding_strategies(self, padding=False, max_length=None):
FILE: transformers/feature_extraction_utils.py
class BatchFeature (line 60) | class BatchFeature(UserDict):
method __init__ (line 75) | def __init__(self, data: Optional[Dict[str, Any]] = None, tensor_type:...
method __getitem__ (line 79) | def __getitem__(self, item: str) -> Union[Any]:
method __getattr__ (line 89) | def __getattr__(self, item: str):
method __getstate__ (line 95) | def __getstate__(self):
method __setstate__ (line 98) | def __setstate__(self, state):
method keys (line 103) | def keys(self):
method values (line 107) | def values(self):
method items (line 111) | def items(self):
method convert_to_tensors (line 114) | def convert_to_tensors(self, tensor_type: Optional[Union[str, TensorTy...
method to (line 187) | def to(self, *args, **kwargs) -> "BatchFeature":
class FeatureExtractionMixin (line 232) | class FeatureExtractionMixin(PushToHubMixin):
method __init__ (line 240) | def __init__(self, **kwargs):
method _set_processor_class (line 252) | def _set_processor_class(self, processor_class: str):
method from_pretrained (line 257) | def from_pretrained(
method save_pretrained (line 342) | def save_pretrained(self, save_directory: Union[str, os.PathLike], pus...
method get_feature_extractor_dict (line 391) | def get_feature_extractor_dict(
method from_dict (line 489) | def from_dict(cls, feature_extractor_dict: Dict[str, Any], **kwargs) -...
method to_dict (line 525) | def to_dict(self) -> Dict[str, Any]:
method from_json_file (line 538) | def from_json_file(cls, json_file: Union[str, os.PathLike]) -> PreTrai...
method to_json_string (line 556) | def to_json_string(self) -> str:
method to_json_file (line 577) | def to_json_file(self, json_file_path: Union[str, os.PathLike]):
method __repr__ (line 588) | def __repr__(self):
method register_for_auto_class (line 592) | def register_for_auto_class(cls, auto_class="AutoFeatureExtractor"):
FILE: transformers/generation/beam_constraints.py
class Constraint (line 5) | class Constraint(ABC):
method __init__ (line 20) | def __init__(self):
method test (line 24) | def test(self):
method advance (line 49) | def advance(self):
method does_advance (line 61) | def does_advance(self, token_id: int):
method update (line 70) | def update(self, token_id: int):
method reset (line 95) | def reset(self):
method remaining (line 105) | def remaining(self):
method copy (line 114) | def copy(self, stateful=False):
class PhrasalConstraint (line 129) | class PhrasalConstraint(Constraint):
method __init__ (line 138) | def __init__(self, token_ids: List[int]):
method advance (line 152) | def advance(self):
method does_advance (line 157) | def does_advance(self, token_id: int):
method update (line 166) | def update(self, token_id: int):
method reset (line 186) | def reset(self):
method remaining (line 190) | def remaining(self):
method copy (line 193) | def copy(self, stateful=False):
class DisjunctiveTrie (line 204) | class DisjunctiveTrie:
method __init__ (line 205) | def __init__(self, nested_token_ids: List[List[int]], no_subsets=True):
method next_tokens (line 228) | def next_tokens(self, current_seq):
method reached_leaf (line 241) | def reached_leaf(self, current_seq):
method count_leaves (line 246) | def count_leaves(self, root):
method has_subsets (line 253) | def has_subsets(self, trie, nested_token_ids):
class DisjunctiveConstraint (line 261) | class DisjunctiveConstraint(Constraint):
method __init__ (line 270) | def __init__(self, nested_token_ids: List[List[int]]):
method advance (line 292) | def advance(self):
method does_advance (line 300) | def does_advance(self, token_id: int):
method update (line 308) | def update(self, token_id: int):
method reset (line 328) | def reset(self):
method remaining (line 332) | def remaining(self):
method copy (line 339) | def copy(self, stateful=False):
class ConstraintListState (line 350) | class ConstraintListState:
method __init__ (line 359) | def __init__(self, constraints: List[Constraint]):
method init_state (line 369) | def init_state(self):
method get_bank (line 374) | def get_bank(self):
method advance (line 382) | def advance(self):
method reset (line 417) | def reset(self, token_ids: Optional[List[int]]):
method add (line 432) | def add(self, token_id: int):
method copy (line 508) | def copy(self, stateful=True):
FILE: transformers/generation/beam_search.py
class BeamScorer (line 87) | class BeamScorer(ABC):
method process (line 95) | def process(
method finalize (line 107) | def finalize(
class BeamSearchScorer (line 119) | class BeamSearchScorer(BeamScorer):
method __init__ (line 158) | def __init__(
method is_done (line 202) | def is_done(self) -> bool:
method process (line 205) | def process(
method finalize (line 302) | def finalize(
class ConstrainedBeamSearchScorer (line 394) | class ConstrainedBeamSearchScorer(BeamScorer):
method __init__ (line 431) | def __init__(
method is_done (line 477) | def is_done(self) -> bool:
method make_constraint_states (line 480) | def make_constraint_states(self, n):
method check_completes_constraints (line 483) | def check_completes_constraints(self, sequence):
method process (line 488) | def process(
method step_sentence_constraint (line 631) | def step_sentence_constraint(
method finalize (line 772) | def finalize(
class BeamHypotheses (line 861) | class BeamHypotheses:
method __init__ (line 862) | def __init__(self, num_beams: int, length_penalty: float, early_stoppi...
method __len__ (line 879) | def __len__(self):
method add (line 885) | def add(self, hyp: torch.LongTensor, sum_logprobs: float, beam_indices...
method is_done (line 899) | def is_done(self, best_sum_logprobs: float, cur_len: int) -> bool:
FILE: transformers/generation/configuration_utils.py
class GenerationConfig (line 38) | class GenerationConfig(PushToHubMixin):
method __init__ (line 224) | def __init__(self, **kwargs):
method __eq__ (line 302) | def __eq__(self, other):
method __repr__ (line 314) | def __repr__(self):
method validate (line 317) | def validate(self):
method save_pretrained (line 325) | def save_pretrained(
method from_pretrained (line 376) | def from_pretrained(
method _dict_from_json_file (line 544) | def _dict_from_json_file(cls, json_file: Union[str, os.PathLike]):
method from_dict (line 550) | def from_dict(cls, config_dict: Dict[str, Any], **kwargs) -> "Generati...
method dict_torch_dtype_to_str (line 583) | def dict_torch_dtype_to_str(self, d: Dict[str, Any]) -> None:
method to_diff_dict (line 595) | def to_diff_dict(self) -> Dict[str, Any]:
method to_dict (line 618) | def to_dict(self) -> Dict[str, Any]:
method to_json_string (line 635) | def to_json_string(self, use_diff: bool = True) -> str:
method to_json_file (line 653) | def to_json_file(self, json_file_path: Union[str, os.PathLike], use_di...
method from_model_config (line 668) | def from_model_config(cls, model_config: PretrainedConfig) -> "Generat...
method update (line 696) | def update(self, **kwargs):
FILE: transformers/generation/flax_logits_process.py
class FlaxLogitsProcessor (line 50) | class FlaxLogitsProcessor:
method __call__ (line 54) | def __call__(self, input_ids: jnp.ndarray, scores: jnp.ndarray) -> jnp...
class FlaxLogitsWarper (line 61) | class FlaxLogitsWarper:
method __call__ (line 65) | def __call__(self, input_ids: jnp.ndarray, scores: jnp.ndarray) -> jnp...
class FlaxLogitsProcessorList (line 72) | class FlaxLogitsProcessorList(list):
method __call__ (line 80) | def __call__(self, input_ids: jnp.ndarray, scores: jnp.ndarray, cur_le...
class FlaxTemperatureLogitsWarper (line 95) | class FlaxTemperatureLogitsWarper(FlaxLogitsWarper):
method __init__ (line 104) | def __init__(self, temperature: float):
method __call__ (line 110) | def __call__(self, input_ids: jnp.ndarray, scores: jnp.ndarray, cur_le...
class FlaxTopPLogitsWarper (line 115) | class FlaxTopPLogitsWarper(FlaxLogitsWarper):
method __init__ (line 129) | def __init__(self, top_p: float, filter_value: float = -float("Inf"), ...
method __call__ (line 137) | def __call__(self, input_ids: jnp.ndarray, scores: jnp.ndarray, cur_le...
class FlaxTopKLogitsWarper (line 157) | class FlaxTopKLogitsWarper(FlaxLogitsWarper):
method __init__ (line 170) | def __init__(self, top_k: int, filter_value: float = -float("Inf"), mi...
method __call__ (line 177) | def __call__(self, input_ids: jnp.ndarray, scores: jnp.ndarray, cur_le...
class FlaxForcedBOSTokenLogitsProcessor (line 192) | class FlaxForcedBOSTokenLogitsProcessor(FlaxLogitsProcessor):
method __init__ (line 201) | def __init__(self, bos_token_id: int):
method __call__ (line 204) | def __call__(self, input_ids: jnp.ndarray, scores: jnp.ndarray, cur_le...
class FlaxForcedEOSTokenLogitsProcessor (line 214) | class FlaxForcedEOSTokenLogitsProcessor(FlaxLogitsProcessor):
method __init__ (line 225) | def __init__(self, max_length: int, eos_token_id: int):
method __call__ (line 229) | def __call__(self, input_ids: jnp.ndarray, scores: jnp.ndarray, cur_le...
class FlaxMinLengthLogitsProcessor (line 239) | class FlaxMinLengthLogitsProcessor(FlaxLogitsProcessor):
method __init__ (line 250) | def __init__(self, min_length: int, eos_token_id: int):
method __call__ (line 260) | def __call__(self, input_ids: jnp.ndarray, scores: jnp.ndarray, cur_le...
class FlaxSuppressTokensAtBeginLogitsProcessor (line 269) | class FlaxSuppressTokensAtBeginLogitsProcessor(FlaxLogitsProcessor):
method __init__ (line 282) | def __init__(self, begin_suppress_tokens, begin_index):
method __call__ (line 286) | def __call__(self, input_ids, scores, cur_len: int):
class FlaxSuppressTokensLogitsProcessor (line 294) | class FlaxSuppressTokensLogitsProcessor(FlaxLogitsProcessor):
method __init__ (line 304) | def __init__(self, suppress_tokens: list):
method __call__ (line 307) | def __call__(self, input_ids: jnp.ndarray, scores: jnp.ndarray, cur_le...
class FlaxForceTokensLogitsProcessor (line 313) | class FlaxForceTokensLogitsProcessor(FlaxLogitsProcessor):
method __init__ (line 324) | def __init__(self, force_token_map):
method __call__ (line 335) | def __call__(self, input_ids: jnp.ndarray, scores: jnp.ndarray, cur_le...
class FlaxWhisperTimeStampLogitsProcessor (line 361) | class FlaxWhisperTimeStampLogitsProcessor(FlaxLogitsProcessor):
method __init__ (line 378) | def __init__(self, generate_config, model_config, decoder_input_length):
method __call__ (line 395) | def __call__(self, input_ids, scores, cur_len):
FILE: transformers/generation/flax_utils.py
class FlaxGreedySearchOutput (line 55) | class FlaxGreedySearchOutput(ModelOutput):
class FlaxSampleOutput (line 69) | class FlaxSampleOutput(ModelOutput):
class FlaxBeamSearchOutput (line 83) | class FlaxBeamSearchOutput(ModelOutput):
class GreedyState (line 100) | class GreedyState:
class SampleState (line 109) | class SampleState:
class BeamSearchState (line 119) | class BeamSearchState:
class FlaxGenerationMixin (line 129) | class FlaxGenerationMixin:
method prepare_inputs_for_generation (line 146) | def prepare_inputs_for_generation(self, *args, **kwargs):
method _run_loop_in_debug (line 152) | def _run_loop_in_debug(cond_fn, body_fn, init_state):
method _prepare_encoder_decoder_kwargs_for_generation (line 161) | def _prepare_encoder_decoder_kwargs_for_generation(self, input_ids, pa...
method _prepare_decoder_input_ids_for_generation (line 170) | def _prepare_decoder_input_ids_for_generation(
method _get_decoder_start_token_id (line 185) | def _get_decoder_start_token_id(self, decoder_start_token_id: int = No...
method _expand_to_num_beams (line 215) | def _expand_to_num_beams(tensor, num_beams):
method _adapt_logits_for_beam_search (line 218) | def _adapt_logits_for_beam_search(self, logits):
method _validate_model_class (line 225) | def _validate_model_class(self):
method _validate_model_kwargs (line 249) | def _validate_model_kwargs(self, model_kwargs: Dict[str, Any]):
method generate (line 267) | def generate(
method _get_logits_warper (line 472) | def _get_logits_warper(self, generation_config: GenerationConfig) -> F...
method _get_logits_processor (line 488) | def _get_logits_processor(
method _merge_criteria_processor_list (line 538) | def _merge_criteria_processor_list(
method _greedy_search (line 559) | def _greedy_search(
method _sample (line 645) | def _sample(
method _beam_search (line 742) | def _beam_search(
FILE: transformers/generation/logits_process.py
class LogitsProcessor (line 51) | class LogitsProcessor:
method __call__ (line 55) | def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTen...
class LogitsWarper (line 62) | class LogitsWarper:
method __call__ (line 66) | def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTen...
class LogitsProcessorList (line 73) | class LogitsProcessorList(list):
method __call__ (line 81) | def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTen...
class MinLengthLogitsProcessor (line 96) | class MinLengthLogitsProcessor(LogitsProcessor):
method __init__ (line 107) | def __init__(self, min_length: int, eos_token_id: Union[int, List[int]]):
method __call__ (line 119) | def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTen...
class MinNewTokensLengthLogitsProcessor (line 127) | class MinNewTokensLengthLogitsProcessor(LogitsProcessor):
method __init__ (line 140) | def __init__(self, prompt_length_to_skip: int, min_new_tokens: int, eo...
method __call__ (line 157) | def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTen...
class TemperatureLogitsWarper (line 166) | class TemperatureLogitsWarper(LogitsWarper):
method __init__ (line 175) | def __init__(self, temperature: float):
method __call__ (line 181) | def __call__(self, input_ids: torch.Tensor, scores: torch.Tensor) -> t...
class RepetitionPenaltyLogitsProcessor (line 186) | class RepetitionPenaltyLogitsProcessor(LogitsProcessor):
method __init__ (line 196) | def __init__(self, penalty: float):
method __call__ (line 202) | def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTen...
class EncoderRepetitionPenaltyLogitsProcessor (line 212) | class EncoderRepetitionPenaltyLogitsProcessor(LogitsProcessor):
method __init__ (line 223) | def __init__(self, penalty: float, encoder_input_ids: torch.LongTensor):
method __call__ (line 230) | def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTen...
class TopPLogitsWarper (line 240) | class TopPLogitsWarper(LogitsWarper):
method __init__ (line 254) | def __init__(self, top_p: float, filter_value: float = -float("Inf"), ...
method __call__ (line 263) | def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTen...
class TopKLogitsWarper (line 279) | class TopKLogitsWarper(LogitsWarper):
method __init__ (line 292) | def __init__(self, top_k: int, filter_value: float = -float("Inf"), mi...
method __call__ (line 299) | def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTen...
class TypicalLogitsWarper (line 307) | class TypicalLogitsWarper(LogitsWarper):
method __init__ (line 321) | def __init__(self, mass: float = 0.9, filter_value: float = -float("In...
method __call__ (line 330) | def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTen...
class EpsilonLogitsWarper (line 355) | class EpsilonLogitsWarper(LogitsWarper):
method __init__ (line 370) | def __init__(self, epsilon: float, filter_value: float = -float("Inf")...
method __call__ (line 385) | def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTen...
class EtaLogitsWarper (line 398) | class EtaLogitsWarper(LogitsWarper):
method __init__ (line 409) | def __init__(self, epsilon: float, filter_value: float = -float("Inf")...
method __call__ (line 424) | def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTen...
function _get_ngrams (line 439) | def _get_ngrams(ngram_size: int, prev_input_ids: torch.Tensor, num_hypos...
function _get_generated_ngrams (line 450) | def _get_generated_ngrams(banned_ngrams, prev_input_ids, ngram_size, cur...
function _calc_banned_ngram_tokens (line 457) | def _calc_banned_ngram_tokens(
class NoRepeatNGramLogitsProcessor (line 474) | class NoRepeatNGramLogitsProcessor(LogitsProcessor):
method __init__ (line 484) | def __init__(self, ngram_size: int):
method __call__ (line 489) | def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTen...
class EncoderNoRepeatNGramLogitsProcessor (line 500) | class EncoderNoRepeatNGramLogitsProcessor(LogitsProcessor):
method __init__ (line 512) | def __init__(self, encoder_ngram_size: int, encoder_input_ids: torch.L...
method __call__ (line 523) | def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTen...
class NoBadWordsLogitsProcessor (line 541) | class NoBadWordsLogitsProcessor(LogitsProcessor):
method __init__ (line 554) | def __init__(self, bad_words_ids: List[List[int]], eos_token_id: Union...
method __call__ (line 589) | def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTen...
method _calc_static_bad_word_mask (line 598) | def _calc_static_bad_word_mask(self, scores: torch.FloatTensor) -> tor...
method _tokens_match (line 603) | def _tokens_match(self, prev_tokens: List[int], tokens: List[int]) -> ...
method _calc_banned_bad_words_ids (line 613) | def _calc_banned_bad_words_ids(self, prev_input_ids: List[List[int]]) ...
method _set_scores_to_inf_for_banned_tokens (line 625) | def _set_scores_to_inf_for_banned_tokens(
class PrefixConstrainedLogitsProcessor (line 675) | class PrefixConstrainedLogitsProcessor(LogitsProcessor):
method __init__ (line 688) | def __init__(self, prefix_allowed_tokens_fn: Callable[[int, torch.Tens...
method __call__ (line 692) | def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTen...
class HammingDiversityLogitsProcessor (line 701) | class HammingDiversityLogitsProcessor(LogitsProcessor):
method __init__ (line 719) | def __init__(self, diversity_penalty: float, num_beams: int, num_beam_...
method __call__ (line 732) | def __call__(
class ForcedBOSTokenLogitsProcessor (line 761) | class ForcedBOSTokenLogitsProcessor(LogitsProcessor):
method __init__ (line 770) | def __init__(self, bos_token_id: int):
method __call__ (line 773) | def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTen...
class ForcedEOSTokenLogitsProcessor (line 782) | class ForcedEOSTokenLogitsProcessor(LogitsProcessor):
method __init__ (line 794) | def __init__(self, max_length: int, eos_token_id: Union[int, List[int]]):
method __call__ (line 800) | def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTen...
class InfNanRemoveLogitsProcessor (line 810) | class InfNanRemoveLogitsProcessor(LogitsProcessor):
method __call__ (line 816) | def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTen...
class ExponentialDecayLengthPenalty (line 826) | class ExponentialDecayLengthPenalty(LogitsProcessor):
method __init__ (line 841) | def __init__(
method __call__ (line 853) | def __call__(self, input_ids: torch.Tensor, scores: torch.Tensor) -> t...
class LogitNormalization (line 861) | class LogitNormalization(LogitsProcessor, LogitsWarper):
method __call__ (line 869) | def __call__(self, input_ids: torch.Tensor, scores: torch.Tensor) -> t...
class SuppressTokensAtBeginLogitsProcessor (line 874) | class SuppressTokensAtBeginLogitsProcessor(LogitsProcessor):
method __init__ (line 881) | def __init__(self, begin_suppress_tokens, begin_index):
method __call__ (line 885) | def __call__(self, input_ids, scores):
class SuppressTokensLogitsProcessor (line 892) | class SuppressTokensLogitsProcessor(LogitsProcessor):
method __init__ (line 896) | def __init__(self, suppress_tokens):
method __call__ (line 899) | def __call__(self, input_ids, scores):
class ForceTokensLogitsProcessor (line 904) | class ForceTokensLogitsProcessor(LogitsProcessor):
method __init__ (line 909) | def __init__(self, force_token_map: List[List[int]]):
method __call__ (line 912) | def __call__(self, input_ids, scores):
class WhisperTimeStampLogitsProcessor (line 921) | class WhisperTimeStampLogitsProcessor(LogitsProcessor):
method __init__ (line 938) | def __init__(self, generate_config): # support for the kwargs
method __call__ (line 948) | def __call__(self, input_ids, scores):
FILE: transformers/generation/stopping_criteria.py
class StoppingCriteria (line 33) | class StoppingCriteria(ABC):
method __call__ (line 37) | def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTen...
class MaxLengthCriteria (line 41) | class MaxLengthCriteria(StoppingCriteria):
method __init__ (line 51) | def __init__(self, max_length: int):
method __call__ (line 55) | def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTen...
class MaxNewTokensCriteria (line 59) | class MaxNewTokensCriteria(StoppingCriteria):
method __init__ (line 72) | def __init__(self, start_length: int, max_new_tokens: int):
method __call__ (line 84) | def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTen...
class MaxTimeCriteria (line 88) | class MaxTimeCriteria(StoppingCriteria):
method __init__ (line 101) | def __init__(self, max_time: float, initial_timestamp: Optional[float]...
method __call__ (line 106) | def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTen...
class StoppingCriteriaList (line 110) | class StoppingCriteriaList(list):
method __call__ (line 112) | def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTen...
method max_length (line 116) | def max_length(self) -> Optional[int]:
function validate_stopping_criteria (line 125) | def validate_stopping_criteria(stopping_criteria: StoppingCriteriaList, ...
FILE: transformers/generation/streamers.py
class BaseStreamer (line 24) | class BaseStreamer:
method put (line 29) | def put(self, value):
method end (line 33) | def end(self):
class TextStreamer (line 38) | class TextStreamer(BaseStreamer):
method __init__ (line 72) | def __init__(self, tokenizer: "AutoTokenizer", skip_prompt: bool = Fal...
method put (line 82) | def put(self, value):
method end (line 116) | def end(self):
method on_finalized_text (line 130) | def on_finalized_text(self, text: str, stream_end: bool = False):
method _is_chinese_char (line 134) | def _is_chinese_char(self, cp):
class TextIteratorStreamer (line 159) | class TextIteratorStreamer(TextStreamer):
method __init__ (line 205) | def __init__(
method on_finalized_text (line 213) | def on_finalized_text(self, text: str, stream_end: bool = False):
method __iter__ (line 219) | def __iter__(self):
method __next__ (line 222) | def __next__(self):
FILE: transformers/generation/tf_logits_process.py
class TFLogitsProcessor (line 53) | class TFLogitsProcessor:
method __call__ (line 57) | def __call__(self, input_ids: tf.Tensor, scores: tf.Tensor, cur_len: i...
class TFLogitsWarper (line 64) | class TFLogitsWarper:
method __call__ (line 68) | def __call__(self, input_ids: tf.Tensor, scores: tf.Tensor, cur_len: i...
class TFLogitsProcessorList (line 75) | class TFLogitsProcessorList(list):
method __call__ (line 83) | def __call__(self, input_ids: tf.Tensor, scores: tf.Tensor, cur_len: i...
class TFTemperatureLogitsWarper (line 98) | class TFTemperatureLogitsWarper(TFLogitsWarper):
method __init__ (line 107) | def __init__(self, temperature: float):
method __call__ (line 113) | def __call__(self, input_ids: tf.Tensor, scores: tf.Tensor, cur_len: i...
class TFTopKLogitsWarper (line 118) | class TFTopKLogitsWarper(TFLogitsWarper):
method __init__ (line 131) | def __init__(self, top_k: int, filter_value: float = -float("Inf"), mi...
method __call__ (line 138) | def __call__(self, input_ids: tf.Tensor, scores: tf.Tensor, cur_len: i...
class TFTopPLogitsWarper (line 146) | class TFTopPLogitsWarper(TFLogitsWarper):
method __init__ (line 160) | def __init__(self, top_p: float, filter_value: float = -float("Inf"), ...
method __call__ (line 168) | def __call__(self, input_ids: tf.Tensor, scores: tf.Tensor, cur_len: i...
class TFMinLengthLogitsProcessor (line 200) | class TFMinLengthLogitsProcessor(TFLogitsProcessor):
method __init__ (line 211) | def __init__(self, min_length: int, eos_token_id: int):
method _apply_eos_token_mask (line 221) | def _apply_eos_token_mask(self, scores: tf.Tensor) -> tf.Tensor:
method __call__ (line 226) | def __call__(self, input_ids: tf.Tensor, scores: tf.Tensor, cur_len: i...
class TFRepetitionPenaltyLogitsProcessor (line 236) | class TFRepetitionPenaltyLogitsProcessor(TFLogitsProcessor):
method __init__ (line 246) | def __init__(self, penalty: float):
method _create_score_penalties (line 252) | def _create_score_penalties(self, input_ids: tf.Tensor, logits: tf.Ten...
method __call__ (line 278) | def __call__(self, input_ids: tf.Tensor, scores: tf.Tensor, cur_len: i...
class TFNoBadWordsLogitsProcessor (line 286) | class TFNoBadWordsLogitsProcessor(TFLogitsProcessor):
method __init__ (line 298) | def __init__(self, bad_words_ids: List[List[int]], eos_token_id: int):
method _calc_row_banned_bad_tokens (line 322) | def _calc_row_banned_bad_tokens(self, row_input_ids: tf.Tensor) -> tf....
method __call__ (line 362) | def __call__(self, input_ids: tf.Tensor, scores: tf.Tensor, cur_len: i...
class TFNoRepeatNGramLogitsProcessor (line 383) | class TFNoRepeatNGramLogitsProcessor(TFLogitsProcessor):
method __init__ (line 393) | def __init__(self, ngram_size: int):
method calc_banned_ngram_tokens (line 398) | def calc_banned_ngram_tokens(self, input_ids, num_hypos, cur_len):
method __call__ (line 422) | def __call__(self, input_ids: tf.Tensor, scores: tf.Tensor, cur_len: i...
class TFForcedBOSTokenLogitsProcessor (line 443) | class TFForcedBOSTokenLogitsProcessor(TFLogitsProcessor):
method __init__ (line 452) | def __init__(self, bos_token_id: int):
method __call__ (line 457) | def __call__(self, input_ids: tf.Tensor, scores: tf.Tensor, cur_len: i...
class TFForcedEOSTokenLogitsProcessor (line 473) | class TFForcedEOSTokenLogitsProcessor(TFLogitsProcessor):
method __init__ (line 484) | def __init__(self, max_length: int, eos_token_id: int):
method __call__ (line 490) | def __call__(self, input_ids: tf.Tensor, scores: tf.Tensor, cur_len: i...
class TFSuppressTokensAtBeginLogitsProcessor (line 506) | class TFSuppressTokensAtBeginLogitsProcessor(TFLogitsProcessor):
method __init__ (line 513) | def __init__(self, begin_suppress_tokens, begin_index):
method __call__ (line 517) | def __call__(self, input_ids: tf.Tensor, scores: tf.Tensor, cur_len: i...
class TFSuppressTokensLogitsProcessor (line 530) | class TFSuppressTokensLogitsProcessor(TFLogitsProcessor):
method __init__ (line 534) | def __init__(self, suppress_tokens):
method __call__ (line 537) | def __call__(self, input_ids: tf.Tensor, scores: tf.Tensor, cur_len: i...
class TFForceTokensLogitsProcessor (line 546) | class TFForceTokensLogitsProcessor(TFLogitsProcessor):
method __init__ (line 551) | def __init__(self, force_token_map: List[List[int]]):
method __call__ (line 562) | def __call__(self, input_ids: tf.Tensor, scores: tf.Tensor, cur_len: i...
FILE: transformers/generation/tf_utils.py
class TFGreedySearchDecoderOnlyOutput (line 58) | class TFGreedySearchDecoderOnlyOutput(ModelOutput):
class TFGreedySearchEncoderDecoderOutput (line 86) | class TFGreedySearchEncoderDecoderOutput(ModelOutput):
class TFSampleDecoderOnlyOutput (line 128) | class TFSampleDecoderOnlyOutput(ModelOutput):
class TFSampleEncoderDecoderOutput (line 156) | class TFSampleEncoderDecoderOutput(ModelOutput):
class TFBeamSearchDecoderOnlyOutput (line 198) | class TFBeamSearchDecoderOnlyOutput(ModelOutput):
class TFBeamSearchEncoderDecoderOutput (line 233) | class TFBeamSearchEncoderDecoderOutput(ModelOutput):
class TFBeamSampleDecoderOnlyOutput (line 283) | class TFBeamSampleDecoderOnlyOutput(ModelOutput):
class TFBeamSampleEncoderDecoderOutput (line 318) | class TFBeamSampleEncoderDecoderOutput(ModelOutput):
class TFContrastiveSearchDecoderOnlyOutput (line 367) | class TFContrastiveSearchDecoderOnlyOutput(ModelOutput):
class TFContrastiveSearchEncoderDecoderOutput (line 394) | class TFContrastiveSearchEncoderDecoderOutput(ModelOutput):
class TFGenerationMixin (line 444) | class TFGenerationMixin:
method seed_generator (line 464) | def seed_generator(self):
method prepare_inputs_for_generation (line 472) | def prepare_inputs_for_generation(self, *args, **kwargs):
method adjust_logits_during_generation (line 477) | def adjust_logits_during_generation(
method compute_transition_scores (line 498) | def compute_transition_scores(
method _validate_model_class (line 618) | def _validate_model_class(self):
method _validate_model_kwargs (line 643) | def _validate_model_kwargs(self, model_kwargs: Dict[str, Any]):
method generate (line 666) | def generate(
method _prepare_attention_mask_for_generation (line 1048) | def _prepare_attention_mask_for_generation(
method _prepare_encoder_decoder_kwargs_for_generation (line 1064) | def _prepare_encoder_decoder_kwargs_for_generation(
method _prepare_decoder_input_ids_for_generation (line 1094) | def _prepare_decoder_input_ids_for_generation(
method _get_decoder_start_token_id (line 1133) | def _get_decoder_start_token_id(self, decoder_start_token_id: int = No...
method _expand_inputs_for_generation (line 1152) | def _expand_inputs_for_generation(
method _prepare_model_inputs (line 1190) | def _prepare_model_inputs(
method _maybe_initialize_input_ids_for_generation (line 1256) | def _maybe_initialize_input_ids_for_generation(
method _extract_past_from_model_output (line 1285) | def _extract_past_from_model_output(outputs: ModelOutput):
method _update_model_kwargs_for_generation (line 1295) | def _update_model_kwargs_for_generation(
method _update_model_kwargs_for_xla_generation (line 1311) | def _update_model_kwargs_for_xla_generation(
method _get_logits_warper (line 1438) | def _get_logits_warper(
method _get_logits_processor (line 1460) | def _get_logits_processor(
method _merge_criteria_processor_list (line 1515) | def _merge_criteria_processor_list(
method greedy_search (line 1536) | def greedy_search(
method sample (line 1798) | def sample(
method _gather_beams (line 2089) | def _gather_beams(nested, beam_indices, batch_axis=0):
method beam_search (line 2109) | def beam_search(
method contrastive_search (line 2672) | def contrastive_search(
function tf_top_k_top_p_filtering (line 3080) | def tf_top_k_top_p_filtering(logits, top_k=0, top_p=1.0, filter_value=-f...
function scatter_values_on_batch_indices (line 3135) | def scatter_values_on_batch_indices(values, batch_indices):
function sample_without_replacement (line 3145) | def sample_without_replacement(logits, num_samples):
function _ranking_fast (line 3155) | def _ranking_fast(
FILE: transformers/generation/utils.py
class GreedySearchDecoderOnlyOutput (line 83) | class GreedySearchDecoderOnlyOutput(ModelOutput):
class ContrastiveSearchEncoderDecoderOutput (line 111) | class ContrastiveSearchEncoderDecoderOutput(ModelOutput):
class ContrastiveSearchDecoderOnlyOutput (line 150) | class ContrastiveSearchDecoderOnlyOutput(ModelOutput):
class GreedySearchEncoderDecoderOutput (line 179) | class GreedySearchEncoderDecoderOutput(ModelOutput):
class SampleDecoderOnlyOutput (line 221) | class SampleDecoderOnlyOutput(ModelOutput):
class SampleEncoderDecoderOutput (line 250) | class SampleEncoderDecoderOutput(ModelOutput):
class BeamSearchDecoderOnlyOutput (line 293) | class BeamSearchDecoderOnlyOutput(ModelOutput):
class BeamSearchEncoderDecoderOutput (line 328) | class BeamSearchEncoderDecoderOutput(ModelOutput):
class BeamSampleDecoderOnlyOutput (line 378) | class BeamSampleDecoderOnlyOutput(ModelOutput):
class BeamSampleEncoderDecoderOutput (line 413) | class BeamSampleEncoderDecoderOutput(ModelOutput):
class GenerationMixin (line 469) | class GenerationMixin:
method prepare_inputs_for_generation (line 493) | def prepare_inputs_for_generation(self, *args, **kwargs):
method _prepare_model_inputs (line 498) | def _prepare_model_inputs(
method adjust_logits_during_generation (line 562) | def adjust_logits_during_generation(self, logits: torch.FloatTensor, *...
method _maybe_initialize_input_ids_for_generation (line 568) | def _maybe_initialize_input_ids_for_generation(
method _prepare_attention_mask_for_generation (line 596) | def _prepare_attention_mask_for_generation(
method _prepare_encoder_decoder_kwargs_for_generation (line 614) | def _prepare_encoder_decoder_kwargs_for_generation(
method _prepare_decoder_input_ids_for_generation (line 646) | def _prepare_decoder_input_ids_for_generation(
method _get_decoder_start_token_id (line 691) | def _get_decoder_start_token_id(self, decoder_start_token_id: int = No...
method _expand_inputs_for_generation (line 708) | def _expand_inputs_for_generation(
method _extract_past_from_model_output (line 734) | def _extract_past_from_model_output(self, outputs: ModelOutput, standa...
method _update_model_kwargs_for_generation (line 749) | def _update_model_kwargs_for_generation(
method _reorder_cache (line 786) | def _reorder_cache(self, past_key_values, beam_idx):
method _get_logits_warper (line 792) | def _get_logits_warper(
method _get_logits_processor (line 830) | def _get_logits_processor(
method _get_stopping_criteria (line 947) | def _get_stopping_criteria(
method _merge_criteria_processor_list (line 958) | def _merge_criteria_processor_list(
method compute_transition_scores (line 979) | def compute_transition_scores(
method _validate_model_class (line 1100) | def _validate_model_class(self):
method _validate_model_kwargs (line 1126) | def _validate_model_kwargs(self, model_kwargs: Dict[str, Any]):
method generate (line 1150) | def generate(
method contrastive_search (line 1798) | def contrastive_search(
method greedy_search (line 2171) | def greedy_search(
method sample (line 2430) | def sample(
method beam_search (line 2712) | def beam_search(
method beam_sample (line 3035) | def beam_sample(
method group_beam_search (line 3368) | def group_beam_search(
method constrained_beam_search (line 3748) | def constrained_beam_search(
method assisted_decoding (line 4073) | def assisted_decoding(
function _crop_past_key_values (line 4497) | def _crop_past_key_values(model, past_key_values, maximum_length):
function _split_model_outputs (line 4539) | def _split_model_outputs(outputs, new_outputs, cur_len, added_len, is_de...
function top_k_top_p_filtering (line 4565) | def top_k_top_p_filtering(
function _ranking_fast (line 4600) | def _ranking_fast(
FILE: transformers/generation_flax_utils.py
class FlaxGenerationMixin (line 22) | class FlaxGenerationMixin(FlaxGenerationMixin):
FILE: transformers/generation_tf_utils.py
class TFGenerationMixin (line 22) | class TFGenerationMixin(TFGenerationMixin):
FILE: transformers/generation_utils.py
class GenerationMixin (line 22) | class GenerationMixin(GenerationMixin):
FILE: transformers/hf_argparser.py
function string_to_bool (line 42) | def string_to_bool(v):
function make_choice_type_function (line 55) | def make_choice_type_function(choices: list) -> Callable[[str], Any]:
function HfArg (line 70) | def HfArg(
class HfArgumentParser (line 117) | class HfArgumentParser(ArgumentParser):
method __init__ (line 128) | def __init__(self, dataclass_types: Union[DataClassType, Iterable[Data...
method _parse_dataclass_field (line 147) | def _parse_dataclass_field(parser: ArgumentParser, field: dataclasses....
method _add_dataclass_arguments (line 239) | def _add_dataclass_arguments(self, dtype: DataClassType):
method parse_args_into_dataclasses (line 273) | def parse_args_into_dataclasses(
method parse_dict (line 359) | def parse_dict(self, args: Dict[str, Any], allow_extra_keys: bool = Fa...
method parse_json_file (line 387) | def parse_json_file(self, json_file: str, allow_extra_keys: bool = Fal...
method parse_yaml_file (line 409) | def parse_yaml_file(self, yaml_file: str, allow_extra_keys: bool = Fal...
FILE: transformers/image_processing_utils.py
class BatchFeature (line 43) | class BatchFeature(BaseBatchFeature):
class ImageProcessingMixin (line 59) | class ImageProcessingMixin(PushToHubMixin):
method __init__ (line 67) | def __init__(self, **kwargs):
method _set_processor_class (line 79) | def _set_processor_class(self, processor_class: str):
method from_pretrained (line 84) | def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os....
method save_pretrained (line 169) | def save_pretrained(self, save_directory: Union[str, os.PathLike], pus...
method get_image_processor_dict (line 218) | def get_image_processor_dict(
method from_dict (line 321) | def from_dict(cls, image_processor_dict: Dict[str, Any], **kwargs):
method to_dict (line 365) | def to_dict(self) -> Dict[str, Any]:
method from_json_file (line 378) | def from_json_file(cls, json_file: Union[str, os.PathLike]):
method to_json_string (line 396) | def to_json_string(self) -> str:
method to_json_file (line 417) | def to_json_file(self, json_file_path: Union[str, os.PathLike]):
method __repr__ (line 428) | def __repr__(self):
method register_for_auto_class (line 432) | def register_for_auto_class(cls, auto_class="AutoImageProcessor"):
class BaseImageProcessor (line 458) | class BaseImageProcessor(ImageProcessingMixin):
method __init__ (line 459) | def __init__(self, **kwargs):
method __call__ (line 462) | def __call__(self, images, **kwargs) -> BatchFeature:
method preprocess (line 466) | def preprocess(self, images, **kwargs) -> BatchFeature:
function is_valid_size_dict (line 473) | def is_valid_size_dict(size_dict):
function convert_to_size_dict (line 484) | def convert_to_size_dict(
function get_size_dict (line 512) | def get_size_dict(
FILE: transformers/image_transforms.py
function to_channel_dimension_format (line 54) | def to_channel_dimension_format(
function rescale (line 91) | def rescale(
function _rescale_for_pil_conversion (line 121) | def _rescale_for_pil_conversion(image):
function to_pil_image (line 148) | def to_pil_image(
function get_resize_output_image_size (line 197) | def get_resize_output_image_size(
function resize (line 261) | def resize(
function normalize (line 327) | def normalize(
function center_crop (line 390) | def center_crop(
function _center_to_corners_format_torch (line 487) | def _center_to_corners_format_torch(bboxes_center: "torch.Tensor") -> "t...
function _center_to_corners_format_numpy (line 497) | def _center_to_corners_format_numpy(bboxes_center: np.ndarray) -> np.nda...
function _center_to_corners_format_tf (line 507) | def _center_to_corners_format_tf(bboxes_center: "tf.Tensor") -> "tf.Tens...
function center_to_corners_format (line 518) | def center_to_corners_format(bboxes_center: TensorType) -> TensorType:
function _corners_to_center_format_torch (line 539) | def _corners_to_center_format_torch(bboxes_corners: "torch.Tensor") -> "...
function _corners_to_center_format_numpy (line 550) | def _corners_to_center_format_numpy(bboxes_corners: np.ndarray) -> np.nd...
function _corners_to_center_format_tf (line 564) | def _corners_to_center_format_tf(bboxes_corners: "tf.Tensor") -> "tf.Ten...
function corners_to_center_format (line 578) | def corners_to_center_format(bboxes_corners: TensorType) -> TensorType:
function rgb_to_id (line 601) | def rgb_to_id(color):
function id_to_rgb (line 612) | def id_to_rgb(id_map):
class PaddingMode (line 631) | class PaddingMode(ExplicitEnum):
function pad (line 642) | def pad(
function convert_to_rgb (line 729) | def convert_to_rgb(image: ImageInput) -> ImageInput:
function flip_channel_order (line 747) | def flip_channel_order(image: np.ndarray, data_format: Optional[ChannelD...
FILE: transformers/image_utils.py
class ChannelDimension (line 62) | class ChannelDimension(ExplicitEnum):
function is_pil_image (line 67) | def is_pil_image(img):
function is_valid_image (line 71) | def is_valid_image(img):
function valid_images (line 81) | def valid_images(imgs):
function is_batched (line 93) | def is_batched(img):
function make_list_of_images (line 99) | def make_list_of_images(images, expected_ndims: int = 3) -> List[ImageIn...
function to_numpy_array (line 138) | def to_numpy_array(img) -> np.ndarray:
function infer_channel_dimension_format (line 147) | def infer_channel_dimension_format(image: np.ndarray) -> ChannelDimension:
function get_channel_dimension_axis (line 172) | def get_channel_dimension_axis(image: np.ndarray) -> int:
function get_image_size (line 191) | def get_image_size(image: np.ndarray, channel_dim: ChannelDimension = No...
function is_valid_annotation_coco_detection (line 215) | def is_valid_annotation_coco_detection(annotation: Dict[str, Union[List,...
function is_valid_annotation_coco_panoptic (line 231) | def is_valid_annotation_coco_panoptic(annotation: Dict[str, Union[List, ...
function valid_coco_detection_annotations (line 248) | def valid_coco_detection_annotations(annotations: Iterable[Dict[str, Uni...
function valid_coco_panoptic_annotations (line 252) | def valid_coco_panoptic_annotations(annotations: Iterable[Dict[str, Unio...
function load_image (line 256) | def load_image(image: Union[str, "PIL.Image.Image"]) -> "PIL.Image.Image":
class ImageFeatureExtractionMixin (line 291) | class ImageFeatureExtractionMixin:
method _ensure_format_supported (line 296) | def _ensure_format_supported(self, image):
method to_pil_image (line 303) | def to_pil_image(self, image, rescale=None):
method convert_rgb (line 333) | def convert_rgb(self, image):
method rescale (line 347) | def rescale(self, image: np.ndarray, scale: Union[float, int]) -> np.n...
method to_numpy_array (line 354) | def to_numpy_array(self, image, rescale=None, channel_first=True):
method expand_dims (line 386) | def expand_dims(self, image):
method normalize (line 406) | def normalize(self, image, mean, std, rescale=False):
method resize (line 452) | def resize(self, image, size, resample=None, default_to_square=True, m...
method center_crop (line 519) | def center_crop(self, image, size):
method flip_channel_order (line 594) | def flip_channel_order(self, image):
method rotate (line 611) | def rotate(self, image, angle, resample=None, expand=0, center=None, t...
FILE: transformers/integrations.py
function is_wandb_available (line 78) | def is_wandb_available():
function is_clearml_available (line 89) | def is_clearml_available():
function is_comet_available (line 93) | def is_comet_available():
function is_tensorboard_available (line 97) | def is_tensorboard_available():
function is_optuna_available (line 101) | def is_optuna_available():
function is_ray_available (line 105) | def is_ray_available():
function is_ray_tune_available (line 109) | def is_ray_tune_available():
function is_sigopt_available (line 115) | def is_sigopt_available():
function is_azureml_available (line 119) | def is_azureml_available():
function is_mlflow_available (line 127) | def is_mlflow_available():
function is_dagshub_available (line 133) | def is_dagshub_available():
function is_fairscale_available (line 137) | def is_fairscale_available():
function is_neptune_available (line 141) | def is_neptune_available():
function is_codecarbon_available (line 145) | def is_codecarbon_available():
function is_flytekit_available (line 149) | def is_flytekit_available():
function is_flyte_deck_standard_available (line 153) | def is_flyte_deck_standard_available():
function hp_params (line 159) | def hp_params(trial):
function default_hp_search_backend (line 180) | def default_hp_search_backend():
function run_hp_search_optuna (line 189) | def run_hp_search_optuna(trainer, n_trials: int, direction: str, **kwarg...
function run_hp_search_ray (line 240) | def run_hp_search_ray(trainer, n_trials: int, direction: str, **kwargs) ...
function run_hp_search_sigopt (line 376) | def run_hp_search_sigopt(trainer, n_trials: int, direction: str, **kwarg...
function run_hp_search_wandb (line 476) | def run_hp_search_wandb(trainer, n_trials: int, direction: str, **kwargs...
function get_available_reporting_integrations (line 545) | def get_available_reporting_integrations():
function rewrite_logs (line 568) | def rewrite_logs(d):
class TensorBoardCallback (line 584) | class TensorBoardCallback(TrainerCallback):
method __init__ (line 593) | def __init__(self, tb_writer=None):
method _init_summary_writer (line 616) | def _init_summary_writer(self, args, log_dir=None):
method on_train_begin (line 621) | def on_train_begin(self, args, state, control, **kwargs):
method on_log (line 643) | def on_log(self, args, state, control, logs=None, **kwargs):
method on_train_end (line 664) | def on_train_end(self, args, state, control, **kwargs):
class WandbCallback (line 670) | class WandbCallback(TrainerCallback):
method __init__ (line 675) | def __init__(self):
method setup (line 695) | def setup(self, args, state, model, **kwargs):
method on_train_begin (line 762) | def on_train_begin(self, args, state, control, model=None, **kwargs):
method on_train_end (line 773) | def on_train_end(self, args, state, control, model=None, tokenizer=Non...
method on_log (line 807) | def on_log(self, args, state, control, model=None, logs=None, **kwargs):
method on_save (line 816) | def on_save(self, args, state, control, **kwargs):
class CometCallback (line 837) | class CometCallback(TrainerCallback):
method __init__ (line 842) | def __init__(self):
method setup (line 848) | def setup(self, args, state, model):
method on_train_begin (line 890) | def on_train_begin(self, args, state, control, model=None, **kwargs):
method on_log (line 894) | def on_log(self, args, state, control, model=None, logs=None, **kwargs):
method on_train_end (line 902) | def on_train_end(self, args, state, control, **kwargs):
class AzureMLCallback (line 914) | class AzureMLCallback(TrainerCallback):
method __init__ (line 919) | def __init__(self, azureml_run=None):
method on_init_end (line 924) | def on_init_end(self, args, state, control, **kwargs):
method on_log (line 930) | def on_log(self, args, state, control, logs=None, **kwargs):
class MLflowCallback (line 937) | class MLflowCallback(TrainerCallback):
method __init__ (line 943) | def __init__(self):
method setup (line 956) | def setup(self, args, state, model):
method on_train_begin (line 1026) | def on_train_begin(self, args, state, control, model=None, **kwargs):
method on_log (line 1030) | def on_log(self, args, state, control, logs, model=None, **kwargs):
method on_train_end (line 1045) | def on_train_end(self, args, state, control, **kwargs):
method on_save (line 1050) | def on_save(self, args, state, control, **kwargs):
method __del__ (line 1061) | def __del__(self):
class DagsHubCallback (line 1072) | class DagsHubCallback(MLflowCallback):
method __init__ (line 1077) | def __init__(self):
method setup (line 1086) | def setup(self, *args, **kwargs):
method on_train_end (line 1113) | def on_train_end(self, args, state, control, **kwargs):
class NeptuneMissingConfiguration (line 1121) | class NeptuneMissingConfiguration(Exception):
method __init__ (line 1122) | def __init__(self):
class NeptuneCallback (line 1132) | class NeptuneCallback(TrainerCallback):
method __init__ (line 1168) | def __init__(
method _stop_run_if_exists (line 1223) | def _stop_run_if_exists(self):
method _initialize_run (line 1229) | def _initialize_run(self, **additional_neptune_kwargs):
method _use_initial_run (line 1245) | def _use_initial_run(self):
method _ensure_run_with_monitoring (line 1251) | def _ensure_run_with_monitoring(self):
method _ensure_at_least_run_without_monitoring (line 1265) | def _ensure_at_least_run_without_monitoring(self):
method run (line 1280) | def run(self):
method _metadata_namespace (line 1286) | def _metadata_namespace(self):
method _log_integration_version (line 1289) | def _log_integration_version(self):
method _log_trainer_parameters (line 1292) | def _log_trainer_parameters(self, args):
method _log_model_parameters (line 1295) | def _log_model_parameters(self, model):
method _log_hyper_param_search_parameters (line 1299) | def _log_hyper_param_search_parameters(self, state):
method _log_model_checkpoint (line 1306) | def _log_model_checkpoint(self, source_directory: str, checkpoint: str):
method on_init_end (line 1330) | def on_init_end(self, args, state, control, **kwargs):
method on_train_begin (line 1338) | def on_train_begin(self, args, state, control, model=None, **kwargs):
method on_train_end (line 1353) | def on_train_end(self, args, state, control, **kwargs):
method __del__ (line 1356) | def __del__(self):
method on_save (line 1362) | def on_save(self, args, state, control, **kwargs):
method on_evaluate (line 1366) | def on_evaluate(self, args, state, control, metrics=None, **kwargs):
method get_run (line 1379) | def get_run(cls, trainer):
method on_log (line 1386) | def on_log(self, args, state, control, logs: Optional[Dict[str, float]...
class CodeCarbonCallback (line 1399) | class CodeCarbonCallback(TrainerCallback):
method __init__ (line 1404) | def __init__(self):
method on_init_end (line 1414) | def on_init_end(self, args, state, control, **kwargs):
method on_train_begin (line 1419) | def on_train_begin(self, args, state, control, model=None, **kwargs):
method on_train_end (line 1423) | def on_train_end(self, args, state, control, **kwargs):
class ClearMLCallback (line 1428) | class ClearMLCallback(TrainerCallback):
method __init__ (line 1441) | def __init__(self):
method setup (line 1454) | def setup(self, args, state, model, tokenizer, **kwargs):
method on_train_begin (line 1482) | def on_train_begin(self, args, state, control, model=None, tokenizer=N...
method on_train_end (line 1490) | def on_train_end(self, args, state, control, model=None, tokenizer=Non...
method on_log (line 1497) | def on_log(self, args, state, control, model=None, tokenizer=None, log...
method on_save (line 1539) | def on_save(self, args, state, control, **kwargs):
class FlyteCallback (line 1547) | class FlyteCallback(TrainerCallback):
method __init__ (line 1574) | def __init__(self, save_log_history: bool = True, sync_checkpoints: bo...
method on_save (line 1592) | def on_save(self, args, state, control, **kwargs):
method on_train_end (line 1600) | def on_train_end(self, args, state, control, **kwargs):
function get_reporting_integration_callbacks (line 1624) | def get_reporting_integration_callbacks(report_to):
FILE: transformers/keras_callbacks.py
class KerasMetricCallback (line 21) | class KerasMetricCallback(Callback):
method __init__ (line 79) | def __init__(
method _concatenate_batches (line 145) | def _concatenate_batches(batches, padding_index=-100):
method _postprocess_predictions_or_labels (line 163) | def _postprocess_predictions_or_labels(self, inputs):
method on_epoch_end (line 185) | def on_epoch_end(self, epoch, logs=None):
class PushToHubCallback (line 268) | class PushToHubCallback(Callback):
method __init__ (line 315) | def __init__(
method on_train_begin (line 352) | def on_train_begin(self, logs=None):
method on_train_batch_end (line 357) | def on_train_batch_end(self, batch, logs=None):
method on_epoch_end (line 368) | def on_epoch_end(self, epoch, logs=None):
method on_train_end (line 395) | def on_train_end(self, logs=None):
FILE: transformers/kernels/deformable_detr/cpu/ms_deform_attn_cpu.cpp
function ms_deform_attn_cpu_forward (line 17) | at::Tensor
function ms_deform_attn_cpu_backward (line 29) | std::vector<at::Tensor>
FILE: transformers/kernels/deformable_detr/ms_deform_attn.h
function im2col_step (line 27) | int im2col_step)
FILE: transformers/kernels/deformable_detr/vision.cpp
function PYBIND11_MODULE (line 13) | PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
FILE: transformers/kernels/rwkv/wkv_op.cpp
function forward (line 12) | void forward(torch::Tensor &w, torch::Tensor &u, torch::Tensor &k, torch...
function forward_bf16 (line 18) | void forward_bf16(torch::Tensor &w, torch::Tensor &u, torch::Tensor &k, ...
function forward_with_state (line 24) | void forward_with_state(torch::Tensor &w, torch::Tensor &u, torch::Tenso...
function forward_with_state_bf16 (line 30) | void forward_with_state_bf16(torch::Tensor &w, torch::Tensor &u, torch::...
function backward (line 36) | void backward(torch::Tensor &w, torch::Tensor &u, torch::Tensor &k, torc...
function backward_bf16 (line 42) | void backward_bf16(torch::Tensor &w, torch::Tensor &u, torch::Tensor &k,...
function PYBIND11_MODULE (line 50) | PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
function TORCH_LIBRARY (line 59) | TORCH_LIBRARY(wkv, m) {
FILE: transformers/kernels/yoso/fast_lsh_cumulation_torch.cpp
function fast_hash (line 7) | std::vector<at::Tensor> fast_hash(
function lsh_cumulation (line 28) | at::Tensor lsh_cumulation(
function lsh_weighted_cumulation (line 49) | at::Tensor lsh_weighted_cumulation(
function PYBIND11_MODULE (line 124) | PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
FILE: transformers/modelcard.py
class ModelCard (line 80) | class ModelCard:
method __init__ (line 91) | def __init__(self, **kwargs):
method save_pretrained (line 114) | def save_pretrained(self, save_directory_or_file):
method from_pretrained (line 126) | def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):
method from_dict (line 222) | def from_dict(cls, json_object):
method from_json_file (line 227) | def from_json_file(cls, json_file):
method __eq__ (line 234) | def __eq__(self, other):
method __repr__ (line 237) | def __repr__(self):
method to_dict (line 240) | def to_dict(self):
method to_json_string (line 245) | def to_json_string(self):
method to_json_file (line 249) | def to_json_file(self, json_file_path):
function _listify (line 300) | def _listify(obj):
function _insert_values_as_list (line 309) | def _insert_values_as_list(metadata, name, values):
function infer_metric_tags_from_eval_results (line 321) | def infer_metric_tags_from_eval_results(eval_results):
function _insert_value (line 333) | def _insert_value(metadata, name, value):
function is_hf_dataset (line 340) | def is_hf_dataset(dataset):
function _get_mapping_values (line 349) | def _get_mapping_values(mapping):
class TrainingSummary (line 360) | class TrainingSummary:
method __post_init__ (line 376) | def __post_init__(self):
method create_model_index (line 392) | def create_model_index(self, metric_mapping):
method create_metadata (line 455) | def create_metadata(self):
method to_model_card (line 468) | def to_model_card(self):
method from_trainer (line 549) | def from_trainer(
method from_keras (line 627) | def from_keras(
function parse_keras_history (line 702) | def parse_keras_history(logs):
function parse_log_history (line 737) | def parse_log_history(log_history):
function extract_hyperparameters_from_keras (line 799) | def extract_hyperparameters_from_keras(model):
function _maybe_round (line 812) | def _maybe_round(v, decimals=4):
function _regular_table_line (line 818) | def _regular_table_line(values, col_widths):
function _second_table_line (line 823) | def _second_table_line(col_widths):
function make_markdown_table (line 828) | def make_markdown_table(lines):
function extract_hyperparameters_from_trainer (line 855) | def extract_hyperparameters_from_trainer(trainer):
FILE: transformers/modeling_flax_outputs.py
class FlaxBaseModelOutput (line 23) | class FlaxBaseModelOutput(ModelOutput):
class FlaxBaseModelOutputWithNoAttention (line 49) | class FlaxBaseModelOutputWithNoAttention(ModelOutput):
class FlaxBaseModelOutputWithPoolingAndNoAttention (line 67) | class FlaxBaseModelOutputWithPoolingAndNoAttention(ModelOutput):
class FlaxImageClassifierOutputWithNoAttention (line 88) | class FlaxImageClassifierOutputWithNoAttention(ModelOutput):
class FlaxBaseModelOutputWithPast (line 107) | class FlaxBaseModelOutputWithPast(ModelOutput):
class FlaxBaseModelOutputWithPooling (line 137) | class FlaxBaseModelOutputWithPooling(ModelOutput):
class FlaxBaseModelOutputWithPoolingAndCrossAttentions (line 168) | class FlaxBaseModelOutputWithPoolingAndCrossAttentions(ModelOutput):
class FlaxBaseModelOutputWithPastAndCrossAttentions (line 217) | class FlaxBaseModelOutputWithPastAndCrossAttentions(ModelOutput):
class FlaxSeq2SeqModelOutput (line 263) | class FlaxSeq2SeqModelOutput(ModelOutput):
class FlaxCausalLMOutputWithCrossAttentions (line 324) | class FlaxCausalLMOutputWithCrossAttentions(ModelOutput):
class FlaxMaskedLMOutput (line 365) | class FlaxMaskedLMOutput(ModelOutput):
class FlaxSeq2SeqLMOutput (line 394) | class FlaxSeq2SeqLMOutput(ModelOutput):
class FlaxNextSentencePredictorOutput (line 451) | class FlaxNextSentencePredictorOutput(ModelOutput):
class FlaxSequenceClassifierOutput (line 478) | class FlaxSequenceClassifierOutput(ModelOutput):
class FlaxSeq2SeqSequenceClassifierOutput (line 504) | class FlaxSeq2SeqSequenceClassifierOutput(ModelOutput):
class FlaxMultipleChoiceModelOutput (line 561) | class FlaxMultipleChoiceModelOutput(ModelOutput):
class FlaxTokenClassifierOutput (line 589) | class FlaxTokenClassifierOutput(ModelOutput):
class FlaxQuestionAnsweringModelOutput (line 615) | class FlaxQuestionAnsweringModelOutput(ModelOutput):
class FlaxSeq2SeqQuestionAnsweringModelOutput (line 644) | class FlaxSeq2SeqQuestionAnsweringModelOutput(ModelOutput):
FILE: transformers/modeling_flax_pytorch_utils.py
function load_pytorch_checkpoint_in_flax_state_dict (line 41) | def load_pytorch_checkpoint_in_flax_state_dict(
function rename_key_and_reshape_tensor (line 69) | def rename_key_and_reshape_tensor(
function convert_pytorch_state_dict_to_flax (line 126) | def convert_pytorch_state_dict_to_flax(pt_state_dict, flax_model):
function convert_pytorch_sharded_state_dict_to_flax (line 204) | def convert_pytorch_sharded_state_dict_to_flax(shard_filenames, flax_mod...
function load_flax_checkpoint_in_pytorch_model (line 284) | def load_flax_checkpoint_in_pytorch_model(model, flax_checkpoint_path):
function load_flax_weights_in_pytorch_model (line 302) | def load_flax_weights_in_pytorch_model(pt_model, flax_state):
FILE: transformers/modeling_flax_utils.py
function quick_gelu (line 61) | def quick_gelu(x):
function dtype_byte_size (line 75) | def dtype_byte_size(dtype):
function flax_shard_checkpoint (line 92) | def flax_shard_checkpoint(params, max_shard_size="10GB"):
class FlaxPreTrainedModel (line 157) | class FlaxPreTrainedModel(PushToHubMixin, FlaxGenerationMixin):
method __init__ (line 179) | def __init__(
method init_weights (line 230) | def init_weights(self, rng: jax.random.PRNGKey, input_shape: Tuple, pa...
method enable_gradient_checkpointing (line 233) | def enable_gradient_checkpointing(self):
method _from_config (line 237) | def _from_config(cls, config, **kwargs):
method framework (line 244) | def framework(self) -> str:
method config (line 251) | def config(self) -> PretrainedConfig:
method module (line 255) | def module(self) -> nn.Module:
method params (line 259) | def params(self) -> Union[Dict, FrozenDict]:
method required_params (line 269) | def required_params(self) -> Set:
method params_shape_tree (line 273) | def params_shape_tree(self) -> Dict:
method params (line 277) | def params(self, params: Union[Dict, FrozenDict]):
method _cast_floating_to (line 295) | def _cast_floating_to(self, params: Union[Dict, FrozenDict], dtype: jn...
method to_bf16 (line 319) | def to_bf16(self, params: Union[Dict, FrozenDict], mask: Any = None):
method to_fp32 (line 358) | def to_fp32(self, params: Union[Dict, FrozenDict], mask: Any = None):
method to_fp16 (line 385) | def to_fp16(self, params: Union[Dict, FrozenDict], mask: Any = None):
method load_flax_sharded_weights (line 425) | def load_flax_sharded_weights(cls, shard_files):
method can_generate (line 471) | def can_generate(self) -> bool:
method from_pretrained (line 482) | def from_pretrained(
method save_pretrained (line 1008) | def save_pretrained(
method register_for_auto_class (line 1112) | def register_for_auto_class(cls, auto_class="FlaxAutoModel"):
function overwrite_call_docstring (line 1146) | def overwrite_call_docstring(model_class, docstring):
function append_call_sample_docstring (line 1155) | def append_call_sample_docstring(model_class, checkpoint, output_type, c...
function append_replace_return_docstrings (line 1165) | def append_replace_return_docstrings(model_class, output_type, config_cl...
FILE: transformers/modeling_outputs.py
class BaseModelOutput (line 25) | class BaseModelOutput(ModelOutput):
class BaseModelOutputWithNoAttention (line 51) | class BaseModelOutputWithNoAttention(ModelOutput):
class BaseModelOutputWithPooling (line 70) | class BaseModelOutputWithPooling(ModelOutput):
class BaseModelOutputWithPoolingAndNoAttention (line 102) | class BaseModelOutputWithPoolingAndNoAttention(ModelOutput):
class BaseModelOutputWithPast (line 124) | class BaseModelOutputWithPast(ModelOutput):
class BaseModelOutputWithCrossAttentions (line 163) | class BaseModelOutputWithCrossAttentions(ModelOutput):
class BaseModelOutputWithPoolingAndCrossAttentions (line 196) | class BaseModelOutputWithPoolingAndCrossAttentions(ModelOutput):
class BaseModelOutputWithPastAndCrossAttentions (line 245) | class BaseModelOutputWithPastAndCrossAttentions(ModelOutput):
class MoECausalLMOutputWithPast (line 291) | class MoECausalLMOutputWithPast(ModelOutput):
class MoEModelOutput (line 340) | class MoEModelOutput(ModelOutput):
class MoEModelOutputWithPastAndCrossAttentions (line 372) | class MoEModelOutputWithPastAndCrossAttentions(ModelOutput):
class Seq2SeqModelOutput (line 425) | class Seq2SeqModelOutput(ModelOutput):
class Seq2SeqMoEModelOutput (line 486) | class Seq2SeqMoEModelOutput(ModelOutput):
class CausalLMOutput (line 558) | class CausalLMOutput(ModelOutput):
class CausalLMOutputWithPast (line 587) | class CausalLMOutputWithPast(ModelOutput):
class CausalLMOutputWithCrossAttentions (line 623) | class CausalLMOutputWithCrossAttentions(ModelOutput):
class SequenceClassifierOutputWithPast (line 667) | class SequenceClassifierOutputWithPast(ModelOutput):
class MaskedLMOutput (line 703) | class MaskedLMOutput(ModelOutput):
class Seq2SeqLMOutput (line 732) | class Seq2SeqLMOutput(ModelOutput):
class Seq2SeqMoEOutput (line 792) | class Seq2SeqMoEOutput(ModelOutput):
class NextSentencePredictorOutput (line 867) | class NextSentencePredictorOutput(ModelOutput):
class SequenceClassifierOutput (line 897) | class SequenceClassifierOutput(ModelOutput):
class Seq2SeqSequenceClassifierOutput (line 926) | class Seq2SeqSequenceClassifierOutput(ModelOutput):
class MultipleChoiceModelOutput (line 986) | class MultipleChoiceModelOutput(ModelOutput):
class TokenClassifierOutput (line 1017) | class TokenClassifierOutput(ModelOutput):
class QuestionAnsweringModelOutput (line 1046) | class QuestionAnsweringModelOutput(ModelOutput):
class Seq2SeqQuestionAnsweringModelOutput (line 1078) | class Seq2SeqQuestionAnsweringModelOutput(ModelOutput):
class SemanticSegmenterOutput (line 1141) | class SemanticSegmenterOutput(ModelOutput):
class ImageClassifierOutput (line 1179) | class ImageClassifierOutput(ModelOutput):
class ImageClassifierOutputWithNoAttention (line 1207) | class ImageClassifierOutputWithNoAttention(ModelOutput):
class DepthEstimatorOutput (line 1228) | class DepthEstimatorOutput(ModelOutput):
class ImageSuperResolutionOutput (line 1258) | class ImageSuperResolutionOutput(ModelOutput):
class Wav2Vec2BaseModelOutput (line 1286) | class Wav2Vec2BaseModelOutput(ModelOutput):
class XVectorOutput (line 1315) | class XVectorOutput(ModelOutput):
class BackboneOutput (line 1347) | class BackboneOutput(ModelOutput):
class BaseModelOutputWithPoolingAndProjection (line 1374) | class BaseModelOutputWithPoolingAndProjection(ModelOutput):
class Seq2SeqSpectrogramOutput (line 1411) | class Seq2SeqSpectrogramOutput(ModelOutput):
class Seq2SeqTSModelOutput (line 1471) | class Seq2SeqTSModelOutput(ModelOutput):
class Seq2SeqTSPredictionOutput (line 1543) | class Seq2SeqTSPredictionOutput(ModelOutput):
class SampleTSPredictionOutput (line 1615) | class SampleTSPredictionOutput(ModelOutput):
class MaskedImageModelingOutput (line 1629) | class MaskedImageModelingOutput(ModelOutput):
method logits (line 1656) | def logits(self):
FILE: transformers/modeling_tf_outputs.py
class TFBaseModelOutput (line 27) | class TFBaseModelOutput(ModelOutput):
class TFBaseModelOutputWithNoAttention (line 53) | class TFBaseModelOutputWithNoAttention(ModelOutput):
class TFBaseModelOutputWithPooling (line 72) | class TFBaseModelOutputWithPooling(ModelOutput):
class TFBaseModelOutputWithPoolingAndNoAttention (line 106) | class TFBaseModelOutputWithPoolingAndNoAttention(ModelOutput):
class TFBaseModelOutputWithPoolingAndCrossAttentions (line 128) | class TFBaseModelOutputWithPoolingAndCrossAttentions(ModelOutput):
class TFBaseModelOutputWithPast (line 176) | class TFBaseModelOutputWithPast(ModelOutput):
class TFBaseModelOutputWithCrossAttentions (line 212) | class TFBaseModelOutputWithCrossAttentions(ModelOutput):
class TFBaseModelOutputWithPastAndCrossAttentions (line 245) | class TFBaseModelOutputWithPastAndCrossAttentions(ModelOutput):
class TFSeq2SeqModelOutput (line 288) | class TFSeq2SeqModelOutput(ModelOutput):
class TFCausalLMOutput (line 348) | class TFCausalLMOutput(ModelOutput):
class TFCausalLMOutputWithPast (line 377) | class TFCausalLMOutputWithPast(ModelOutput):
class TFCausalLMOutputWithCrossAttentions (line 413) | class TFCausalLMOutputWithCrossAttentions(ModelOutput):
class TFMaskedLMOutput (line 456) | class TFMaskedLMOutput(ModelOutput):
class TFSeq2SeqLMOutput (line 485) | class TFSeq2SeqLMOutput(ModelOutput):
class TFNextSentencePredictorOutput (line 544) | class TFNextSentencePredictorOutput(ModelOutput):
class TFSequenceClassifierOutput (line 574) | class TFSequenceClassifierOutput(ModelOutput):
class TFSeq2SeqSequenceClassifierOutput (line 603) | class TFSeq2SeqSequenceClassifierOutput(ModelOutput):
class TFSemanticSegmenterOutput (line 659) | class TFSemanticSegmenterOutput(ModelOutput):
class TFSemanticSegmenterOutputWithNoAttention (line 696) | class TFSemanticSegmenterOutputWithNoAttention(ModelOutput):
class TFImageClassifierOutput (line 727) | class TFImageClassifierOutput(ModelOutput):
class TFMultipleChoiceModelOutput (line 754) | class TFMultipleChoiceModelOutput(ModelOutput):
class TFTokenClassifierOutput (line 785) | class TFTokenClassifierOutput(ModelOutput):
class TFQuestionAnsweringModelOutput (line 814) | class TFQuestionAnsweringModelOutput(ModelOutput):
class TFSeq2SeqQuestionAnsweringModelOutput (line 846) | class TFSeq2SeqQuestionAnsweringModelOutput(ModelOutput):
class TFSequenceClassifierOutputWithPast (line 901) | class TFSequenceClassifierOutputWithPast(ModelOutput):
class TFImageClassifierOutputWithNoAttention (line 937) | class TFImageClassifierOutputWithNoAttention(ModelOutput):
class TFMaskedImageModelingOutput (line 958) | class TFMaskedImageModelingOutput(ModelOutput):
method logits (line 985) | def logits(self):
FILE: transformers/modeling_tf_pytorch_utils.py
class TransposeType (line 31) | class TransposeType(ExplicitEnum):
function convert_tf_weight_name_to_pt_weight_name (line 42) | def convert_tf_weight_name_to_pt_weight_name(
function apply_transpose (line 114) | def apply_transpose(transpose: TransposeType, weight, match_shape=None, ...
function load_pytorch_checkpoint_in_tf2_model (line 156) | def load_pytorch_checkpoint_in_tf2_model(
function load_pytorch_model_in_tf2_model (line 200) | def load_pytorch_model_in_tf2_model(tf_model, pt_model, tf_inputs=None, ...
function load_pytorch_weights_in_tf2_model (line 209) | def load_pytorch_weights_in_tf2_model(
function load_pytorch_state_dict_in_tf2_model (line 241) | def load_pytorch_state_dict_in_tf2_model(
function load_tf2_checkpoint_in_pytorch_model (line 410) | def load_tf2_checkpoint_in_pytorch_model(
function load_tf2_model_in_pytorch_model (line 451) | def load_tf2_model_in_pytorch_model(pt_model, tf_model, allow_missing_ke...
function load_tf2_weights_in_pytorch_model (line 460) | def load_tf2_weights_in_pytorch_model(pt_model, tf_weights, allow_missin...
function load_tf2_state_dict_in_pytorch_model (line 478) | def load_tf2_state_dict_in_pytorch_model(pt_model, tf_state_dict, allow_...
FILE: transformers/modeling_tf_utils.py
function dummy_loss (line 113) | def dummy_loss(y_true, y_pred):
class TFModelUtilsMixin (line 121) | class TFModelUtilsMixin:
method num_parameters (line 126) | def num_parameters(self, only_trainable: bool = False) -> int:
function keras_serializable (line 143) | def keras_serializable(cls):
class TFCausalLanguageModelingLoss (line 208) | class TFCausalLanguageModelingLoss:
method hf_compute_loss (line 219) | def hf_compute_loss(self, labels, logits):
class TFQuestionAnsweringLoss (line 239) | class TFQuestionAnsweringLoss:
method hf_compute_loss (line 244) | def hf_compute_loss(self, labels, logits):
class TFTokenClassificationLoss (line 254) | class TFTokenClassificationLoss:
method hf_compute_loss (line 265) | def hf_compute_loss(self, labels, logits):
class TFSequenceClassificationLoss (line 298) | class TFSequenceClassificationLoss:
method hf_compute_loss (line 303) | def hf_compute_loss(self, labels, logits):
class TFMultipleChoiceLoss (line 317) | class TFMultipleChoiceLoss:
method hf_compute_loss (line 320) | def hf_compute_loss(self, labels, logits):
class TFMaskedLanguageModelingLoss (line 327) | class TFMaskedLanguageModelingLoss(TFCausalLanguageModelingLoss):
class TFNextSentencePredictionLoss (line 339) | class TFNextSentencePredictionLoss:
method hf_compute_loss (line 350) | def hf_compute_loss(self, labels, logits):
function booleans_processing (line 375) | def booleans_processing(config, **kwargs):
function unpack_inputs (line 408) | def unpack_inputs(func):
function input_processing (line 452) | def input_processing(func, config, **kwargs):
function dtype_byte_size (line 625) | def dtype_byte_size(dtype):
function format_weight_name (line 645) | def format_weight_name(name, _prefix=None):
function tf_shard_checkpoint (line 653) | def tf_shard_checkpoint(weights, max_shard_size="10GB"):
function load_tf_sharded_weights (line 719) | def load_tf_sharded_weights(model, shard_files, ignore_mismatched_sizes=...
function load_tf_shard (line 786) | def load_tf_shard(model, model_layer_map, resolved_archive_file, ignore_...
function load_tf_weights (line 874) | def load_tf_weights(model, resolved_archive_file, ignore_mismatched_size...
function load_tf_weights_from_h5 (line 901) | def load_tf_weights_from_h5(model, resolved_archive_file, ignore_mismatc...
function load_tf_weights_from_safetensors (line 1001) | def load_tf_weights_from_safetensors(model, resolved_archive_file, ignor...
function init_copy_embeddings (line 1042) | def init_copy_embeddings(old_embeddings, new_num_tokens):
class TFPreTrainedModel (line 1082) | class TFPreTrainedModel(tf.keras.Model, TFModelUtilsMixin, TFGenerationM...
method dummy_inputs (line 1117) | def dummy_inputs(self) -> Dict[str, tf.Tensor]:
method framework (line 1149) | def framework(self) -> str:
method build (line 1155) | def build(self, input_shape=None):
method __init__ (line 1162) | def __init__(self, config, *inputs, **kwargs):
method get_config (line 1181) | def get_config(self):
method from_config (line 1185) | def from_config(cls, config, **kwargs):
method _from_config (line 1191) | def _from_config(cls, config, **kwargs):
method get_head_mask (line 1197) | def get_head_mask(self, head_mask: tf.Tensor | None, num_hidden_layers...
method _convert_head_mask_to_5d (line 1218) | def _convert_head_mask_to_5d(self, head_mask, num_hidden_layers):
method eager_serving (line 1229) | def eager_serving(self, inputs):
method input_signature (line 1243) | def input_signature(self) -> Dict[str, tf.TensorSpec]:
method _prune_signature (line 1290) | def _prune_signature(self, signature):
method serving_output (line 1295) | def serving_output(self, output):
method can_generate (line 1319) | def can_generate(self) -> bool:
method get_input_embeddings (line 1331) | def get_input_embeddings(self) -> tf.keras.layers.Layer:
method _save_checkpoint (line 1345) | def _save_checkpoint(self, checkpoint_dir, epoch):
method load_repo_checkpoint (line 1358) | def load_repo_checkpoint(self, repo_path_or_name):
method prepare_tf_dataset (line 1412) | def prepare_tf_dataset(
method compile (line 1517) | def compile(
method compute_loss (line 1571) | def compute_loss(self, *args, **kwargs):
method get_label_to_output_name_mapping (line 1585) | def get_label_to_output_name_mapping(self):
method train_step (line 1600) | def train_step(self, data):
method test_step (line 1708) | def test_step(self, data):
method create_model_card (line 1812) | def create_model_card(
method set_input_embeddings (line 1872) | def set_input_embeddings(self, value):
method get_output_embeddings (line 1892) | def get_output_embeddings(self) -> Union[None, tf.keras.layers.Layer]:
method set_output_embeddings (line 1912) | def set_output_embeddings(self, value):
method get_output_layer_with_bias (line 1929) | def get_output_layer_with_bias(self) -> Union[None, tf.keras.layers.La...
method get_prefix_bias_name (line 1942) | def get_prefix_bias_name(self) -> Union[None, str]:
method get_bias (line 1952) | def get_bias(self) -> Union[None, Dict[str, tf.Variable]]:
method set_bias (line 1969) | def set_bias(self, value):
method get_lm_head (line 1985) | def get_lm_head(self) -> tf.keras.layers.Layer:
method resize_token_embeddings (line 1994) | def resize_token_embeddings(
method _v2_resized_token_embeddings (line 2027) | def _v2_resized_token_embeddings(self, new_num_tokens: Optional[int] =...
method _get_word_embedding_weight (line 2050) | def _get_word_embedding_weight(model, embedding_layer):
method _resize_token_embeddings (line 2081) | def _resize_token_embeddings(self, new_num_tokens):
method _v2_resize_token_embeddings (line 2104) | def _v2_resize_token_embeddings(self, new_num_tokens):
method _get_resized_lm_head_bias (line 2125) | def _get_resized_lm_head_bias(self, old_lm_head_bias, new_num_tokens):
method _v2_get_resized_lm_head_bias (line 2178) | def _v2_get_resized_lm_head_bias(
method _get_resized_lm_head_decoder (line 2212) | def _get_resized_lm_head_decoder(self, old_lm_head_decoder, new_num_to...
method _get_resized_embeddings (line 2250) | def _get_resized_embeddings(self, old_embeddings, new_num_tokens=None)...
method _v2_get_resized_embeddings (line 2285) | def _v2_get_resized_embeddings(
method prune_heads (line 2332) | def prune_heads(self, heads_to_prune):
method save_pretrained (line 2344) | def save_pretrained(
method from_pretrained (line 2501) | def from_pretrained(cls, pretrained_model_name_or_path, *model_args, *...
method push_to_hub (line 3029) | def push_to_hub(
method register_for_auto_class (line 3126) | def register_for_auto_class(cls, auto_class="TFAutoModel"):
class TFConv1D (line 3152) | class TFConv1D(tf.keras.layers.Layer):
method __init__ (line 3169) | def __init__(self, nf, nx, initializer_range=0.02, **kwargs):
method build (line 3175) | def build(self, input_shape):
method call (line 3181) | def call(self, x):
class TFSharedEmbeddings (line 3192) | class TFSharedEmbeddings(tf.keras.layers.Layer):
method __init__ (line 3212) | def __init__(self, vocab_size: int, hidden_size: int, initializer_rang...
method build (line 3222) | def build(self, input_shape):
method get_config (line 3232) | def get_config(self):
method call (line 3242) | def call(self, inputs: tf.Tensor, mode: str = "embedding") -> tf.Tensor:
method _embedding (line 3274) | def _embedding(self, input_ids):
method _linear (line 3278) | def _linear(self, inputs):
class TFSequenceSummary (line 3295) | class TFSequenceSummary(tf.keras.layers.Layer):
method __init__ (line 3325) | def __init__(self, config: PretrainedConfig, initializer_range: float ...
method call (line 3359) | def call(self, inputs, cls_index=None, training=False):
function get_initializer (line 3411) | def get_initializer(initializer_range: float = 0.02) -> tf.keras.initial...
FILE: transformers/modeling_utils.py
function no_init_weights (line 127) | def no_init_weights(_enable=True):
class Identity (line 147) | class Identity(nn.Module):
method __init__ (line 150) | def __init__(self, *args, **kwargs):
method forward (line 153) | def forward(self, input):
function get_parameter_device (line 157) | def get_parameter_device(parameter: Union[nn.Module, GenerationMixin, "M...
function get_first_parameter_dtype (line 172) | def get_first_parameter_dtype(parameter: Union[nn.Module, GenerationMixi...
function get_parameter_dtype (line 190) | def get_parameter_dtype(parameter: Union[nn.Module, GenerationMixin, "Mo...
function get_state_dict_float_dtype (line 239) | def get_state_dict_float_dtype(state_dict):
function get_state_dict_dtype (line 250) | def get_state_dict_dtype(state_dict):
function dtype_byte_size (line 263) | def dtype_byte_size(dtype):
function shard_checkpoint (line 283) | def shard_checkpoint(
function load_sharded_checkpoint (line 360) | def load_sharded_checkpoint(model, folder, strict=True, prefer_safe=True):
function load_state_dict (line 444) | def load_state_dict(checkpoint_file: Union[str, os.PathLike]):
function set_initialized_submodules (line 486) | def set_initialized_submodules(model, state_dict_keys):
function _load_state_dict_into_model (line 497) | def _load_state_dict_into_model(model_to_load, state_dict, start_prefix):
function find_submodule_and_param_name (line 558) | def find_submodule_and_param_name(model, long_key, start_prefix):
function _move_model_to_meta (line 581) | def _move_model_to_meta(model, loaded_state_dict_keys, start_prefix):
function _load_state_dict_into_meta_model (line 607) | def _load_state_dict_into_meta_model(
function _add_variant (line 735) | def _add_variant(weights_name: str, variant: Optional[str] = None) -> str:
class ModuleUtilsMixin (line 744) | class ModuleUtilsMixin:
method _hook_rss_memory_pre_forward (line 750) | def _hook_rss_memory_pre_forward(module, *args, **kwargs):
method _hook_rss_memory_post_forward (line 762) | def _hook_rss_memory_post_forward(module, *args, **kwargs):
method add_memory_hooks (line 775) | def add_memory_hooks(self):
method reset_memory_hooks_state (line 787) | def reset_memory_hooks_state(self):
method device (line 797) | def device(self) -> torch.device:
method dtype (line 805) | def dtype(self) -> torch.dtype:
method invert_attention_mask (line 811) | def invert_attention_mask(self, encoder_attention_mask: Tensor) -> Ten...
method create_extended_attention_mask_for_decoder (line 836) | def create_extended_attention_mask_for_decoder(input_shape, attention_...
method get_extended_attention_mask (line 863) | def get_extended_attention_mask(
method get_head_mask (line 915) | def get_head_mask(
method _convert_head_mask_to_5d (line 942) | def _convert_head_mask_to_5d(self, head_mask, num_hidden_layers):
method num_parameters (line 953) | def num_parameters(self, only_trainable: bool = False, exclude_embeddi...
method estimate_tokens (line 979) | def estimate_tokens(self, input_dict: Dict[str, Union[torch.Tensor, An...
method floating_point_ops (line 1000) | def floating_point_ops(
class PreTrainedModel (line 1027) | class PreTrainedModel(nn.Module, ModuleUtilsMixin, GenerationMixin, Push...
method dummy_inputs (line 1077) | def dummy_inputs(self) -> Dict[str, torch.Tensor]:
method framework (line 1084) | def framework(self) -> str:
method __init__ (line 1090) | def __init__(self, config: PretrainedConfig, *inputs, **kwargs):
method post_init (line 1104) | def post_init(self):
method _backward_compatibility_gradient_checkpointing (line 1112) | def _backward_compatibility_gradient_checkpointing(self):
method _from_config (line 1119) | def _from_config(cls, config, **kwargs):
method _set_default_torch_dtype (line 1152) | def _set_default_torch_dtype(cls, dtype: torch.dtype) -> torch.dtype:
method base_model (line 1179) | def base_model(self) -> nn.Module:
method can_generate (line 1185) | def can_generate(self) -> bool:
method enable_input_require_grads (line 1197) | def enable_input_require_grads(self):
method disable_input_require_grads (line 1208) | def disable_input_require_grads(self):
method get_input_embeddings (line 1214) | def get_input_embeddings(self) -> nn.Module:
method set_input_embeddings (line 1227) | def set_input_embeddings(self, value: nn.Module):
method get_output_embeddings (line 1240) | def get_output_embeddings(self) -> nn.Module:
method _init_weights (line 1249) | def _init_weights(self, module):
method _initialize_weights (line 1255) | def _initialize_weights(self, module):
method tie_weights (line 1264) | def tie_weights(self):
method _tie_encoder_decoder_weights (line 1286) | def _tie_encoder_decoder_weights(encoder: nn.Module, decoder: nn.Modul...
method _tie_or_clone_weights (line 1360) | def _tie_or_clone_weights(self, output_embeddings, input_embeddings):
method resize_token_embeddings (line 1380) | def resize_token_embeddings(self, new_num_tokens: Optional[int] = None...
method _resize_token_embeddings (line 1408) | def _resize_token_embeddings(self, new_num_tokens):
method _get_resized_embeddings (line 1421) | def _get_resized_embeddings(
method _get_resized_lm_head (line 1485) | def _get_resized_lm_head(
method resize_position_embeddings (line 1577) | def resize_position_embeddings(self, new_num_position_embeddings: int):
method get_position_embeddings (line 1583) | def get_position_embeddings(self) -> Union[nn.Embedding, Tuple[nn.Embe...
method init_weights (line 1589) | def init_weights(self):
method prune_heads (line 1606) | def prune_heads(self, heads_to_prune: Dict[int, List[int]]):
method gradient_checkpointing_enable (line 1623) | def gradient_checkpointing_enable(self):
method gradient_checkpointing_disable (line 1634) | def gradient_checkpointing_disable(self):
method is_gradient_checkpointing (line 1645) | def is_gradient_checkpointing(self) -> bool:
method save_pretrained (line 1654) | def save_pretrained(
method get_memory_footprint (line 1876) | def get_memory_footprint(self, return_buffers=True):
method to (line 1894) | def to(self, *args, **kwargs):
method half (line 1904) | def half(self, *args):
method float (line 1914) | def float(self, *args):
method from_pretrained (line 1925) | def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union...
method _load_pretrained_model (line 2952) | def _load_pretrained_model(
method retrieve_modules_from_names (line 3325) | def retrieve_modules_from_names(self, names, add_prefix=False, remove_...
method _load_pretrained_model_low_mem (line 3349) | def _load_pretrained_model_low_mem(model, loaded_state_dict_keys, reso...
method register_for_auto_class (line 3373) | def register_for_auto_class(cls, auto_class="AutoModel"):
method to_bettertransformer (line 3398) | def to_bettertransformer(self) -> "PreTrainedModel":
method reverse_bettertransformer (line 3426) | def reverse_bettertransformer(self):
class PoolerStartLogits (line 3456) | class PoolerStartLogits(nn.Module):
method __init__ (line 3465) | def __init__(self, config: PretrainedConfig):
method forward (line 3469) | def forward(
class PoolerEndLogits (line 3494) | class PoolerEndLogits(nn.Module):
method __init__ (line 3504) | def __init__(self, config: PretrainedConfig):
method forward (line 3511) | def forward(
class PoolerAnswerClass (line 3563) | class PoolerAnswerClass(nn.Module):
method __init__ (line 3572) | def __init__(self, config):
method forward (line 3578) | def forward(
class SquadHeadOutput (line 3629) | class SquadHeadOutput(ModelOutput):
class SQuADHead (line 3659) | class SQuADHead(nn.Module):
method __init__ (line 3669) | def __init__(self, config):
method forward (line 3679) | def forward(
class SequenceSummary (line 3776) | class SequenceSummary(nn.Module):
method __init__ (line 3802) | def __init__(self, config: PretrainedConfig):
method forward (line 3831) | def forward(
function unwrap_model (line 3875) | def unwrap_model(model: nn.Module) -> nn.Module:
function expand_device_map (line 3889) | def expand_device_map(device_map, param_names):
function get_disk_only_shard_files (line 3899) | def get_disk_only_shard_files(device_map, sharded_metadata):
FILE: transformers/models/albert/configuration_albert.py
class AlbertConfig (line 36) | class AlbertConfig(PretrainedConfig):
method __init__ (line 113) | def __init__(
class AlbertOnnxConfig (line 159) | class AlbertOnnxConfig(OnnxConfig):
method inputs (line 161) | def inputs(self) -> Mapping[str, Mapping[int, str]]:
FILE: transformers/models/albert/convert_albert_original_tf_checkpoint_to_pytorch.py
function convert_tf_checkpoint_to_pytorch (line 29) | def convert_tf_checkpoint_to_pytorch(tf_checkpoint_path, albert_config_f...
FILE: transformers/models/albert/modeling_albert.py
function load_tf_weights_in_albert (line 68) | def load_tf_weights_in_albert(model, config, tf_checkpoint_path):
class AlbertEmbeddings (line 194) | class AlbertEmbeddings(nn.Module):
method __init__ (line 199) | def __init__(self, config: AlbertConfig):
method forward (line 218) | def forward(
class AlbertAttention (line 260) | class AlbertAttention(nn.Module):
method __init__ (line 261) | def __init__(self, config: AlbertConfig):
method transpose_for_scores (line 290) | def transpose_for_scores(self, x: torch.Tensor) -> torch.Tensor:
method prune_heads (line 295) | def prune_heads(self, heads: List[int]) -> None:
method forward (line 313) | def forward(
class AlbertLayer (line 372) | class AlbertLayer(nn.Module):
method __init__ (line 373) | def __init__(self, config: AlbertConfig):
method forward (line 386) | def forward(
method ff_chunk (line 406) | def ff_chunk(self, attention_output: torch.Tensor) -> torch.Tensor:
class AlbertLayerGroup (line 413) | class AlbertLayerGroup(nn.Module):
method __init__ (line 414) | def __init__(self, config: AlbertConfig):
method forward (line 419) | def forward(
class AlbertTransformer (line 448) | class AlbertTransformer(nn.Module):
method __init__ (line 449) | def __init__(self, config: AlbertConfig):
method forward (line 456) | def forward(
class AlbertPreTrainedModel (line 501) | class AlbertPreTrainedModel(PreTrainedModel):
method _init_weights (line 512) | def _init_weights(self, module):
class AlbertForPreTrainingOutput (line 530) | class AlbertForPreTrainingOutput(ModelOutput):
class AlbertModel (line 633) | class AlbertModel(AlbertPreTrainedModel):
method __init__ (line 637) | def __init__(self, config: AlbertConfig, add_pooling_layer: bool = True):
method get_input_embeddings (line 653) | def get_input_embeddings(self) -> nn.Embedding:
method set_input_embeddings (line 656) | def set_input_embeddings(self, value: nn.Embedding) -> None:
method _prune_heads (line 659) | def _prune_heads(self, heads_to_prune: Dict[int, List[int]]) -> None:
method forward (line 682) | def forward(
class AlbertForPreTraining (line 761) | class AlbertForPreTraining(AlbertPreTrainedModel):
method __init__ (line 768) | def __init__(self, config: AlbertConfig):
method get_output_embeddings (line 778) | def get_output_embeddings(self) -> nn.Linear:
method set_output_embeddings (line 781) | def set_output_embeddings(self, new_embeddings: nn.Linear) -> None:
method get_input_embeddings (line 784) | def get_input_embeddings(self) -> nn.Embedding:
method forward (line 789) | def forward(
class AlbertMLMHead (line 870) | class AlbertMLMHead(nn.Module):
method __init__ (line 871) | def __init__(self, config: AlbertConfig):
method forward (line 881) | def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
method _tie_weights (line 891) | def _tie_weights(self) -> None:
class AlbertSOPHead (line 896) | class AlbertSOPHead(nn.Module):
method __init__ (line 897) | def __init__(self, config: AlbertConfig):
method forward (line 903) | def forward(self, pooled_output: torch.Tensor) -> torch.Tensor:
class AlbertForMaskedLM (line 913) | class AlbertForMaskedLM(AlbertPreTrainedModel):
method __init__ (line 921) | def __init__(self, config):
method get_output_embeddings (line 930) | def get_output_embeddings(self) -> nn.Linear:
method set_output_embeddings (line 933) | def set_output_embeddings(self, new_embeddings: nn.Linear) -> None:
method get_input_embeddings (line 936) | def get_input_embeddings(self) -> nn.Embedding:
method forward (line 941) | def forward(
class AlbertForSequenceClassification (line 1032) | class AlbertForSequenceClassification(AlbertPreTrainedModel):
method __init__ (line 1033) | def __init__(self, config: AlbertConfig):
method forward (line 1053) | def forward(
class AlbertForTokenClassification (line 1133) | class AlbertForTokenClassification(AlbertPreTrainedModel):
method __init__ (line 1136) | def __init__(self, config: AlbertConfig):
method forward (line 1158) | def forward(
class AlbertForQuestionAnswering (line 1218) | class AlbertForQuestionAnswering(AlbertPreTrainedModel):
method __init__ (line 1221) | def __init__(self, config: AlbertConfig):
method forward (line 1241) | def forward(
class AlbertForMultipleChoice (line 1323) | class AlbertForMultipleChoice(AlbertPreTrainedModel):
method __init__ (line 1324) | def __init__(self, config: AlbertConfig):
method forward (line 1340) | def forward(
FILE: transformers/models/albert/modeling_flax_albert.py
class FlaxAlbertForPreTrainingOutput (line 55) | class FlaxAlbertForPreTrainingOutput(ModelOutput):
class FlaxAlbertEmbeddings (line 151) | class FlaxAlbertEmbeddings(nn.Module):
method setup (line 157) | def setup(self):
method __call__ (line 177) | def __call__(self, input_ids, token_type_ids, position_ids, determinis...
class FlaxAlbertSelfAttention (line 192) | class FlaxAlbertSelfAttention(nn.Module):
method setup (line 196) | def setup(self):
method __call__ (line 226) | def __call__(self, hidden_states, attention_mask, deterministic=True, ...
class FlaxAlbertLayer (line 277) | class FlaxAlbertLayer(nn.Module):
method setup (line 281) | def setup(self):
method __call__ (line 297) | def __call__(
class FlaxAlbertLayerCollection (line 321) | class FlaxAlbertLayerCollection(nn.Module):
method setup (line 325) | def setup(self):
method __call__ (line 330) | def __call__(
class FlaxAlbertLayerCollections (line 364) | class FlaxAlbertLayerCollections(nn.Module):
method setup (line 369) | def setup(self):
method __call__ (line 372) | def __call__(
class FlaxAlbertLayerGroups (line 390) | class FlaxAlbertLayerGroups(nn.Module):
method setup (line 394) | def setup(self):
method __call__ (line 400) | def __call__(
class FlaxAlbertEncoder (line 437) | class FlaxAlbertEncoder(nn.Module):
method setup (line 441) | def setup(self):
method __call__ (line 449) | def __call__(
class FlaxAlbertOnlyMLMHead (line 468) | class FlaxAlbertOnlyMLMHead(nn.Module):
method setup (line 473) | def setup(self):
method __call__ (line 480) | def __call__(self, hidden_states, shared_embedding=None):
class FlaxAlbertSOPHead (line 494) | class FlaxAlbertSOPHead(nn.Module):
method setup (line 498) | def setup(self):
method __call__ (line 502) | def __call__(self, pooled_output, deterministic=True):
class FlaxAlbertPreTrainedModel (line 508) | class FlaxAlbertPreTrainedModel(FlaxPreTrainedModel):
method __init__ (line 518) | def __init__(
method init_weights (line 530) | def init_weights(self, rng: jax.random.PRNGKey, input_shape: Tuple, pa...
method __call__ (line 555) | def __call__(
class FlaxAlbertModule (line 603) | class FlaxAlbertModule(nn.Module):
method setup (line 608) | def setup(self):
method __call__ (line 623) | def __call__(
class FlaxAlbertModel (line 677) | class FlaxAlbertModel(FlaxAlbertPreTrainedModel):
class FlaxAlbertForPreTrainingModule (line 684) | class FlaxAlbertForPreTrainingModule(nn.Module):
method setup (line 688) | def setup(self):
method __call__ (line 693) | def __call__(
class FlaxAlbertForPreTraining (line 745) | class FlaxAlbertForPreTraining(FlaxAlbertPreTrainedModel):
class FlaxAlbertForMaskedLMModule (line 777) | class FlaxAlbertForMaskedLMModule(nn.Module):
method setup (line 781) | def setup(self):
method __call__ (line 785) | def __call__(
class FlaxAlbertForMaskedLM (line 828) | class FlaxAlbertForMaskedLM(FlaxAlbertPreTrainedModel):
class FlaxAlbertForSequenceClassificationModule (line 835) | class FlaxAlbertForSequenceClassificationModule(nn.Module):
method setup (line 839) | def setup(self):
method __call__ (line 852) | def __call__(
class FlaxAlbertForSequenceClassification (line 896) | class FlaxAlbertForSequenceClassification(FlaxAlbertPreTrainedModel):
class FlaxAlbertForMultipleChoiceModule (line 908) | class FlaxAlbertForMultipleChoiceModule(nn.Module):
method setup (line 912) | def setup(self):
method __call__ (line 917) | def __call__(
class FlaxAlbertForMultipleChoice (line 969) | class FlaxAlbertForMultipleChoice(FlaxAlbertPreTrainedModel):
class FlaxAlbertForTokenClassificationModule (line 984) | class FlaxAlbertForTokenClassificationModule(nn.Module):
method setup (line 988) | def setup(self):
method __call__ (line 998) | def __call__(
class FlaxAlbertForTokenClassification (line 1042) | class FlaxAlbertForTokenClassification(FlaxAlbertPreTrainedModel):
class FlaxAlbertForQuestionAnsweringModule (line 1054) | class FlaxAlbertForQuestionAnsweringModule(nn.Module):
method setup (line 1058) | def setup(self):
method __call__ (line 1062) | def __call__(
class FlaxAlbertForQuestionAnswering (line 1110) | class FlaxAlbertForQuestionAnswering(FlaxAlbertPreTrainedModel):
FILE: transformers/models/albert/modeling_tf_albert.py
class TFAlbertPreTrainingLoss (line 80) | class TFAlbertPreTrainingLoss:
method hf_compute_loss (line 86) | def hf_compute_loss(self, labels: tf.Tensor, logits: tf.Tensor) -> tf....
class TFAlbertEmbeddings (line 136) | class TFAlbertEmbeddings(tf.keras.layers.Layer):
method __init__ (line 139) | def __init__(self, config: AlbertConfig, **kwargs):
method build (line 149) | def build(self, input_shape: tf.TensorShape):
method call (line 174) | def call(
class TFAlbertAttention (line 215) | class TFAlbertAttention(tf.keras.layers.Layer):
method __init__ (line 218) | def __init__(self, config: AlbertConfig, **kwargs):
method transpose_for_scores (line 250) | def transpose_for_scores(self, tensor: tf.Tensor, batch_size: int) -> ...
method call (line 257) | def call(
class TFAlbertLayer (line 311) | class TFAlbertLayer(tf.keras.layers.Layer):
method __init__ (line 312) | def __init__(self, config: AlbertConfig, **kwargs):
method call (line 333) | def call(
class TFAlbertLayerGroup (line 360) | class TFAlbertLayerGroup(tf.keras.layers.Layer):
method __init__ (line 361) | def __init__(self, config: AlbertConfig, **kwargs):
method call (line 368) | def call(
class TFAlbertTransformer (line 403) | class TFAlbertTransformer(tf.keras.layers.Layer):
method __init__ (line 404) | def __init__(self, config: AlbertConfig, **kwargs):
method call (line 420) | def call(
class TFAlbertPreTrainedModel (line 461) | class TFAlbertPreTrainedModel(TFPreTrainedModel):
class TFAlbertMLMHead (line 471) | class TFAlbertMLMHead(tf.keras.layers.Layer):
method __init__ (line 472) | def __init__(self, config: AlbertConfig, input_embeddings: tf.keras.la...
method build (line 491) | def build(self, input_shape: tf.TensorShape):
method get_output_embeddings (line 499) | def get_output_embeddings(self) -> tf.keras.layers.Layer:
method set_output_embeddings (line 502) | def set_output_embeddings(self, value: tf.Variable):
method get_bias (line 506) | def get_bias(self) -> Dict[str, tf.Variable]:
method set_bias (line 509) | def set_bias(self, value: tf.Variable):
method call (line 514) | def call(self, hidden_states: tf.Tensor) -> tf.Tensor:
class TFAlbertMainLayer (line 528) | class TFAlbertMainLayer(tf.keras.layers.Layer):
method __init__ (line 531) | def __init__(self, config: AlbertConfig, add_pooling_layer: bool = Tru...
method get_input_embeddings (line 549) | def get_input_embeddings(self) -> tf.keras.layers.Layer:
method set_input_embeddings (line 552) | def set_input_embeddings(self, value: tf.Variable):
method _prune_heads (line 556) | def _prune_heads(self, heads_to_prune):
method call (line 564) | def call(
class TFAlbertForPreTrainingOutput (line 655) | class TFAlbertForPreTrainingOutput(ModelOutput):
class TFAlbertModel (line 787) | class TFAlbertModel(TFAlbertPreTrainedModel):
method __init__ (line 788) | def __init__(self, config: AlbertConfig, *inputs, **kwargs):
method call (line 800) | def call(
class TFAlbertForPreTraining (line 836) | class TFAlbertForPreTraining(TFAlbertPreTrainedModel, TFAlbertPreTrainin...
method __init__ (line 840) | def __init__(self, config: AlbertConfig, *inputs, **kwargs):
method get_lm_head (line 849) | def get_lm_head(self) -> tf.keras.layers.Layer:
method call (line 855) | def call(
class TFAlbertSOPHead (line 925) | class TFAlbertSOPHead(tf.keras.layers.Layer):
method __init__ (line 926) | def __init__(self, config: AlbertConfig, **kwargs):
method call (line 936) | def call(self, pooled_output: tf.Tensor, training: bool) -> tf.Tensor:
class TFAlbertForMaskedLM (line 944) | class TFAlbertForMaskedLM(TFAlbertPreTrainedModel, TFMaskedLanguageModel...
method __init__ (line 948) | def __init__(self, config: AlbertConfig, *inputs, **kwargs):
method get_lm_head (line 954) | def get_lm_head(self) -> tf.keras.layers.Layer:
method call (line 960) | def call(
class TFAlbertForSequenceClassification (line 1046) | class TFAlbertForSequenceClassification(TFAlbertPreTrainedModel, TFSeque...
method __init__ (line 1051) | def __init__(self, config: AlbertConfig, *inputs, **kwargs):
method call (line 1071) | def call(
class TFAlbertForTokenClassification (line 1128) | class TFAlbertForTokenClassification(TFAlbertPreTrainedModel, TFTokenCla...
method __init__ (line 1133) | def __init__(self, config: AlbertConfig, *inputs, **kwargs):
method call (line 1156) | def call(
class TFAlbertForQuestionAnswering (line 1211) | class TFAlbertForQuestionAnswering(TFAlbertPreTrainedModel, TFQuestionAn...
method __init__ (line 1215) | def __init__(self, config: AlbertConfig, *inputs, **kwargs):
method call (line 1236) | def call(
class TFAlbertForMultipleChoice (line 1306) | class TFAlbertForMultipleChoice(TFAlbertPreTrainedModel, TFMultipleChoic...
method __init__ (line 1311) | def __init__(self, config: AlbertConfig, *inputs, **kwargs):
method call (line 1327) | def call(
FILE: transformers/models/albert/tokenization_albert.py
class AlbertTokenizer (line 59) | class AlbertTokenizer(PreTrainedTokenizer):
method __init__ (line 136) | def __init__(
method vocab_size (line 186) | def vocab_size(self):
method get_vocab (line 189) | def get_vocab(self):
method __getstate__ (line 194) | def __getstate__(self):
method __setstate__ (line 199) | def __setstate__(self, d):
method preprocess_text (line 209) | def preprocess_text(self, inputs):
method _tokenize (line 224) | def _tokenize(self, text: str) -> List[str]:
method _convert_token_to_id (line 244) | def _convert_token_to_id(self, token):
method _convert_id_to_token (line 248) | def _convert_id_to_token(self, index):
method convert_tokens_to_string (line 252) | def convert_tokens_to_string(self, tokens):
method build_inputs_with_special_tokens (line 271) | def build_inputs_with_special_tokens(
method get_special_tokens_mask (line 296) | def get_special_tokens_mask(
method create_token_type_ids_from_sequences (line 324) | def create_token_type_ids_from_sequences(
method save_vocabulary (line 354) | def save_vocabulary(self, save_directory: str, filename_prefix: Option...
FILE: transformers/models/albert/tokenization_albert_fast.py
class AlbertTokenizerFast (line 72) | class AlbertTokenizerFast(PreTrainedTokenizerFast):
method __init__ (line 124) | def __init__(
method build_inputs_with_special_tokens (line 170) | def build_inputs_with_special_tokens(
method create_token_type_ids_from_sequences (line 195) | def create_token_type_ids_from_sequences(
method save_vocabulary (line 225) | def save_vocabulary(self, save_directory: str, filename_prefix: Option...
FILE: transformers/models/align/configuration_align.py
class AlignTextConfig (line 36) | class AlignTextConfig(PretrainedConfig):
method __init__ (line 103) | def __init__(
method from_pretrained (line 141) | def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os....
class AlignVisionConfig (line 157) | class AlignVisionConfig(PretrainedConfig):
method __init__ (line 231) | def __init__(
method from_pretrained (line 282) | def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os....
class AlignConfig (line 298) | class AlignConfig(PretrainedConfig):
method __init__ (line 349) | def __init__(
method from_text_vision_configs (line 376) | def from_text_vision_configs(cls, text_config: AlignTextConfig, vision...
method to_dict (line 387) | def to_dict(self):
FILE: transformers/models/align/convert_align_tf_to_hf.py
function preprocess (line 44) | def preprocess(image):
function get_align_config (line 50) | def get_align_config():
function prepare_img (line 66) | def prepare_img():
function get_processor (line 72) | def get_processor():
function rename_keys (line 88) | def rename_keys(original_param_names):
function replace_params (line 260) | def replace_params(hf_params, tf_params, key_mapping):
function convert_align_checkpoint (line 288) | def convert_align_checkpoint(checkpoint_path, pytorch_dump_folder_path, ...
FILE: transformers/models/align/modeling_align.py
class AlignVisionModelOutput (line 190) | class AlignVisionModelOutput(ModelOutput):
class AlignTextModelOutput (line 212) | class AlignTextModelOutput(ModelOutput):
class AlignOutput (line 241) | class AlignOutput(ModelOutput):
method to_tuple (line 270) | def to_tuple(self) -> Tuple[Any]:
function contrastive_loss (line 279) | def contrastive_loss(logits: torch.Tensor) -> torch.Tensor:
function align_loss (line 283) | def align_loss(similarity: torch.Tensor) -> torch.Tensor:
function round_filters (line 290) | def round_filters(config: AlignVisionConfig, num_channels: int):
function correct_pad (line 306) | def correct_pad(kernel_size: Union[int, Tuple], adjust: bool = True):
class AlignVisionEmbeddings (line 327) | class AlignVisionEmbeddings(nn.Module):
method __init__ (line 332) | def __init__(self, config: AlignVisionConfig):
method forward (line 343) | def forward(self, pixel_values: torch.Tensor) -> torch.Tensor:
class AlignVisionDepthwiseConv2d (line 353) | class AlignVisionDepthwiseConv2d(nn.Conv2d):
method __init__ (line 354) | def __init__(
class AlignVisionExpansionLayer (line 380) | class AlignVisionExpansionLayer(nn.Module):
method __init__ (line 385) | def __init__(self, config: AlignVisionConfig, in_dim: int, out_dim: in...
method forward (line 397) | def forward(self, hidden_states: torch.FloatTensor) -> torch.Tensor:
class AlignVisionDepthwiseLayer (line 407) | class AlignVisionDepthwiseLayer(nn.Module):
method __init__ (line 412) | def __init__(
method forward (line 434) | def forward(self, hidden_states: torch.FloatTensor) -> torch.Tensor:
class AlignVisionSqueezeExciteLayer (line 447) | class AlignVisionSqueezeExciteLayer(nn.Module):
method __init__ (line 452) | def __init__(self, config: AlignVisionConfig, in_dim: int, expand_dim:...
method forward (line 473) | def forward(self, hidden_states: torch.FloatTensor) -> torch.Tensor:
class AlignVisionFinalBlockLayer (line 486) | class AlignVisionFinalBlockLayer(nn.Module):
method __init__ (line 491) | def __init__(
method forward (line 508) | def forward(self, embeddings: torch.FloatTensor, hidden_states: torch....
class AlignVisionBlock (line 519) | class AlignVisionBlock(nn.Module):
method __init__ (line 546) | def __init__(
method forward (line 587) | def forward(self, hidden_states: torch.FloatTensor) -> torch.Tensor:
class AlignVisionEncoder (line 600) | class AlignVisionEncoder(nn.Module):
method __init__ (line 609) | def __init__(self, config: AlignVisionConfig):
method forward (line 652) | def forward(
class AlignTextEmbeddings (line 675) | class AlignTextEmbeddings(nn.Module):
method __init__ (line 678) | def __init__(self, config):
method forward (line 695) | def forward(
class AlignTextSelfAttention (line 738) | class AlignTextSelfAttention(nn.Module):
method __init__ (line 739) | def __init__(self, config, position_embedding_type=None):
method transpose_for_scores (line 765) | def transpose_for_scores(self, x: torch.Tensor) -> torch.Tensor:
method forward (line 770) | def forward(
class AlignTextSelfOutput (line 873) | class AlignTextSelfOutput(nn.Module):
method __init__ (line 874) | def __init__(self, config):
method forward (line 880) | def forward(self, hidden_states: torch.Tensor, input_tensor: torch.Ten...
class AlignTextAttention (line 888) | class AlignTextAttention(nn.Module):
method __init__ (line 889) | def __init__(self, config, position_embedding_type=None):
method prune_heads (line 895) | def prune_heads(self, heads):
method forward (line 913) | def forward(
class AlignTextIntermediate (line 938) | class AlignTextIntermediate(nn.Module):
method __init__ (line 939) | def __init__(self, config):
method forward (line 947) | def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
class AlignTextOutput (line 954) | class AlignTextOutput(nn.Module):
method __init__ (line 955) | def __init__(self, config):
method forward (line 961) | def forward(self, hidden_states: torch.Tensor, input_tensor: torch.Ten...
class AlignTextLayer (line 969) | class AlignTextLayer(nn.Module):
method __init__ (line 970) | def __init__(self, config):
method forward (line 984) | def forward(
method feed_forward_chunk (line 1049) | def feed_forward_chunk(self, attention_output):
class AlignTextEncoder (line 1056) | class AlignTextEncoder(nn.Module):
method __init__ (line 1057) | def __init__(self, config):
method forward (line 1063) | def forward(
class AlignTextPooler (line 1155) | class AlignTextPooler(nn.Module):
method __init__ (line 1156) | def __init__(self, config):
method forward (line 1161) | def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
class AlignPreTrainedModel (line 1170) | class AlignPreTrainedModel(PreTrainedModel):
method _init_weights (line 1181) | def _init_weights(self, module):
method _set_gradient_checkpointing (line 1199) | def _set_gradient_checkpointing(self, module, value=False):
class AlignTextModel (line 1208) | class AlignTextModel(AlignPreTrainedModel):
method __init__ (line 1211) | def __init__(self, config: AlignTextConfig, add_pooling_layer: bool = ...
method get_input_embeddings (line 1223) | def get_input_embeddings(self):
method set_input_embeddings (line 1226) | def set_input_embeddings(self, value):
method forward (line 1231) | def forward(
class AlignVisionModel (line 1333) | class AlignVisionModel(AlignPreTrainedModel):
method __init__ (line 1337) | def __init__(self, config: AlignVisionConfig):
method get_input_embeddings (line 1354) | def get_input_embeddings(self) -> nn.Module:
method forward (line 1359) | def forward(
class AlignModel (line 1418) | class AlignModel(AlignPreTrainedModel):
method __init__ (line 1421) | def __init__(self, config: AlignConfig):
method get_text_features (line 1452) | def get_text_features(
method get_image_features (line 1505) | def get_image_features(
method forward (line 1551) | def forward(
FILE: transformers/models/align/processing_align.py
class AlignProcessor (line 24) | class AlignProcessor(ProcessorMixin):
method __init__ (line 42) | def __init__(self, image_processor, tokenizer):
method __call__ (line 45) | def __call__(self, text=None, images=None, padding="max_length", max_l...
method batch_decode (line 104) | def batch_decode(self, *args, **kwargs):
method decode (line 111) | def decode(self, *args, **kwargs):
method model_input_names (line 119) | def model_input_names(self):
FILE: transformers/models/altclip/configuration_altclip.py
class AltCLIPTextConfig (line 32) | class AltCLIPTextConfig(PretrainedConfig):
method __init__ (line 99) | def __init__(
class AltCLIPVisionConfig (line 142) | class AltCLIPVisionConfig(PretrainedConfig):
method __init__ (line 196) | def __init__(
method from_pretrained (line 230) | def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os....
class AltCLIPConfig (line 246) | class AltCLIPConfig(PretrainedConfig):
method __init__ (line 294) | def __init__(
method from_text_vision_configs (line 383) | def from_text_vision_configs(cls, text_config: AltCLIPTextConfig, visi...
method to_dict (line 394) | def to_dict(self):
FILE: transformers/models/altclip/modeling_altclip.py
function contrastive_loss (line 151) | def contrastive_loss(logits: torch.Tensor) -> torch.Tensor:
function clip_loss (line 155) | def clip_loss(similarity: torch.Tensor) -> torch.Tensor:
class AltCLIPOutput (line 163) | class AltCLIPOutput(ModelOutput):
method to_tuple (line 193) | def to_tuple(self) -> Tuple[Any]:
class AltRobertaEmbeddings (line 201) | class AltRobertaEmbeddings(nn.Module):
method __init__ (line 207) | def __init__(self, config):
method forward (line 230) | def forward(
method create_position_ids_from_inputs_embeds (line 270) | def create_position_ids_from_inputs_embeds(self, inputs_embeds):
class AltRobertaSelfAttention (line 289) | class AltRobertaSelfAttention(nn.Module):
method __init__ (line 290) | def __init__(self, config, position_embedding_type=None):
method transpose_for_scores (line 316) | def transpose_for_scores(self, x: torch.Tensor) -> torch.Tensor:
method forward (line 321) | def forward(
class AltRobertaSelfOutput (line 424) | class AltRobertaSelfOutput(nn.Module):
method __init__ (line 425) | def __init__(self, config):
method forward (line 431) | def forward(self, hidden_states: torch.Tensor, input_tensor: torch.Ten...
class AltRobertaAttention (line 439) | class AltRobertaAttention(nn.Module):
method __init__ (line 440) | def __init__(self, config, position_embedding_type=None):
method prune_heads (line 446) | def prune_heads(self, heads):
method forward (line 464) | def forward(
class AltRobertaIntermediate (line 489) | class AltRobertaIntermediate(nn.Module):
method __init__ (line 490) | def __init__(self, config):
method forward (line 498) | def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
class AltRobertaOutput (line 505) | class AltRobertaOutput(nn.Module):
method __init__ (line 506) | def __init__(self, config):
method forward (line 512) | def forward(self, hidden_states: torch.Tensor, input_tensor: torch.Ten...
class AltRobertaLayer (line 520) | class AltRobertaLayer(nn.Module):
method __init__ (line 521) | def __init__(self, config):
method forward (line 535) | def forward(
method feed_forward_chunk (line 600) | def feed_forward_chunk(self, attention_output):
class AltRobertaEncoder (line 607) | class AltRobertaEncoder(nn.Module):
method __init__ (line 608) | def __init__(self, config):
method forward (line 614) | def forward(
class AltRobertaPooler (line 706) | class AltRobertaPooler(nn.Module):
method __init__ (line 707) | def __init__(self, config):
method forward (line 712) | def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
class AltCLIPAttention (line 722) | class AltCLIPAttention(nn.Module):
method __init__ (line 725) | def __init__(self, config):
method _shape (line 744) | def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int):
method forward (line 747) | def forward(
class AltCLIPMLP (line 827) | class AltCLIPMLP(nn.Module):
method __init__ (line 828) | def __init__(self, config):
method forward (line 835) | def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
class AltCLIPEncoderLayer (line 843) | class AltCLIPEncoderLayer(nn.Module):
method __init__ (line 844) | def __init__(self, config: AltCLIPConfig):
method forward (line 852) | def forward(
class AltCLIPEncoder (line 894) | class AltCLIPEncoder(nn.Module):
method __init__ (line 903) | def __init__(self, config: AltCLIPConfig):
method forward (line 909) | def forward(
class AltCLIPVisionEmbeddings (line 998) | class AltCLIPVisionEmbeddings(nn.Module):
method __init__ (line 999) | def __init__(self, config: AltCLIPVisionConfig):
method forward (line 1021) | def forward(self, pixel_values: torch.FloatTensor) -> torch.Tensor:
class AltCLIPPreTrainedModel (line 1032) | class AltCLIPPreTrainedModel(PreTrainedModel):
method _init_weights (line 1043) | def _init_weights(self, module):
method _set_gradient_checkpointing (line 1090) | def _set_gradient_checkpointing(self, module, value=False):
class AltCLIPVisionTransformer (line 1098) | class AltCLIPVisionTransformer(nn.Module):
method __init__ (line 1099) | def __init__(self, config: AltCLIPVisionConfig):
method forward (line 1111) | def forward(
class AltCLIPVisionModel (line 1156) | class AltCLIPVisionModel(AltCLIPPreTrainedModel):
method __init__ (line 1160) | def __init__(self, config: AltCLIPVisionConfig):
method get_input_embeddings (line 1166) | def get_input_embeddings(self) -> nn.Module:
method forward (line 1171) | def forward(
class AltRobertaModel (line 1210) | class AltRobertaModel(AltCLIPPreTrainedModel):
method __init__ (line 1229) | def __init__(self, config, add_pooling_layer=True):
method get_input_embeddings (line 1241) | def get_input_embeddings(self):
method set_input_embeddings (line 1244) | def set_input_embeddings(self, value):
method _prune_heads (line 1247) | def _prune_heads(self, heads_to_prune):
method forward (line 1256) | def forward(
class AltCLIPTextModel (line 1386) | class AltCLIPTextModel(AltCLIPPreTrainedModel):
method __init__ (line 1389) | def __init__(self, config):
method get_input_embeddings (line 1396) | def get_input_embeddings(self) -> nn.Module:
method set_input_embeddings (line 1399) | def set_input_embeddings(self, value: nn.Embedding) -> None:
method resize_token_embeddings (line 1402) | def resize_token_embeddings(self, new_num_tokens: Optional[int] = None...
method forward (line 1407) | def forward(
class AltCLIPModel (line 1478) | class AltCLIPModel(AltCLIPPreTrainedModel):
method __init__ (line 1481) | def __init__(self, config: AltCLIPConfig):
method get_text_features (line 1513) | def get_text_features(
method get_image_features (line 1560) | def get_image_features(
method forward (line 1607) | def forward(
function create_position_ids_from_input_ids (line 1699) | def create_position_ids_from_input_ids(input_ids, padding_idx, past_key_...
FILE: transformers/models/altclip/processing_altclip.py
class AltCLIPProcessor (line 24) | class AltCLIPProcessor(ProcessorMixin):
method __init__ (line 42) | def __init__(self, image_processor=None, tokenizer=None, **kwargs):
method __call__ (line 59) | def __call__(self, text=None, images=None, return_tensors=None, **kwar...
method batch_decode (line 112) | def batch_decode(self, *args, **kwargs):
method decode (line 119) | def decode(self, *args, **kwargs):
method model_input_names (line 127) | def model_input_names(self):
FILE: transformers/models/audio_spectrogram_transformer/configuration_audio_spectrogram_transformer.py
class ASTConfig (line 31) | class ASTConfig(PretrainedConfig):
method __init__ (line 91) | def __init__(
FILE: transformers/models/audio_spectrogram_transformer/convert_audio_spectrogram_transformer_original_to_pytorch.py
function get_audio_spectrogram_transformer_config (line 35) | def get_audio_spectrogram_transformer_config(model_name):
function rename_key (line 70) | def rename_key(name):
function convert_state_dict (line 108) | def convert_state_dict(orig_state_dict, config):
function remove_keys (line 142) | def remove_keys(state_dict):
function convert_audio_spectrogram_transformer_checkpoint (line 154) | def convert_audio_spectrogram_transformer_checkpoint(model_name, pytorch...
FILE: transformers/models/audio_spectrogram_transformer/feature_extraction_audio_spectrogram_transformer.py
class ASTFeatureExtractor (line 33) | class ASTFeatureExtractor(SequenceFeatureExtractor):
method __init__ (line 65) | def __init__(
method _extract_fbank_features (line 86) | def _extract_fbank_features(
method normalize (line 122) | def normalize(self, input_values: np.ndarray) -> np.ndarray:
method __call__ (line 125) | def __call__(
FILE: transformers/models/audio_spectrogram_transformer/modeling_audio_spectrogram_transformer.py
class ASTEmbeddings (line 54) | class ASTEmbeddings(nn.Module):
method __init__ (line 59) | def __init__(self, config: ASTConfig) -> None:
method get_shape (line 72) | def get_shape(self, config):
method forward (line 80) | def forward(self, input_values: torch.Tensor) -> torch.Tensor:
class ASTPatchEmbeddings (line 93) | class ASTPatchEmbeddings(nn.Module):
method __init__ (line 99) | def __init__(self, config):
method forward (line 110) | def forward(self, input_values: torch.Tensor) -> torch.Tensor:
class ASTSelfAttention (line 118) | class ASTSelfAttention(nn.Module):
method __init__ (line 119) | def __init__(self, config: ASTConfig) -> None:
method transpose_for_scores (line 137) | def transpose_for_scores(self, x: torch.Tensor) -> torch.Tensor:
method forward (line 142) | def forward(
class ASTSelfOutput (line 179) | class ASTSelfOutput(nn.Module):
method __init__ (line 185) | def __init__(self, config: ASTConfig) -> None:
method forward (line 190) | def forward(self, hidden_states: torch.Tensor, input_tensor: torch.Ten...
class ASTAttention (line 198) | class ASTAttention(nn.Module):
method __init__ (line 199) | def __init__(self, config: ASTConfig) -> None:
method prune_heads (line 205) | def prune_heads(self, heads: Set[int]) -> None:
method forward (line 223) | def forward(
class ASTIntermediate (line 238) | class ASTIntermediate(nn.Module):
method __init__ (line 239) | def __init__(self, config: ASTConfig) -> None:
method forward (line 247) | def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
class ASTOutput (line 255) | class ASTOutput(nn.Module):
method __init__ (line 256) | def __init__(self, config: ASTConfig) -> None:
method forward (line 261) | def forward(self, hidden_states: torch.Tensor, input_tensor: torch.Ten...
class ASTLayer (line 271) | class ASTLayer(nn.Module):
method __init__ (line 274) | def __init__(self, config: ASTConfig) -> None:
method forward (line 284) | def forward(
class ASTEncoder (line 314) | class ASTEncoder(nn.Module):
method __init__ (line 315) | def __init__(self, config: ASTConfig) -> None:
method forward (line 321) | def forward(
class ASTPreTrainedModel (line 371) | class ASTPreTrainedModel(PreTrainedModel):
method _init_weights (line 383) | def _init_weights(self, module: Union[nn.Linear, nn.Conv2d, nn.LayerNo...
method _set_gradient_checkpointing (line 398) | def _set_gradient_checkpointing(self, module: ASTEncoder, value: bool ...
class ASTModel (line 445) | class ASTModel(ASTPreTrainedModel):
method __init__ (line 446) | def __init__(self, config: ASTConfig):
method get_input_embeddings (line 458) | def get_input_embeddings(self) -> ASTPatchEmbeddings:
method _prune_heads (line 461) | def _prune_heads(self, heads_to_prune: Dict[int, List[int]]) -> None:
method forward (line 477) | def forward(
class ASTMLPHead (line 526) | class ASTMLPHead(nn.Module):
method __init__ (line 527) | def __init__(self, config: ASTConfig):
method forward (line 532) | def forward(self, hidden_state):
class ASTForAudioClassification (line 545) | class ASTForAudioClassification(ASTPreTrainedModel):
method __init__ (line 546) | def __init__(self, config: ASTConfig) -> None:
method forward (line 567) | def forward(
FILE: transformers/models/auto/auto_factory.py
function _get_model_class (line 374) | def _get_model_class(config, model_mapping):
class _BaseAutoModelClass (line 394) | class _BaseAutoModelClass:
method __init__ (line 398) | def __init__(self, *args, **kwargs):
method from_config (line 406) | def from_config(cls, config, **kwargs):
method from_pretrained (line 433) | def from_pretrained(cls, pretrained_model_name_or_path, *model_args, *...
method register (line 493) | def register(cls, config_class, model_class):
class _BaseAutoBackboneClass (line 512) | class _BaseAutoBackboneClass(_BaseAutoModelClass):
method _load_timm_backbone_from_pretrained (line 517) | def _load_timm_backbone_from_pretrained(cls, pretrained_model_name_or_...
method from_pretrained (line 547) | def from_pretrained(cls, pretrained_model_name_or_path, *model_args, *...
function insert_head_doc (line 554) | def insert_head_doc(docstring, head_doc=""):
function auto_class_update (line 565) | def auto_class_update(cls, checkpoint_for_example="bert-base-cased", hea...
function get_values (line 600) | def get_values(model_mapping):
function getattribute_from_module (line 611) | def getattribute_from_module(module, attr):
class _LazyAutoMapping (line 631) | class _LazyAutoMapping(OrderedDict):
method __init__ (line 640) | def __init__(self, config_mapping, model_mapping):
method __len__ (line 647) | def __len__(self):
method __getitem__ (line 651) | def __getitem__(self, key):
method _load_attr_from_module (line 667) | def _load_attr_from_module(self, model_type, attr):
method keys (line 673) | def keys(self):
method get (line 681) | def get(self, key, default):
method __bool__ (line 687) | def __bool__(self):
method values (line 690) | def values(self):
method items (line 698) | def items(self):
method __iter__ (line 709) | def __iter__(self):
method __contains__ (line 712) | def __contains__(self, item):
method register (line 720) | def register(self, key, value):
FILE: transformers/models/auto/configuration_auto.py
function model_type_to_module_name (line 636) | def model_type_to_module_name(key):
function config_class_to_model_type (line 645) | def config_class_to_model_type(config):
class _LazyConfigMapping (line 657) | class _LazyConfigMapping(OrderedDict):
method __init__ (line 662) | def __init__(self, mapping):
method __getitem__ (line 667) | def __getitem__(self, key):
method keys (line 684) | def keys(self):
method values (line 687) | def values(self):
method items (line 690) | def items(self):
method __iter__ (line 693) | def __iter__(self):
method __contains__ (line 696) | def __contains__(self, item):
method register (line 699) | def register(self, key, value):
class _LazyLoadAllMappings (line 711) | class _LazyLoadAllMappings(OrderedDict):
method __init__ (line 720) | def __init__(self, mapping):
method _initialize (line 725) | def _initialize(self):
method __getitem__ (line 742) | def __getitem__(self, key):
method keys (line 746) | def keys(self):
method values (line 750) | def values(self):
method items (line 754) | def items(self):
method __iter__ (line 758) | def __iter__(self):
method __contains__ (line 762) | def __contains__(self, item):
function _get_class_name (line 770) | def _get_class_name(model_class: Union[str, List[str]]):
function _list_model_options (line 776) | def _list_model_options(indent, config_to_class=None, use_model_types=Tr...
function replace_list_option_in_docstrings (line 809) | def replace_list_option_in_docstrings(config_to_class=None, use_model_ty...
class AutoConfig (line 833) | class AutoConfig:
method __init__ (line 841) | def __init__(self):
method for_model (line 848) | def for_model(cls, model_type: str, *args, **kwargs):
method from_pretrained (line 858) | def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):
method register (line 973) | def register(model_type, config):
FILE: transformers/models/auto/feature_extraction_auto.py
function feature_extractor_class_from_name (line 107) | def feature_extractor_class_from_name(class_name: str):
function get_feature_extractor_config (line 131) | def get_feature_extractor_config(
class AutoFeatureExtractor (line 221) | class AutoFeatureExtractor:
method __init__ (line 229) | def __init__(self):
method from_pretrained (line 237) | def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):
method register (line 357) | def register(config_class, feature_extractor_class):
FILE: transformers/models/auto/image_processing_auto.py
function image_processor_class_from_name (line 114) | def image_processor_class_from_name(class_name: str):
function get_image_processor_config (line 138) | def get_image_processor_config(
class AutoImageProcessor (line 228) | class AutoImageProcessor:
method __init__ (line 236) | def __init__(self):
method from_pretrained (line 244) | def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):
method register (line 382) | def register(config_class, image_processor_class):
FILE: transformers/models/auto/modeling_auto.py
class AutoModelForMaskGeneration (line 1091) | class AutoModelForMaskGeneration(_BaseAutoModelClass):
class AutoModel (line 1095) | class AutoModel(_BaseAutoModelClass):
class AutoModelForPreTraining (line 1102) | class AutoModelForPreTraining(_BaseAutoModelClass):
class _AutoModelWithLMHead (line 1110) | class _AutoModelWithLMHead(_BaseAutoModelClass):
class AutoModelForCausalLM (line 1117) | class AutoModelForCausalLM(_BaseAutoModelClass):
class AutoModelForMaskedLM (line 1124) | class AutoModelForMaskedLM(_BaseAutoModelClass):
class AutoModelForSeq2SeqLM (line 1131) | class AutoModelForSeq2SeqLM(_BaseAutoModelClass):
class AutoModelForSequenceClassification (line 1140) | class AutoModelForSequenceClassification(_BaseAutoModelClass):
class AutoModelForQuestionAnswering (line 1149) | class AutoModelForQuestionAnswering(_BaseAutoModelClass):
class AutoModelForTableQuestionAnswering (line 1156) | class AutoModelForTableQuestionAnswering(_BaseAutoModelClass):
class AutoModelForVisualQuestionAnswering (line 1167) | class AutoModelForVisualQuestionAnswering(_BaseAutoModelClass):
class AutoModelForDocumentQuestionAnswering (line 1178) | class AutoModelForDocumentQuestionAnswering(_BaseAutoModelClass):
class AutoModelForTokenClassification (line 1189) | class AutoModelForTokenClassification(_BaseAutoModelClass):
class AutoModelForMultipleChoice (line 1196) | class AutoModelForMultipleChoice(_BaseAutoModelClass):
class AutoModelForNextSentencePrediction (line 1203) | class AutoModelForNextSentencePrediction(_BaseAutoModelClass):
class AutoModelForImageClassification (line 1212) | class AutoModelForImageClassification(_BaseAutoModelClass):
class AutoModelForZeroShotImageClassification (line 1219) | class AutoModelForZeroShotImageClassification(_BaseAutoModelClass):
class AutoModelForImageSegmentation (line 1228) | class AutoModelForImageSegmentation(_BaseAutoModelClass):
class AutoModelForSemanticSegmentation (line 1235) | class AutoModelForSemanticSegmentation(_BaseAutoModelClass):
class AutoModelForUniversalSegmentation (line 1244) | class AutoModelForUniversalSegmentation(_BaseAutoModelClass):
class AutoModelForInstanceSegmentation (line 1253) | class AutoModelForInstanceSegmentation(_BaseAutoModelClass):
class AutoModelForObjectDetection (line 1262) | class AutoModelForObjectDetection(_BaseAutoModelClass):
class AutoModelForZeroShotObjectDetection (line 1269) | class AutoModelForZeroShotObjectDetection(_BaseAutoModelClass):
class AutoModelForDepthEstimation (line 1278) | class AutoModelForDepthEstimation(_BaseAutoModelClass):
class AutoModelForVideoClassification (line 1285) | class AutoModelForVideoClassification(_BaseAutoModelClass):
class AutoModelForVision2Seq (line 1292) | class AutoModelForVision2Seq(_BaseAutoModelClass):
class AutoModelForAudioClassification (line 1299) | class AutoModelForAudioClassification(_BaseAutoModelClass):
class AutoModelForCTC (line 1306) | class AutoModelForCTC(_BaseAutoModelClass):
class AutoModelForSpeechSeq2Seq (line 1313) | class AutoModelForSpeechSeq2Seq(_BaseAutoModelClass):
class AutoModelForAudioFrameClassification (line 1322) | class AutoModelForAudioFrameClassification(_BaseAutoModelClass):
class AutoModelForAudioXVector (line 1331) | class AutoModelForAudioXVector(_BaseAutoModelClass):
class AutoBackbone (line 1335) | class AutoBackbone(_BaseAutoBackboneClass):
class AutoModelForMaskedImageModeling (line 1342) | class AutoModelForMaskedImageModeling(_BaseAutoModelClass):
class AutoModelWithLMHead (line 1349) | class AutoModelWithLMHead(_AutoModelWithLMHead):
method from_config (line 1351) | def from_config(cls, config):
method from_pretrained (line 1361) | def from_pretrained(cls, pretrained_model_name_or_path, *model_args, *...
FILE: transformers/models/auto/modeling_flax_auto.py
class FlaxAutoModel (line 269) | class FlaxAutoModel(_BaseAutoModelClass):
class FlaxAutoModelForPreTraining (line 276) | class FlaxAutoModelForPreTraining(_BaseAutoModelClass):
class FlaxAutoModelForCausalLM (line 283) | class FlaxAutoModelForCausalLM(_BaseAutoModelClass):
class FlaxAutoModelForMaskedLM (line 290) | class FlaxAutoModelForMaskedLM(_BaseAutoModelClass):
class FlaxAutoModelForSeq2SeqLM (line 297) | class FlaxAutoModelForSeq2SeqLM(_BaseAutoModelClass):
class FlaxAutoModelForSequenceClassification (line 306) | class FlaxAutoModelForSequenceClassification(_BaseAutoModelClass):
class FlaxAutoModelForQuestionAnswering (line 315) | class FlaxAutoModelForQuestionAnswering(_BaseAutoModelClass):
class FlaxAutoModelForTokenClassification (line 322) | class FlaxAutoModelForTokenClassification(_BaseAutoModelClass):
class FlaxAutoModelForMultipleChoice (line 331) | class FlaxAutoModelForMultipleChoice(_BaseAutoModelClass):
class FlaxAutoModelForNextSentencePrediction (line 338) | class FlaxAutoModelForNextSentencePrediction(_BaseAutoModelClass):
class FlaxAutoModelForImageClassification (line 347) | class FlaxAutoModelForImageClassification(_BaseAutoModelClass):
class FlaxAutoModelForVision2Seq (line 356) | class FlaxAutoModelForVision2Seq(_BaseAutoModelClass):
class FlaxAutoModelForSpeechSeq2Seq (line 3
Copy disabled (too large)
Download .json
Condensed preview — 1362 files, each showing path, character count, and a content snippet. Download the .json file for the full structured content (36,348K chars).
[
{
"path": ".gitignore",
"chars": 142,
"preview": "output/\nwandb/\nrun_moe.sh\nrun_multi_node.sh\nsingle_lora_run_sft.sh\nrun_loramoe_bak.sh\ndata/tiny_data/test_1024\ndata/tiny"
},
{
"path": "README.md",
"chars": 2716,
"preview": "# LoRAMoE: Revolutionizing Mixture of Experts for Maintaining World Knowledge in Language Model Alignment\n\n\nThis is the "
},
{
"path": "__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "build_dataset.py",
"chars": 4143,
"preview": "import logging\nimport os\nfrom dataclasses import dataclass\nfrom typing import Dict, Sequence, Union, List\nimport dataset"
},
{
"path": "data/tiny_data/test.json",
"chars": 1526757,
"preview": "[\n {\n \"instruction\": \"Read the following document and then please summarize this document. \\n Document:\\nSo often repea"
},
{
"path": "data/tiny_data/train/train.json",
"chars": 1554930,
"preview": "[\n {\n \"instruction\": \"Read the following document and then please summarize this document. \\n Document:\\nSo often"
},
{
"path": "ds_zero2_no_offload.json",
"chars": 530,
"preview": "{\n \"bf16\": {\n \"enabled\": true\n },\n \"zero_optimization\": {\n \"stage\": 2,\n \"allgather_partiti"
},
{
"path": "ds_zero3_nvme_offload.json",
"chars": 790,
"preview": "{\n \"bf16\": {\n \"enabled\": true\n },\n \"zero_optimization\": {\n \"stage\": 3,\n \"offload_optimizer"
},
{
"path": "ds_zero3_offload.json",
"chars": 788,
"preview": "{\n \"bf16\": {\n \"enabled\": true\n },\n \"zero_optimization\": {\n \"stage\": 3,\n \"offload_optimizer"
},
{
"path": "environment.yml",
"chars": 10347,
"preview": "name: newloramoe\nchannels:\n - pytorch\n - huggingface\n - nvidia\n - https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs"
},
{
"path": "flash_attn_patch.py",
"chars": 4342,
"preview": "# Below code is based on https://github.com/lm-sys/FastChat/blob/main/fastchat/train/llama_flash_attn_monkey_patch.py.\nf"
},
{
"path": "peft/__init__.py",
"chars": 1676,
"preview": "# flake8: noqa\n# There's no way to ignore \"F401 '...' imported but unused\" warnings in this\n# module, but to preserve ot"
},
{
"path": "peft/mapping.py",
"chars": 5692,
"preview": "# coding=utf-8\n# Copyright 2023-present the HuggingFace Inc. team.\n#\n# Licensed under the Apache License, Version 2.0 (t"
},
{
"path": "peft/peft_model.py",
"chars": 42647,
"preview": "# coding=utf-8\n# Copyright 2023-present the HuggingFace Inc. team.\n#\n# Licensed under the Apache License, Version 2.0 (t"
},
{
"path": "peft/tuners/__init__.py",
"chars": 1060,
"preview": "# flake8: noqa\n# There's no way to ignore \"F401 '...' imported but unused\" warnings in this\n# module, but to preserve ot"
},
{
"path": "peft/tuners/lora.py",
"chars": 15034,
"preview": "# coding=utf-8\n# Copyright 2023-present the HuggingFace Inc. team.\n#\n# Licensed under the Apache License, Version 2.0 (t"
},
{
"path": "peft/tuners/p_tuning.py",
"chars": 6690,
"preview": "# coding=utf-8\n# Copyright 2023-present the HuggingFace Inc. team.\n#\n# Licensed under the Apache License, Version 2.0 (t"
},
{
"path": "peft/tuners/prefix_tuning.py",
"chars": 3714,
"preview": "# coding=utf-8\n# Copyright 2023-present the HuggingFace Inc. team.\n#\n# Licensed under the Apache License, Version 2.0 (t"
},
{
"path": "peft/tuners/prompt_tuning.py",
"chars": 4863,
"preview": "# coding=utf-8\n# Copyright 2023-present the HuggingFace Inc. team.\n#\n# Licensed under the Apache License, Version 2.0 (t"
},
{
"path": "peft/utils/__init__.py",
"chars": 1218,
"preview": "# flake8: noqa\n# There's no way to ignore \"F401 '...' imported but unused\" warnings in this\n# module, but to preserve ot"
},
{
"path": "peft/utils/adapters_utils.py",
"chars": 730,
"preview": "# coding=utf-8\n# Copyright 2023-present the HuggingFace Inc. team.\n#\n# Licensed under the Apache License, Version 2.0 (t"
},
{
"path": "peft/utils/config.py",
"chars": 6507,
"preview": "# coding=utf-8\n# Copyright 2023-present the HuggingFace Inc. team.\n#\n# Licensed under the Apache License, Version 2.0 (t"
},
{
"path": "peft/utils/other.py",
"chars": 3794,
"preview": "# coding=utf-8\n# Copyright 2023-present the HuggingFace Inc. team.\n#\n# Licensed under the Apache License, Version 2.0 (t"
},
{
"path": "peft/utils/save_and_load.py",
"chars": 3188,
"preview": "# coding=utf-8\n# Copyright 2023-present the HuggingFace Inc. team.\n#\n# Licensed under the Apache License, Version 2.0 (t"
},
{
"path": "requirements.txt",
"chars": 10644,
"preview": "absl-py @ file:///croot/absl-py_1686852429912/work\naccelerate @ file:///home/conda/feedstock_root/build_artifacts/accele"
},
{
"path": "run_loramoe.py",
"chars": 23693,
"preview": "#!/usr/bin/env python\n# coding=utf-8\n# Copyright 2020 The HuggingFace Inc. team. All rights reserved.\n#\n# Licensed under"
},
{
"path": "run_loramoe.sh",
"chars": 2380,
"preview": "export CUDA_HOME=/usr/local/cuda-11.8\nexport LD_LIBRARY_PATH=${CUDA_HOME}/lib64\nexport PATH=${CUDA_HOME}/bin:${PATH}\n\n# "
},
{
"path": "transformers/__init__.py",
"chars": 271993,
"preview": "# Copyright 2020 The HuggingFace Team. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lic"
},
{
"path": "transformers/activations.py",
"chars": 8783,
"preview": "# Copyright 2020 The HuggingFace Team. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lic"
},
{
"path": "transformers/activations_tf.py",
"chars": 4313,
"preview": "# Copyright 2020 The HuggingFace Team. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lic"
},
{
"path": "transformers/audio_utils.py",
"chars": 30874,
"preview": "# coding=utf-8\n# Copyright 2023 The HuggingFace Inc. team and the librosa & torchaudio authors.\n#\n# Licensed under the A"
},
{
"path": "transformers/benchmark/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "transformers/benchmark/benchmark.py",
"chars": 10752,
"preview": "# coding=utf-8\n# Copyright 2018 The HuggingFace Inc. team.\n# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserve"
},
{
"path": "transformers/benchmark/benchmark_args.py",
"chars": 3890,
"preview": "# coding=utf-8\n# Copyright 2018 The HuggingFace Inc. team.\n# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserve"
},
{
"path": "transformers/benchmark/benchmark_args_tf.py",
"chars": 4735,
"preview": "# coding=utf-8\n# Copyright 2018 The HuggingFace Inc. team.\n# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserve"
},
{
"path": "transformers/benchmark/benchmark_args_utils.py",
"chars": 6425,
"preview": "# coding=utf-8\n# Copyright 2018 The HuggingFace Inc. team.\n# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserve"
},
{
"path": "transformers/benchmark/benchmark_tf.py",
"chars": 13065,
"preview": "# coding=utf-8\n# Copyright 2018 The HuggingFace Inc. team.\n# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserve"
},
{
"path": "transformers/benchmark/benchmark_utils.py",
"chars": 37567,
"preview": "# This file is adapted from the AllenNLP library at https://github.com/allenai/allennlp\n\n# Copyright 2020 The HuggingFac"
},
{
"path": "transformers/commands/__init__.py",
"chars": 923,
"preview": "# Copyright 2020 The HuggingFace Team. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lic"
},
{
"path": "transformers/commands/add_new_model.py",
"chars": 11064,
"preview": "# Copyright 2020 The HuggingFace Team. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lic"
},
{
"path": "transformers/commands/add_new_model_like.py",
"chars": 70937,
"preview": "# Copyright 2021 The HuggingFace Team. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lic"
},
{
"path": "transformers/commands/convert.py",
"chars": 7852,
"preview": "# Copyright 2020 The HuggingFace Team. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lic"
},
{
"path": "transformers/commands/download.py",
"chars": 1860,
"preview": "# Copyright 2020 The HuggingFace Team. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lic"
},
{
"path": "transformers/commands/env.py",
"chars": 3807,
"preview": "# Copyright 2020 The HuggingFace Team. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lic"
},
{
"path": "transformers/commands/lfs.py",
"chars": 7998,
"preview": "\"\"\"\nImplementation of a custom transfer agent for the transfer type \"multipart\" for git-lfs.\n\nInspired by: github.com/cb"
},
{
"path": "transformers/commands/pt_to_tf.py",
"chars": 20540,
"preview": "# Copyright 2022 The HuggingFace Team. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lic"
},
{
"path": "transformers/commands/run.py",
"chars": 4249,
"preview": "# Copyright 2020 The HuggingFace Team. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lic"
},
{
"path": "transformers/commands/serving.py",
"chars": 8026,
"preview": "# Copyright 2020 The HuggingFace Team. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lic"
},
{
"path": "transformers/commands/train.py",
"chars": 6329,
"preview": "# Copyright 2020 The HuggingFace Team. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lic"
},
{
"path": "transformers/commands/transformers_cli.py",
"chars": 2047,
"preview": "#!/usr/bin/env python\n# Copyright 2020 The HuggingFace Team. All rights reserved.\n#\n# Licensed under the Apache License,"
},
{
"path": "transformers/commands/user.py",
"chars": 7124,
"preview": "# Copyright 2020 The HuggingFace Team. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lic"
},
{
"path": "transformers/configuration_utils.py",
"chars": 50333,
"preview": "# coding=utf-8\n# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team.\n# Copyright (c) 2018,"
},
{
"path": "transformers/convert_graph_to_onnx.py",
"chars": 20903,
"preview": "# Copyright 2020 The HuggingFace Team. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lic"
},
{
"path": "transformers/convert_pytorch_checkpoint_to_tf2.py",
"chars": 16631,
"preview": "# coding=utf-8\n# Copyright 2018 The HuggingFace Inc. team.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lice"
},
{
"path": "transformers/convert_slow_tokenizer.py",
"chars": 51697,
"preview": "# coding=utf-8\n# Copyright 2018 The HuggingFace Inc. team.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lice"
},
{
"path": "transformers/convert_slow_tokenizers_checkpoints_to_fast.py",
"chars": 4982,
"preview": "# coding=utf-8\n# Copyright 2018 The HuggingFace Inc. team.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lice"
},
{
"path": "transformers/convert_tf_hub_seq_to_seq_bert_to_pytorch.py",
"chars": 2899,
"preview": "# coding=utf-8\n# Copyright 2020 The HuggingFace Inc. team.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lice"
},
{
"path": "transformers/data/__init__.py",
"chars": 1423,
"preview": "# Copyright 2020 The HuggingFace Team. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lic"
},
{
"path": "transformers/data/data_collator.py",
"chars": 76840,
"preview": "# Copyright 2020 The HuggingFace Team. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lic"
},
{
"path": "transformers/data/datasets/__init__.py",
"chars": 909,
"preview": "# Copyright 2020 The HuggingFace Team. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lic"
},
{
"path": "transformers/data/datasets/glue.py",
"chars": 6160,
"preview": "# Copyright 2020 The HuggingFace Team. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lic"
},
{
"path": "transformers/data/datasets/language_modeling.py",
"chars": 23718,
"preview": "# Copyright 2020 The HuggingFace Team. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lic"
},
{
"path": "transformers/data/datasets/squad.py",
"chars": 9219,
"preview": "# Copyright 2020 The HuggingFace Team. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lic"
},
{
"path": "transformers/data/metrics/__init__.py",
"chars": 3607,
"preview": "# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with "
},
{
"path": "transformers/data/metrics/squad_metrics.py",
"chars": 29698,
"preview": "# Copyright 2020 The HuggingFace Team. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lic"
},
{
"path": "transformers/data/processors/__init__.py",
"chars": 1014,
"preview": "# Copyright 2020 The HuggingFace Team. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lic"
},
{
"path": "transformers/data/processors/glue.py",
"chars": 23216,
"preview": "# coding=utf-8\n# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team.\n# Copyright (c) 2018,"
},
{
"path": "transformers/data/processors/squad.py",
"chars": 33153,
"preview": "# Copyright 2020 The HuggingFace Team. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lic"
},
{
"path": "transformers/data/processors/utils.py",
"chars": 13829,
"preview": "# coding=utf-8\n# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team.\n# Copyright (c) 2018,"
},
{
"path": "transformers/data/processors/xnli.py",
"chars": 3489,
"preview": "# coding=utf-8\n# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team.\n# Copyright (c) 2018,"
},
{
"path": "transformers/debug_utils.py",
"chars": 12907,
"preview": "# Copyright 2020 The HuggingFace Team. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lic"
},
{
"path": "transformers/deepspeed.py",
"chars": 16966,
"preview": "# Copyright 2020 The HuggingFace Team. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lic"
},
{
"path": "transformers/dependency_versions_check.py",
"chars": 1756,
"preview": "# Copyright 2020 The HuggingFace Team. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lic"
},
{
"path": "transformers/dependency_versions_table.py",
"chars": 3164,
"preview": "# THIS FILE HAS BEEN AUTOGENERATED. To update:\n# 1. modify the `_deps` dict in setup.py\n# 2. run `make deps_table_update"
},
{
"path": "transformers/dynamic_module_utils.py",
"chars": 24475,
"preview": "# coding=utf-8\n# Copyright 2021 The HuggingFace Inc. team.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lice"
},
{
"path": "transformers/feature_extraction_sequence_utils.py",
"chars": 18307,
"preview": "# coding=utf-8\n# Copyright 2021 The HuggingFace Inc. team.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lice"
},
{
"path": "transformers/feature_extraction_utils.py",
"chars": 27128,
"preview": "# coding=utf-8\n# Copyright 2021 The HuggingFace Inc. team.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lice"
},
{
"path": "transformers/file_utils.py",
"chars": 3522,
"preview": "# Copyright 2020 The HuggingFace Team. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lic"
},
{
"path": "transformers/generation/__init__.py",
"chars": 9435,
"preview": "# Copyright 2022 The HuggingFace Team. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lic"
},
{
"path": "transformers/generation/beam_constraints.py",
"chars": 19089,
"preview": "from abc import ABC, abstractmethod\nfrom typing import List, Optional\n\n\nclass Constraint(ABC):\n r\"\"\"Abstract base cla"
},
{
"path": "transformers/generation/beam_search.py",
"chars": 44725,
"preview": "# coding=utf-8\n# Copyright 2020 The HuggingFace Inc. team\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "transformers/generation/configuration_utils.py",
"chars": 38948,
"preview": "# coding=utf-8\n# Copyright 2022 The HuggingFace Inc. team.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lice"
},
{
"path": "transformers/generation/flax_logits_process.py",
"chars": 18995,
"preview": "# coding=utf-8\n# Copyright 2021 The HuggingFace Inc. team\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "transformers/generation/flax_utils.py",
"chars": 49064,
"preview": "# coding=utf-8\n# Copyright 2021 The Google AI Flax Team Authors, and The HuggingFace Inc. team.\n# Copyright (c) 2020, NV"
},
{
"path": "transformers/generation/logits_process.py",
"chars": 44516,
"preview": "# coding=utf-8\n# Copyright 2020 The HuggingFace Inc. team\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "transformers/generation/stopping_criteria.py",
"chars": 5510,
"preview": "import time\nimport warnings\nfrom abc import ABC\nfrom copy import deepcopy\nfrom typing import Optional\n\nimport torch\n\nfro"
},
{
"path": "transformers/generation/streamers.py",
"chars": 9144,
"preview": "# coding=utf-8\n# Copyright 2023 The HuggingFace Inc. team.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lice"
},
{
"path": "transformers/generation/tf_logits_process.py",
"chars": 27575,
"preview": "# coding=utf-8\n# Copyright 2022 The HuggingFace Inc. team\n#\n# Licensed under the Apache License, Version 2.0 (the \"Licen"
},
{
"path": "transformers/generation/tf_utils.py",
"chars": 177938,
"preview": "# coding=utf-8\n# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team.\n# Copyright (c) 2018,"
},
{
"path": "transformers/generation/utils.py",
"chars": 250412,
"preview": "# coding=utf-8\n# Copyright 2020 The Google AI Language Team Authors, Facebook AI Research authors and The HuggingFace In"
},
{
"path": "transformers/generation_flax_utils.py",
"chars": 1118,
"preview": "# coding=utf-8\n# Copyright 2021 The Google AI Flax Team Authors, and The HuggingFace Inc. team.\n# Copyright (c) 2020, NV"
},
{
"path": "transformers/generation_tf_utils.py",
"chars": 1109,
"preview": "# coding=utf-8\n# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team.\n# Copyright (c) 2018,"
},
{
"path": "transformers/generation_utils.py",
"chars": 1126,
"preview": "# coding=utf-8\n# Copyright 2020 The Google AI Language Team Authors, Facebook AI Research authors and The HuggingFace In"
},
{
"path": "transformers/hf_argparser.py",
"chars": 19929,
"preview": "# Copyright 2020 The HuggingFace Team. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lic"
},
{
"path": "transformers/image_processing_utils.py",
"chars": 25795,
"preview": "# coding=utf-8\n# Copyright 2022 The HuggingFace Inc. team.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lice"
},
{
"path": "transformers/image_transforms.py",
"chars": 31733,
"preview": "# coding=utf-8\n# Copyright 2022 The HuggingFace Inc. team.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lice"
},
{
"path": "transformers/image_utils.py",
"chars": 24006,
"preview": "# coding=utf-8\n# Copyright 2021 The HuggingFace Inc. team.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lice"
},
{
"path": "transformers/integrations.py",
"chars": 71856,
"preview": "# Copyright 2020 The HuggingFace Team. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lic"
},
{
"path": "transformers/keras_callbacks.py",
"chars": 20775,
"preview": "import logging\nimport os\nfrom pathlib import Path\nfrom time import sleep\nfrom typing import Callable, List, Optional, Un"
},
{
"path": "transformers/kernels/deformable_detr/cpu/ms_deform_attn_cpu.cpp",
"chars": 1255,
"preview": "/*!\n**************************************************************************************************\n* Deformable DETR"
},
{
"path": "transformers/kernels/deformable_detr/cpu/ms_deform_attn_cpu.h",
"chars": 1138,
"preview": "/*!\n**************************************************************************************************\n* Deformable DETR"
},
{
"path": "transformers/kernels/deformable_detr/cuda/ms_deform_attn_cuda.cu",
"chars": 7360,
"preview": "/*!\n**************************************************************************************************\n* Deformable DETR"
},
{
"path": "transformers/kernels/deformable_detr/cuda/ms_deform_attn_cuda.cuh",
"chars": 61433,
"preview": "/*!\n**************************************************************************************************\n* Deformable DETR"
},
{
"path": "transformers/kernels/deformable_detr/cuda/ms_deform_attn_cuda.h",
"chars": 1139,
"preview": "/*!\n**************************************************************************************************\n* Deformable DETR"
},
{
"path": "transformers/kernels/deformable_detr/cuda/ms_deform_im2col_cuda.cuh",
"chars": 54695,
"preview": "/*!\n**************************************************************************\n* Deformable DETR\n* Copyright (c) 2020 Se"
},
{
"path": "transformers/kernels/deformable_detr/ms_deform_attn.h",
"chars": 1837,
"preview": "/*!\n**************************************************************************************************\n* Deformable DETR"
},
{
"path": "transformers/kernels/deformable_detr/vision.cpp",
"chars": 798,
"preview": "/*!\n**************************************************************************************************\n* Deformable DETR"
},
{
"path": "transformers/kernels/rwkv/wkv_cuda.cu",
"chars": 6219,
"preview": "#include <stdio.h>\n#include <assert.h>\n\n#define MIN_VALUE (-1e38)\n\ntemplate <typename F>\n__global__ void kernel_forward("
},
{
"path": "transformers/kernels/rwkv/wkv_cuda_bf16.cu",
"chars": 6633,
"preview": "#include <stdio.h>\n#include <assert.h>\n#include \"ATen/ATen.h\"\n#define MIN_VALUE (-1e38)\ntypedef at::BFloat16 bf16;\n\n__gl"
},
{
"path": "transformers/kernels/rwkv/wkv_op.cpp",
"chars": 4022,
"preview": "#include <torch/extension.h>\n#include \"ATen/ATen.h\"\ntypedef at::BFloat16 bf16;\n\nvoid cuda_forward(int B, int T, int C, f"
},
{
"path": "transformers/kernels/yoso/common.h",
"chars": 273,
"preview": "\n#define min(a, b) ((a)<(b)?(a):(b))\n#define max(a, b) ((a)>(b)?(a):(b))\n#define ceil_divide(a, b) ((a)/(b)+((a)%(b)!=0)"
},
{
"path": "transformers/kernels/yoso/common_cuda.h",
"chars": 258,
"preview": "\n#define MAX_THREADS_PER_BLOCK 1024\n#define OPTIMAL_THREADS_PER_BLOCK 256\n#define WARP_SIZE 32\n#define MAX_NUM_BLOCK_X 2"
},
{
"path": "transformers/kernels/yoso/common_cuda_device.h",
"chars": 2063,
"preview": "\n#include \"common.h\"\n\ntemplate<typename T>\n__device__ int set_insert(T *set, int set_size, T value) {\n int slot = value"
},
{
"path": "transformers/kernels/yoso/fast_lsh_cumulation.cu",
"chars": 19061,
"preview": "// File from https://github.com/mlpen/YOSO/blob/main/encoders/backbones/efficient_attentions/yoso/yoso_v1/cuda/fast_lsh_"
},
{
"path": "transformers/kernels/yoso/fast_lsh_cumulation.h",
"chars": 1575,
"preview": "#include <torch/extension.h>\n#include <ATen/ATen.h>\n#include <vector>\n\nstd::vector<at::Tensor> fast_hash_ver1_kernel(\n "
},
{
"path": "transformers/kernels/yoso/fast_lsh_cumulation_cuda.cu",
"chars": 32870,
"preview": "// File from https://github.com/mlpen/YOSO/blob/main/encoders/backbones/efficient_attentions/yoso/yoso_v1/cuda/fast_lsh_"
},
{
"path": "transformers/kernels/yoso/fast_lsh_cumulation_cuda.h",
"chars": 5490,
"preview": "__global__ void fast_hash_ver1_cuda_kernel(\n int *mask, // [batch_size, num_vector]\n float *vector, // [batc"
},
{
"path": "transformers/kernels/yoso/fast_lsh_cumulation_torch.cpp",
"chars": 3154,
"preview": "#include <torch/extension.h>\n#include <ATen/ATen.h>\n#include \"fast_lsh_cumulation.h\"\n#include \"common_cuda.h\"\n#include <"
},
{
"path": "transformers/modelcard.py",
"chars": 35054,
"preview": "# coding=utf-8\n# Copyright 2018 The HuggingFace Inc. team.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lice"
},
{
"path": "transformers/modeling_flax_outputs.py",
"chars": 41961,
"preview": "# Copyright 2021 The HuggingFace Team. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lic"
},
{
"path": "transformers/modeling_flax_pytorch_utils.py",
"chars": 18210,
"preview": "# coding=utf-8\n# Copyright 2021 The HuggingFace Inc. team.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lice"
},
{
"path": "transformers/modeling_flax_utils.py",
"chars": 56041,
"preview": "# coding=utf-8\n# Copyright 2021 The Google Flax Team Authors and The HuggingFace Inc. team.\n#\n# Licensed under the Apach"
},
{
"path": "transformers/modeling_outputs.py",
"chars": 106127,
"preview": "# Copyright 2020 The HuggingFace Team. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lic"
},
{
"path": "transformers/modeling_tf_outputs.py",
"chars": 56074,
"preview": "# Copyright 2020 The HuggingFace Team. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lic"
},
{
"path": "transformers/modeling_tf_pytorch_utils.py",
"chars": 23690,
"preview": "# coding=utf-8\n# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team.\n# Copyright (c) 2018,"
},
{
"path": "transformers/modeling_tf_utils.py",
"chars": 160223,
"preview": "# coding=utf-8\n# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team.\n# Copyright (c) 2018,"
},
{
"path": "transformers/modeling_utils.py",
"chars": 187447,
"preview": "# coding=utf-8\n# Copyright 2018 The Google AI Language Team Authors, Facebook AI Research authors and The HuggingFace In"
},
{
"path": "transformers/models/__init__.py",
"chars": 3550,
"preview": "# Copyright 2020 The HuggingFace Team. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lic"
},
{
"path": "transformers/models/albert/__init__.py",
"chars": 5482,
"preview": "# Copyright 2020 The HuggingFace Team. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lic"
},
{
"path": "transformers/models/albert/configuration_albert.py",
"chars": 8568,
"preview": "# coding=utf-8\n# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team.\n# Copyright (c) 2018,"
},
{
"path": "transformers/models/albert/convert_albert_original_tf_checkpoint_to_pytorch.py",
"chars": 2162,
"preview": "# coding=utf-8\n# Copyright 2018 The HuggingFace Inc. team.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lice"
},
{
"path": "transformers/models/albert/modeling_albert.py",
"chars": 60935,
"preview": "# coding=utf-8\n# Copyright 2018 Google AI, Google Brain and the HuggingFace Inc. team.\n#\n# Licensed under the Apache Lic"
},
{
"path": "transformers/models/albert/modeling_flax_albert.py",
"chars": 40772,
"preview": "# coding=utf-8\n# Copyright 2021 Google AI, Google Brain and the HuggingFace Inc. team.\n#\n# Licensed under the Apache Lic"
},
{
"path": "transformers/models/albert/modeling_tf_albert.py",
"chars": 61857,
"preview": "# coding=utf-8\n# Copyright 2018 The OpenAI Team Authors and HuggingFace Inc. team.\n# Copyright (c) 2018, NVIDIA CORPORAT"
},
{
"path": "transformers/models/albert/tokenization_albert.py",
"chars": 15427,
"preview": "# coding=utf-8\n# Copyright 2018 Google AI, Google Brain and the HuggingFace Inc. team.\n#\n# Licensed under the Apache Lic"
},
{
"path": "transformers/models/albert/tokenization_albert_fast.py",
"chars": 10813,
"preview": "# coding=utf-8\n# Copyright 2018 Google AI, Google Brain and the HuggingFace Inc. team.\n#\n# Licensed under the Apache Lic"
},
{
"path": "transformers/models/align/__init__.py",
"chars": 2064,
"preview": "# Copyright 2023 The HuggingFace Team. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lic"
},
{
"path": "transformers/models/align/configuration_align.py",
"chars": 18932,
"preview": "# coding=utf-8\n# Copyright 2023 The HuggingFace Inc. team. All rights reserved.\n#\n# Licensed under the Apache License, V"
},
{
"path": "transformers/models/align/convert_align_tf_to_hf.py",
"chars": 15478,
"preview": "# coding=utf-8\n# Copyright 2023 The HuggingFace Inc. team.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lice"
},
{
"path": "transformers/models/align/modeling_align.py",
"chars": 72079,
"preview": "# coding=utf-8\n# Copyright 2023 The Google Research Team Authors and The HuggingFace Team. All rights reserved.\n#\n# Lice"
},
{
"path": "transformers/models/align/processing_align.py",
"chars": 6216,
"preview": "# coding=utf-8\n# Copyright 2023 The HuggingFace Inc. team.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lice"
},
{
"path": "transformers/models/altclip/__init__.py",
"chars": 2126,
"preview": "# Copyright 2020 The HuggingFace Team. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lic"
},
{
"path": "transformers/models/altclip/configuration_altclip.py",
"chars": 19622,
"preview": "# coding=utf-8\n# Copyright 2022 WenXiang ZhongzhiCheng LedellWu LiuGuang BoWenZhang and The HuggingFace Inc. team. All r"
},
{
"path": "transformers/models/altclip/modeling_altclip.py",
"chars": 78707,
"preview": "# coding=utf-8\n# Copyright 2022 The BAAI Teams Authors and The HuggingFace Inc. team. All rights reserved.\n#\n# Licensed "
},
{
"path": "transformers/models/altclip/processing_altclip.py",
"chars": 6444,
"preview": "# coding=utf-8\n# Copyright 2022 WenXiang ZhongzhiCheng LedellWu LiuGuang BoWenZhang The HuggingFace Inc. team. All right"
},
{
"path": "transformers/models/audio_spectrogram_transformer/__init__.py",
"chars": 2507,
"preview": "# Copyright 2021 The HuggingFace Team. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lic"
},
{
"path": "transformers/models/audio_spectrogram_transformer/configuration_audio_spectrogram_transformer.py",
"chars": 5650,
"preview": "# coding=utf-8\n# Copyright 2022 Google AI and The HuggingFace Inc. team. All rights reserved.\n#\n# Licensed under the Apa"
},
{
"path": "transformers/models/audio_spectrogram_transformer/convert_audio_spectrogram_transformer_original_to_pytorch.py",
"chars": 11046,
"preview": "# coding=utf-8\n# Copyright 2022 The HuggingFace Inc. team.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lice"
},
{
"path": "transformers/models/audio_spectrogram_transformer/feature_extraction_audio_spectrogram_transformer.py",
"chars": 8614,
"preview": "# coding=utf-8\n# Copyright 2022 The HuggingFace Inc. team.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lice"
},
{
"path": "transformers/models/audio_spectrogram_transformer/modeling_audio_spectrogram_transformer.py",
"chars": 26449,
"preview": "# coding=utf-8\n# Copyright 2022 MIT and The HuggingFace Inc. team. All rights reserved.\n#\n# Licensed under the Apache Li"
},
{
"path": "transformers/models/auto/__init__.py",
"chars": 14814,
"preview": "# Copyright 2020 The HuggingFace Team. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lic"
},
{
"path": "transformers/models/auto/auto_factory.py",
"chars": 39763,
"preview": "# coding=utf-8\n# Copyright 2021 The HuggingFace Inc. team.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lice"
},
{
"path": "transformers/models/auto/configuration_auto.py",
"chars": 43156,
"preview": "# coding=utf-8\n# Copyright 2018 The HuggingFace Inc. team.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lice"
},
{
"path": "transformers/models/auto/feature_extraction_auto.py",
"chars": 18325,
"preview": "# coding=utf-8\n# Copyright 2021 The HuggingFace Inc. team.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lice"
},
{
"path": "transformers/models/auto/image_processing_auto.py",
"chars": 19799,
"preview": "# coding=utf-8\n# Copyright 2022 The HuggingFace Inc. team.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lice"
},
{
"path": "transformers/models/auto/modeling_auto.py",
"chars": 55592,
"preview": "# coding=utf-8\n# Copyright 2018 The HuggingFace Inc. team.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lice"
},
{
"path": "transformers/models/auto/modeling_flax_auto.py",
"chars": 13986,
"preview": "# coding=utf-8\n# Copyright 2018 The Google Flax Team Authors and The HuggingFace Inc. team.\n#\n# Licensed under the Apach"
},
{
"path": "transformers/models/auto/modeling_tf_auto.py",
"chars": 26791,
"preview": "# coding=utf-8\n# Copyright 2018 The HuggingFace Inc. team.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lice"
},
{
"path": "transformers/models/auto/processing_auto.py",
"chars": 15021,
"preview": "# coding=utf-8\n# Copyright 2021 The HuggingFace Inc. team.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lice"
},
{
"path": "transformers/models/auto/tokenization_auto.py",
"chars": 39251,
"preview": "# coding=utf-8\n# Copyright 2018 The HuggingFace Inc. team.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lice"
},
{
"path": "transformers/models/autoformer/__init__.py",
"chars": 1914,
"preview": "# Copyright 2023 The HuggingFace Team. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lic"
},
{
"path": "transformers/models/autoformer/configuration_autoformer.py",
"chars": 12325,
"preview": "# coding=utf-8\n# Copyright 2023 The HuggingFace Inc. team. All rights reserved.\n#\n# Licensed under the Apache License, V"
},
{
"path": "transformers/models/autoformer/modeling_autoformer.py",
"chars": 109394,
"preview": "# coding=utf-8\n# Copyright (c) 2021 THUML @ Tsinghua University\n# Copyright 2023 Amazon.com, Inc. or its affiliates. All"
},
{
"path": "transformers/models/bart/__init__.py",
"chars": 4333,
"preview": "# Copyright 2020 The HuggingFace Team. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lic"
},
{
"path": "transformers/models/bart/configuration_bart.py",
"chars": 18993,
"preview": "# coding=utf-8\n# Copyright 2021 The Fairseq Authors and The HuggingFace Inc. team. All rights reserved.\n#\n# Licensed und"
},
{
"path": "transformers/models/bart/convert_bart_original_pytorch_checkpoint_to_pytorch.py",
"chars": 5645,
"preview": "# coding=utf-8\n# Copyright 2020 The HuggingFace Inc. team.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lice"
},
{
"path": "transformers/models/bart/modeling_bart.py",
"chars": 90101,
"preview": "# coding=utf-8\n# Copyright 2021 The Fairseq Authors and The HuggingFace Inc. team. All rights reserved.\n#\n# Licensed und"
},
{
"path": "transformers/models/bart/modeling_flax_bart.py",
"chars": 82723,
"preview": "# coding=utf-8\n# Copyright 2021 The Fairseq Authors and The Google Flax Team Authors And The HuggingFace Inc. team. All "
},
{
"path": "transformers/models/bart/modeling_tf_bart.py",
"chars": 74717,
"preview": "# coding=utf-8\n# Copyright 2021 The Fairseq Authors and The HuggingFace Inc. team. All rights reserved.\n#\n# Licensed und"
},
{
"path": "transformers/models/bart/tokenization_bart.py",
"chars": 17977,
"preview": "# coding=utf-8\n# Copyright 2020 The Facebook AI Research Team Authors and The HuggingFace Inc. team.\n#\n# Licensed under "
},
{
"path": "transformers/models/bart/tokenization_bart_fast.py",
"chars": 13812,
"preview": "# coding=utf-8\n# Copyright 2020 The Facebook AI Research Team Authors and The HuggingFace Inc. team.\n#\n# Licensed under "
},
{
"path": "transformers/models/barthez/__init__.py",
"chars": 1848,
"preview": "# Copyright 2020 The HuggingFace Team. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lic"
},
{
"path": "transformers/models/barthez/tokenization_barthez.py",
"chars": 13191,
"preview": "# coding=utf-8\n# Copyright 2020 Ecole Polytechnique and the HuggingFace Inc. team.\n#\n# Licensed under the Apache License"
},
{
"path": "transformers/models/barthez/tokenization_barthez_fast.py",
"chars": 8898,
"preview": "# coding=utf-8\n# Copyright 2020 Ecole Polytechnique and the HuggingFace Inc. team.\n#\n# Licensed under the Apache License"
},
{
"path": "transformers/models/bartpho/__init__.py",
"chars": 1362,
"preview": "# Copyright 2021 The HuggingFace Team. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lic"
},
{
"path": "transformers/models/bartpho/tokenization_bartpho.py",
"chars": 14217,
"preview": "# coding=utf-8\n# Copyright 2021 VinAI Research and the HuggingFace Inc. team.\n#\n# Licensed under the Apache License, Ver"
},
{
"path": "transformers/models/beit/__init__.py",
"chars": 3289,
"preview": "# Copyright 2021 The HuggingFace Team. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lic"
},
{
"path": "transformers/models/beit/configuration_beit.py",
"chars": 9641,
"preview": "# coding=utf-8\n# Copyright Microsoft Research and The HuggingFace Inc. team. All rights reserved.\n#\n# Licensed under the"
},
{
"path": "transformers/models/beit/convert_beit_unilm_to_pytorch.py",
"chars": 16496,
"preview": "# coding=utf-8\n# Copyright 2021 The HuggingFace Inc. team.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lice"
},
{
"path": "transformers/models/beit/feature_extraction_beit.py",
"chars": 1172,
"preview": "# coding=utf-8\n# Copyright 2021 The HuggingFace Inc. team. All rights reserved.\n#\n# Licensed under the Apache License, V"
},
{
"path": "transformers/models/beit/image_processing_beit.py",
"chars": 24698,
"preview": "# coding=utf-8\n# Copyright 2022 The HuggingFace Inc. team. All rights reserved.\n#\n# Licensed under the Apache License, V"
},
{
"path": "transformers/models/beit/modeling_beit.py",
"chars": 54250,
"preview": "# coding=utf-8\n# Copyright 2021 Microsoft Research and The HuggingFace Inc. team. All rights reserved.\n#\n# Licensed unde"
},
{
"path": "transformers/models/beit/modeling_flax_beit.py",
"chars": 36989,
"preview": "# coding=utf-8\n# Copyright 2021 Microsoft Research and the HuggingFace Inc. team.\n#\n# Licensed under the Apache License,"
},
{
"path": "transformers/models/bert/__init__.py",
"chars": 6057,
"preview": "# Copyright 2020 The HuggingFace Team. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lic"
},
{
"path": "transformers/models/bert/configuration_bert.py",
"chars": 10150,
"preview": "# coding=utf-8\n# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team.\n# Copyright (c) 2018,"
},
{
"path": "transformers/models/bert/convert_bert_original_tf2_checkpoint_to_pytorch.py",
"chars": 10490,
"preview": "# Copyright 2020 The HuggingFace Team. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lic"
},
{
"path": "transformers/models/bert/convert_bert_original_tf_checkpoint_to_pytorch.py",
"chars": 2159,
"preview": "# coding=utf-8\n# Copyright 2018 The HuggingFace Inc. team.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lice"
},
{
"path": "transformers/models/bert/convert_bert_pytorch_checkpoint_to_original_tf.py",
"chars": 4100,
"preview": "# coding=utf-8\n# Copyright 2018 The HuggingFace Inc. team.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lice"
},
{
"path": "transformers/models/bert/convert_bert_token_dropping_original_tf2_checkpoint_to_pytorch.py",
"chars": 7606,
"preview": "# Copyright 2022 The HuggingFace Team. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lic"
},
{
"path": "transformers/models/bert/modeling_bert.py",
"chars": 84191,
"preview": "# coding=utf-8\n# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team.\n# Copyright (c) 2018,"
},
{
"path": "transformers/models/bert/modeling_flax_bert.py",
"chars": 63604,
"preview": "# coding=utf-8\n# Copyright 2021 The Google Flax Team Authors and The HuggingFace Inc. team.\n#\n# Licensed under the Apach"
},
{
"path": "transformers/models/bert/modeling_tf_bert.py",
"chars": 85690,
"preview": "# coding=utf-8\n# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team.\n# Copyright (c) 2018,"
},
{
"path": "transformers/models/bert/tokenization_bert.py",
"chars": 24516,
"preview": "# coding=utf-8\n# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team.\n#\n# Licensed under th"
},
{
"path": "transformers/models/bert/tokenization_bert_fast.py",
"chars": 14871,
"preview": "# coding=utf-8\n# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team.\n#\n# Licensed under th"
},
{
"path": "transformers/models/bert/tokenization_bert_tf.py",
"chars": 11551,
"preview": "import os\nfrom typing import List, Union\n\nimport tensorflow as tf\nfrom tensorflow_text import BertTokenizer as BertToken"
},
{
"path": "transformers/models/bert_generation/__init__.py",
"chars": 2275,
"preview": "# Copyright 2020 The HuggingFace Team. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lic"
},
{
"path": "transformers/models/bert_generation/configuration_bert_generation.py",
"chars": 6228,
"preview": "# coding=utf-8\n# Copyright 2020 The Google AI Language Team Authors and The HuggingFace Inc. team.\n#\n# Licensed under th"
},
{
"path": "transformers/models/bert_generation/modeling_bert_generation.py",
"chars": 47930,
"preview": "# coding=utf-8\n# Copyright 2020 The Google AI Language Team Authors and The HuggingFace Inc. team.\n#\n# Licensed under th"
},
{
"path": "transformers/models/bert_generation/tokenization_bert_generation.py",
"chars": 7140,
"preview": "# coding=utf-8\n# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.\n#\n# Licensed under the Apache License, Ve"
},
{
"path": "transformers/models/bert_japanese/__init__.py",
"chars": 1053,
"preview": "# Copyright 2020 The HuggingFace Team. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"Lic"
}
]
// ... and 1162 more files (download for full content)
About this extraction
This page contains the full source code of the Ablustrund/LoRAMoE GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 1362 files (33.5 MB), approximately 8.9M tokens, and a symbol index with 30522 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.