Showing preview only (4,106K chars total). The displayed content is truncated. Use the JSON API for full output.
Repository: allenai/allennlp
Branch: main
Commit: 80fb6061e568
Files: 727
Total size: 3.8 MB
Directory structure:
gitextract_g9t09ufj/
├── .coveragerc
├── .dockerignore
├── .flake8
├── .github/
│ ├── ISSUE_TEMPLATE/
│ │ ├── bug_report.md
│ │ ├── feature_request.md
│ │ └── question.md
│ ├── pull_request_template.md
│ └── workflows/
│ ├── cffconvert.yml
│ ├── ci.yml
│ └── issues.yml
├── .gitignore
├── CHANGELOG.md
├── CITATION.cff
├── CONTRIBUTING.md
├── Dockerfile
├── Dockerfile.test
├── LICENSE
├── MANIFEST.in
├── Makefile
├── README.md
├── RELEASE_PROCESS.md
├── STYLE.md
├── allennlp/
│ ├── __init__.py
│ ├── __main__.py
│ ├── commands/
│ │ ├── __init__.py
│ │ ├── _checklist_internal.py
│ │ ├── build_vocab.py
│ │ ├── cached_path.py
│ │ ├── checklist.py
│ │ ├── count_instances.py
│ │ ├── diff.py
│ │ ├── evaluate.py
│ │ ├── find_learning_rate.py
│ │ ├── predict.py
│ │ ├── print_results.py
│ │ ├── push_to_hf.py
│ │ ├── subcommand.py
│ │ ├── test_install.py
│ │ └── train.py
│ ├── common/
│ │ ├── __init__.py
│ │ ├── cached_transformers.py
│ │ ├── checks.py
│ │ ├── file_utils.py
│ │ ├── from_params.py
│ │ ├── lazy.py
│ │ ├── logging.py
│ │ ├── meta.py
│ │ ├── model_card.py
│ │ ├── params.py
│ │ ├── plugins.py
│ │ ├── push_to_hf.py
│ │ ├── registrable.py
│ │ ├── sequences.py
│ │ ├── task_card.py
│ │ ├── testing/
│ │ │ ├── __init__.py
│ │ │ ├── checklist_test.py
│ │ │ ├── confidence_check_test.py
│ │ │ ├── distributed_test.py
│ │ │ ├── interpret_test.py
│ │ │ ├── model_test_case.py
│ │ │ └── test_case.py
│ │ ├── tqdm.py
│ │ └── util.py
│ ├── confidence_checks/
│ │ ├── __init__.py
│ │ ├── normalization_bias_verification.py
│ │ ├── task_checklists/
│ │ │ ├── __init__.py
│ │ │ ├── question_answering_suite.py
│ │ │ ├── sentiment_analysis_suite.py
│ │ │ ├── task_suite.py
│ │ │ ├── textual_entailment_suite.py
│ │ │ └── utils.py
│ │ └── verification_base.py
│ ├── data/
│ │ ├── __init__.py
│ │ ├── batch.py
│ │ ├── data_loaders/
│ │ │ ├── __init__.py
│ │ │ ├── data_collator.py
│ │ │ ├── data_loader.py
│ │ │ ├── multiprocess_data_loader.py
│ │ │ ├── multitask_data_loader.py
│ │ │ ├── multitask_epoch_sampler.py
│ │ │ ├── multitask_scheduler.py
│ │ │ └── simple_data_loader.py
│ │ ├── dataset_readers/
│ │ │ ├── __init__.py
│ │ │ ├── babi.py
│ │ │ ├── conll2003.py
│ │ │ ├── dataset_reader.py
│ │ │ ├── dataset_utils/
│ │ │ │ ├── __init__.py
│ │ │ │ └── span_utils.py
│ │ │ ├── interleaving_dataset_reader.py
│ │ │ ├── multitask.py
│ │ │ ├── sequence_tagging.py
│ │ │ ├── sharded_dataset_reader.py
│ │ │ └── text_classification_json.py
│ │ ├── fields/
│ │ │ ├── __init__.py
│ │ │ ├── adjacency_field.py
│ │ │ ├── array_field.py
│ │ │ ├── field.py
│ │ │ ├── flag_field.py
│ │ │ ├── index_field.py
│ │ │ ├── label_field.py
│ │ │ ├── list_field.py
│ │ │ ├── metadata_field.py
│ │ │ ├── multilabel_field.py
│ │ │ ├── namespace_swapping_field.py
│ │ │ ├── sequence_field.py
│ │ │ ├── sequence_label_field.py
│ │ │ ├── span_field.py
│ │ │ ├── tensor_field.py
│ │ │ ├── text_field.py
│ │ │ └── transformer_text_field.py
│ │ ├── image_loader.py
│ │ ├── instance.py
│ │ ├── samplers/
│ │ │ ├── __init__.py
│ │ │ ├── batch_sampler.py
│ │ │ ├── bucket_batch_sampler.py
│ │ │ └── max_tokens_batch_sampler.py
│ │ ├── token_indexers/
│ │ │ ├── __init__.py
│ │ │ ├── elmo_indexer.py
│ │ │ ├── pretrained_transformer_indexer.py
│ │ │ ├── pretrained_transformer_mismatched_indexer.py
│ │ │ ├── single_id_token_indexer.py
│ │ │ ├── spacy_indexer.py
│ │ │ ├── token_characters_indexer.py
│ │ │ └── token_indexer.py
│ │ ├── tokenizers/
│ │ │ ├── __init__.py
│ │ │ ├── character_tokenizer.py
│ │ │ ├── letters_digits_tokenizer.py
│ │ │ ├── pretrained_transformer_tokenizer.py
│ │ │ ├── sentence_splitter.py
│ │ │ ├── spacy_tokenizer.py
│ │ │ ├── token_class.py
│ │ │ ├── tokenizer.py
│ │ │ └── whitespace_tokenizer.py
│ │ └── vocabulary.py
│ ├── evaluation/
│ │ ├── __init__.py
│ │ ├── evaluator.py
│ │ ├── postprocessors/
│ │ │ └── __init__.py
│ │ └── serializers/
│ │ ├── __init__.py
│ │ └── serializers.py
│ ├── fairness/
│ │ ├── __init__.py
│ │ ├── adversarial_bias_mitigator.py
│ │ ├── bias_direction.py
│ │ ├── bias_direction_wrappers.py
│ │ ├── bias_metrics.py
│ │ ├── bias_mitigator_applicator.py
│ │ ├── bias_mitigator_wrappers.py
│ │ ├── bias_mitigators.py
│ │ ├── bias_utils.py
│ │ └── fairness_metrics.py
│ ├── interpret/
│ │ ├── __init__.py
│ │ ├── attackers/
│ │ │ ├── __init__.py
│ │ │ ├── attacker.py
│ │ │ ├── hotflip.py
│ │ │ ├── input_reduction.py
│ │ │ └── utils.py
│ │ ├── influence_interpreters/
│ │ │ ├── __init__.py
│ │ │ ├── influence_interpreter.py
│ │ │ └── simple_influence.py
│ │ └── saliency_interpreters/
│ │ ├── __init__.py
│ │ ├── integrated_gradient.py
│ │ ├── saliency_interpreter.py
│ │ ├── simple_gradient.py
│ │ └── smooth_gradient.py
│ ├── models/
│ │ ├── __init__.py
│ │ ├── archival.py
│ │ ├── basic_classifier.py
│ │ ├── heads/
│ │ │ ├── __init__.py
│ │ │ ├── classifier_head.py
│ │ │ └── head.py
│ │ ├── model.py
│ │ ├── multitask.py
│ │ └── simple_tagger.py
│ ├── modules/
│ │ ├── __init__.py
│ │ ├── attention/
│ │ │ ├── __init__.py
│ │ │ ├── additive_attention.py
│ │ │ ├── attention.py
│ │ │ ├── bilinear_attention.py
│ │ │ ├── cosine_attention.py
│ │ │ ├── dot_product_attention.py
│ │ │ ├── linear_attention.py
│ │ │ └── scaled_dot_product_attention.py
│ │ ├── augmented_lstm.py
│ │ ├── backbones/
│ │ │ ├── __init__.py
│ │ │ ├── backbone.py
│ │ │ ├── pretrained_transformer_backbone.py
│ │ │ └── vilbert_backbone.py
│ │ ├── bimpm_matching.py
│ │ ├── conditional_random_field/
│ │ │ ├── __init__.py
│ │ │ ├── conditional_random_field.py
│ │ │ ├── conditional_random_field_wemission.py
│ │ │ ├── conditional_random_field_wlannoy.py
│ │ │ └── conditional_random_field_wtrans.py
│ │ ├── elmo.py
│ │ ├── elmo_lstm.py
│ │ ├── encoder_base.py
│ │ ├── feedforward.py
│ │ ├── gated_sum.py
│ │ ├── highway.py
│ │ ├── input_variational_dropout.py
│ │ ├── layer_norm.py
│ │ ├── lstm_cell_with_projection.py
│ │ ├── masked_layer_norm.py
│ │ ├── matrix_attention/
│ │ │ ├── __init__.py
│ │ │ ├── bilinear_matrix_attention.py
│ │ │ ├── cosine_matrix_attention.py
│ │ │ ├── dot_product_matrix_attention.py
│ │ │ ├── linear_matrix_attention.py
│ │ │ ├── matrix_attention.py
│ │ │ └── scaled_dot_product_matrix_attention.py
│ │ ├── maxout.py
│ │ ├── residual_with_layer_dropout.py
│ │ ├── sampled_softmax_loss.py
│ │ ├── scalar_mix.py
│ │ ├── seq2seq_encoders/
│ │ │ ├── __init__.py
│ │ │ ├── compose_encoder.py
│ │ │ ├── feedforward_encoder.py
│ │ │ ├── gated_cnn_encoder.py
│ │ │ ├── pass_through_encoder.py
│ │ │ ├── pytorch_seq2seq_wrapper.py
│ │ │ ├── pytorch_transformer_wrapper.py
│ │ │ └── seq2seq_encoder.py
│ │ ├── seq2vec_encoders/
│ │ │ ├── __init__.py
│ │ │ ├── bert_pooler.py
│ │ │ ├── boe_encoder.py
│ │ │ ├── cls_pooler.py
│ │ │ ├── cnn_encoder.py
│ │ │ ├── cnn_highway_encoder.py
│ │ │ ├── pytorch_seq2vec_wrapper.py
│ │ │ └── seq2vec_encoder.py
│ │ ├── softmax_loss.py
│ │ ├── span_extractors/
│ │ │ ├── __init__.py
│ │ │ ├── bidirectional_endpoint_span_extractor.py
│ │ │ ├── endpoint_span_extractor.py
│ │ │ ├── max_pooling_span_extractor.py
│ │ │ ├── self_attentive_span_extractor.py
│ │ │ ├── span_extractor.py
│ │ │ └── span_extractor_with_span_width_embedding.py
│ │ ├── stacked_alternating_lstm.py
│ │ ├── stacked_bidirectional_lstm.py
│ │ ├── text_field_embedders/
│ │ │ ├── __init__.py
│ │ │ ├── basic_text_field_embedder.py
│ │ │ └── text_field_embedder.py
│ │ ├── time_distributed.py
│ │ ├── token_embedders/
│ │ │ ├── __init__.py
│ │ │ ├── bag_of_word_counts_token_embedder.py
│ │ │ ├── elmo_token_embedder.py
│ │ │ ├── embedding.py
│ │ │ ├── empty_embedder.py
│ │ │ ├── pass_through_token_embedder.py
│ │ │ ├── pretrained_transformer_embedder.py
│ │ │ ├── pretrained_transformer_mismatched_embedder.py
│ │ │ ├── token_characters_encoder.py
│ │ │ └── token_embedder.py
│ │ ├── transformer/
│ │ │ ├── __init__.py
│ │ │ ├── activation_layer.py
│ │ │ ├── attention_module.py
│ │ │ ├── bimodal_attention.py
│ │ │ ├── bimodal_connection_layer.py
│ │ │ ├── bimodal_encoder.py
│ │ │ ├── layer_norm.py
│ │ │ ├── output_layer.py
│ │ │ ├── positional_encoding.py
│ │ │ ├── t5.py
│ │ │ ├── transformer_embeddings.py
│ │ │ ├── transformer_layer.py
│ │ │ ├── transformer_module.py
│ │ │ ├── transformer_pooler.py
│ │ │ ├── transformer_stack.py
│ │ │ └── util.py
│ │ ├── util.py
│ │ └── vision/
│ │ ├── __init__.py
│ │ ├── grid_embedder.py
│ │ ├── image2image.py
│ │ └── region_detector.py
│ ├── nn/
│ │ ├── __init__.py
│ │ ├── activations.py
│ │ ├── beam_search.py
│ │ ├── checkpoint/
│ │ │ ├── __init__.py
│ │ │ ├── checkpoint_wrapper.py
│ │ │ └── fairscale_checkpoint_wrapper.py
│ │ ├── chu_liu_edmonds.py
│ │ ├── initializers.py
│ │ ├── module.py
│ │ ├── parallel/
│ │ │ ├── __init__.py
│ │ │ ├── ddp_accelerator.py
│ │ │ ├── fairscale_fsdp_accelerator.py
│ │ │ └── sharded_module_mixin.py
│ │ ├── regularizers/
│ │ │ ├── __init__.py
│ │ │ ├── regularizer.py
│ │ │ ├── regularizer_applicator.py
│ │ │ └── regularizers.py
│ │ └── util.py
│ ├── predictors/
│ │ ├── __init__.py
│ │ ├── multitask.py
│ │ ├── predictor.py
│ │ ├── sentence_tagger.py
│ │ └── text_classifier.py
│ ├── py.typed
│ ├── sanity_checks/
│ │ ├── __init__.py
│ │ └── task_checklists/
│ │ └── __init__.py
│ ├── tools/
│ │ ├── EVALB/
│ │ │ ├── .gitignore
│ │ │ ├── COLLINS.prm
│ │ │ ├── LICENSE
│ │ │ ├── Makefile
│ │ │ ├── README
│ │ │ ├── bug/
│ │ │ │ ├── bug.gld
│ │ │ │ ├── bug.rsl-new
│ │ │ │ ├── bug.rsl-old
│ │ │ │ └── bug.tst
│ │ │ ├── evalb.c
│ │ │ ├── new.prm
│ │ │ ├── sample/
│ │ │ │ ├── sample.gld
│ │ │ │ ├── sample.prm
│ │ │ │ ├── sample.rsl
│ │ │ │ └── sample.tst
│ │ │ └── tgrep_proc.prl
│ │ ├── __init__.py
│ │ ├── archive_surgery.py
│ │ ├── create_elmo_embeddings_from_vocab.py
│ │ └── inspect_cache.py
│ ├── training/
│ │ ├── __init__.py
│ │ ├── callbacks/
│ │ │ ├── __init__.py
│ │ │ ├── backward.py
│ │ │ ├── callback.py
│ │ │ ├── confidence_checks.py
│ │ │ ├── console_logger.py
│ │ │ ├── log_writer.py
│ │ │ ├── should_validate.py
│ │ │ ├── tensorboard.py
│ │ │ ├── track_epoch.py
│ │ │ └── wandb.py
│ │ ├── checkpointer.py
│ │ ├── gradient_descent_trainer.py
│ │ ├── learning_rate_schedulers/
│ │ │ ├── __init__.py
│ │ │ ├── combined.py
│ │ │ ├── cosine.py
│ │ │ ├── learning_rate_scheduler.py
│ │ │ ├── linear_with_warmup.py
│ │ │ ├── noam.py
│ │ │ ├── polynomial_decay.py
│ │ │ ├── pytorch_lr_schedulers.py
│ │ │ └── slanted_triangular.py
│ │ ├── metric_tracker.py
│ │ ├── metrics/
│ │ │ ├── __init__.py
│ │ │ ├── attachment_scores.py
│ │ │ ├── auc.py
│ │ │ ├── average.py
│ │ │ ├── bleu.py
│ │ │ ├── boolean_accuracy.py
│ │ │ ├── categorical_accuracy.py
│ │ │ ├── covariance.py
│ │ │ ├── entropy.py
│ │ │ ├── evalb_bracketing_scorer.py
│ │ │ ├── f1_measure.py
│ │ │ ├── fbeta_measure.py
│ │ │ ├── fbeta_multi_label_measure.py
│ │ │ ├── fbeta_verbose_measure.py
│ │ │ ├── mean_absolute_error.py
│ │ │ ├── metric.py
│ │ │ ├── pearson_correlation.py
│ │ │ ├── perplexity.py
│ │ │ ├── rouge.py
│ │ │ ├── sequence_accuracy.py
│ │ │ ├── span_based_f1_measure.py
│ │ │ ├── spearman_correlation.py
│ │ │ └── unigram_recall.py
│ │ ├── momentum_schedulers/
│ │ │ ├── __init__.py
│ │ │ ├── inverted_triangular.py
│ │ │ └── momentum_scheduler.py
│ │ ├── moving_average.py
│ │ ├── no_op_trainer.py
│ │ ├── optimizers.py
│ │ ├── scheduler.py
│ │ ├── trainer.py
│ │ └── util.py
│ └── version.py
├── benchmarks/
│ ├── __init__.py
│ ├── data/
│ │ ├── __init__.py
│ │ └── tokenizers/
│ │ ├── __init__.py
│ │ └── character_tokenizer_bench.py
│ ├── nn/
│ │ └── util_bench.py
│ └── pytest.ini
├── codecov.yml
├── constraints.txt
├── dev-requirements.txt
├── docs/
│ └── css/
│ └── extra.css
├── mkdocs-skeleton.yml
├── mypy.ini
├── pyproject.toml
├── pytest.ini
├── requirements.in
├── requirements.txt
├── scripts/
│ ├── 24hr_diff.sh
│ ├── ai2_internal/
│ │ ├── resumable_train.sh
│ │ ├── resume_daemon.py
│ │ └── run_with_beaker.py
│ ├── build_docs.sh
│ ├── build_docs_config.py
│ ├── check_large_files.sh
│ ├── check_links.py
│ ├── check_torch_version.py
│ ├── close_stale_issues.py
│ ├── get_version.py
│ ├── ping_issue_assignees.py
│ ├── py2md.py
│ ├── release_notes.py
│ ├── tests/
│ │ ├── ai2_internal/
│ │ │ └── resume_daemon_test.py
│ │ └── py2md/
│ │ ├── basic_example.py
│ │ ├── basic_example_expected_output.md
│ │ └── py2md_test.py
│ └── train_fixtures.py
├── setup.py
├── test_fixtures/
│ ├── __init__.py
│ ├── basic_classifier/
│ │ ├── common.jsonnet
│ │ ├── experiment_from_archive.jsonnet
│ │ ├── experiment_seq2seq.jsonnet
│ │ ├── experiment_seq2vec.jsonnet
│ │ ├── parameters_inspection.json
│ │ └── serialization/
│ │ ├── best.th
│ │ └── vocabulary/
│ │ ├── labels.txt
│ │ ├── non_padded_namespaces.txt
│ │ └── tokens.txt
│ ├── common/
│ │ └── .gitignore
│ ├── data/
│ │ ├── babi.txt
│ │ ├── brown_corpus.txt
│ │ ├── conll2003.txt
│ │ ├── sequence_tagging.tsv
│ │ ├── shards/
│ │ │ ├── sequence_tagging_00.tsv
│ │ │ ├── sequence_tagging_01.tsv
│ │ │ └── sequence_tagging_02.tsv
│ │ └── text_classification_json/
│ │ ├── ag_news_corpus.jsonl
│ │ ├── ag_news_corpus_fake_sentiment_labels.jsonl
│ │ ├── imdb_corpus.jsonl
│ │ ├── imdb_corpus2.jsonl
│ │ └── integer_labels.jsonl
│ ├── elmo/
│ │ ├── config/
│ │ │ └── characters_token_embedder.json
│ │ ├── elmo_token_embeddings.hdf5
│ │ ├── lm_embeddings_0.hdf5
│ │ ├── lm_embeddings_1.hdf5
│ │ ├── lm_embeddings_2.hdf5
│ │ ├── lm_weights.hdf5
│ │ ├── options.json
│ │ ├── sentences.json
│ │ └── vocab_test.txt
│ ├── embeddings/
│ │ ├── fake_embeddings.5d.txt
│ │ ├── fake_embeddings.5d.txt.bz2
│ │ └── fake_embeddings.5d.txt.xz
│ ├── fairness/
│ │ ├── bias_embeddings.json
│ │ ├── definitional_pairs.json
│ │ ├── equalize_pairs.json
│ │ └── gender_specific_full.json
│ ├── plugins/
│ │ ├── .allennlp_plugins
│ │ └── d/
│ │ ├── __init__.py
│ │ └── d.py
│ ├── simple_tagger/
│ │ ├── experiment.json
│ │ ├── experiment_with_regularization.json
│ │ ├── model_test_case.jsonnet
│ │ ├── serialization/
│ │ │ ├── best.th
│ │ │ └── vocabulary/
│ │ │ ├── labels.txt
│ │ │ ├── non_padded_namespaces.txt
│ │ │ └── tokens.txt
│ │ └── serialization_full/
│ │ ├── best.th
│ │ ├── config.json
│ │ ├── meta.json
│ │ ├── metrics.json
│ │ ├── metrics_epoch_0.json
│ │ ├── model_state_e1_b0.th
│ │ ├── out.log
│ │ ├── training_state_e1_b0.th
│ │ └── vocabulary/
│ │ ├── labels.txt
│ │ ├── non_padded_namespaces.txt
│ │ └── tokens.txt
│ ├── simple_tagger_with_elmo/
│ │ ├── experiment.json
│ │ └── serialization/
│ │ ├── best.th
│ │ └── vocabulary/
│ │ ├── labels.txt
│ │ └── non_padded_namespaces.txt
│ ├── simple_tagger_with_span_f1/
│ │ ├── experiment.json
│ │ └── serialization/
│ │ ├── best.th
│ │ └── vocabulary/
│ │ ├── labels.txt
│ │ ├── non_padded_namespaces.txt
│ │ ├── test_tokens.txt
│ │ └── tokens.txt
│ └── utf-8_sample/
│ ├── archives/
│ │ ├── utf-8.tar.bz2
│ │ └── utf-8.tar.xz
│ └── utf-8_sample.txt
└── tests/
├── __init__.py
├── commands/
│ ├── __init__.py
│ ├── build_vocab_test.py
│ ├── cached_path_test.py
│ ├── checklist_test.py
│ ├── diff_test.py
│ ├── evaluate_test.py
│ ├── find_learning_rate_test.py
│ ├── main_test.py
│ ├── no_op_train_test.py
│ ├── predict_test.py
│ ├── print_results_test.py
│ ├── test_install_test.py
│ └── train_test.py
├── common/
│ ├── __init__.py
│ ├── cached_transformers_test.py
│ ├── file_utils_test.py
│ ├── from_params_test.py
│ ├── logging_test.py
│ ├── model_card_test.py
│ ├── params_test.py
│ ├── plugins_test.py
│ ├── push_to_hub_test.py
│ ├── registrable_test.py
│ ├── sequences_test.py
│ ├── task_card_test.py
│ ├── testing.py
│ └── util_test.py
├── confidence_checks/
│ ├── normalization_bias_verification_test.py
│ └── task_checklists/
│ ├── __init__.py
│ ├── sentiment_analysis_suite_test.py
│ ├── task_suite_test.py
│ └── utils_test.py
├── data/
│ ├── __init__.py
│ ├── data_loaders/
│ │ ├── __init__.py
│ │ ├── multiprocess_data_loader_test.py
│ │ ├── multitask_data_loader_test.py
│ │ └── multitask_scheduler_test.py
│ ├── dataset_readers/
│ │ ├── __init__.py
│ │ ├── babi_reader_test.py
│ │ ├── conll2003_test.py
│ │ ├── dataset_reader_test.py
│ │ ├── dataset_utils/
│ │ │ └── span_utils_test.py
│ │ ├── interleaving_dataset_reader_test.py
│ │ ├── sequence_tagging_test.py
│ │ ├── sharded_dataset_reader_test.py
│ │ └── text_classification_json_test.py
│ ├── dataset_test.py
│ ├── fields/
│ │ ├── __init__.py
│ │ ├── adjacency_field_test.py
│ │ ├── field_test.py
│ │ ├── flag_field_test.py
│ │ ├── index_field_test.py
│ │ ├── label_field_test.py
│ │ ├── list_field_test.py
│ │ ├── metadata_field_test.py
│ │ ├── multilabel_field_test.py
│ │ ├── sequence_label_field_test.py
│ │ ├── span_field_test.py
│ │ ├── tensor_field_test.py
│ │ ├── text_field_test.py
│ │ └── transformer_text_field_test.py
│ ├── image_loader_test.py
│ ├── instance_test.py
│ ├── samplers/
│ │ ├── __init__.py
│ │ ├── bucket_batch_sampler_test.py
│ │ ├── max_tokens_batch_sampler_test.py
│ │ └── sampler_test.py
│ ├── token_indexers/
│ │ ├── __init__.py
│ │ ├── character_token_indexer_test.py
│ │ ├── elmo_indexer_test.py
│ │ ├── pretrained_transformer_indexer_test.py
│ │ ├── pretrained_transformer_mismatched_indexer_test.py
│ │ ├── single_id_token_indexer_test.py
│ │ └── spacy_indexer_test.py
│ ├── tokenizers/
│ │ ├── __init__.py
│ │ ├── character_tokenizer_test.py
│ │ ├── letters_digits_tokenizer_test.py
│ │ ├── pretrained_transformer_tokenizer_test.py
│ │ ├── sentence_splitter_test.py
│ │ └── spacy_tokenizer_test.py
│ └── vocabulary_test.py
├── evaluation/
│ ├── __init__.py
│ ├── evaluator_tests.py
│ └── serializers/
│ ├── __init__.py
│ └── serializer_test.py
├── fairness/
│ ├── __init__.py
│ ├── bias_direction_test.py
│ ├── bias_metrics_test.py
│ ├── bias_mitigators_test.py
│ ├── bias_utils_test.py
│ └── fairness_metrics_test.py
├── interpret/
│ ├── __init__.py
│ ├── hotflip_test.py
│ ├── input_reduction_test.py
│ ├── integrated_gradient_test.py
│ ├── simple_gradient_test.py
│ ├── simple_influence_test.py
│ └── smooth_gradient_test.py
├── models/
│ ├── __init__.py
│ ├── archival_test.py
│ ├── basic_classifier_test.py
│ ├── model_test.py
│ ├── multitask_test.py
│ ├── simple_tagger_test.py
│ └── test_model_test_case.py
├── modules/
│ ├── attention/
│ │ ├── __init__.py
│ │ ├── additive_attention_test.py
│ │ ├── attention_test.py
│ │ ├── bilinear_attention_test.py
│ │ ├── cosine_attention_test.py
│ │ ├── dot_product_attention_test.py
│ │ ├── linear_attention_test.py
│ │ └── scaled_dot_product_attention_test.py
│ ├── augmented_lstm_test.py
│ ├── bimpm_matching_test.py
│ ├── conditional_random_field_test.py
│ ├── elmo_test.py
│ ├── encoder_base_test.py
│ ├── feedforward_test.py
│ ├── gated_sum_test.py
│ ├── highway_test.py
│ ├── lstm_cell_with_projection_test.py
│ ├── masked_layer_norm_test.py
│ ├── matrix_attention/
│ │ ├── __init__.py
│ │ ├── bilinear_matrix_attention_test.py
│ │ ├── cosine_matrix_attention_test.py
│ │ ├── dot_product_matrix_attention_test.py
│ │ ├── linear_matrix_attention_test.py
│ │ ├── matrix_attention_test.py
│ │ └── scaled_dot_product_matrix_attention_test.py
│ ├── maxout_test.py
│ ├── residual_with_layer_dropout_test.py
│ ├── sampled_softmax_loss_test.py
│ ├── scalar_mix_test.py
│ ├── seq2seq_encoder_test.py
│ ├── seq2seq_encoders/
│ │ ├── __init__.py
│ │ ├── compose_encoder_test.py
│ │ ├── feedforward_encoder_test.py
│ │ ├── gated_cnn_encoder_test.py
│ │ ├── pass_through_encoder_test.py
│ │ ├── pytorch_seq2seq_wrapper_test.py
│ │ └── pytorch_transformer_wrapper_test.py
│ ├── seq2vec_encoder_test.py
│ ├── seq2vec_encoders/
│ │ ├── __init__.py
│ │ ├── bert_pooler_test.py
│ │ ├── boe_encoder_test.py
│ │ ├── cls_pooler_test.py
│ │ ├── cnn_encoder_test.py
│ │ ├── cnn_highway_encoder_test.py
│ │ └── pytorch_seq2vec_wrapper_test.py
│ ├── span_extractors/
│ │ ├── __init__.py
│ │ ├── bidirectional_endpoint_span_extractor_test.py
│ │ ├── endpoint_span_extractor_test.py
│ │ ├── max_pooling_span_extractor_test.py
│ │ └── self_attentive_span_extractor_test.py
│ ├── stacked_alternating_lstm_test.py
│ ├── stacked_bidirectional_lstm_test.py
│ ├── stacked_elmo_lstm_test.py
│ ├── text_field_embedders/
│ │ ├── __init__.py
│ │ └── basic_text_field_embedder_test.py
│ ├── time_distributed_test.py
│ ├── token_embedders/
│ │ ├── __init__.py
│ │ ├── bag_of_word_counts_token_embedder_test.py
│ │ ├── elmo_token_embedder_test.py
│ │ ├── embedding_test.py
│ │ ├── pass_through_embedder_test.py
│ │ ├── pretrained_transformer_embedder_test.py
│ │ ├── pretrained_transformer_mismatched_embedder_test.py
│ │ └── token_characters_encoder_test.py
│ ├── transformer/
│ │ ├── activation_layer_test.py
│ │ ├── bimodal_attention_test.py
│ │ ├── bimodal_encoder_test.py
│ │ ├── output_layer_test.py
│ │ ├── positional_encoding_test.py
│ │ ├── self_attention_test.py
│ │ ├── t5_self_attention_test.py
│ │ ├── t5_test.py
│ │ ├── toolkit_test.py
│ │ ├── transformer_embeddings_test.py
│ │ ├── transformer_layer_test.py
│ │ ├── transformer_module_test.py
│ │ ├── transformer_pooler_test.py
│ │ └── transformer_stack_test.py
│ └── vision/
│ ├── __init__.py
│ ├── grid_embedder_test.py
│ └── region_detector_test.py
├── nn/
│ ├── __init__.py
│ ├── beam_search_test.py
│ ├── checkpoint/
│ │ ├── __init__.py
│ │ ├── checkpoint_wrapper_test.py
│ │ └── fairscale_checkpoint_wrapper_test.py
│ ├── chu_liu_edmonds_test.py
│ ├── initializers_test.py
│ ├── parallel/
│ │ ├── __init__.py
│ │ └── fairscale_fsdp_accelerator_test.py
│ ├── pretrained_model_initializer_test.py
│ ├── regularizers_test.py
│ └── util_test.py
├── predictors/
│ ├── __init__.py
│ ├── predictor_test.py
│ ├── sentence_tagger_test.py
│ └── text_classifier_test.py
├── training/
│ ├── __init__.py
│ ├── checkpointer_test.py
│ ├── learning_rate_schedulers/
│ │ ├── __init__.py
│ │ ├── combined_test.py
│ │ ├── cosine_test.py
│ │ ├── learning_rate_scheduler_test.py
│ │ └── slanted_triangular_test.py
│ ├── metrics/
│ │ ├── __init__.py
│ │ ├── attachment_scores_test.py
│ │ ├── auc_test.py
│ │ ├── average_test.py
│ │ ├── bleu_test.py
│ │ ├── boolean_accuracy_test.py
│ │ ├── categorical_accuracy_test.py
│ │ ├── covariance_test.py
│ │ ├── entropy_test.py
│ │ ├── evalb_bracketing_scorer_test.py
│ │ ├── f1_measure_test.py
│ │ ├── fbeta_measure_test.py
│ │ ├── fbeta_multi_label_measure_test.py
│ │ ├── fbeta_verbose_measure_test.py
│ │ ├── mean_absolute_error_test.py
│ │ ├── pearson_correlation_test.py
│ │ ├── rouge_test.py
│ │ ├── sequence_accuracy_test.py
│ │ ├── span_based_f1_measure_test.py
│ │ ├── spearman_correlation_test.py
│ │ └── unigram_recall_test.py
│ ├── momentum_schedulers/
│ │ ├── __init__.py
│ │ └── inverted_triangular_test.py
│ ├── moving_average_test.py
│ ├── no_op_trainer_test.py
│ ├── optimizer_test.py
│ ├── trainer_test.py
│ └── util_test.py
├── tutorials/
│ ├── __init__.py
│ └── tagger/
│ ├── __init__.py
│ └── basic_allennlp_test.py
└── version_test.py
================================================
FILE CONTENTS
================================================
================================================
FILE: .coveragerc
================================================
[run]
omit = tests/*
================================================
FILE: .dockerignore
================================================
.dockerignore
**.pyc
**/__pycache__
.gitignore
.git
.coverage
.benchmarks
.mypy_cache
================================================
FILE: .flake8
================================================
[flake8]
max-line-length = 115
ignore =
# these rules don't play well with black
E203 # whitespace before :
W503 # line break before binary operator
exclude =
build/**
doc/**
tutorials/tagger/**
per-file-ignores =
# __init__.py files are allowed to have unused imports and lines-too-long
*/__init__.py:F401
*/**/**/__init__.py:F401,E501
# tests don't have to respect
# E731: do not assign a lambda expression, use a def
tests/**:E731
# scripts don't have to respect
# E402: imports not at top of file (because we mess with sys.path)
scripts/**:E402
================================================
FILE: .github/ISSUE_TEMPLATE/bug_report.md
================================================
---
name: Bug report
about: Create a report to help us improve
title: ''
labels: 'bug'
assignees: ''
---
<!--
Please fill this template entirely and do not erase any of it.
We reserve the right to close without a response bug reports which are incomplete.
If you have a question rather than a bug, please ask on [Stack Overflow](https://stackoverflow.com/questions/tagged/allennlp) rather than posting an issue here.
-->
## Checklist
<!-- To check an item on the list replace [ ] with [x]. -->
- [ ] I have verified that the issue exists against the `main` branch of AllenNLP.
- [ ] I have read the relevant section in the [contribution guide](https://github.com/allenai/allennlp/blob/main/CONTRIBUTING.md#bug-fixes-and-new-features) on reporting bugs.
- [ ] I have checked the [issues list](https://github.com/allenai/allennlp/issues) for similar or identical bug reports.
- [ ] I have checked the [pull requests list](https://github.com/allenai/allennlp/pulls) for existing proposed fixes.
- [ ] I have checked the [CHANGELOG](https://github.com/allenai/allennlp/blob/main/CHANGELOG.md) and the [commit log](https://github.com/allenai/allennlp/commits/main) to find out if the bug was already fixed in the main branch.
- [ ] I have included in the "Description" section below a traceback from any exceptions related to this bug.
- [ ] I have included in the "Related issues or possible duplicates" section beloew all related issues and possible duplicate issues (If there are none, check this box anyway).
- [ ] I have included in the "Environment" section below the name of the operating system and Python version that I was using when I discovered this bug.
- [ ] I have included in the "Environment" section below the output of `pip freeze`.
- [ ] I have included in the "Steps to reproduce" section below a minimally reproducible example.
## Description
<!-- Please provide a clear and concise description of what the bug is here. -->
<details>
<summary><b>Python traceback:</b></summary>
<p>
<!-- Paste the traceback from any exception (if there was one) in between the next two lines below -->
```
```
</p>
</details>
## Related issues or possible duplicates
- None
## Environment
<!-- Provide the name of operating system below (e.g. OS X, Linux) -->
OS:
<!-- Provide the Python version you were using (e.g. 3.7.1) -->
Python version:
<details>
<summary><b>Output of <code>pip freeze</code>:</b></summary>
<p>
<!-- Paste the output of `pip freeze` in between the next two lines below -->
```
```
</p>
</details>
## Steps to reproduce
<details>
<summary><b>Example source:</b></summary>
<p>
<!-- Add a fully runnable example in between the next two lines below that will reproduce the bug -->
```
```
</p>
</details>
================================================
FILE: .github/ISSUE_TEMPLATE/feature_request.md
================================================
---
name: Feature request
about: Suggest an idea for this project
title: ''
labels: 'Feature request'
assignees: ''
---
**Is your feature request related to a problem? Please describe.**
A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
**Describe the solution you'd like**
A clear and concise description of what you want to happen.
**Describe alternatives you've considered**
A clear and concise description of any alternative solutions or features you've considered.
**Additional context**
Add any other context or screenshots about the feature request here.
================================================
FILE: .github/ISSUE_TEMPLATE/question.md
================================================
---
name: Question
about: Ask a question
title: ''
labels: 'question'
assignees: ''
---
Please ask questions on [Stack Overflow](https://stackoverflow.com/questions/tagged/allennlp) rather than on GitHub. We monitor and triage questions on Stack Overflow with the AllenNLP label and questions there are more easily searchable for others.
================================================
FILE: .github/pull_request_template.md
================================================
<!-- To ensure we can review your pull request promptly please complete this template entirely. -->
<!-- Please reference the issue number here. You can replace "Fixes" with "Closes" if it makes more sense. -->
Fixes # .
Changes proposed in this pull request:
<!-- Please list all changes/additions here. -->
-
## Before submitting
<!-- Please complete this checklist BEFORE submitting your PR to speed along the review process. -->
- [ ] I've read and followed all steps in the [Making a pull request](https://github.com/allenai/allennlp/blob/main/CONTRIBUTING.md#making-a-pull-request)
section of the `CONTRIBUTING` docs.
- [ ] I've updated or added any relevant docstrings following the syntax described in the
[Writing docstrings](https://github.com/allenai/allennlp/blob/main/CONTRIBUTING.md#writing-docstrings) section of the `CONTRIBUTING` docs.
- [ ] If this PR fixes a bug, I've added a test that will fail without my fix.
- [ ] If this PR adds a new feature, I've added tests that sufficiently cover my new functionality.
## After submitting
<!-- Please complete this checklist AFTER submitting your PR to speed along the review process. -->
- [ ] All GitHub Actions jobs for my pull request have passed.
- [ ] **`codecov/patch`** reports high test coverage (at least 90%).
You can find this under the "Actions" tab of the pull request once the other checks have finished.
================================================
FILE: .github/workflows/cffconvert.yml
================================================
name: cffconvert
on:
pull_request:
paths:
- CITATION.cff
push:
paths:
- CITATION.cff
jobs:
validate:
name: "validate"
runs-on: ubuntu-latest
steps:
- name: Check out a copy of the repository
uses: actions/checkout@v3
- name: Check whether the citation metadata from CITATION.cff is valid
uses: citation-file-format/cffconvert-github-action@2.0.0
with:
args: "--validate"
================================================
FILE: .github/workflows/ci.yml
================================================
name: CI
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
on:
pull_request:
branches:
- main
push:
branches:
- main
release:
types: [published]
schedule:
- cron: '37 11 * * 1,2,3,4,5' # early morning (11:37 UTC / 4:37 AM PDT) Monday - Friday
env:
# NOTE: Need to update `TORCH_VERSION`, and `TORCH_*_INSTALL` for new torch releases.
TORCH_VERSION: 1.12.0
# TORCH_CPU_INSTALL: conda install pytorch torchvision torchaudio cpuonly -c pytorch
# TORCH_GPU_INSTALL: conda install pytorch torchvision torchaudio cudatoolkit=11.3 -c pytorch
TORCH_CPU_INSTALL: pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu -c constraints.txt
TORCH_GPU_INSTALL: pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu113 -c constraints.txt
# Change this to invalidate existing cache.
CACHE_PREFIX: v11
# Disable tokenizers parallelism because this doesn't help, and can cause issues in distributed tests.
TOKENIZERS_PARALLELISM: 'false'
# Disable multithreading with OMP because this can lead to dead-locks in distributed tests.
OMP_NUM_THREADS: '1'
# See https://github.com/pytorch/pytorch/issues/37377#issuecomment-677851112.
MKL_THREADING_LAYER: 'GNU'
DEFAULT_PYTHON_VERSION: 3.8
# For debugging GPU tests.
CUDA_LAUNCH_BLOCKING: '1'
defaults:
run:
shell: bash -l {0}
jobs:
changelog:
name: CHANGELOG
runs-on: ubuntu-latest
if: github.event_name == 'pull_request'
steps:
- uses: actions/checkout@v3
- name: Check if source files have changed
run: |
git diff --name-only $(git merge-base origin/main HEAD) | grep '^allennlp/.*\.py$' && echo "source_files_changed=true" >> $GITHUB_ENV || echo "source_files_changed=false" >> $GITHUB_ENV
- name: Check that CHANGELOG has been updated
if: env.source_files_changed == 'true'
run: |
# If this step fails, this means you haven't updated the CHANGELOG.md
# file with notes on your contribution.
git diff --name-only $(git merge-base origin/main HEAD) | grep '^CHANGELOG.md$' && echo "Thanks for helping keep our CHANGELOG up-to-date!"
style:
name: Style
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Setup Python
uses: actions/setup-python@v4
with:
python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
- name: Install requirements
run: |
grep -E '^black' dev-requirements.txt | xargs pip install
- name: Debug info
run: |
pip freeze
- name: Run black
run: |
black --check .
checks:
name: ${{ matrix.task.name }}
runs-on: ${{ matrix.task.runs_on }}
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
task:
- name: Lint
runs_on: ubuntu-latest
coverage_report: false
torch_platform: cpu
run: |
make flake8
make typecheck
- name: CPU Tests
runs_on: ubuntu-latest
coverage_report: true
torch_platform: cpu
run: make test
- name: GPU Tests
runs_on: [self-hosted, GPU, Multi GPU]
coverage_report: true
torch_platform: gpu
run: make gpu-tests
- name: Model Tests
runs_on: ubuntu-latest
coverage_report: true
torch_platform: cpu
run: |
cd allennlp-models
make test-with-cov COV=allennlp
mv coverage.xml ../
steps:
- uses: actions/checkout@v3
- uses: conda-incubator/setup-miniconda@v2
with:
miniconda-version: "latest"
python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
- name: Set build variables
run: |
# Get the exact Python version to use in the cache key.
echo "PYTHON_VERSION=$(python --version)" >> $GITHUB_ENV
echo "RUNNER_ARCH=$(uname -m)" >> $GITHUB_ENV
# Use week number in cache key so we can refresh the cache weekly.
echo "WEEK_NUMBER=$(date +%V)" >> $GITHUB_ENV
- name: Set build variables (CPU only)
if: matrix.task.torch_platform == 'cpu'
run: |
echo "TORCH_INSTALL=$TORCH_CPU_INSTALL" >> $GITHUB_ENV
- name: Set build variables (GPU only)
if: matrix.task.torch_platform == 'gpu'
run: |
echo "TORCH_INSTALL=$TORCH_GPU_INSTALL" >> $GITHUB_ENV
- uses: actions/cache@v3
id: virtualenv-cache
with:
path: .venv
key: ${{ env.CACHE_PREFIX }}-${{ env.WEEK_NUMBER }}-${{ runner.os }}-${{ env.RUNNER_ARCH }}-${{ env.PYTHON_VERSION }}-${{ matrix.task.torch_platform }}-${{ hashFiles('setup.py') }}-${{ hashFiles('*requirements.txt', 'constraints.txt') }}
- name: Setup virtual environment (no cache hit)
if: steps.virtualenv-cache.outputs.cache-hit != 'true'
run: |
python${{ env.DEFAULT_PYTHON_VERSION }} -m venv .venv
source .venv/bin/activate
make install TORCH_INSTALL="$TORCH_INSTALL"
- name: Setup virtual environment (cache hit)
if: steps.virtualenv-cache.outputs.cache-hit == 'true'
run: |
source .venv/bin/activate
pip install --no-deps -e .[all]
make download-extras
- name: Pull and install models repo
if: matrix.task.name == 'Model Tests'
env:
ALLENNLP_VERSION_OVERRIDE: "" # Don't replace the core library.
run: |
source .venv/bin/activate
git clone https://github.com/allenai/allennlp-models.git
cd allennlp-models
# git checkout dependabot/pip/torch-gte-1.7.0-and-lt-1.13.0
pip install -e .[dev,all]
- name: Debug info
run: |
source .venv/bin/activate
pip freeze
- name: Ensure torch up-to-date
run: |
source .venv/bin/activate
python scripts/check_torch_version.py
- name: ${{ matrix.task.name }}
run: |
source .venv/bin/activate
${{ matrix.task.run }}
- name: Prepare coverage report
if: matrix.task.coverage_report
run: |
mkdir coverage
mv coverage.xml coverage/
- name: Save coverage report
if: matrix.task.coverage_report
uses: actions/upload-artifact@v3
with:
name: ${{ matrix.task.name }}-coverage
path: ./coverage
- name: Clean up
if: always()
run: |
# Could run into issues with the cache if we don't uninstall the editable.
# See https://github.com/pypa/pip/issues/4537.
source .venv/bin/activate
pip uninstall --yes allennlp allennlp-models
upload_coverage:
name: Upload Coverage Report
timeout-minutes: 5
if: github.repository == 'allenai/allennlp' && (github.event_name == 'push' || github.event_name == 'pull_request')
runs-on: ubuntu-latest
needs: [checks]
steps:
# Need to checkout code to get the coverage config.
- uses: actions/checkout@v3
- name: Download coverage report from CPU tests
uses: actions/download-artifact@v3
with:
name: CPU Tests-coverage
path: coverage/cpu_tests
- name: Download coverage report from GPU Tests
uses: actions/download-artifact@v3
with:
name: GPU Tests-coverage
path: coverage/gpu_tests
- name: Download coverage report from model tests
uses: actions/download-artifact@v3
with:
name: Model Tests-coverage
path: coverage/model_tests
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v3
with:
files: coverage/cpu_tests/coverage.xml,coverage/gpu_tests/coverage.xml,coverage/model_tests/coverage.xml
# Ignore codecov failures as the codecov server is not
# very reliable but we don't want to report a failure
# in the github UI just because the coverage report failed to
# be published.
fail_ci_if_error: false
# Builds package distribution files for PyPI.
build_package:
name: Build Package
timeout-minutes: 18
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: conda-incubator/setup-miniconda@v2
with:
miniconda-version: "latest"
python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
- name: Set build variables
run: |
echo "PYTHON_VERSION=$(python --version)" >> $GITHUB_ENV
echo "RUNNER_ARCH=$(uname -m)" >> $GITHUB_ENV
echo "WEEK_NUMBER=$(date +%V)" >> $GITHUB_ENV
echo "TORCH_INSTALL=$TORCH_CPU_INSTALL" >> $GITHUB_ENV
- uses: actions/cache@v3
id: virtualenv-cache
with:
path: .venv
key: ${{ env.CACHE_PREFIX }}-${{ env.WEEK_NUMBER }}-${{ runner.os }}-${{ env.RUNNER_ARCH }}-${{ env.PYTHON_VERSION }}-cpu-${{ hashFiles('setup.py') }}-${{ hashFiles('*requirements.txt', 'constraints.txt') }}
- name: Setup virtual environment (no cache hit)
if: steps.virtualenv-cache.outputs.cache-hit != 'true'
run: |
python${{ env.DEFAULT_PYTHON_VERSION }} -m venv .venv
source .venv/bin/activate
make install TORCH_INSTALL="$TORCH_INSTALL"
- name: Setup virtual environment (cache hit)
if: steps.virtualenv-cache.outputs.cache-hit == 'true'
run: |
source .venv/bin/activate
pip install --no-deps -e .[all]
make download-extras
- name: Debug info
run: |
source .venv/bin/activate
pip freeze
- name: Check and set nightly version
if: github.event_name == 'schedule'
run: |
# Verify that current version is ahead of the last release.
source .venv/bin/activate
LATEST=$(scripts/get_version.py latest)
CURRENT=$(scripts/get_version.py current)
if [ "$CURRENT" == "$LATEST" ]; then
echo "Current version needs to be ahead of latest release in order to build nightly release";
exit 1;
fi
# This is somewhat bizarre, but you can't set env variables to bash
# commands in the action workflow - so we have to use this odd way of
# exporting a variable instead.
echo "ALLENNLP_VERSION_SUFFIX=dev$(date -u +%Y%m%d)" >> $GITHUB_ENV
- name: Check version and release tag match
if: github.event_name == 'release'
run: |
# Remove 'refs/tags/' to get the actual tag from the release.
source .venv/bin/activate
TAG=${GITHUB_REF#refs/tags/};
VERSION=$(scripts/get_version.py current)
if [ "$TAG" != "$VERSION" ]; then
echo "Bad tag or version. Tag $TAG does not match $VERSION";
exit 1;
fi
- name: Build core package
run: |
# Just print out the version for debugging.
source .venv/bin/activate
make version
python setup.py bdist_wheel sdist
- name: Save core package
uses: actions/upload-artifact@v3
with:
name: core-package
path: dist
- name: Clean up
if: always()
run: |
source .venv/bin/activate
pip uninstall --yes allennlp
# Tests installing from the distribution files.
test_package:
name: Test Package
timeout-minutes: 10
needs: [build_package] # needs the package artifact created from 'build_package' job.
runs-on: ubuntu-latest
strategy:
matrix:
python: ['3.7', '3.8', '3.9']
# check that CLI remains working for all package flavors;
# currently allennlp[checklist]==allennlp[all], so avoiding duplication on this
flavor: ['', '[all]']
steps:
- uses: actions/checkout@v3
- name: Setup Python
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python }}
- name: Install requirements
run: |
pip install --upgrade pip setuptools wheel
- name: Download core package
uses: actions/download-artifact@v3
with:
name: core-package
path: dist
- name: Install core package
run: |
pip install $(ls dist/*.whl)${{ matrix.flavor }} -c constraints.txt
- name: Download NLTK prerequisites
run: |
make download-extras
- name: Cleanup workspace
run: |
rm -rf allennlp/ setup.py tests/ test_fixtures/
- name: Pip freeze
run: |
pip freeze
- name: Test install
run: |
allennlp test-install
# Builds Docker image from the core distribution files and uploads to Docker Hub.
docker:
name: Docker (CUDA ${{ matrix.cuda }})
timeout-minutes: 30
if: github.repository == 'allenai/allennlp'
runs-on: ubuntu-latest
strategy:
matrix:
cuda: ['11.3']
steps:
- uses: actions/checkout@v3
- name: Set image name and torch version
env:
CUDA: ${{ matrix.cuda }}
run: |
echo "DOCKER_TORCH_VERSION=${TORCH_VERSION}-cuda${CUDA}-python3.8" >> $GITHUB_ENV;
if [[ $GITHUB_EVENT_NAME == 'release' ]]; then
echo "DOCKER_IMAGE_NAME=allennlp/allennlp:${GITHUB_REF#refs/tags/}-cuda${CUDA}" >> $GITHUB_ENV;
else
echo "DOCKER_IMAGE_NAME=allennlp/commit:${GITHUB_SHA}-cuda${CUDA}" >> $GITHUB_ENV;
fi
- name: Build image
run: |
make docker-image DOCKER_IMAGE_NAME="$DOCKER_IMAGE_NAME" DOCKER_TORCH_VERSION="$DOCKER_TORCH_VERSION"
- name: Test image
run: |
docker run --rm $DOCKER_IMAGE_NAME test-install
- name: Authenticate to Docker Hub
if: github.event_name == 'release' || github.event_name == 'push'
run: |
docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}
- name: Upload image
if: github.event_name == 'release' || github.event_name == 'push'
run: |
docker push $DOCKER_IMAGE_NAME
- name: Upload default commit image
# CUDA 11.3 is currently our default.
if: github.event_name == 'push' && matrix.cuda == '11.3'
run: |
docker tag $DOCKER_IMAGE_NAME allennlp/commit:${GITHUB_SHA}
docker push allennlp/commit:${GITHUB_SHA}
- name: Upload latest image
# CUDA 11.3 is currently our default.
if: github.event_name == 'release' && matrix.cuda == '11.3'
run: |
docker tag $DOCKER_IMAGE_NAME allennlp/allennlp:latest
docker push allennlp/allennlp:latest
# Builds the API documentation and pushes it to the appropriate folder in the
# allennlp-docs repo.
docs:
name: Docs
timeout-minutes: 15
# Don't run for forks.
if: github.repository == 'allenai/allennlp'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Setup SSH Client 🔑
if: github.event_name == 'release' || github.event_name == 'push'
uses: webfactory/ssh-agent@v0.5.4
with:
ssh-private-key: ${{ secrets.DOCS_DEPLOY_KEY }}
- uses: conda-incubator/setup-miniconda@v2
with:
miniconda-version: "latest"
python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
- name: Set build variables
run: |
echo "PYTHON_VERSION=$(python --version)" >> $GITHUB_ENV
echo "RUNNER_ARCH=$(uname -m)" >> $GITHUB_ENV
echo "WEEK_NUMBER=$(date +%V)" >> $GITHUB_ENV
echo "TORCH_INSTALL=$TORCH_CPU_INSTALL" >> $GITHUB_ENV
- uses: actions/cache@v3
id: virtualenv-cache
with:
path: .venv
key: ${{ env.CACHE_PREFIX }}-${{ env.WEEK_NUMBER }}-${{ runner.os }}-${{ env.RUNNER_ARCH }}-${{ env.PYTHON_VERSION }}-cpu-${{ hashFiles('setup.py') }}-${{ hashFiles('*requirements.txt', 'constraints.txt') }}
- name: Setup virtual environment (no cache hit)
if: steps.virtualenv-cache.outputs.cache-hit != 'true'
run: |
python${{ env.DEFAULT_PYTHON_VERSION }} -m venv .venv
source .venv/bin/activate
make install TORCH_INSTALL="$TORCH_INSTALL"
- name: Setup virtual environment (cache hit)
if: steps.virtualenv-cache.outputs.cache-hit == 'true'
run: |
source .venv/bin/activate
pip install --no-deps -e .[all]
make download-extras
- name: Debug info
run: |
source .venv/bin/activate
pip freeze
- name: Prepare environment
run: |
if [[ $GITHUB_EVENT_NAME == 'release' ]]; then
echo "DOCS_FOLDER=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV;
echo "BASE_SOURCE_LINK=https://github.com/allenai/allennlp/blob/${GITHUB_REF#refs/tags/}/allennlp/" >> $GITHUB_ENV;
else
echo "DOCS_FOLDER=main" >> $GITHUB_ENV;
echo "BASE_SOURCE_LINK=https://github.com/allenai/allennlp/blob/main/allennlp/" >> $GITHUB_ENV;
fi
- name: Build docs
run: |
source .venv/bin/activate
env PYTHONPATH=. ./scripts/build_docs.sh
- name: Print the ref
run: |
echo ${{ github.ref }}
- name: Configure Git
if: github.event_name == 'release' || (github.event_name == 'push' && github.ref == 'refs/heads/main')
run: |
git config --global user.email "ai2service@allenai.org"
git config --global user.name "ai2service"
git config --global push.default simple
- name: Stage docs
if: github.event_name == 'release' || (github.event_name == 'push' && github.ref == 'refs/heads/main')
run: |
echo "Staging docs to $DOCS_FOLDER"
# Checkout allennlp-docs to /allennlp-docs
git clone git@github.com:allenai/allennlp-docs.git ~/allennlp-docs
# Copy the generated docs to the checked out docs repo
rm -rf ~/allennlp-docs/$DOCS_FOLDER/
mkdir -p ~/allennlp-docs/$DOCS_FOLDER
cp -r site/* ~/allennlp-docs/$DOCS_FOLDER
- name: Update shortcuts
if: github.event_name == 'release'
run: |
# Fail immediately if any step fails.
set -e
source .venv/bin/activate
LATEST=$(./scripts/get_version.py latest)
STABLE=$(./scripts/get_version.py stable)
cd ~/allennlp-docs/
echo "Updating latest/index.html to point to $LATEST"
mkdir -p latest
cat >latest/index.html << EOL
<!DOCTYPE html>
<html>
<head>
<meta http-equiv="Refresh" content="0; url=/${LATEST}/" />
</head>
<body>
<p>Please follow <a href="/${LATEST}/">this link</a>.</p>
</body>
</html>
EOL
echo "Updating stable/index.html to point to $STABLE"
mkdir -p stable
cat >stable/index.html << EOL
<!DOCTYPE html>
<html>
<head>
<meta http-equiv="Refresh" content="0; url=/${STABLE}/" />
</head>
<body>
<p>Please follow <a href="/${STABLE}/">this link</a>.</p>
</body>
</html>
EOL
- name: Deploy docs
if: github.event_name == 'release' || (github.event_name == 'push' && github.ref == 'refs/heads/main')
run: |
# And push them up to GitHub
cd ~/allennlp-docs/
git add -A
git commit -m "automated update of the docs"
git push
- name: Re-write docs commit history
if: github.event_name == 'release'
run: |
cd ~/allennlp-docs/
git checkout --orphan latest_branch
git add -A
git commit -m "Re-write commit history"
git branch -D master # remove old master branch
git branch -m master # rename clean new branch to master branch
git push -f origin master
- name: Clean up
if: always()
run: |
source .venv/bin/activate
pip uninstall --yes allennlp
# Publish the core distribution files to PyPI.
publish:
name: PyPI
timeout-minutes: 10
needs: [style, checks, build_package, test_package, docker, docs]
# Only publish to PyPI on releases and nightly builds to "allenai/allennlp" (not forks).
if: github.repository == 'allenai/allennlp' && (github.event_name == 'release' || github.event_name == 'schedule')
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Setup Python
uses: actions/setup-python@v4
with:
python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
- name: Install requirements
run: |
pip install --upgrade pip setuptools wheel twine
- name: Download core package
uses: actions/download-artifact@v3
with:
name: core-package
path: dist
- name: Publish core package
run: |
twine upload -u allennlp -p ${{ secrets.PYPI_PASSWORD }} dist/*
================================================
FILE: .github/workflows/issues.yml
================================================
name: Issues Bot
on:
# Uncomment this PR trigger for testing.
# pull_request:
# branches:
# - main
schedule:
- cron: '5 16 * * 1,2,3,4,5' # runs at 16:05 UTC Monday - Friday
jobs:
close_stale_issues:
name: Close Stale Issues
if: github.repository == 'allenai/allennlp'
runs-on: ubuntu-latest
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
steps:
- uses: actions/checkout@v3
- name: Setup Python
uses: actions/setup-python@v4
with:
python-version: 3.7
- name: Install requirements
run: |
pip install PyGithub
- name: Close stale issues
run: |
python scripts/close_stale_issues.py
ping_assignees:
name: Ping Inactive Assignees
if: github.repository == 'allenai/allennlp'
runs-on: ubuntu-latest
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
steps:
- uses: actions/checkout@v3
- name: Setup Python
uses: actions/setup-python@v4
with:
python-version: 3.7
- name: Install requirements
run: |
pip install PyGithub
- name: Ping inactive assignees
run: |
python scripts/ping_issue_assignees.py
================================================
FILE: .gitignore
================================================
# build artifacts
.eggs/
.mypy_cache
allennlp.egg-info/
build/
dist/
pip-wheel-metadata/
context.tar.gz
# dev tools
.envrc
.python-version
.idea
.venv/
.vscode/
/*.iml
# jupyter notebooks
.ipynb_checkpoints
# miscellaneous
.cache/
allennlp/tools/EVALB/evalb.dSYM/
doc/_build/
*.swp
.DS_Store
# python
*.pyc
*.pyo
__pycache__
# testing and continuous integration
.coverage
.pytest_cache/
.benchmarks
# documentation build artifacts
docs/*.md
docs/api
site/
mkdocs.yml
================================================
FILE: CHANGELOG.md
================================================
# Changelog
All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [v2.10.1](https://github.com/allenai/allennlp/releases/tag/v2.10.1) - 2022-10-18
### Fixed
- Updated dependencies
## [v2.10.0](https://github.com/allenai/allennlp/releases/tag/v2.10.0) - 2022-07-14
### Added
- Added metric `FBetaVerboseMeasure` which extends `FBetaMeasure` to ensure compatibility with logging plugins and add some options.
- Added three sample weighting techniques to `ConditionalRandomField` by supplying three new subclasses: `ConditionalRandomFieldWeightEmission`, `ConditionalRandomFieldWeightTrans`, and `ConditionalRandomFieldWeightLannoy`.
### Fixed
- Fix error from `cached-path` version update.
## [v2.9.3](https://github.com/allenai/allennlp/releases/tag/v2.9.3) - 2022-04-13
### Added
- Added `verification_tokens` argument to `TestPretrainedTransformerTokenizer`.
### Fixed
- Updated various dependencies
## [v2.9.2](https://github.com/allenai/allennlp/releases/tag/v2.9.2) - 2022-03-21
### Fixed
- Removed unnecessary dependencies
- Restore functionality of CLI in absence of now-optional checklist-package
## [v2.9.1](https://github.com/allenai/allennlp/releases/tag/v2.9.1) - 2022-03-09
### Fixed
- Updated dependencies, especially around doc creation.
- Running the test suite out-of-tree (e.g. after installation) is now possible by pointing the environment variable `ALLENNLP_SRC_DIR` to the sources.
- Silenced a warning that happens when you inappropriately clone a tensor.
- Adding more clarification to the `Vocabulary` documentation around `min_pretrained_embeddings` and `only_include_pretrained_words`.
- Fixed bug with type mismatch caused by latest release of `cached-path` that now returns a `Path` instead of a `str`.
### Added
- We can now transparently read compressed input files during prediction.
- LZMA compression is now supported.
- Added a way to give JSON blobs as input to dataset readers in the `evaluate` command.
- Added the argument `sub_module` in `PretrainedTransformerMismatchedEmbedder`
- Updated the docs for `PytorchSeq2VecWrapper` to specify that `mask` is required rather than sequence lengths for clarity.
### Changed
- You can automatically include all words from a pretrained file when building a vocabulary by setting the value in `min_pretrained_embeddings` to `-1`
for that particular namespace.
## [v2.9.0](https://github.com/allenai/allennlp/releases/tag/v2.9.0) - 2022-01-27
### Added
- Added an `Evaluator` class to make comparing source, target, and predictions easier.
- Added a way to resize the vocabulary in the T5 module
- Added an argument `reinit_modules` to `cached_transformers.get()` that allows you to re-initialize the pretrained weights of a transformer model, using layer indices or regex strings.
- Added attribute `_should_validate_this_epoch` to `GradientDescentTrainer` that controls whether validation is run at the end of each epoch.
- Added `ShouldValidateCallback` that can be used to configure the frequency of validation during training.
- Added a `MaxPoolingSpanExtractor`. This `SpanExtractor` represents each span by a component wise max-pooling-operation.
- Added support for `dist_metric` kwarg in initializing fairness metrics, which allows optionally using `wasserstein` distance (previously only KL divergence was supported).
### Fixed
- Fixed the docstring information for the `FBetaMultiLabelMeasure` metric.
- Various fixes for Python 3.9
- Fixed the name that the `push-to-hf` command uses to store weights.
- `FBetaMultiLabelMeasure` now works with multiple dimensions
- Support for inferior operating systems when making hardlinks
- Use `,` as a separator for filenames in the `evaluate` command, thus allowing for URLs (eg. `gs://...`) as input files.
- Removed a spurious error message "'torch.cuda' has no attribute '_check_driver'" that would be appear in the logs
when a `ConfigurationError` for missing GPU was raised.
- Load model on CPU post training to save GPU memory.
- Fixed a bug in `ShouldValidateCallback` that leads to validation occuring after the first epoch regardless of `validation_start` value.
- Fixed a bug in `ShouldValidateCallback` that leads to validation occuring every `validation_interval + 1` epochs, instead of every `validation_interval` epochs.
- Fixed a bug in `ShouldValidateCallback` that leads to validation never occuring at the end of training.
### Removed
- Removed dependency on the overrides package
- Removed Tango components, since they now live at https://github.com/allenai/tango.
### Changed
- Make `checklist` an optional dependency.
## [v2.8.0](https://github.com/allenai/allennlp/releases/tag/v2.8.0) - 2021-11-01
### Added
- Added support to push models directly to the [Hugging Face Hub](https://huggingface.co/) with the command `allennlp push-to-hf`.
- More default tests for the `TextualEntailmentSuite`.
### Changed
- The behavior of `--overrides` has changed. Previously the final configuration params were simply taken as the union over the original params and the `--overrides` params.
But now you can use `--overrides` to completely replace any part of the original config. For example, passing `--overrides '{"model":{"type":"foo"}}'` will completely
replace the "model" part of the original config. However, when you just want to change a single field in the JSON structure without removing / replacing adjacent fields,
you can still use the "dot" syntax. For example, `--overrides '{"model.num_layers":3}'` will only change the `num_layers` parameter to the "model" part of the config, leaving
everything else unchanged.
- Integrated [`cached_path`](https://github.com/allenai/cached_path) library to replace existing functionality in `common.file_utils`. This introduces some improvements without
any breaking changes.
### Fixed
- Fixed the implementation of `PairedPCABiasDirection` in `allennlp.fairness.bias_direction`, where the difference vectors should not be centered when performing the PCA.
- Fixed the docstring of `ExponentialMovingAverage`, which was causing its argument descriptions to render inccorrectly in the docs.
## [v2.7.0](https://github.com/allenai/allennlp/releases/tag/v2.7.0) - 2021-09-01
### Added
- Added in a default behavior to the `_to_params` method of `Registrable` so that in the case it is not implemented by the child class, it will still produce _a parameter dictionary_.
- Added in `_to_params` implementations to all tokenizers.
- Added support to evaluate mutiple datasets and produce corresponding output files in the `evaluate` command.
- Added more documentation to the learning rate schedulers to include a sample config object for how to use it.
- Moved the pytorch learning rate schedulers wrappers to their own file called `pytorch_lr_schedulers.py` so that they will have their own documentation page.
- Added a module `allennlp.nn.parallel` with a new base class, `DdpAccelerator`, which generalizes
PyTorch's `DistributedDataParallel` wrapper to support other implementations. Two implementations of
this class are provided. The default is `TorchDdpAccelerator` (registered at "torch"), which is just a thin wrapper around
`DistributedDataParallel`. The other is `FairScaleFsdpAccelerator`, which wraps FairScale's
[`FullyShardedDataParallel`](https://fairscale.readthedocs.io/en/latest/api/nn/fsdp.html).
You can specify the `DdpAccelerator` in the "distributed" section of a configuration file under the key "ddp_accelerator".
- Added a module `allennlp.nn.checkpoint` with a new base class, `CheckpointWrapper`, for implementations
of activation/gradient checkpointing. Two implentations are provided. The default implementation is `TorchCheckpointWrapper` (registered as "torch"),
which exposes [PyTorch's checkpoint functionality](https://pytorch.org/docs/stable/checkpoint.html).
The other is `FairScaleCheckpointWrapper` which exposes the more flexible
[checkpointing funtionality from FairScale](https://fairscale.readthedocs.io/en/latest/api/nn/checkpoint/checkpoint_activations.html).
- The `Model` base class now takes a `ddp_accelerator` parameter (an instance of `DdpAccelerator`) which will be available as
`self.ddp_accelerator` during distributed training. This is useful when, for example, instantiating submodules in your
model's `__init__()` method by wrapping them with `self.ddp_accelerator.wrap_module()`. See the `allennlp.modules.transformer.t5`
for an example.
- We now log batch metrics to tensorboard and wandb.
- Added Tango components, to be explored in detail in a later post
- Added `ScaledDotProductMatrixAttention`, and converted the transformer toolkit to use it
- Added tests to ensure that all `Attention` and `MatrixAttention` implementations are interchangeable
- Added a way for AllenNLP Tango to read and write datasets lazily.
- Added a way to remix datasets flexibly
- Added `from_pretrained_transformer_and_instances` constructor to `Vocabulary`
- `TransformerTextField` now supports `__len__`.
### Fixed
- Fixed a bug in `ConditionalRandomField`: `transitions` and `tag_sequence` tensors were not initialized on the desired device causing high CPU usage (see https://github.com/allenai/allennlp/issues/2884)
- Fixed a mispelling: the parameter `contructor_extras` in `Lazy()` is now correctly called `constructor_extras`.
- Fixed broken links in `allennlp.nn.initializers` docs.
- Fixed bug in `BeamSearch` where `last_backpointers` was not being passed to any `Constraint`s.
- `TransformerTextField` can now take tensors of shape `(1, n)` like the tensors produced from a HuggingFace tokenizer.
- `tqdm` lock is now set inside `MultiProcessDataLoading` when new workers are spawned to avoid contention when writing output.
- `ConfigurationError` is now pickleable.
- Checkpointer cleaning was fixed to work on Windows Paths
- Multitask models now support `TextFieldTensor` in heads, not just in the backbone.
- Fixed the signature of `ScaledDotProductAttention` to match the other `Attention` classes
- `allennlp` commands will now catch `SIGTERM` signals and handle them similar to `SIGINT` (keyboard interrupt).
- The `MultiProcessDataLoader` will properly shutdown its workers when a `SIGTERM` is received.
- Fixed the way names are applied to Tango `Step` instances.
- Fixed a bug in calculating loss in the distributed setting.
- Fixed a bug when extending a sparse sequence by 0 items.
### Changed
- The type of the `grad_norm` parameter of `GradientDescentTrainer` is now `Union[float, bool]`,
with a default value of `False`. `False` means gradients are not rescaled and the gradient
norm is never even calculated. `True` means the gradients are still not rescaled but the gradient
norm is calculated and passed on to callbacks. A `float` value means gradients are rescaled.
- `TensorCache` now supports more concurrent readers and writers.
- We no longer log parameter statistics to tensorboard or wandb by default.
## [v2.6.0](https://github.com/allenai/allennlp/releases/tag/v2.6.0) - 2021-07-19
### Added
- Added `on_backward` training callback which allows for control over backpropagation and gradient manipulation.
- Added `AdversarialBiasMitigator`, a Model wrapper to adversarially mitigate biases in predictions produced by a pretrained model for a downstream task.
- Added `which_loss` parameter to `ensure_model_can_train_save_and_load` in `ModelTestCase` to specify which loss to test.
- Added `**kwargs` to `Predictor.from_path()`. These key-word argument will be passed on to the `Predictor`'s constructor.
- The activation layer in the transformer toolkit now can be queried for its output dimension.
- `TransformerEmbeddings` now takes, but ignores, a parameter for the attention mask. This is needed for compatibility with some other modules that get called the same way and use the mask.
- `TransformerPooler` can now be instantiated from a pretrained transformer module, just like the other modules in the transformer toolkit.
- `TransformerTextField`, for cases where you don't care about AllenNLP's advanced text handling capabilities.
- Added `TransformerModule._post_load_pretrained_state_dict_hook()` method. Can be used to modify `missing_keys` and `unexpected_keys` after
loading a pretrained state dictionary. This is useful when tying weights, for example.
- Added an end-to-end test for the Transformer Toolkit.
- Added `vocab` argument to `BeamSearch`, which is passed to each contraint in `constraints` (if provided).
### Fixed
- Fixed missing device mapping in the `allennlp.modules.conditional_random_field.py` file.
- Fixed Broken link in `allennlp.fairness.fairness_metrics.Separation` docs
- Ensured all `allennlp` submodules are imported with `allennlp.common.plugins.import_plugins()`.
- Fixed `IndexOutOfBoundsException` in `MultiOptimizer` when checking if optimizer received any parameters.
- Removed confusing zero mask from VilBERT.
- Ensured `ensure_model_can_train_save_and_load` is consistently random.
- Fixed weight tying logic in `T5` transformer module. Previously input/output embeddings were always tied. Now this is optional,
and the default behavior is taken from the `config.tie_word_embeddings` value when instantiating `from_pretrained_module()`.
- Implemented slightly faster label smoothing.
- Fixed the docs for `PytorchTransformerWrapper`
- Fixed recovering training jobs with models that expect `get_metrics()` to not be called until they have seen at least one batch.
- Made the Transformer Toolkit compatible with transformers that don't start their positional embeddings at 0.
- Weights & Biases training callback ("wandb") now works when resuming training jobs.
### Changed
- Changed behavior of `MultiOptimizer` so that while a default optimizer is still required, an error is not thrown if the default optimizer receives no parameters.
- Made the epsilon parameter for the layer normalization in token embeddings configurable.
### Removed
- Removed `TransformerModule._tied_weights`. Weights should now just be tied directly in the `__init__()` method.
You can also override `TransformerModule._post_load_pretrained_state_dict_hook()` to remove keys associated with tied weights from `missing_keys`
after loading a pretrained state dictionary.
## [v2.5.0](https://github.com/allenai/allennlp/releases/tag/v2.5.0) - 2021-06-03
### Added
- Added `TaskSuite` base class and command line functionality for running [`checklist`](https://github.com/marcotcr/checklist) test suites, along with implementations for `SentimentAnalysisSuite`, `QuestionAnsweringSuite`, and `TextualEntailmentSuite`. These can be found in the `allennlp.confidence_checks.task_checklists` module.
- Added `BiasMitigatorApplicator`, which wraps any Model and mitigates biases by finetuning
on a downstream task.
- Added `allennlp diff` command to compute a diff on model checkpoints, analogous to what `git diff` does on two files.
- Meta data defined by the class `allennlp.common.meta.Meta` is now saved in the serialization directory and archive file
when training models from the command line. This is also now part of the `Archive` named tuple that's returned from `load_archive()`.
- Added `nn.util.distributed_device()` helper function.
- Added `allennlp.nn.util.load_state_dict` helper function.
- Added a way to avoid downloading and loading pretrained weights in modules that wrap transformers
such as the `PretrainedTransformerEmbedder` and `PretrainedTransformerMismatchedEmbedder`.
You can do this by setting the parameter `load_weights` to `False`.
See [PR #5172](https://github.com/allenai/allennlp/pull/5172) for more details.
- Added `SpanExtractorWithSpanWidthEmbedding`, putting specific span embedding computations into the `_embed_spans` method and leaving the common code in `SpanExtractorWithSpanWidthEmbedding` to unify the arguments, and modified `BidirectionalEndpointSpanExtractor`, `EndpointSpanExtractor` and `SelfAttentiveSpanExtractor` accordingly. Now, `SelfAttentiveSpanExtractor` can also embed span widths.
- Added a `min_steps` parameter to `BeamSearch` to set a minimum length for the predicted sequences.
- Added the `FinalSequenceScorer` abstraction to calculate the final scores of the generated sequences in `BeamSearch`.
- Added `shuffle` argument to `BucketBatchSampler` which allows for disabling shuffling.
- Added `allennlp.modules.transformer.attention_module` which contains a generalized `AttentionModule`. `SelfAttention` and `T5Attention` both inherit from this.
- Added a `Constraint` abstract class to `BeamSearch`, which allows for incorporating constraints on the predictions found by `BeamSearch`,
along with a `RepeatedNGramBlockingConstraint` constraint implementation, which allows for preventing repeated n-grams in the output from `BeamSearch`.
- Added `DataCollator` for dynamic operations for each batch.
### Changed
- Use `dist_reduce_sum` in distributed metrics.
- Allow Google Cloud Storage paths in `cached_path` ("gs://...").
- Renamed `nn.util.load_state_dict()` to `read_state_dict` to avoid confusion with `torch.nn.Module.load_state_dict()`.
- `TransformerModule.from_pretrained_module` now only accepts a pretrained model ID (e.g. "bert-base-case") instead of
an actual `torch.nn.Module`. Other parameters to this method have changed as well.
- Print the first batch to the console by default.
- Renamed `sanity_checks` to `confidence_checks` (`sanity_checks` is deprecated and will be removed in AllenNLP 3.0).
- Trainer callbacks can now store and restore state in case a training run gets interrupted.
- VilBERT backbone now rolls and unrolls extra dimensions to handle input with > 3 dimensions.
- `BeamSearch` is now a `Registrable` class.
### Fixed
- When `PretrainedTransformerIndexer` folds long sequences, it no longer loses the information from token type ids.
- Fixed documentation for `GradientDescentTrainer.cuda_device`.
- Re-starting a training run from a checkpoint in the middle of an epoch now works correctly.
- When using the "moving average" weights smoothing feature of the trainer, training checkpoints would also get smoothed, with strange results for resuming a training job. This has been fixed.
- When re-starting an interrupted training job, the trainer will now read out the data loader even for epochs and batches that can be skipped. We do this to try to get any random number generators used by the reader or data loader into the same state as they were the first time the training job ran.
- Fixed the potential for a race condition with `cached_path()` when extracting archives. Although the race condition
is still possible if used with `force_extract=True`.
- Fixed `wandb` callback to work in distributed training.
- Fixed `tqdm` logging into multiple files with `allennlp-optuna`.
## [v2.4.0](https://github.com/allenai/allennlp/releases/tag/v2.4.0) - 2021-04-22
### Added
- Added a T5 implementation to `modules.transformers`.
### Changed
- Weights & Biases callback can now work in anonymous mode (i.e. without the `WANDB_API_KEY` environment variable).
### Fixed
- The `GradientDescentTrainer` no longer leaves stray model checkpoints around when it runs out of patience.
- Fixed `cached_path()` for "hf://" files.
- Improved the error message for the `PolynomialDecay` LR scheduler when `num_steps_per_epoch` is missing.
## [v2.3.1](https://github.com/allenai/allennlp/releases/tag/v2.3.1) - 2021-04-20
### Added
- Added support for the HuggingFace Hub as an alternative way to handle loading files. Hub downloads should be made through the `hf://` URL scheme.
- Add new dimension to the `interpret` module: influence functions via the `InfluenceInterpreter` base class, along with a concrete implementation: `SimpleInfluence`.
- Added a `quiet` parameter to the `MultiProcessDataLoading` that disables `Tqdm` progress bars.
- The test for distributed metrics now takes a parameter specifying how often you want to run it.
- Created the fairness module and added three fairness metrics: `Independence`, `Separation`, and `Sufficiency`.
- Added four bias metrics to the fairness module: `WordEmbeddingAssociationTest`, `EmbeddingCoherenceTest`, `NaturalLanguageInference`, and `AssociationWithoutGroundTruth`.
- Added four bias direction methods (`PCABiasDirection`, `PairedPCABiasDirection`, `TwoMeansBiasDirection`, `ClassificationNormalBiasDirection`) and four bias mitigation methods (`LinearBiasMitigator`, `HardBiasMitigator`, `INLPBiasMitigator`, `OSCaRBiasMitigator`).
### Changed
- Updated CONTRIBUTING.md to remind reader to upgrade pip setuptools to avoid spaCy installation issues.
### Fixed
- Fixed a bug with the `ShardedDatasetReader` when used with multi-process data loading (https://github.com/allenai/allennlp/issues/5132).
## [v2.3.0](https://github.com/allenai/allennlp/releases/tag/v2.3.0) - 2021-04-14
### Added
- Ported the following Huggingface `LambdaLR`-based schedulers: `ConstantLearningRateScheduler`, `ConstantWithWarmupLearningRateScheduler`, `CosineWithWarmupLearningRateScheduler`, `CosineHardRestartsWithWarmupLearningRateScheduler`.
- Added new `sub_token_mode` parameter to `pretrained_transformer_mismatched_embedder` class to support first sub-token embedding
- Added a way to run a multi task model with a dataset reader as part of `allennlp predict`.
- Added new `eval_mode` in `PretrainedTransformerEmbedder`. If it is set to `True`, the transformer is _always_ run in evaluation mode, which, e.g., disables dropout and does not update batch normalization statistics.
- Added additional parameters to the W&B callback: `entity`, `group`, `name`, `notes`, and `wandb_kwargs`.
### Changed
- Sanity checks in the `GradientDescentTrainer` can now be turned off by setting the `run_sanity_checks` parameter to `False`.
- Allow the order of examples in the task cards to be specified explicitly
- `histogram_interval` parameter is now deprecated in `TensorboardWriter`, please use `distribution_interval` instead.
- Memory usage is not logged in tensorboard during training now. `ConsoleLoggerCallback` should be used instead.
- If you use the `min_count` parameter of the Vocabulary, but you specify a namespace that does not exist, the vocabulary creation will raise a `ConfigurationError`.
- Documentation updates made to SoftmaxLoss regarding padding and the expected shapes of the input and output tensors of `forward`.
- Moved the data preparation script for coref into allennlp-models.
- If a transformer is not in cache but has override weights, the transformer's pretrained weights are no longer downloaded, that is, only its `config.json` file is downloaded.
- `SanityChecksCallback` now raises `SanityCheckError` instead of `AssertionError` when a check fails.
- `jsonpickle` removed from dependencies.
- Improved the error message from `Registrable.by_name()` when the name passed does not match any registered subclassess.
The error message will include a suggestion if there is a close match between the name passed and a registered name.
### Fixed
- Fixed a bug where some `Activation` implementations could not be pickled due to involving a lambda function.
- Fixed `__str__()` method on `ModelCardInfo` class.
- Fixed a stall when using distributed training and gradient accumulation at the same time
- Fixed an issue where using the `from_pretrained_transformer` `Vocabulary` constructor in distributed training via the `allennlp train` command
would result in the data being iterated through unnecessarily.
- Fixed a bug regarding token indexers with the `InterleavingDatasetReader` when used with multi-process data loading.
- Fixed a warning from `transformers` when using `max_length` in the `PretrainedTransformerTokenizer`.
### Removed
- Removed the `stride` parameter to `PretrainedTransformerTokenizer`. This parameter had no effect.
## [v2.2.0](https://github.com/allenai/allennlp/releases/tag/v2.2.0) - 2021-03-26
### Added
- Add new method on `Field` class: `.human_readable_repr() -> Any`
- Add new method on `Instance` class: `.human_readable_dict() -> JsonDict`.
- Added `WandBCallback` class for [Weights & Biases](https://wandb.ai) integration, registered as a callback under
the name "wandb".
- Added `TensorBoardCallback` to replace the `TensorBoardWriter`. Registered as a callback
under the name "tensorboard".
- Added `NormalizationBiasVerification` and `SanityChecksCallback` for model sanity checks.
- `SanityChecksCallback` runs by default from the `allennlp train` command.
It can be turned off by setting `trainer.enable_default_callbacks` to `false` in your config.
### Changed
- Use attributes of `ModelOutputs` object in `PretrainedTransformerEmbedder` instead of indexing.
- Added support for PyTorch version 1.8 and `torchvision` version 0.9 .
- `Model.get_parameters_for_histogram_tensorboard_logging` is deprecated in favor of
`Model.get_parameters_for_histogram_logging`.
### Fixed
- Makes sure tensors that are stored in `TensorCache` always live on CPUs
- Fixed a bug where `FromParams` objects wrapped in `Lazy()` couldn't be pickled.
- Fixed a bug where the `ROUGE` metric couldn't be picked.
- Fixed a bug reported by https://github.com/allenai/allennlp/issues/5036. We keeps our spacy POS tagger on.
### Removed
- Removed `TensorBoardWriter`. Please use the `TensorBoardCallback` instead.
## [v2.1.0](https://github.com/allenai/allennlp/releases/tag/v2.1.0) - 2021-02-24
### Changed
- `coding_scheme` parameter is now deprecated in `Conll2003DatasetReader`, please use `convert_to_coding_scheme` instead.
- Support spaCy v3
### Added
- Added `ModelUsage` to `ModelCard` class.
- Added a way to specify extra parameters to the predictor in an `allennlp predict` call.
- Added a way to initialize a `Vocabulary` from transformers models.
- Added the ability to use `Predictors` with multitask models through the new `MultiTaskPredictor`.
- Added an example for fields of type `ListField[TextField]` to `apply_token_indexers` API docs.
- Added `text_key` and `label_key` parameters to `TextClassificationJsonReader` class.
- Added `MultiOptimizer`, which allows you to use different optimizers for different parts of your model.
- Added a clarification to `predictions_to_labeled_instances` API docs for attack from json
### Fixed
- `@Registrable.register(...)` decorator no longer masks the decorated class's annotations
- Ensured that `MeanAbsoluteError` always returns a `float` metric value instead of a `Tensor`.
- Learning rate schedulers that rely on metrics from the validation set were broken in v2.0.0. This
brings that functionality back.
- Fixed a bug where the `MultiProcessDataLoading` would crash when `num_workers > 0`, `start_method = "spawn"`, `max_instances_in_memory not None`, and `batches_per_epoch not None`.
- Fixed documentation and validation checks for `FBetaMultiLabelMetric`.
- Fixed handling of HTTP errors when fetching remote resources with `cached_path()`. Previously the content would be cached even when
certain errors - like 404s - occurred. Now an `HTTPError` will be raised whenever the HTTP response is not OK.
- Fixed a bug where the `MultiTaskDataLoader` would crash when `num_workers > 0`
- Fixed an import error that happens when PyTorch's distributed framework is unavailable on the system.
## [v2.0.1](https://github.com/allenai/allennlp/releases/tag/v2.0.1) - 2021-01-29
### Added
- Added `tokenizer_kwargs` and `transformer_kwargs` arguments to `PretrainedTransformerBackbone`
- Resize transformers word embeddings layer for `additional_special_tokens`
### Changed
- GradientDescentTrainer makes `serialization_dir` when it's instantiated, if it doesn't exist.
### Fixed
- `common.util.sanitize` now handles sets.
## [v2.0.0](https://github.com/allenai/allennlp/releases/tag/v2.0.0) - 2021-01-27
### Added
- The `TrainerCallback` constructor accepts `serialization_dir` provided by `Trainer`. This can be useful for `Logger` callbacks those need to store files in the run directory.
- The `TrainerCallback.on_start()` is fired at the start of the training.
- The `TrainerCallback` event methods now accept `**kwargs`. This may be useful to maintain backwards-compability of callbacks easier in the future. E.g. we may decide to pass the exception/traceback object in case of failure to `on_end()` and this older callbacks may simply ignore the argument instead of raising a `TypeError`.
- Added a `TensorBoardCallback` which wraps the `TensorBoardWriter`.
### Changed
- The `TrainerCallack.on_epoch()` does not fire with `epoch=-1` at the start of the training.
Instead, `TrainerCallback.on_start()` should be used for these cases.
- `TensorBoardBatchMemoryUsage` is converted from `BatchCallback` into `TrainerCallback`.
- `TrackEpochCallback` is converted from `EpochCallback` into `TrainerCallback`.
- `Trainer` can accept callbacks simply with name `callbacks` instead of `trainer_callbacks`.
- `TensorboardWriter` renamed to `TensorBoardWriter`, and removed as an argument to the `GradientDescentTrainer`.
In order to enable TensorBoard logging during training, you should utilize the `TensorBoardCallback` instead.
### Removed
- Removed `EpochCallback`, `BatchCallback` in favour of `TrainerCallback`.
The metaclass-wrapping implementation is removed as well.
- Removed the `tensorboard_writer` parameter to `GradientDescentTrainer`. You should use the `TensorBoardCallback` now instead.
### Fixed
- Now Trainer always fires `TrainerCallback.on_end()` so all the resources can be cleaned up properly.
- Fixed the misspelling, changed `TensoboardBatchMemoryUsage` to `TensorBoardBatchMemoryUsage`.
- We set a value to `epoch` so in case of firing `TrainerCallback.on_end()` the variable is bound.
This could have lead to an error in case of trying to recover a run after it was finished training.
## [v2.0.0rc1](https://github.com/allenai/allennlp/releases/tag/v2.0.0rc1) - 2021-01-21
### Added
- Added `TensorCache` class for caching tensors on disk
- Added abstraction and concrete implementation for image loading
- Added abstraction and concrete implementation for `GridEmbedder`
- Added abstraction and demo implementation for an image augmentation module.
- Added abstraction and concrete implementation for region detectors.
- A new high-performance default `DataLoader`: `MultiProcessDataLoading`.
- A `MultiTaskModel` and abstractions to use with it, including `Backbone` and `Head`. The
`MultiTaskModel` first runs its inputs through the `Backbone`, then passes the result (and
whatever other relevant inputs it got) to each `Head` that's in use.
- A `MultiTaskDataLoader`, with a corresponding `MultiTaskDatasetReader`, and a couple of new
configuration objects: `MultiTaskEpochSampler` (for deciding what proportion to sample from each
dataset at every epoch) and a `MultiTaskScheduler` (for ordering the instances within an epoch).
- Transformer toolkit to plug and play with modular components of transformer architectures.
- Added a command to count the number of instances we're going to be training with
- Added a `FileLock` class to `common.file_utils`. This is just like the `FileLock` from the `filelock` library, except that
it adds an optional flag `read_only_ok: bool`, which when set to `True` changes the behavior so that a warning will be emitted
instead of an exception when lacking write permissions on an existing file lock.
This makes it possible to use the `FileLock` class on a read-only file system.
- Added a new learning rate scheduler: `CombinedLearningRateScheduler`. This can be used to combine different LR schedulers, using one after the other.
- Added an official CUDA 10.1 Docker image.
- Moving `ModelCard` and `TaskCard` abstractions into the main repository.
- Added a util function `allennlp.nn.util.dist_reduce(...)` for handling distributed reductions.
This is especially useful when implementing a distributed `Metric`.
- Added a `FileLock` class to `common.file_utils`. This is just like the `FileLock` from the `filelock` library, except that
it adds an optional flag `read_only_ok: bool`, which when set to `True` changes the behavior so that a warning will be emitted
instead of an exception when lacking write permissions on an existing file lock.
This makes it possible to use the `FileLock` class on a read-only file system.
- Added a new learning rate scheduler: `CombinedLearningRateScheduler`. This can be used to combine different LR schedulers, using one after the other.
- Moving `ModelCard` and `TaskCard` abstractions into the main repository.
### Changed
- `DatasetReader`s are now always lazy. This means there is no `lazy` parameter in the base
class, and the `_read()` method should always be a generator.
- The `DataLoader` now decides whether to load instances lazily or not.
With the `PyTorchDataLoader` this is controlled with the `lazy` parameter, but with
the `MultiProcessDataLoading` this is controlled by the `max_instances_in_memory` setting.
- `ArrayField` is now called `TensorField`, and implemented in terms of torch tensors, not numpy.
- Improved `nn.util.move_to_device` function by avoiding an unnecessary recursive check for tensors and
adding a `non_blocking` optional argument, which is the same argument as in `torch.Tensor.to()`.
- If you are trying to create a heterogeneous batch, you now get a better error message.
- Readers using the new vision features now explicitly log how they are featurizing images.
- `master_addr` and `master_port` renamed to `primary_addr` and `primary_port`, respectively.
- `is_master` parameter for training callbacks renamed to `is_primary`.
- `master` branch renamed to `main`
- Torch version bumped to 1.7.1 in Docker images.
- 'master' branch renamed to 'main'
- Torch version bumped to 1.7.1 in Docker images.
### Removed
- Removed `nn.util.has_tensor`.
### Fixed
- The `build-vocab` command no longer crashes when the resulting vocab file is
in the current working directory.
- VQA models now use the `vqa_score` metric for early stopping. This results in
much better scores.
- Fixed typo with `LabelField` string representation: removed trailing apostrophe.
- `Vocabulary.from_files` and `cached_path` will issue a warning, instead of failing, when a lock on an existing resource
can't be acquired because the file system is read-only.
- `TrackEpochCallback` is now a `EpochCallback`.
## [v1.3.0](https://github.com/allenai/allennlp/releases/tag/v1.3.0) - 2020-12-15
### Added
- Added links to source code in docs.
- Added `get_embedding_layer` and `get_text_field_embedder` to the `Predictor` class; to specify embedding layers for non-AllenNLP models.
- Added [Gaussian Error Linear Unit (GELU)](https://pytorch.org/docs/stable/generated/torch.nn.GELU.html) as an Activation.
### Changed
- Renamed module `allennlp.data.tokenizers.token` to `allennlp.data.tokenizers.token_class` to avoid
[this bug](https://github.com/allenai/allennlp/issues/4819).
- `transformers` dependency updated to version 4.0.1.
- `BasicClassifier`'s forward method now takes a metadata field.
### Fixed
- Fixed a lot of instances where tensors were first created and then sent to a device
with `.to(device)`. Instead, these tensors are now created directly on the target device.
- Fixed issue with `GradientDescentTrainer` when constructed with `validation_data_loader=None` and `learning_rate_scheduler!=None`.
- Fixed a bug when removing all handlers in root logger.
- `ShardedDatasetReader` now inherits parameters from `base_reader` when required.
- Fixed an issue in `FromParams` where parameters in the `params` object used to a construct a class
were not passed to the constructor if the value of the parameter was equal to the default value.
This caused bugs in some edge cases where a subclass that takes `**kwargs` needs to inspect
`kwargs` before passing them to its superclass.
- Improved the band-aid solution for segmentation faults and the "ImportError: dlopen: cannot load any more object with static TLS"
by adding a `transformers` import.
- Added safety checks for extracting tar files
- Turned superfluous warning to info when extending the vocab in the embedding matrix, if no pretrained file was provided
## [v1.2.2](https://github.com/allenai/allennlp/releases/tag/v1.2.2) - 2020-11-17
### Added
- Added Docker builds for other torch-supported versions of CUDA.
- Adds [`allennlp-semparse`](https://github.com/allenai/allennlp-semparse) as an official, default plugin.
### Fixed
- `GumbelSampler` now sorts the beams by their true log prob.
## [v1.2.1](https://github.com/allenai/allennlp/releases/tag/v1.2.1) - 2020-11-10
### Added
- Added an optional `seed` parameter to `ModelTestCase.set_up_model` which sets the random
seed for `random`, `numpy`, and `torch`.
- Added support for a global plugins file at `~/.allennlp/plugins`.
- Added more documentation about plugins.
- Added sampler class and parameter in beam search for non-deterministic search, with several
implementations, including `MultinomialSampler`, `TopKSampler`, `TopPSampler`, and
`GumbelSampler`. Utilizing `GumbelSampler` will give [Stochastic Beam Search](https://api.semanticscholar.org/CorpusID:76662039).
### Changed
- Pass batch metrics to `BatchCallback`.
### Fixed
- Fixed a bug where forward hooks were not cleaned up with saliency interpreters if there
was an exception.
- Fixed the computation of saliency maps in the Interpret code when using mismatched indexing.
Previously, we would compute gradients from the top of the transformer, after aggregation from
wordpieces to tokens, which gives results that are not very informative. Now, we compute gradients
with respect to the embedding layer, and aggregate wordpieces to tokens separately.
- Fixed the heuristics for finding embedding layers in the case of RoBERTa. An update in the
`transformers` library broke our old heuristic.
- Fixed typo with registered name of ROUGE metric. Previously was `rogue`, fixed to `rouge`.
- Fixed default masks that were erroneously created on the CPU even when a GPU is available.
- Fixed pretrained embeddings for transformers that don't use end tokens.
- Fixed the transformer tokenizer cache when the tokenizers are initialized with custom kwargs.
## [v1.2.0](https://github.com/allenai/allennlp/releases/tag/v1.2.0) - 2020-10-29
### Changed
- Enforced stricter typing requirements around the use of `Optional[T]` types.
- Changed the behavior of `Lazy` types in `from_params` methods. Previously, if you defined a `Lazy` parameter like
`foo: Lazy[Foo] = None` in a custom `from_params` classmethod, then `foo` would actually never be `None`.
This behavior is now different. If no params were given for `foo`, it will be `None`.
You can also now set default values for foo like `foo: Lazy[Foo] = Lazy(Foo)`.
Or, if you want you want a default value but also want to allow for `None` values, you can
write it like this: `foo: Optional[Lazy[Foo]] = Lazy(Foo)`.
- Added support for PyTorch version 1.7.
### Fixed
- Made it possible to instantiate `TrainerCallback` from config files.
- Fixed the remaining broken internal links in the API docs.
- Fixed a bug where Hotflip would crash with a model that had multiple TokenIndexers and the input
used rare vocabulary items.
- Fixed a bug where `BeamSearch` would fail if `max_steps` was equal to 1.
- Fixed `BasicTextFieldEmbedder` to not raise ConfigurationError if it has embedders that are empty and not in input
## [v1.2.0rc1](https://github.com/allenai/allennlp/releases/tag/v1.2.0rc1) - 2020-10-22
### Added
- Added a warning when `batches_per_epoch` for the validation data loader is inherited from
the train data loader.
- Added a `build-vocab` subcommand that can be used to build a vocabulary from a training config file.
- Added `tokenizer_kwargs` argument to `PretrainedTransformerMismatchedIndexer`.
- Added `tokenizer_kwargs` and `transformer_kwargs` arguments to `PretrainedTransformerMismatchedEmbedder`.
- Added official support for Python 3.8.
- Added a script: `scripts/release_notes.py`, which automatically prepares markdown release notes from the
CHANGELOG and commit history.
- Added a flag `--predictions-output-file` to the `evaluate` command, which tells AllenNLP to write the
predictions from the given dataset to the file as JSON lines.
- Added the ability to ignore certain missing keys when loading a model from an archive. This is done
by adding a class-level variable called `authorized_missing_keys` to any PyTorch module that a `Model` uses.
If defined, `authorized_missing_keys` should be a list of regex string patterns.
- Added `FBetaMultiLabelMeasure`, a multi-label Fbeta metric. This is a subclass of the existing `FBetaMeasure`.
- Added ability to pass additional key word arguments to `cached_transformers.get()`, which will be passed on to `AutoModel.from_pretrained()`.
- Added an `overrides` argument to `Predictor.from_path()`.
- Added a `cached-path` command.
- Added a function `inspect_cache` to `common.file_utils` that prints useful information about the cache. This can also
be used from the `cached-path` command with `allennlp cached-path --inspect`.
- Added a function `remove_cache_entries` to `common.file_utils` that removes any cache entries matching the given
glob patterns. This can used from the `cached-path` command with `allennlp cached-path --remove some-files-*`.
- Added logging for the main process when running in distributed mode.
- Added a `TrainerCallback` object to support state sharing between batch and epoch-level training callbacks.
- Added support for .tar.gz in PretrainedModelInitializer.
- Made `BeamSearch` instantiable `from_params`.
- Pass `serialization_dir` to `Model` and `DatasetReader`.
- Added an optional `include_in_archive` parameter to the top-level of configuration files. When specified, `include_in_archive` should be a list of paths relative to the serialization directory which will be bundled up with the final archived model from a training run.
### Changed
- Subcommands that don't require plugins will no longer cause plugins to be loaded or have an `--include-package` flag.
- Allow overrides to be JSON string or `dict`.
- `transformers` dependency updated to version 3.1.0.
- When `cached_path` is called on a local archive with `extract_archive=True`, the archive is now extracted into a unique subdirectory of the cache root instead of a subdirectory of the archive's directory. The extraction directory is also unique to the modification time of the archive, so if the file changes, subsequent calls to `cached_path` will know to re-extract the archive.
- Removed the `truncation_strategy` parameter to `PretrainedTransformerTokenizer`. The way we're calling the tokenizer, the truncation strategy takes no effect anyways.
- Don't use initializers when loading a model, as it is not needed.
- Distributed training will now automatically search for a local open port if the `master_port` parameter is not provided.
- In training, save model weights before evaluation.
- `allennlp.common.util.peak_memory_mb` renamed to `peak_cpu_memory`, and `allennlp.common.util.gpu_memory_mb` renamed to `peak_gpu_memory`,
and they both now return the results in bytes as integers. Also, the `peak_gpu_memory` function now utilizes PyTorch functions to find the memory
usage instead of shelling out to the `nvidia-smi` command. This is more efficient and also more accurate because it only takes
into account the tensor allocations of the current PyTorch process.
- Make sure weights are first loaded to the cpu when using PretrainedModelInitializer, preventing wasted GPU memory.
- Load dataset readers in `load_archive`.
- Updated `AllenNlpTestCase` docstring to remove reference to `unittest.TestCase`
### Removed
- Removed `common.util.is_master` function.
### Fixed
- Fix CUDA/CPU device mismatch bug during distributed training for categorical accuracy metric.
- Fixed a bug where the reported `batch_loss` metric was incorrect when training with gradient accumulation.
- Class decorators now displayed in API docs.
- Fixed up the documentation for the `allennlp.nn.beam_search` module.
- Ignore `*args` when constructing classes with `FromParams`.
- Ensured some consistency in the types of the values that metrics return.
- Fix a PyTorch warning by explicitly providing the `as_tuple` argument (leaving
it as its default value of `False`) to `Tensor.nonzero()`.
- Remove temporary directory when extracting model archive in `load_archive`
at end of function rather than via `atexit`.
- Fixed a bug where using `cached_path()` offline could return a cached resource's lock file instead
of the cache file.
- Fixed a bug where `cached_path()` would fail if passed a `cache_dir` with the user home shortcut `~/`.
- Fixed a bug in our doc building script where markdown links did not render properly
if the "href" part of the link (the part inside the `()`) was on a new line.
- Changed how gradients are zeroed out with an optimization. See [this video from NVIDIA](https://www.youtube.com/watch?v=9mS1fIYj1So)
at around the 9 minute mark.
- Fixed a bug where parameters to a `FromParams` class that are dictionaries wouldn't get logged
when an instance is instantiated `from_params`.
- Fixed a bug in distributed training where the vocab would be saved from every worker, when it should have been saved by only the local master process.
- Fixed a bug in the calculation of rouge metrics during distributed training where the total sequence count was not being aggregated across GPUs.
- Fixed `allennlp.nn.util.add_sentence_boundary_token_ids()` to use `device` parameter of input tensor.
- Be sure to close the TensorBoard writer even when training doesn't finish.
- Fixed the docstring for `PyTorchSeq2VecWrapper`.
- Fixed a bug in the cnn_encoder where activations involving masked tokens could be picked up by the max
- Fix intra word tokenization for `PretrainedTransformerTokenizer` when disabling fast tokenizer.
## [v1.1.0](https://github.com/allenai/allennlp/releases/tag/v1.1.0) - 2020-09-08
### Fixed
- Fixed handling of some edge cases when constructing classes with `FromParams` where the class
accepts `**kwargs`.
- Fixed division by zero error when there are zero-length spans in the input to a
`PretrainedTransformerMismatchedIndexer`.
- Improved robustness of `cached_path` when extracting archives so that the cache won't be corrupted
if a failure occurs during extraction.
- Fixed a bug with the `average` and `evalb_bracketing_score` metrics in distributed training.
### Added
- `Predictor.capture_model_internals()` now accepts a regex specifying which modules to capture.
## [v1.1.0rc4](https://github.com/allenai/allennlp/releases/tag/v1.1.0rc4) - 2020-08-20
### Added
- Added a workflow to GitHub Actions that will automatically close unassigned stale issues and
ping the assignees of assigned stale issues.
### Fixed
- Fixed a bug in distributed metrics that caused nan values due to repeated addition of an accumulated variable.
## [v1.1.0rc3](https://github.com/allenai/allennlp/releases/tag/v1.1.0rc3) - 2020-08-12
### Fixed
- Fixed how truncation was handled with `PretrainedTransformerTokenizer`.
Previously, if `max_length` was set to `None`, the tokenizer would still do truncation if the
transformer model had a default max length in its config.
Also, when `max_length` was set to a non-`None` value, several warnings would appear
for certain transformer models around the use of the `truncation` parameter.
- Fixed evaluation of all metrics when using distributed training.
- Added a `py.typed` marker. Fixed type annotations in `allennlp.training.util`.
- Fixed problem with automatically detecting whether tokenization is necessary.
This affected primarily the Roberta SST model.
- Improved help text for using the --overrides command line flag.
## [v1.1.0rc2](https://github.com/allenai/allennlp/releases/tag/v1.1.0rc2) - 2020-07-31
### Changed
- Upgraded PyTorch requirement to 1.6.
- Replaced the NVIDIA Apex AMP module with torch's native AMP module. The default trainer (`GradientDescentTrainer`)
now takes a `use_amp: bool` parameter instead of the old `opt_level: str` parameter.
### Fixed
- Removed unnecessary warning about deadlocks in `DataLoader`.
- Fixed testing models that only return a loss when they are in training mode.
- Fixed a bug in `FromParams` that caused silent failure in case of the parameter type being `Optional[Union[...]]`.
- Fixed a bug where the program crashes if `evaluation_data_loader` is a `AllennlpLazyDataset`.
### Added
- Added the option to specify `requires_grad: false` within an optimizer's parameter groups.
- Added the `file-friendly-logging` flag back to the `train` command. Also added this flag to the `predict`, `evaluate`, and `find-learning-rate` commands.
- Added an `EpochCallback` to track current epoch as a model class member.
- Added the option to enable or disable gradient checkpointing for transformer token embedders via boolean parameter `gradient_checkpointing`.
### Removed
- Removed the `opt_level` parameter to `Model.load` and `load_archive`. In order to use AMP with a loaded
model now, just run the model's forward pass within torch's [`autocast`](https://pytorch.org/docs/stable/amp.html#torch.cuda.amp.autocast)
context.
## [v1.1.0rc1](https://github.com/allenai/allennlp/releases/tag/v1.1.0rc1) - 2020-07-14
### Fixed
- Reduced the amount of log messages produced by `allennlp.common.file_utils`.
- Fixed a bug where `PretrainedTransformerEmbedder` parameters appeared to be trainable
in the log output even when `train_parameters` was set to `False`.
- Fixed a bug with the sharded dataset reader where it would only read a fraction of the instances
in distributed training.
- Fixed checking equality of `TensorField`s.
- Fixed a bug where `NamespaceSwappingField` did not work correctly with `.empty_field()`.
- Put more sensible defaults on the `huggingface_adamw` optimizer.
- Simplified logging so that all logging output always goes to one file.
- Fixed interaction with the python command line debugger.
- Log the grad norm properly even when we're not clipping it.
- Fixed a bug where `PretrainedModelInitializer` fails to initialize a model with a 0-dim tensor
- Fixed a bug with the layer unfreezing schedule of the `SlantedTriangular` learning rate scheduler.
- Fixed a regression with logging in the distributed setting. Only the main worker should write log output to the terminal.
- Pinned the version of boto3 for package managers (e.g. poetry).
- Fixed issue #4330 by updating the `tokenizers` dependency.
- Fixed a bug in `TextClassificationPredictor` so that it passes tokenized inputs to the `DatasetReader`
in case it does not have a tokenizer.
- `reg_loss` is only now returned for models that have some regularization penalty configured.
- Fixed a bug that prevented `cached_path` from downloading assets from GitHub releases.
- Fixed a bug that erroneously increased last label's false positive count in calculating fbeta metrics.
- `Tqdm` output now looks much better when the output is being piped or redirected.
- Small improvements to how the API documentation is rendered.
- Only show validation progress bar from main process in distributed training.
### Added
- Adjust beam search to support multi-layer decoder.
- A method to ModelTestCase for running basic model tests when you aren't using config files.
- Added some convenience methods for reading files.
- Added an option to `file_utils.cached_path` to automatically extract archives.
- Added the ability to pass an archive file instead of a local directory to `Vocab.from_files`.
- Added the ability to pass an archive file instead of a glob to `ShardedDatasetReader`.
- Added a new `"linear_with_warmup"` learning rate scheduler.
- Added a check in `ShardedDatasetReader` that ensures the base reader doesn't implement manual
distributed sharding itself.
- Added an option to `PretrainedTransformerEmbedder` and `PretrainedTransformerMismatchedEmbedder` to use a
scalar mix of all hidden layers from the transformer model instead of just the last layer. To utilize
this, just set `last_layer_only` to `False`.
- `cached_path()` can now read files inside of archives.
- Training metrics now include `batch_loss` and `batch_reg_loss` in addition to aggregate loss across number of batches.
### Changed
- Not specifying a `cuda_device` now automatically determines whether to use a GPU or not.
- Discovered plugins are logged so you can see what was loaded.
- `allennlp.data.DataLoader` is now an abstract registrable class. The default implementation
remains the same, but was renamed to `allennlp.data.PyTorchDataLoader`.
- `BertPooler` can now unwrap and re-wrap extra dimensions if necessary.
- New `transformers` dependency. Only version >=3.0 now supported.
## [v1.0.0](https://github.com/allenai/allennlp/releases/tag/v1.0.0) - 2020-06-16
### Fixed
- Lazy dataset readers now work correctly with multi-process data loading.
- Fixed race conditions that could occur when using a dataset cache.
### Added
- A bug where where all datasets would be loaded for vocab creation even if not needed.
- A parameter to the `DatasetReader` class: `manual_multi_process_sharding`. This is similar
to the `manual_distributed_sharding` parameter, but applies when using a multi-process
`DataLoader`.
## [v1.0.0rc6](https://github.com/allenai/allennlp/releases/tag/v1.0.0rc6) - 2020-06-11
### Fixed
- A bug where `TextField`s could not be duplicated since some tokenizers cannot be deep-copied.
See https://github.com/allenai/allennlp/issues/4270.
- Our caching mechanism had the potential to introduce race conditions if multiple processes
were attempting to cache the same file at once. This was fixed by using a lock file tied to each
cached file.
- `get_text_field_mask()` now supports padding indices that are not `0`.
- A bug where `predictor.get_gradients()` would return an empty dictionary if an embedding layer had trainable set to false
- Fixes `PretrainedTransformerMismatchedIndexer` in the case where a token consists of zero word pieces.
- Fixes a bug when using a lazy dataset reader that results in a `UserWarning` from PyTorch being printed at
every iteration during training.
- Predictor names were inconsistently switching between dashes and underscores. Now they all use underscores.
- `Predictor.from_path` now automatically loads plugins (unless you specify `load_plugins=False`) so
that you don't have to manually import a bunch of modules when instantiating predictors from
an archive path.
- `allennlp-server` automatically found as a plugin once again.
### Added
- A `duplicate()` method on `Instance`s and `Field`s, to be used instead of `copy.deepcopy()`
- A batch sampler that makes sure each batch contains approximately the same number of tokens (`MaxTokensBatchSampler`)
- Functions to turn a sequence of token indices back into tokens
- The ability to use Huggingface encoder/decoder models as token embedders
- Improvements to beam search
- ROUGE metric
- Polynomial decay learning rate scheduler
- A `BatchCallback` for logging CPU and GPU memory usage to tensorboard. This is mainly for debugging
because using it can cause a significant slowdown in training.
- Ability to run pretrained transformers as an embedder without training the weights
- Add Optuna Integrated badge to README.md
### Changed
- Similar to our caching mechanism, we introduced a lock file to the vocab to avoid race
conditions when saving/loading the vocab from/to the same serialization directory in different processes.
- Changed the `Token`, `Instance`, and `Batch` classes along with all `Field` classes to "slots" classes. This dramatically reduces the size in memory of instances.
- SimpleTagger will no longer calculate span-based F1 metric when `calculate_span_f1` is `False`.
- CPU memory for every worker is now reported in the logs and the metrics. Previously this was only reporting the CPU memory of the master process, and so it was only
correct in the non-distributed setting.
- To be consistent with PyTorch `IterableDataset`, `AllennlpLazyDataset` no longer implements `__len__()`.
Previously it would always return 1.
- Removed old tutorials, in favor of [the new AllenNLP Guide](https://guide.allennlp.org)
- Changed the vocabulary loading to consider new lines for Windows/Linux and Mac.
## [v1.0.0rc5](https://github.com/allenai/allennlp/releases/tag/v1.0.0rc5) - 2020-05-26
### Fixed
- Fix bug where `PretrainedTransformerTokenizer` crashed with some transformers (#4267)
- Make `cached_path` work offline.
- Tons of docstring inconsistencies resolved.
- Nightly builds no longer run on forks.
- Distributed training now automatically figures out which worker should see which instances
- A race condition bug in distributed training caused from saving the vocab to file from the master process while other processing might be reading those files.
- Unused dependencies in `setup.py` removed.
### Added
- Additional CI checks to ensure docstrings are consistently formatted.
- Ability to train on CPU with multiple processes by setting `cuda_devices` to a list of negative integers in your training config. For example: `"distributed": {"cuda_devices": [-1, -1]}`. This is mainly to make it easier to test and debug distributed training code..
- Documentation for when parameters don't need config file entries.
### Changed
- The `allennlp test-install` command now just ensures the core submodules can
be imported successfully, and prints out some other useful information such as the version, PyTorch version,
and the number of GPU devices available.
- All of the tests moved from `allennlp/tests` to `tests` at the root level, and
`allennlp/tests/fixtures` moved to `test_fixtures` at the root level. The PyPI source and wheel distributions will no longer include tests and fixtures.
## [v1.0.0rc4](https://github.com/allenai/allennlp/releases/tag/v1.0.0rc4) - 2020-05-14
We first introduced this `CHANGELOG` after release `v1.0.0rc4`, so please refer to the GitHub release
notes for this and earlier releases.
================================================
FILE: CITATION.cff
================================================
# YAML 1.2
---
authors:
-
affiliation: "Allen Institute for Artificial Intelligence"
family-names: Gardner
given-names: Matt
-
affiliation: "Allen Institute for Artificial Intelligence"
family-names: Grus
given-names: Joel
-
affiliation: "Allen Institute for Artificial Intelligence"
family-names: Neumann
given-names: Mark
-
affiliation: "Allen Institute for Artificial Intelligence"
family-names: Tafjord
given-names: Oyvind
-
affiliation: "Allen Institute for Artificial Intelligence"
family-names: Dasigi
given-names: Pradeep
-
affiliation: "Allen Institute for Artificial Intelligence"
family-names: Liu
given-names: Nelson
-
affiliation: "Allen Institute for Artificial Intelligence"
family-names: Peters
given-names: Matthew
-
affiliation: "Allen Institute for Artificial Intelligence"
family-names: Schmitz
given-names: Michael
-
affiliation: "Allen Institute for Artificial Intelligence"
family-names: Zettlemoyer
given-names: Luke
cff-version: "1.2.0"
license: "Apache-2.0"
message: "If you use this software, please cite it using this metadata."
repository-code: "https://github.com/allenai/allennlp"
title: "AllenNLP: A Deep Semantic Natural Language Processing Platform"
version: "2.10.1"
doi: "10.18653/v1/W18-2501"
...
================================================
FILE: CONTRIBUTING.md
================================================
# Contributing
Thanks for considering contributing! We want AllenNLP to be *the way* to do cutting-edge NLP research, but we cannot
get there without community support.
## How Can I Contribute?
### Bug fixes and new features
**Did you find a bug?**
First, do [a quick search](https://github.com/allenai/allennlp/issues) to see whether your issue has already been reported.
If your issue has already been reported, please comment on the existing issue.
Otherwise, open [a new GitHub issue](https://github.com/allenai/allennlp/issues). Be sure to include a clear title
and description. The description should include as much relevant information as possible. The description should
explain how to reproduce the erroneous behavior as well as the behavior you expect to see. Ideally you would include a
code sample or an executable test case demonstrating the expected behavior.
**Do you have a suggestion for an enhancement?**
We use GitHub issues to track enhancement requests. Before you create an enhancement request:
* Make sure you have a clear idea of the enhancement you would like. If you have a vague idea, consider discussing
it first on a GitHub issue.
* Check the documentation to make sure your feature does not already exist.
* Do [a quick search](https://github.com/allenai/allennlp/issues) to see whether your enhancement has already been suggested.
When creating your enhancement request, please:
* Provide a clear title and description.
* Explain why the enhancement would be useful. It may be helpful to highlight the feature in other libraries.
* Include code examples to demonstrate how the enhancement would be used.
### Making a pull request
When you're ready to contribute code to address an open issue, please follow these guidelines to help us be able to review your pull request (PR) quickly.
1. **Initial setup** (only do this once)
<details><summary>Expand details 👇</summary><br/>
If you haven't already done so, please [fork](https://help.github.com/en/enterprise/2.13/user/articles/fork-a-repo) this repository on GitHub.
Then clone your fork locally with
git clone https://github.com/USERNAME/allennlp.git
or
git clone git@github.com:USERNAME/allennlp.git
At this point the local clone of your fork only knows that it came from *your* repo, github.com/USERNAME/allennlp.git, but doesn't know anything the *main* repo, [https://github.com/allenai/allennlp.git](https://github.com/allenai/allennlp). You can see this by running
git remote -v
which will output something like this:
origin https://github.com/USERNAME/allennlp.git (fetch)
origin https://github.com/USERNAME/allennlp.git (push)
This means that your local clone can only track changes from your fork, but not from the main repo, and so you won't be able to keep your fork up-to-date with the main repo over time. Therefor you'll need to add another "remote" to your clone that points to [https://github.com/allenai/allennlp.git](https://github.com/allenai/allennlp). To do this, run the following:
git remote add upstream https://github.com/allenai/allennlp.git
Now if you do `git remote -v` again, you'll see
origin https://github.com/USERNAME/allennlp.git (fetch)
origin https://github.com/USERNAME/allennlp.git (push)
upstream https://github.com/allenai/allennlp.git (fetch)
upstream https://github.com/allenai/allennlp.git (push)
Finally, you'll need to create a Python 3 virtual environment suitable for working on AllenNLP. There a number of tools out there that making working with virtual environments easier, but the most direct way is with the [`venv` module](https://docs.python.org/3.7/library/venv.html) in the standard library.
Once your virtual environment is activated, you can install your local clone in "editable mode" with
pip install -U pip setuptools wheel
pip install -e .[dev,all]
The "editable mode" comes from the `-e` argument to `pip`, and essential just creates a symbolic link from the site-packages directory of your virtual environment to the source code in your local clone. That way any changes you make will be immediately reflected in your virtual environment.
</details>
2. **Ensure your fork is up-to-date**
<details><summary>Expand details 👇</summary><br/>
Once you've added an "upstream" remote pointing to [https://github.com/allenai/allennlp.git](https://github.com/allenai/allennlp), keeping your fork up-to-date is easy:
git checkout main # if not already on main
git pull --rebase upstream main
git push
</details>
3. **Create a new branch to work on your fix or enhancement**
<details><summary>Expand details 👇</summary><br/>
Commiting directly to the main branch of your fork is not recommended. It will be easier to keep your fork clean if you work on a seperate branch for each contribution you intend to make.
You can create a new branch with
# replace BRANCH with whatever name you want to give it
git checkout -b BRANCH
git push -u origin BRANCH
</details>
4. **Test your changes**
<details><summary>Expand details 👇</summary><br/>
Our continuous integration (CI) testing runs [a number of checks](https://github.com/allenai/allennlp/actions?query=workflow%3APR) for each pull request on [GitHub Actions](https://github.com/features/actions). You can run most of these tests locally, which is something you should do *before* opening a PR to help speed up the review process and make it easier for us.
First, you should run [`black`](https://github.com/psf/black) to make sure you code is formatted consistently. Many IDEs support code formatters as plugins, so you may be able to setup black to run automatically everytime you save. [`black.vim`](https://github.com/psf/black/tree/master/plugin) will give you this functionality in Vim, for example. But `black` is also easy to run directly from the command line. Just run this from the root of your clone:
black .
Our CI also uses [`flake8`](https://github.com/allenai/allennlp/tree/main/tests) to lint the code base and [`mypy`](http://mypy-lang.org/) for type-checking. You should run both of these next with
flake8 .
and
make typecheck
We also strive to maintain high test coverage, so most contributions should include additions to [the unit tests](https://github.com/allenai/allennlp/tree/main/tests). These tests are run with [`pytest`](https://docs.pytest.org/en/latest/), which you can use to locally run any test modules that you've added or changed.
For example, if you've fixed a bug in `allennlp/nn/util.py`, you can run the tests specific to that module with
pytest -v tests/nn/util_test.py
Our CI will automatically check that test coverage stays above a certain threshold (around 90%). To check the coverage locally in this example, you could run
pytest -v --cov allennlp.nn.util tests/nn/util_test.py
If your contribution involves additions to any public part of the API, we require that you write docstrings
for each function, method, class, or module that you add.
See the [Writing docstrings](#writing-docstrings) section below for details on the syntax.
You should test to make sure the API documentation can build without errors by running
make build-docs
If the build fails, it's most likely due to small formatting issues. If the error message isn't clear, feel free to comment on this in your pull request.
You can also serve and view the docs locally with
make serve-docs
And finally, please update the [CHANGELOG](https://github.com/allenai/allennlp/blob/main/CHANGELOG.md) with notes on your contribution in the "Unreleased" section at the top.
After all of the above checks have passed, you can now open [a new GitHub pull request](https://github.com/allenai/allennlp/pulls).
Make sure you have a clear description of the problem and the solution, and include a link to relevant issues.
We look forward to reviewing your PR!
</details>
### Writing docstrings
Our docstrings are written in a syntax that is essentially just Markdown with additional special syntax for writing parameter descriptions.
Class docstrings should start with a description of the class, followed by a `# Parameters` section
that lists the names, types, and purpose of all parameters to the class's `__init__()` method.
Parameter descriptions should look like:
```
name : `type`
Description of the parameter, indented by four spaces.
```
Optional parameters can also be written like this:
```
name : `type`, optional (default = `default_value`)
Description of the parameter, indented by four spaces.
```
Sometimes you can omit the description if the parameter is self-explanatory.
Method and function docstrings are similar, but should also include a `# Returns`
section when the return value is not obvious. Other valid sections are
- `# Attributes`, for listing class attributes. These should be formatted in the same
way as parameters.
- `# Raises`, for listing any errors that the function or method might intentionally raise.
- `# Examples`, where you can include code snippets.
Here is an example of what the docstrings should look like in a class:
```python
class SentenceClassifier(Model):
"""
A model for classifying sentences.
This is based on [this paper](link-to-paper). The input is a sentence and
the output is a score for each target label.
# Parameters
vocab : `Vocabulary`
text_field_embedder : `TextFieldEmbedder`
The text field embedder that will be used to create a representation of the
source tokens.
seq2vec_encoder : `Seq2VeqEncoder`
This encoder will take the embeddings from the `text_field_embedder` and
encode them into a vector which represents the un-normalized scores
for the target labels.
dropout : `Optional[float]`, optional (default = `None`)
Optional dropout to apply to the text field embeddings before passing through
the `seq2vec_encoder`.
"""
def __init__(
self,
vocab: Vocabulary,
text_field_embedder: TextFieldEmbedder,
seq2vec_encoder: Seq2SeqEncoder,
dropout: Optional[float] = None,
) -> None:
pass
def forward(
self,
tokens: TextFieldTensors,
labels: Optional[Tensor] = None,
) -> Dict[str, Tensor]:
"""
Runs a forward pass of the model, computing the predicted logits and also the loss
when `labels` is provided.
# Parameters
tokens : `TextFieldTensors`
The tokens corresponding to the source sequence.
labels : `Optional[Tensor]`, optional (default = `None`)
The target labels.
# Returns
`Dict[str, Tensor]`
An output dictionary with keys for the `loss` and `logits`.
"""
pass
```
### New models
**Do you have a new state-of-the-art model?**
We are always looking for new models to add to our collection. The most popular models are usually added to the official [AllenNLP Models](https://github.com/allenai/allennlp-models) repository, and in some cases to the [AllenNLP Demo](https://demo.allennlp.org/).
If you think your model should be part of AllenNLP Models, please [create a pull request](https://github.com/allenai/allennlp-models/pulls) in the models repo that includes:
* Any code changes needed to support your new model.
* A link to the model itself. Please do not check your model into the GitHub repository, but instead upload it in the
PR conversation or provide a link to it at an external location.
In the description of your PR, please clearly explain the task your model performs along with the relevant metrics on an established dataset.
================================================
FILE: Dockerfile
================================================
# This Dockerfile creates an environment suitable for downstream usage of AllenNLP.
# It's built from a wheel installation of allennlp using the base images from
# https://github.com/allenai/docker-images/pkgs/container/pytorch
ARG TORCH=1.12.0-cuda11.3-python3.8
FROM ghcr.io/allenai/pytorch:${TORCH}
WORKDIR /stage/allennlp
# Installing AllenNLP's dependencies is the most time-consuming part of building
# this Docker image, so we make use of layer caching here by adding the minimal files
# necessary to install the dependencies.
COPY allennlp/version.py allennlp/version.py
COPY setup.py .
COPY requirements.txt .
COPY dev-requirements.txt .
COPY constraints.txt .
RUN touch allennlp/__init__.py \
&& touch README.md \
&& pip install --no-cache-dir -c constraints.txt -e .[all]
# Now add the full package source and re-install just the package.
COPY allennlp allennlp
RUN pip install --no-cache-dir --no-deps -e .[all]
COPY Makefile .
RUN make download-extras
WORKDIR /app/
# Copy wrapper script to allow beaker to run resumable training workloads.
COPY scripts/ai2_internal/resumable_train.sh .
LABEL maintainer="allennlp-contact@allenai.org"
ENTRYPOINT ["allennlp"]
================================================
FILE: Dockerfile.test
================================================
# Used to build an image for running tests.
ARG TORCH=1.12.0-cuda11.3-python3.8
FROM ghcr.io/allenai/pytorch:${TORCH}
# These environment variables are helpful for debugging.
# See https://pytorch.org/docs/stable/distributed.html#common-environment-variables for more info.
ENV NCCL_DEBUG INFO
ENV NCCL_DEBUG_SUBSYS ALL
WORKDIR /stage/allennlp
# Installing AllenNLP's dependencies is the most time-consuming part of building
# this Docker image, so we make use of layer caching here by adding the minimal files
# necessary to install the dependencies.
COPY allennlp/version.py allennlp/version.py
COPY setup.py .
COPY dev-requirements.txt .
COPY constraints.txt .
RUN touch allennlp/__init__.py \
&& touch README.md \
&& pip install --no-cache-dir -c constraints.txt -e . -r dev-requirements.txt
# Now add the full package source and re-install just the package.
COPY . .
RUN pip install --no-cache-dir --no-deps -e .
ENTRYPOINT ["make"]
================================================
FILE: LICENSE
================================================
Apache License
Version 2.0, January 2004
https://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "{}"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright {yyyy} {name of copyright owner}
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
================================================
FILE: MANIFEST.in
================================================
include LICENSE
include README.md
recursive-include allennlp *
recursive-include scripts *
global-exclude .DS_Store *.py[cod]
prune **/__pycache__
prune **/.mypy_cache
================================================
FILE: Makefile
================================================
SRC = allennlp
MD_DOCS_ROOT = docs/
MD_DOCS_API_ROOT = $(MD_DOCS_ROOT)api/
MD_DOCS_SRC = $(filter-out $(SRC)/__main__.py %/__init__.py $(SRC)/version.py,$(shell find $(SRC) -type f -name '*.py' | grep -v -E 'tests/'))
MD_DOCS = $(subst .py,.md,$(subst $(SRC)/,$(MD_DOCS_API_ROOT),$(MD_DOCS_SRC)))
MD_DOCS_CMD = python scripts/py2md.py
MD_DOCS_CONF = mkdocs.yml
MD_DOCS_CONF_SRC = mkdocs-skeleton.yml
MD_DOCS_TGT = site/
MD_DOCS_EXTRAS = $(addprefix $(MD_DOCS_ROOT),README.md CHANGELOG.md CONTRIBUTING.md)
TORCH_INSTALL = pip install torch torchvision -c constraints.txt
DOCKER_TORCH_VERSION = 1.12.0-cuda11.3-python3.8
DOCKER_TEST_TORCH_VERSION = 1.12.0-cuda11.3-python3.8
DOCKER_TAG = latest
DOCKER_IMAGE_NAME = allennlp/allennlp:$(DOCKER_TAG)
DOCKER_TEST_IMAGE_NAME = allennlp/test:$(DOCKER_TAG)
DOCKER_RUN_CMD = docker run --rm \
-v $$HOME/.allennlp:/root/.allennlp \
-v $$HOME/.cache/huggingface:/root/.cache/huggingface \
-v $$HOME/nltk_data:/root/nltk_data
# These nltk packages are used by the 'checklist' module. They are downloaded automatically
# if not found when `checklist` is imported, but it's good to download the ahead of time
# to avoid potential race conditions.
NLTK_DOWNLOAD_CMD = python -c 'import nltk; [nltk.download(p) for p in ("wordnet", "wordnet_ic", "sentiwordnet", "omw", "omw-1.4")]'
ifeq ($(shell uname),Darwin)
ifeq ($(shell which gsed),)
$(error Please install GNU sed with 'brew install gnu-sed')
else
SED = gsed
endif
else
SED = sed
endif
.PHONY : version
version :
@python -c 'from allennlp.version import VERSION; print(f"AllenNLP v{VERSION}")'
.PHONY : check-for-cuda
check-for-cuda :
@python -c 'import torch; assert torch.cuda.is_available(); print("Cuda is available")'
#
# Testing helpers.
#
.PHONY : flake8
flake8 :
flake8 allennlp tests scripts
.PHONY : format
format :
black --check allennlp tests scripts
.PHONY : typecheck
typecheck :
mypy allennlp tests scripts --cache-dir=/dev/null
.PHONY : test
test :
pytest --color=yes -v -rf --durations=40 \
--cov-config=.coveragerc \
--cov=$(SRC) \
--cov-report=xml
.PHONY : test-without-checklist
test-without-checklist :
pytest --color=yes -v -rf --durations=40 \
--cov-config=.coveragerc \
--cov=$(SRC) \
--cov-report=xml \
--ignore-glob=*checklist*
.PHONY : test-checklist
test-checklist :
pytest --color=yes -v -rf --durations=40 \
--cov-config=.coveragerc \
--cov=$(SRC) \
--cov-report=xml \
tests/ \
-k checklist
.PHONY : gpu-tests
gpu-tests : check-for-cuda
pytest --color=yes -v -rf --durations=20 \
--cov-config=.coveragerc \
--cov=$(SRC) \
--cov-report=xml \
-m gpu
.PHONY : benchmarks
benchmarks :
pytest -c benchmarks/pytest.ini benchmarks/
#
# Setup helpers
#
.PHONY : download-extras
download-extras :
$(NLTK_DOWNLOAD_CMD)
.PHONY : install
install :
# Due to a weird thing with pip, we may need egg-info before running `pip install -e`.
# See https://github.com/pypa/pip/issues/4537.
# python setup.py install_egg_info
# Install torch ecosystem first.
$(TORCH_INSTALL)
pip install --upgrade pip
pip install pip-tools
pip-compile requirements.in -o final_requirements.txt --allow-unsafe --rebuild --verbose
pip install -e . -r final_requirements.txt
# These nltk packages are used by the 'checklist' module.
$(NLTK_DOWNLOAD_CMD)
#
# Documention helpers.
#
.PHONY : build-all-api-docs
build-all-api-docs :
@$(MD_DOCS_CMD) $(subst /,.,$(subst .py,,$(MD_DOCS_SRC))) -o $(MD_DOCS)
.PHONY : build-docs
build-docs : build-all-api-docs $(MD_DOCS_CONF) $(MD_DOCS) $(MD_DOCS_EXTRAS)
mkdocs build
.PHONY : serve-docs
serve-docs : build-all-api-docs $(MD_DOCS_CONF) $(MD_DOCS) $(MD_DOCS_EXTRAS)
mkdocs serve --dirtyreload
.PHONY : update-docs
update-docs : $(MD_DOCS) $(MD_DOCS_EXTRAS)
$(MD_DOCS_ROOT)README.md : README.md
cp $< $@
# Alter the relative path of the README image for the docs.
$(SED) -i '1s/docs/./' $@
# Alter external doc links to relative links.
$(SED) -i 's|https://docs.allennlp.org/main/api/|api/|' $@
$(MD_DOCS_ROOT)%.md : %.md
cp $< $@
$(MD_DOCS_CONF) : $(MD_DOCS_CONF_SRC) $(MD_DOCS)
python scripts/build_docs_config.py $@ $(MD_DOCS_CONF_SRC) $(MD_DOCS_ROOT) $(MD_DOCS_API_ROOT)
$(MD_DOCS_API_ROOT)%.md : $(SRC)/%.py scripts/py2md.py
mkdir -p $(shell dirname $@)
$(MD_DOCS_CMD) $(subst /,.,$(subst .py,,$<)) --out $@
.PHONY : clean
clean :
rm -rf $(MD_DOCS_TGT)
rm -rf $(MD_DOCS_API_ROOT)
rm -f $(MD_DOCS_ROOT)*.md
rm -rf .pytest_cache/
rm -rf allennlp.egg-info/
rm -rf dist/
rm -rf build/
find . | grep -E '(\.mypy_cache|__pycache__|\.pyc|\.pyo$$)' | xargs rm -rf
#
# Docker helpers.
#
.PHONY : docker-image
docker-image :
docker build \
--pull \
-f Dockerfile \
--build-arg TORCH=$(DOCKER_TORCH_VERSION) \
-t $(DOCKER_IMAGE_NAME) .
DOCKER_GPUS = --gpus all
.PHONY : docker-run
docker-run :
$(DOCKER_RUN_CMD) $(DOCKER_GPUS) $(DOCKER_IMAGE_NAME) $(ARGS)
.PHONY : docker-test-image
docker-test-image :
docker build \
--pull \
-f Dockerfile.test \
--build-arg TORCH=$(DOCKER_TEST_TORCH_VERSION) \
-t $(DOCKER_TEST_IMAGE_NAME) .
.PHONY : docker-test-run
docker-test-run :
$(DOCKER_RUN_CMD) --shm-size 2G $(DOCKER_GPUS) $(DOCKER_TEST_IMAGE_NAME) $(ARGS)
================================================
FILE: README.md
================================================
<div align="center">
<br>
<img src="https://raw.githubusercontent.com/allenai/allennlp/main/docs/img/allennlp-logo-dark.png" width="400"/>
<p>
An Apache 2.0 NLP research library, built on PyTorch, for developing state-of-the-art deep learning models on a wide variety of linguistic tasks.
</p>
<hr/>
</div>
<p align="center">
<a href="https://github.com/allenai/allennlp/actions">
<img alt="CI" src="https://github.com/allenai/allennlp/workflows/CI/badge.svg?event=push&branch=main">
</a>
<a href="https://pypi.org/project/allennlp/">
<img alt="PyPI" src="https://img.shields.io/pypi/v/allennlp">
</a>
<a href="https://github.com/allenai/allennlp/blob/main/LICENSE">
<img alt="License" src="https://img.shields.io/github/license/allenai/allennlp.svg?color=blue&cachedrop">
</a>
<a href="https://codecov.io/gh/allenai/allennlp">
<img alt="Codecov" src="https://codecov.io/gh/allenai/allennlp/branch/main/graph/badge.svg">
</a>
<a href="https://optuna.org">
<img alt="Optuna" src="https://img.shields.io/badge/Optuna-integrated-blue">
</a>
<br/>
</p>
⚠️ **NOTICE:** The AllenNLP library is now in maintenance mode. That means we are no longer adding new features or upgrading dependencies. We will still respond to questions and address bugs as they arise up until December 16th, 2022. If you have any concerns or are interested in maintaining AllenNLP going forward, please open an issue on this repository.
AllenNLP has been a big success, but as the field is advancing quickly it's time to focus on new initiatives. We're working hard to make [AI2 Tango](https://github.com/allenai/tango) the best way to organize research codebases. If you are an active user of AllenNLP, here are some suggested alternatives:
* If you like the trainer, the configuration language, or are simply looking for a better way to manage your experiments, check out [AI2 Tango](https://github.com/allenai/tango).
* If you like AllenNLP's `modules` and `nn` packages, check out [delmaksym/allennlp-light](https://github.com/delmaksym/allennlp-light). It's even compatible with [AI2 Tango](https://github.com/allenai/tango)!
* If you like the framework aspect of AllenNLP, check out [flair](https://github.com/flairNLP/flair). It has multiple state-of-art NLP models and allows you to easily use pretrained embeddings such as those from transformers.
* If you like the AllenNLP metrics package, check out [torchmetrics](https://torchmetrics.readthedocs.io/en/stable/). It has the same API as AllenNLP, so it should be a quick learning curve to make the switch.
* If you want to vectorize text, try [the transformers library](https://github.com/huggingface/transformers).
* If you want to maintain the AllenNLP Fairness or Interpret components, please get in touch. There is no alternative to it, so we are looking for a dedicated maintainer.
* If you are concerned about other AllenNLP functionality, please create an issue. Maybe we can find another way to continue supporting your use case.
## Quick Links
- ↗️ [Website](https://allennlp.org/)
- 🔦 [Guide](https://guide.allennlp.org/)
- 🖼 [Gallery](https://gallery.allennlp.org)
- 💻 [Demo](https://demo.allennlp.org)
- 📓 [Documentation](https://docs.allennlp.org/) ( [latest](https://docs.allennlp.org/latest/) | [stable](https://docs.allennlp.org/stable/) | [commit](https://docs.allennlp.org/main/) )
- ⬆️ [Upgrade Guide from 1.x to 2.0](https://github.com/allenai/allennlp/discussions/4933)
- ❓ [Stack Overflow](https://stackoverflow.com/questions/tagged/allennlp)
- ✋ [Contributing Guidelines](CONTRIBUTING.md)
- 🤖 [Officially Supported Models](https://github.com/allenai/allennlp-models)
- [Pretrained Models](https://github.com/allenai/allennlp-models/blob/main/allennlp_models/pretrained.py)
- [Documentation](https://docs.allennlp.org/models/) ( [latest](https://docs.allennlp.org/models/latest/) | [stable](https://docs.allennlp.org/models/stable/) | [commit](https://docs.allennlp.org/models/main/) )
- ⚙️ [Continuous Build](https://github.com/allenai/allennlp/actions)
- 🌙 [Nightly Releases](https://pypi.org/project/allennlp/#history)
## In this README
- [Getting Started Using the Library](#getting-started-using-the-library)
- [Plugins](#plugins)
- [Package Overview](#package-overview)
- [Installation](#installation)
- [Installing via pip](#installing-via-pip)
- [Installing using Docker](#installing-using-docker)
- [Installing from source](#installing-from-source)
- [Running AllenNLP](#running-allennlp)
- [Issues](#issues)
- [Contributions](#contributions)
- [Citing](#citing)
- [Team](#team)
## Getting Started Using the Library
If you're interested in using AllenNLP for model development, we recommend you check out the
[AllenNLP Guide](https://guide.allennlp.org) for a thorough introduction to the library, followed by our more advanced guides
on [GitHub Discussions](https://github.com/allenai/allennlp/discussions/categories/guides).
When you're ready to start your project, we've created a couple of template repositories that you can use as a starting place:
* If you want to use `allennlp train` and config files to specify experiments, use [this
template](https://github.com/allenai/allennlp-template-config-files). We recommend this approach.
* If you'd prefer to use python code to configure your experiments and run your training loop, use
[this template](https://github.com/allenai/allennlp-template-python-script). There are a few
things that are currently a little harder in this setup (loading a saved model, and using
distributed training), but otherwise it's functionality equivalent to the config files
setup.
In addition, there are external tutorials:
* [Hyperparameter optimization for AllenNLP using Optuna](https://medium.com/optuna/hyperparameter-optimization-for-allennlp-using-optuna-54b4bfecd78b)
* [Training with multiple GPUs in AllenNLP](https://medium.com/ai2-blog/tutorial-how-to-train-with-multiple-gpus-in-allennlp-c4d7c17eb6d6)
* [Training on larger batches with less memory in AllenNLP](https://medium.com/ai2-blog/tutorial-training-on-larger-batches-with-less-memory-in-allennlp-1cd2047d92ad)
* [How to upload transformer weights and tokenizers from AllenNLP to HuggingFace](https://medium.com/ai2-blog/tutorial-how-to-upload-transformer-weights-and-tokenizers-from-allennlp-to-huggingface-ecf6c0249bf)
And others on the [AI2 AllenNLP blog](https://medium.com/ai2-blog/allennlp/home).
## Plugins
AllenNLP supports loading "plugins" dynamically. A plugin is just a Python package that
provides custom registered classes or additional `allennlp` subcommands.
There is ecosystem of open source plugins, some of which are maintained by the AllenNLP
team here at AI2, and some of which are maintained by the broader community.
<table>
<tr>
<td><b> Plugin </b></td>
<td><b> Maintainer </b></td>
<td><b> CLI </b></td>
<td><b> Description </b></td>
</tr>
<tr>
<td> <a href="https://github.com/allenai/allennlp-models"><b>allennlp-models</b></a> </td>
<td> AI2 </td>
<td> No </td>
<td> A collection of state-of-the-art models </td>
</tr>
<tr>
<td> <a href="https://github.com/allenai/allennlp-semparse"><b>allennlp-semparse</b></a> </td>
<td> AI2 </td>
<td> No </td>
<td> A framework for building semantic parsers </td>
</tr>
<tr>
<td> <a href="https://github.com/allenai/allennlp-server"><b>allennlp-server</b></a> </td>
<td> AI2 </td>
<td> Yes </td>
<td> A simple demo server for serving models </td>
</tr>
<tr>
<td> <a href="https://github.com/himkt/allennlp-optuna"><b>allennlp-optuna</b></a> </td>
<td> <a href="https://himkt.github.io/profile/">Makoto Hiramatsu</a> </td>
<td> Yes </td>
<td> <a href="https://optuna.org/">Optuna</a> integration for hyperparameter optimization </td>
</tr>
</table>
AllenNLP will automatically find any official AI2-maintained plugins that you have installed,
but for AllenNLP to find personal or third-party plugins you've installed,
you also have to create either a local plugins file named `.allennlp_plugins`
in the directory where you run the `allennlp` command, or a global plugins file at `~/.allennlp/plugins`.
The file should list the plugin modules that you want to be loaded, one per line.
To test that your plugins can be found and imported by AllenNLP, you can run the `allennlp test-install` command.
Each discovered plugin will be logged to the terminal.
For more information about plugins, see the [plugins API docs](https://docs.allennlp.org/main/api/common/plugins/). And for information on how to create a custom subcommand
to distribute as a plugin, see the [subcommand API docs](https://docs.allennlp.org/main/api/commands/subcommand/).
## Package Overview
<table>
<tr>
<td><b> allennlp </b></td>
<td> An open-source NLP research library, built on PyTorch </td>
</tr>
<tr>
<td><b> allennlp.commands </b></td>
<td> Functionality for the CLI </td>
</tr>
<tr>
<td><b> allennlp.common </b></td>
<td> Utility modules that are used across the library </td>
</tr>
<tr>
<td><b> allennlp.data </b></td>
<td> A data processing module for loading datasets and encoding strings as integers for representation in matrices </td>
</tr>
<tr>
<td><b> allennlp.fairness </b></td>
<td> A module for bias mitigation and fairness algorithms and metrics </td>
</tr>
<tr>
<td><b> allennlp.modules </b></td>
<td> A collection of PyTorch modules for use with text </td>
</tr>
<tr>
<td><b> allennlp.nn </b></td>
<td> Tensor utility functions, such as initializers and activation functions </td>
</tr>
<tr>
<td><b> allennlp.training </b></td>
<td> Functionality for training models </td>
</tr>
</table>
## Installation
AllenNLP requires Python 3.6.1 or later and [PyTorch](https://pytorch.org/).
We support AllenNLP on Mac and Linux environments. We presently do not support Windows but are open to contributions.
### Installing via conda-forge
The simplest way to install AllenNLP is using conda (you can choose a different python version):
```
conda install -c conda-forge python=3.8 allennlp
```
To install optional packages, such as `checklist`, use
```
conda install -c conda-forge allennlp-checklist
```
or simply install `allennlp-all` directly. The plugins mentioned above are similarly installable, e.g.
```
conda install -c conda-forge allennlp-models allennlp-semparse allennlp-server allennlp-optuna
```
### Installing via pip
It's recommended that you install the PyTorch ecosystem **before** installing AllenNLP by following the instructions on [pytorch.org](https://pytorch.org/).
After that, just run `pip install allennlp`.
> ⚠️ If you're using Python 3.7 or greater, you should ensure that you don't have the PyPI version of `dataclasses` installed after running the above command, as this could cause issues on certain platforms. You can quickly check this by running `pip freeze | grep dataclasses`. If you see something like `dataclasses=0.6` in the output, then just run `pip uninstall -y dataclasses`.
If you need pointers on setting up an appropriate Python environment or would like to install AllenNLP using a different method, see below.
#### Setting up a virtual environment
[Conda](https://conda.io/) can be used set up a virtual environment with the
version of Python required for AllenNLP. If you already have a Python 3
environment you want to use, you can skip to the 'installing via pip' section.
1. [Download and install Conda](https://conda.io/projects/conda/en/latest/user-guide/install/index.html).
2. Create a Conda environment with Python 3.8 (3.7 or 3.9 would work as well):
```
conda create -n allennlp_env python=3.8
```
3. Activate the Conda environment. You will need to activate the Conda environment in each terminal in which you want to use AllenNLP:
```
conda activate allennlp_env
```
#### Installing the library and dependencies
Installing the library and dependencies is simple using `pip`.
```bash
pip install allennlp
```
To install the optional dependencies, such as `checklist`, run
```bash
pip install allennlp[checklist]
```
Or you can just install all optional dependencies with `pip install allennlp[all]`.
*Looking for bleeding edge features? You can install nightly releases directly from [pypi](https://pypi.org/project/allennlp/#history)*
AllenNLP installs a script when you install the python package, so you can run allennlp commands just by typing `allennlp` into a terminal. For example, you can now test your installation with `allennlp test-install`.
You may also want to install `allennlp-models`, which contains the NLP constructs to train and run our officially
supported models, many of which are hosted at [https://demo.allennlp.org](https://demo.allennlp.org).
```bash
pip install allennlp-models
```
### Installing using Docker
Docker provides a virtual machine with everything set up to run AllenNLP--
whether you will leverage a GPU or just run on a CPU. Docker provides more
isolation and consistency, and also makes it easy to distribute your
environment to a compute cluster.
AllenNLP provides [official Docker images](https://hub.docker.com/r/allennlp/allennlp) with the library and all of its dependencies installed.
Once you have [installed Docker](https://docs.docker.com/engine/installation/),
you should also install the [NVIDIA Container Toolkit](https://github.com/NVIDIA/nvidia-docker)
if you have GPUs available.
Then run the following command to get an environment that will run on GPU:
```bash
mkdir -p $HOME/.allennlp/
docker run --rm --gpus all -v $HOME/.allennlp:/root/.allennlp allennlp/allennlp:latest
```
You can test the Docker environment with
```bash
docker run --rm --gpus all -v $HOME/.allennlp:/root/.allennlp allennlp/allennlp:latest test-install
```
If you don't have GPUs available, just omit the `--gpus all` flag.
#### Building your own Docker image
For various reasons you may need to create your own AllenNLP Docker image, such as if you need a different version
of PyTorch. To do so, just run `make docker-image` from the root of your local clone of AllenNLP.
By default this builds an image with the tag `allennlp/allennlp`, but you can change this to anything you want
by setting the `DOCKER_IMAGE_NAME` flag when you call `make`. For example,
`make docker-image DOCKER_IMAGE_NAME=my-allennlp`.
If you want to use a different version of Python or PyTorch, set the flags `DOCKER_PYTHON_VERSION` and `DOCKER_TORCH_VERSION` to something like
`3.9` and `1.9.0-cuda10.2`, respectively. These flags together determine the base image that is used. You can see the list of valid
combinations in this GitHub Container Registry: [github.com/allenai/docker-images/pkgs/container/pytorch](https://github.com/allenai/docker-images/pkgs/container/pytorch).
After building the image you should be able to see it listed by running `docker images allennlp`.
```
REPOSITORY TAG IMAGE ID CREATED SIZE
allennlp/allennlp latest b66aee6cb593 5 minutes ago 2.38GB
```
### Installing from source
You can also install AllenNLP by cloning our git repository:
```bash
git clone https://github.com/allenai/allennlp.git
```
Create a Python 3.7 or 3.8 virtual environment, and install AllenNLP in `editable` mode by running:
```bash
pip install -U pip setuptools wheel
pip install --editable .[dev,all]
```
This will make `allennlp` available on your system but it will use the sources from the local clone
you made of the source repository.
You can test your installation with `allennlp test-install`.
See [https://github.com/allenai/allennlp-models](https://github.com/allenai/allennlp-models)
for instructions on installing `allennlp-models` from source.
## Running AllenNLP
Once you've installed AllenNLP, you can run the command-line interface
with the `allennlp` command (whether you installed from `pip` or from source).
`allennlp` has various subcommands such as `train`, `evaluate`, and `predict`.
To see the full usage information, run `allennlp --help`.
You can test your installation by running `allennlp test-install`.
## Issues
Everyone is welcome to file issues with either feature requests, bug reports, or general questions. As a small team with our own internal goals, we may ask for contributions if a prompt fix doesn't fit into our roadmap. To keep things tidy we will often close issues we think are answered, but don't hesitate to follow up if further discussion is needed.
## Contributions
The AllenNLP team at AI2 ([@allenai](https://github.com/allenai)) welcomes contributions from the community.
If you're a first time contributor, we recommend you start by reading our [CONTRIBUTING.md](https://github.com/allenai/allennlp/blob/main/CONTRIBUTING.md) guide.
Then have a look at our issues with the tag [**`Good First Issue`**](https://github.com/allenai/allennlp/issues?q=is%3Aissue+is%3Aopen+label%3A%22Good+First+Issue%22).
If you would like to contribute a larger feature, we recommend first creating an issue with a proposed design for discussion. This will prevent you from spending significant time on an implementation which has a technical limitation someone could have pointed out early on. Small contributions can be made directly in a pull request.
Pull requests (PRs) must have one approving review and no requested changes before they are merged. As AllenNLP is primarily driven by AI2 we reserve the right to reject or revert contributions that we don't think are good additions.
## Citing
If you use AllenNLP in your research, please cite [AllenNLP: A Deep Semantic Natural Language Processing Platform](https://www.semanticscholar.org/paper/AllenNLP%3A-A-Deep-Semantic-Natural-Language-Platform-Gardner-Grus/a5502187140cdd98d76ae711973dbcdaf1fef46d).
```bibtex
@inproceedings{Gardner2017AllenNLP,
title={AllenNLP: A Deep Semantic Natural Language Processing Platform},
author={Matt Gardner and Joel Grus and Mark Neumann and Oyvind Tafjord
and Pradeep Dasigi and Nelson F. Liu and Matthew Peters and
Michael Schmitz and Luke S. Zettlemoyer},
year={2017},
Eprint = {arXiv:1803.07640},
}
```
## Team
AllenNLP is an open-source project backed by [the Allen Institute for Artificial Intelligence (AI2)](https://allenai.org/).
AI2 is a non-profit institute with the mission to contribute to humanity through high-impact AI research and engineering.
To learn more about who specifically contributed to this codebase, see [our contributors](https://github.com/allenai/allennlp/graphs/contributors) page.
================================================
FILE: RELEASE_PROCESS.md
================================================
# AllenNLP GitHub and PyPI Release Process
This document describes the procedure for releasing new versions of the core library.
> ❗️ This assumes you are using a clone of the main repo with the remote `origin` pointed
to `git@github.com:allenai/allennlp.git` (or the `HTTPS` equivalent).
## Steps
1. Set the environment variable `TAG`, which should be of the form `v{VERSION}`.
For example, if the version of the release is `1.0.0`, you should set `TAG` to `v1.0.0`:
```bash
export TAG='v1.0.0'
```
Or if you use `fish`:
```fish
set -x TAG 'v1.0.0'
```
2. Update `allennlp/version.py` with the correct version. Then check that the output of
```
python scripts/get_version.py current
```
matches the `TAG` environment variable.
3. Update the `CHANGELOG.md` so that everything under the "Unreleased" section is now under a section corresponding to this release.
4. Update the `CITATION.cff` file to refer to the right version.
4. Commit and push these changes with:
```
git commit -a -m "Prepare for release $TAG" && git push
```
5. Then add the tag in git to mark the release:
```
git tag $TAG -m "Release $TAG" && git push --tags
```
6. Find the tag you just pushed [on GitHub](https://github.com/allenai/allennlp/tags), click edit, then copy over the output of:
```
python scripts/release_notes.py
```
On a Mac, for example, you can just pipe the above command into `pbcopy`.
7. Check the box "This is a pre-release" if the release is a release candidate (ending with `rc*`). Otherwise leave it unchecked.
8. Click "Publish Release". GitHub Actions will then handle the rest, including publishing the package to PyPI the Docker image to Docker Hub.
9. After the [GitHub Actions workflow](https://github.com/allenai/allennlp/actions?query=workflow%3AMaster+event%3Arelease) finishes, follow the same process to publish a release for the `allennlp-models` repo.
## Fixing a failed release
If for some reason the GitHub Actions release workflow failed with an error that needs to be fixed, you'll have to delete both the tag and corresponding release from GitHub. After you've pushed a fix, delete the tag from your local clone with
```bash
git tag -l | xargs git tag -d && git fetch -t
```
Then repeat the steps above.
================================================
FILE: STYLE.md
================================================
# AllenNLP Style Guide
Our highest priority for code style is that our code be easily readable to someone new to the
codebase. Deep learning is easy to get wrong, and we want our code to be easy enough to read that
someone looking at it can be thinking about our modeling decisions, not trying to understand what
is going on.
To that end, we use descriptive names, we use type annotations, and we write coherent docstrings.
In code that manipulates tensors, most lines that compute a tensor have a comment describing the
tensor's shape. When there's an interesting or important modeling decision in the code, we write
a comment about it (either in-line or in an appropriate docstring).
## Docstrings
All reasonably complex public methods should have docstrings describing their basic function, their
inputs and their outputs. Private methods should also most often have docstrings, so that people
who read your code know what the method is supposed to do. The basic outline we use for docstrings
is: (1) a brief description of what the method does, sometimes also including how or why the method
does it, (2) the parameters / arguments to the method, (3) the return value of the method, if any.
If the method is particularly simple or the arguments are obvious, (2) and (3) can be omitted. Our
docs use Markdown formatting, so function arguments and return values should be formatted as Markdown
headers (e.g `# Parameters`), seen in just about any model or module in the codebase. We treat the
class docstring as the documentation for `__init__` methods, giving parameters there and omitting
any docstring on the constructor itself. For model / module constructors and methods like
`forward`, _always_ include the parameters and return values (when there is one) in the docstring.
## Code format
We use `flake8`, `black` and `mypy` to enforce some basic consistency in formatting. Those
formatting guidelines roughly follow [Google's python style
guide](https://google.github.io/styleguide/pyguide.html#Python_Style_Rules), with a few notable
exceptions. In particular, because we use type annotations and descriptive variable names, we use
100-character lines instead of 80-character lines, and it's ok to go over sometimes in code.
Additionally, we use `mkdocs` for building our docs, so Google's docstring formats don't apply.
## Naming
We follow Google's [general naming
rules](https://google.github.io/styleguide/cppguide.html#General_Naming_Rules), and their
[definition of camel case](https://google.github.io/styleguide/javaguide.html#s5.3-camel-case).
## Module layout and imports
To keep files from getting too big, we typically have one class per file, though small classes
that are inseparable from a companion class can also go in the same file (often these will be
private classes).
To avoid verbosity when importing classes structured this way, classes should be imported from
their module's `__init__.py`. For example, the `Batch` class is in `allennlp/data/batch.py`,
but `allennlp/data/__init__.py` imports the class, so that you can just do `from allennlp.data
import Batch`.
Abstract classes typically go in a module containing the abstract class and all built-in
implementations. This includes things like `Field` (in `allennlp.data.fields`), `Seq2SeqEncoder`
(in `allennlp.modules.seq2seq_encoders`), and many others. In these cases, the abstract class
should be imported into the module _above_, so that you can do, e.g., `from allennlp.data import
Field`. Concrete implementations follow the same layout as above: `from allennlp.data.fields
import TextField`.
Imports should be formatted at the top of the file, following [PEP 8's
recommendations](https://www.python.org/dev/peps/pep-0008/#imports): three sections (standard
library, third-party libraries, internal imports), each sorted and separated by a blank line.
## Conclusion
Some of the conventions we've adopted are arbitrary (e.g., other definitions of camel case are
also valid), but we stick to them to keep a consistent style throughout the codebase, which makes
it easier to read and maintain.
================================================
FILE: allennlp/__init__.py
================================================
# We get a lot of these spurious warnings,
# see https://github.com/ContinuumIO/anaconda-issues/issues/6678
import warnings # noqa
warnings.filterwarnings("ignore", message="numpy.dtype size changed")
warnings.filterwarnings("ignore", message="numpy.ufunc size changed")
try:
# On some systems this prevents the dreaded
# ImportError: dlopen: cannot load any more object with static TLS
import transformers, spacy, torch, numpy # noqa
except ModuleNotFoundError:
print(
"Using AllenNLP requires the python packages Spacy, "
"Pytorch and Numpy to be installed. Please see "
"https://github.com/allenai/allennlp for installation instructions."
)
raise
from allennlp.version import VERSION as __version__ # noqa
================================================
FILE: allennlp/__main__.py
================================================
#!/usr/bin/env python
import logging
import os
import sys
if os.environ.get("ALLENNLP_DEBUG"):
LEVEL = logging.DEBUG
else:
level_name = os.environ.get("ALLENNLP_LOG_LEVEL", "INFO")
LEVEL = logging._nameToLevel.get(level_name, logging.INFO)
sys.path.insert(0, os.path.dirname(os.path.abspath(os.path.join(__file__, os.pardir))))
logging.basicConfig(format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", level=LEVEL)
# filelock emits too many messages, so tell it to be quiet unless it has something
# important to say.
logging.getLogger("filelock").setLevel(logging.WARNING)
# transformers emits an annoying log message everytime it's imported, so we filter that
# one message out specifically.
def _transformers_log_filter(record):
if record.msg.startswith("PyTorch version"):
return False
return True
logging.getLogger("transformers.file_utils").addFilter(_transformers_log_filter)
def run():
from allennlp.commands import main # noqa
from allennlp.common.util import install_sigterm_handler
# We want to be able to catch SIGTERM signals in addition to SIGINT (keyboard interrupt).
install_sigterm_handler()
main(prog="allennlp")
if __name__ == "__main__":
run()
================================================
FILE: allennlp/commands/__init__.py
================================================
import argparse
import logging
import sys
from typing import Any, Optional, Tuple, Set
import warnings
from allennlp import __version__
from allennlp.commands.build_vocab import BuildVocab
from allennlp.commands.cached_path import CachedPath
from allennlp.commands.checklist import CheckList
from allennlp.commands.diff import Diff
from allennlp.commands.evaluate import Evaluate
from allennlp.commands.find_learning_rate import FindLearningRate
from allennlp.commands.predict import Predict
from allennlp.commands.print_results import PrintResults
from allennlp.commands.subcommand import Subcommand
from allennlp.commands.test_install import TestInstall
from allennlp.commands.train import Train
from allennlp.commands.push_to_hf import PushToHf
from allennlp.commands.count_instances import CountInstances
from allennlp.common.plugins import import_plugins
from allennlp.common.util import import_module_and_submodules
logger = logging.getLogger(__name__)
class ArgumentParserWithDefaults(argparse.ArgumentParser):
"""
Custom argument parser that will display the default value for an argument
in the help message.
"""
_action_defaults_to_ignore = {"help", "store_true", "store_false", "store_const"}
@staticmethod
def _is_empty_default(default: Any) -> bool:
if default is None:
return True
if isinstance(default, (str, list, tuple, set)):
return not bool(default)
return False
def add_argument(self, *args, **kwargs):
# Add default value to the help message when the default is meaningful.
default = kwargs.get("default")
if kwargs.get(
"action"
) not in self._action_defaults_to_ignore and not self._is_empty_default(default):
description = kwargs.get("help", "")
kwargs["help"] = f"{description} (default = {default})"
super().add_argument(*args, **kwargs)
def parse_args(prog: Optional[str] = None) -> Tuple[argparse.ArgumentParser, argparse.Namespace]:
"""
Creates the argument parser for the main program and uses it to parse the args.
"""
parser = ArgumentParserWithDefaults(description="Run AllenNLP", prog=prog)
parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
subparsers = parser.add_subparsers(title="Commands", metavar="")
subcommands: Set[str] = set()
def add_subcommands():
for subcommand_name in sorted(Subcommand.list_available()):
if subcommand_name in subcommands:
continue
subcommands.add(subcommand_name)
subcommand_class = Subcommand.by_name(subcommand_name)
subcommand = subcommand_class()
subparser = subcommand.add_subparser(subparsers)
if subcommand_class.requires_plugins:
subparser.add_argument(
"--include-package",
type=str,
action="append",
default=[],
help="additional packages to include",
)
# Add all default registered subcommands first.
add_subcommands()
# If we need to print the usage/help, or the subcommand is unknown,
# we'll call `import_plugins()` to register any plugin subcommands first.
argv = sys.argv[1:]
plugins_imported: bool = False
if not argv or argv == ["--help"] or argv[0] not in subcommands:
import_plugins()
plugins_imported = True
# Add subcommands again in case one of the plugins has a registered subcommand.
add_subcommands()
# Now we can parse the arguments.
args = parser.parse_args()
if not plugins_imported and Subcommand.by_name(argv[0]).requires_plugins: # type: ignore
import_plugins()
return parser, args
def main(prog: Optional[str] = None) -> None:
"""
The [`run`](./train.md#run) command only knows about the registered classes in the ``allennlp``
codebase. In particular, once you start creating your own `Model` s and so forth, it won't
work for them, unless you use the ``--include-package`` flag or you make your code available
as a plugin (see [`plugins`](./plugins.md)).
"""
parser, args = parse_args(prog)
# If a subparser is triggered, it adds its work as `args.func`.
# So if no such attribute has been added, no subparser was triggered,
# so give the user some help.
if "func" in dir(args):
# Import any additional modules needed (to register custom classes).
for package_name in getattr(args, "include_package", []):
import_module_and_submodules(package_name)
args.func(args)
else:
parser.print_help()
================================================
FILE: allennlp/commands/_checklist_internal.py
================================================
"""
The `checklist` subcommand allows you to conduct behavioural
testing for your model's predictions using a trained model and its
[`Predictor`](../predictors/predictor.md#predictor) wrapper.
"""
from typing import Optional, Dict, Any, List
import argparse
import sys
import json
import logging
from allennlp.commands.subcommand import Subcommand
from allennlp.common.checks import check_for_gpu, ConfigurationError
from allennlp.models.archival import load_archive
from allennlp.predictors.predictor import Predictor
logger = logging.getLogger(__name__)
try:
from allennlp.confidence_checks.task_checklists.task_suite import TaskSuite
except ImportError:
raise
@Subcommand.register("checklist")
class CheckList(Subcommand):
def add_subparser(self, parser: argparse._SubParsersAction) -> argparse.ArgumentParser:
description = """Run the specified model through a checklist suite."""
subparser = parser.add_parser(
self.name,
description=description,
help="Run a trained model through a checklist suite.",
)
subparser.add_argument(
"archive_file", type=str, help="The archived model to make predictions with"
)
subparser.add_argument("task", type=str, help="The name of the task suite")
subparser.add_argument("--checklist-suite", type=str, help="The checklist suite path")
subparser.add_argument(
"--capabilities",
nargs="+",
default=[],
help=('An optional list of strings of capabilities. Eg. "[Vocabulary, Robustness]"'),
)
subparser.add_argument(
"--max-examples",
type=int,
default=None,
help="Maximum number of examples to check per test.",
)
subparser.add_argument(
"--task-suite-args",
type=str,
default="",
help=(
"An optional JSON structure used to provide additional parameters to the task suite"
),
)
subparser.add_argument(
"--print-summary-args",
type=str,
default="",
help=(
"An optional JSON structure used to provide additional "
"parameters for printing test summary"
),
)
subparser.add_argument("--output-file", type=str, help="Path to output file")
subparser.add_argument(
"--cuda-device", type=int, default=-1, help="ID of GPU to use (if any)"
)
subparser.add_argument(
"--predictor", type=str, help="Optionally specify a specific predictor to use"
)
subparser.add_argument(
"--predictor-args",
type=str,
default="",
help=(
"An optional JSON structure used to provide additional parameters to the predictor"
),
)
subparser.set_defaults(func=_run_suite)
return subparser
def _get_predictor(args: argparse.Namespace) -> Predictor:
check_for_gpu(args.cuda_device)
archive = load_archive(
args.archive_file,
cuda_device=args.cuda_device,
)
predictor_args = args.predictor_args.strip()
if len(predictor_args) <= 0:
predictor_args = {}
else:
predictor_args = json.loads(predictor_args)
return Predictor.from_archive(
archive,
args.predictor,
extra_args=predictor_args,
)
def _get_task_suite(args: argparse.Namespace) -> TaskSuite:
available_tasks = TaskSuite.list_available()
if args.task in available_tasks:
suite_name = args.task
else:
raise ConfigurationError(
f"'{args.task}' is not a recognized task suite. "
f"Available tasks are: {available_tasks}."
)
file_path = args.checklist_suite
task_suite_args = args.task_suite_args.strip()
if len(task_suite_args) <= 0:
task_suite_args = {}
else:
task_suite_args = json.loads(task_suite_args)
return TaskSuite.constructor(
name=suite_name,
suite_file=file_path,
extra_args=task_suite_args,
)
class _CheckListManager:
def __init__(
self,
task_suite: TaskSuite,
predictor: Predictor,
capabilities: Optional[List[str]] = None,
max_examples: Optional[int] = None,
output_file: Optional[str] = None,
print_summary_args: Optional[Dict[str, Any]] = None,
) -> None:
self._task_suite = task_suite
self._predictor = predictor
self._capabilities = capabilities
self._max_examples = max_examples
self._output_file = None if output_file is None else open(output_file, "w")
self._print_summary_args = print_summary_args or {}
if capabilities:
self._print_summary_args["capabilities"] = capabilities
def run(self) -> None:
self._task_suite.run(
self._predictor, capabilities=self._capabilities, max_examples=self._max_examples
)
# We pass in an IO object.
output_file = self._output_file or sys.stdout
self._task_suite.summary(file=output_file, **self._print_summary_args)
# If `_output_file` was None, there would be nothing to close.
if self._output_file is not None:
self._output_file.close()
def _run_suite(args: argparse.Namespace) -> None:
task_suite = _get_task_suite(args)
predictor = _get_predictor(args)
print_summary_args = args.print_summary_args.strip()
if len(print_summary_args) <= 0:
print_summary_args = {}
else:
print_summary_args = json.loads(print_summary_args)
capabilities = args.capabilities
max_examples = args.max_examples
manager = _CheckListManager(
task_suite,
predictor,
capabilities,
max_examples,
args.output_file,
print_summary_args,
)
manager.run()
================================================
FILE: allennlp/commands/build_vocab.py
================================================
"""
Subcommand for building a vocabulary from a training config.
"""
import argparse
import json
import logging
import os
import tarfile
import tempfile
from allennlp.commands.subcommand import Subcommand
from allennlp.common.file_utils import CacheFile
from allennlp.common.params import Params
from allennlp.training.util import make_vocab_from_params
logger = logging.getLogger(__name__)
@Subcommand.register("build-vocab")
class BuildVocab(Subcommand):
def add_subparser(self, parser: argparse._SubParsersAction) -> argparse.ArgumentParser:
description = """Build a vocabulary from an experiment config file."""
subparser = parser.add_parser(self.name, description=description, help=description)
subparser.add_argument("param_path", type=str, help="path to an experiment config file")
subparser.add_argument(
"output_path", type=str, help="path to save the vocab tar.gz file to"
)
subparser.add_argument(
"-f",
"--force",
action="store_true",
help="force write if the output_path already exists",
)
subparser.add_argument(
"-o",
"--overrides",
type=str,
default="",
help=(
"a json(net) structure used to override the experiment configuration, e.g., "
"'{\"vocabulary.min_count.labels\": 10}'. Nested parameters can be specified either"
" with nested dictionaries or with dot syntax."
),
)
subparser.set_defaults(func=build_vocab_from_args)
return subparser
def build_vocab_from_args(args: argparse.Namespace):
if not args.output_path.endswith(".tar.gz"):
raise ValueError("param 'output_path' should end with '.tar.gz'")
if os.path.exists(args.output_path) and not args.force:
raise RuntimeError(f"{args.output_path} already exists. Use --force to overwrite.")
output_directory = os.path.dirname(args.output_path)
if len(output_directory) > 0:
os.makedirs(output_directory, exist_ok=True)
params = Params.from_file(args.param_path)
with tempfile.TemporaryDirectory() as temp_dir:
# Serializes the vocab to 'tempdir/vocabulary'.
make_vocab_from_params(params, temp_dir)
# The CacheFile context manager gives us a temporary file to write to.
# On a successful exit from the context, it will rename the temp file to
# the target `output_path`.
with CacheFile(args.output_path, suffix=".tar.gz") as temp_archive:
logger.info("Archiving vocabulary to %s", args.output_path)
with tarfile.open(temp_archive.name, "w:gz") as archive:
vocab_dir = os.path.join(temp_dir, "vocabulary")
for fname in os.listdir(vocab_dir):
if fname.endswith(".lock"):
continue
archive.add(os.path.join(vocab_dir, fname), arcname=fname)
print(f"Success! Vocab saved to {args.output_path}")
print('You can now set the "vocabulary" entry of your training config to:')
print(json.dumps({"type": "from_files", "directory": os.path.abspath(args.output_path)}))
================================================
FILE: allennlp/commands/cached_path.py
================================================
"""
CLI to the the caching mechanism in `common.file_utils`.
"""
import argparse
import logging
from allennlp.commands.subcommand import Subcommand
from allennlp.common.file_utils import (
cached_path,
CACHE_DIRECTORY,
inspect_cache,
remove_cache_entries,
)
logger = logging.getLogger(__name__)
@Subcommand.register("cached-path")
class CachedPath(Subcommand):
requires_plugins: bool = False
def add_subparser(self, parser: argparse._SubParsersAction) -> argparse.ArgumentParser:
description = """Cache remote files to the AllenNLP cache."""
subparser = parser.add_parser(
self.name,
description=description,
help=description,
)
subparser.set_defaults(func=_cached_path)
subparser.add_argument(
"resources",
type=str,
help="""The URLs or paths to the resources.
If using the --inspect or --remove flag, this can also contain glob patterns.""",
nargs="*",
)
subparser.add_argument(
"-d",
"--cache-dir",
type=str,
help="""Use a custom cache directory.""",
default=CACHE_DIRECTORY,
)
subparser.add_argument(
"-x",
"--extract-archive",
action="store_true",
help="""Automatically extract zip or tar.gz archive files.""",
)
subparser.add_argument(
"-f",
"--force-extract",
action="store_true",
help="""Extract archives regardless of whether or not they already exist.""",
)
subparser.add_argument(
"--inspect",
action="store_true",
help="""Print some useful information about the cache.""",
)
subparser.add_argument(
"--remove",
action="store_true",
help="""Remove any cache entries matching the given resource patterns.""",
)
return subparser
def _cached_path(args: argparse.Namespace):
logger.info("Cache directory: %s", args.cache_dir)
if args.inspect:
if args.extract_archive or args.force_extract or args.remove:
raise RuntimeError(
"cached-path cannot accept --extract-archive, --force-extract, or --remove "
"options when --inspect flag is used."
)
inspect_cache(patterns=args.resources, cache_dir=args.cache_dir)
elif args.remove:
from allennlp.common.util import format_size
if args.extract_archive or args.force_extract or args.inspect:
raise RuntimeError(
"cached-path cannot accept --extract-archive, --force-extract, or --inspect "
"options when --remove flag is used."
)
if not args.resources:
raise RuntimeError(
"Missing positional argument(s) 'resources'. 'resources' is required when using "
"the --remove option. If you really want to remove everything, pass '*' for 'resources'."
)
reclaimed_space = remove_cache_entries(args.resources, cache_dir=args.cache_dir)
print(f"Reclaimed {format_size(reclaimed_space)} of space")
else:
for resource in args.resources:
print(
cached_path(
resource,
cache_dir=args.cache_dir,
extract_archive=args.extract_archive,
force_extract=args.force_extract,
)
)
================================================
FILE: allennlp/commands/checklist.py
================================================
"""
The `checklist` subcommand allows you to conduct behavioural
testing for your model's predictions using a trained model and its
[`Predictor`](../predictors/predictor.md#predictor) wrapper.
It is based on the optional checklist package; if this is not
available, the command will be replaced by a dummy.
"""
import argparse
import logging
from allennlp.commands.subcommand import Subcommand
logger = logging.getLogger(__name__)
try:
from allennlp.commands._checklist_internal import CheckList
except ImportError:
# create dummy command that tells users how to
# install the necessary dependency
def _dummy_output(args: argparse.Namespace):
logger.info(
"The checklist integration of allennlp is optional; if you're using conda, "
"it can be installed with `conda install allennlp-checklist`, "
"otherwise use `pip install allennlp[checklist]`."
)
# need to work around https://github.com/python/mypy/issues/1153
@Subcommand.register("checklist")
class CheckList(Subcommand): # type: ignore
def add_subparser(self, parser: argparse._SubParsersAction) -> argparse.ArgumentParser:
description = """Dummy command because checklist is not installed."""
subparser = parser.add_parser(
self.name,
description=description,
help="Run a trained model through a checklist suite.",
)
subparser.set_defaults(func=_dummy_output)
return subparser
================================================
FILE: allennlp/commands/count_instances.py
================================================
"""
Subcommand for counting the number of instances from a training config.
"""
import argparse
import logging
from allennlp.commands.subcommand import Subcommand
from allennlp.common.params import Params
logger = logging.getLogger(__name__)
@Subcommand.register("count-instances")
class CountInstances(Subcommand):
def add_subparser(self, parser: argparse._SubParsersAction) -> argparse.ArgumentParser:
description = """Count the number of training instances in an experiment config file."""
subparser = parser.add_parser(self.name, description=description, help=description)
subparser.add_argument("param_path", type=str, help="path to an experiment config file")
subparser.add_argument(
"-o",
"--overrides",
type=str,
default="",
help=(
"a json(net) structure used to override the experiment configuration, e.g., "
"'{\"vocabulary.min_count.labels\": 10}'. Nested parameters can be specified either"
" with nested dictionaries or with dot syntax."
),
)
subparser.set_defaults(func=count_instances_from_args)
return subparser
def count_instances_from_args(args: argparse.Namespace):
from allennlp.training.util import data_loaders_from_params
params = Params.from_file(args.param_path)
data_loaders = data_loaders_from_params(params, train=True, validation=False, test=False)
instances = sum(
1 for data_loader in data_loaders.values() for _ in data_loader.iter_instances()
)
print(f"Success! One epoch of training contains {instances} instances.")
================================================
FILE: allennlp/commands/diff.py
================================================
"""
# Examples
```bash
allennlp diff \
hf://roberta-large/pytorch_model.bin \
https://storage.googleapis.com/allennlp-public-models/transformer-qa-2020-10-03.tar.gz \
--strip-prefix-1 'roberta.' \
--strip-prefix-2 '_text_field_embedder.token_embedder_tokens.transformer_model.'
```
"""
import argparse
import logging
from typing import Union, Dict, List, Tuple, NamedTuple, cast
import termcolor
import torch
from allennlp.commands.subcommand import Subcommand
from allennlp.common.file_utils import cached_path
from allennlp.nn.util import read_state_dict
logger = logging.getLogger(__name__)
@Subcommand.register("diff")
class Diff(Subcommand):
requires_plugins: bool = False
def add_subparser(self, parser: argparse._SubParsersAction) -> argparse.ArgumentParser:
description = """Display a diff between two model checkpo
gitextract_g9t09ufj/
├── .coveragerc
├── .dockerignore
├── .flake8
├── .github/
│ ├── ISSUE_TEMPLATE/
│ │ ├── bug_report.md
│ │ ├── feature_request.md
│ │ └── question.md
│ ├── pull_request_template.md
│ └── workflows/
│ ├── cffconvert.yml
│ ├── ci.yml
│ └── issues.yml
├── .gitignore
├── CHANGELOG.md
├── CITATION.cff
├── CONTRIBUTING.md
├── Dockerfile
├── Dockerfile.test
├── LICENSE
├── MANIFEST.in
├── Makefile
├── README.md
├── RELEASE_PROCESS.md
├── STYLE.md
├── allennlp/
│ ├── __init__.py
│ ├── __main__.py
│ ├── commands/
│ │ ├── __init__.py
│ │ ├── _checklist_internal.py
│ │ ├── build_vocab.py
│ │ ├── cached_path.py
│ │ ├── checklist.py
│ │ ├── count_instances.py
│ │ ├── diff.py
│ │ ├── evaluate.py
│ │ ├── find_learning_rate.py
│ │ ├── predict.py
│ │ ├── print_results.py
│ │ ├── push_to_hf.py
│ │ ├── subcommand.py
│ │ ├── test_install.py
│ │ └── train.py
│ ├── common/
│ │ ├── __init__.py
│ │ ├── cached_transformers.py
│ │ ├── checks.py
│ │ ├── file_utils.py
│ │ ├── from_params.py
│ │ ├── lazy.py
│ │ ├── logging.py
│ │ ├── meta.py
│ │ ├── model_card.py
│ │ ├── params.py
│ │ ├── plugins.py
│ │ ├── push_to_hf.py
│ │ ├── registrable.py
│ │ ├── sequences.py
│ │ ├── task_card.py
│ │ ├── testing/
│ │ │ ├── __init__.py
│ │ │ ├── checklist_test.py
│ │ │ ├── confidence_check_test.py
│ │ │ ├── distributed_test.py
│ │ │ ├── interpret_test.py
│ │ │ ├── model_test_case.py
│ │ │ └── test_case.py
│ │ ├── tqdm.py
│ │ └── util.py
│ ├── confidence_checks/
│ │ ├── __init__.py
│ │ ├── normalization_bias_verification.py
│ │ ├── task_checklists/
│ │ │ ├── __init__.py
│ │ │ ├── question_answering_suite.py
│ │ │ ├── sentiment_analysis_suite.py
│ │ │ ├── task_suite.py
│ │ │ ├── textual_entailment_suite.py
│ │ │ └── utils.py
│ │ └── verification_base.py
│ ├── data/
│ │ ├── __init__.py
│ │ ├── batch.py
│ │ ├── data_loaders/
│ │ │ ├── __init__.py
│ │ │ ├── data_collator.py
│ │ │ ├── data_loader.py
│ │ │ ├── multiprocess_data_loader.py
│ │ │ ├── multitask_data_loader.py
│ │ │ ├── multitask_epoch_sampler.py
│ │ │ ├── multitask_scheduler.py
│ │ │ └── simple_data_loader.py
│ │ ├── dataset_readers/
│ │ │ ├── __init__.py
│ │ │ ├── babi.py
│ │ │ ├── conll2003.py
│ │ │ ├── dataset_reader.py
│ │ │ ├── dataset_utils/
│ │ │ │ ├── __init__.py
│ │ │ │ └── span_utils.py
│ │ │ ├── interleaving_dataset_reader.py
│ │ │ ├── multitask.py
│ │ │ ├── sequence_tagging.py
│ │ │ ├── sharded_dataset_reader.py
│ │ │ └── text_classification_json.py
│ │ ├── fields/
│ │ │ ├── __init__.py
│ │ │ ├── adjacency_field.py
│ │ │ ├── array_field.py
│ │ │ ├── field.py
│ │ │ ├── flag_field.py
│ │ │ ├── index_field.py
│ │ │ ├── label_field.py
│ │ │ ├── list_field.py
│ │ │ ├── metadata_field.py
│ │ │ ├── multilabel_field.py
│ │ │ ├── namespace_swapping_field.py
│ │ │ ├── sequence_field.py
│ │ │ ├── sequence_label_field.py
│ │ │ ├── span_field.py
│ │ │ ├── tensor_field.py
│ │ │ ├── text_field.py
│ │ │ └── transformer_text_field.py
│ │ ├── image_loader.py
│ │ ├── instance.py
│ │ ├── samplers/
│ │ │ ├── __init__.py
│ │ │ ├── batch_sampler.py
│ │ │ ├── bucket_batch_sampler.py
│ │ │ └── max_tokens_batch_sampler.py
│ │ ├── token_indexers/
│ │ │ ├── __init__.py
│ │ │ ├── elmo_indexer.py
│ │ │ ├── pretrained_transformer_indexer.py
│ │ │ ├── pretrained_transformer_mismatched_indexer.py
│ │ │ ├── single_id_token_indexer.py
│ │ │ ├── spacy_indexer.py
│ │ │ ├── token_characters_indexer.py
│ │ │ └── token_indexer.py
│ │ ├── tokenizers/
│ │ │ ├── __init__.py
│ │ │ ├── character_tokenizer.py
│ │ │ ├── letters_digits_tokenizer.py
│ │ │ ├── pretrained_transformer_tokenizer.py
│ │ │ ├── sentence_splitter.py
│ │ │ ├── spacy_tokenizer.py
│ │ │ ├── token_class.py
│ │ │ ├── tokenizer.py
│ │ │ └── whitespace_tokenizer.py
│ │ └── vocabulary.py
│ ├── evaluation/
│ │ ├── __init__.py
│ │ ├── evaluator.py
│ │ ├── postprocessors/
│ │ │ └── __init__.py
│ │ └── serializers/
│ │ ├── __init__.py
│ │ └── serializers.py
│ ├── fairness/
│ │ ├── __init__.py
│ │ ├── adversarial_bias_mitigator.py
│ │ ├── bias_direction.py
│ │ ├── bias_direction_wrappers.py
│ │ ├── bias_metrics.py
│ │ ├── bias_mitigator_applicator.py
│ │ ├── bias_mitigator_wrappers.py
│ │ ├── bias_mitigators.py
│ │ ├── bias_utils.py
│ │ └── fairness_metrics.py
│ ├── interpret/
│ │ ├── __init__.py
│ │ ├── attackers/
│ │ │ ├── __init__.py
│ │ │ ├── attacker.py
│ │ │ ├── hotflip.py
│ │ │ ├── input_reduction.py
│ │ │ └── utils.py
│ │ ├── influence_interpreters/
│ │ │ ├── __init__.py
│ │ │ ├── influence_interpreter.py
│ │ │ └── simple_influence.py
│ │ └── saliency_interpreters/
│ │ ├── __init__.py
│ │ ├── integrated_gradient.py
│ │ ├── saliency_interpreter.py
│ │ ├── simple_gradient.py
│ │ └── smooth_gradient.py
│ ├── models/
│ │ ├── __init__.py
│ │ ├── archival.py
│ │ ├── basic_classifier.py
│ │ ├── heads/
│ │ │ ├── __init__.py
│ │ │ ├── classifier_head.py
│ │ │ └── head.py
│ │ ├── model.py
│ │ ├── multitask.py
│ │ └── simple_tagger.py
│ ├── modules/
│ │ ├── __init__.py
│ │ ├── attention/
│ │ │ ├── __init__.py
│ │ │ ├── additive_attention.py
│ │ │ ├── attention.py
│ │ │ ├── bilinear_attention.py
│ │ │ ├── cosine_attention.py
│ │ │ ├── dot_product_attention.py
│ │ │ ├── linear_attention.py
│ │ │ └── scaled_dot_product_attention.py
│ │ ├── augmented_lstm.py
│ │ ├── backbones/
│ │ │ ├── __init__.py
│ │ │ ├── backbone.py
│ │ │ ├── pretrained_transformer_backbone.py
│ │ │ └── vilbert_backbone.py
│ │ ├── bimpm_matching.py
│ │ ├── conditional_random_field/
│ │ │ ├── __init__.py
│ │ │ ├── conditional_random_field.py
│ │ │ ├── conditional_random_field_wemission.py
│ │ │ ├── conditional_random_field_wlannoy.py
│ │ │ └── conditional_random_field_wtrans.py
│ │ ├── elmo.py
│ │ ├── elmo_lstm.py
│ │ ├── encoder_base.py
│ │ ├── feedforward.py
│ │ ├── gated_sum.py
│ │ ├── highway.py
│ │ ├── input_variational_dropout.py
│ │ ├── layer_norm.py
│ │ ├── lstm_cell_with_projection.py
│ │ ├── masked_layer_norm.py
│ │ ├── matrix_attention/
│ │ │ ├── __init__.py
│ │ │ ├── bilinear_matrix_attention.py
│ │ │ ├── cosine_matrix_attention.py
│ │ │ ├── dot_product_matrix_attention.py
│ │ │ ├── linear_matrix_attention.py
│ │ │ ├── matrix_attention.py
│ │ │ └── scaled_dot_product_matrix_attention.py
│ │ ├── maxout.py
│ │ ├── residual_with_layer_dropout.py
│ │ ├── sampled_softmax_loss.py
│ │ ├── scalar_mix.py
│ │ ├── seq2seq_encoders/
│ │ │ ├── __init__.py
│ │ │ ├── compose_encoder.py
│ │ │ ├── feedforward_encoder.py
│ │ │ ├── gated_cnn_encoder.py
│ │ │ ├── pass_through_encoder.py
│ │ │ ├── pytorch_seq2seq_wrapper.py
│ │ │ ├── pytorch_transformer_wrapper.py
│ │ │ └── seq2seq_encoder.py
│ │ ├── seq2vec_encoders/
│ │ │ ├── __init__.py
│ │ │ ├── bert_pooler.py
│ │ │ ├── boe_encoder.py
│ │ │ ├── cls_pooler.py
│ │ │ ├── cnn_encoder.py
│ │ │ ├── cnn_highway_encoder.py
│ │ │ ├── pytorch_seq2vec_wrapper.py
│ │ │ └── seq2vec_encoder.py
│ │ ├── softmax_loss.py
│ │ ├── span_extractors/
│ │ │ ├── __init__.py
│ │ │ ├── bidirectional_endpoint_span_extractor.py
│ │ │ ├── endpoint_span_extractor.py
│ │ │ ├── max_pooling_span_extractor.py
│ │ │ ├── self_attentive_span_extractor.py
│ │ │ ├── span_extractor.py
│ │ │ └── span_extractor_with_span_width_embedding.py
│ │ ├── stacked_alternating_lstm.py
│ │ ├── stacked_bidirectional_lstm.py
│ │ ├── text_field_embedders/
│ │ │ ├── __init__.py
│ │ │ ├── basic_text_field_embedder.py
│ │ │ └── text_field_embedder.py
│ │ ├── time_distributed.py
│ │ ├── token_embedders/
│ │ │ ├── __init__.py
│ │ │ ├── bag_of_word_counts_token_embedder.py
│ │ │ ├── elmo_token_embedder.py
│ │ │ ├── embedding.py
│ │ │ ├── empty_embedder.py
│ │ │ ├── pass_through_token_embedder.py
│ │ │ ├── pretrained_transformer_embedder.py
│ │ │ ├── pretrained_transformer_mismatched_embedder.py
│ │ │ ├── token_characters_encoder.py
│ │ │ └── token_embedder.py
│ │ ├── transformer/
│ │ │ ├── __init__.py
│ │ │ ├── activation_layer.py
│ │ │ ├── attention_module.py
│ │ │ ├── bimodal_attention.py
│ │ │ ├── bimodal_connection_layer.py
│ │ │ ├── bimodal_encoder.py
│ │ │ ├── layer_norm.py
│ │ │ ├── output_layer.py
│ │ │ ├── positional_encoding.py
│ │ │ ├── t5.py
│ │ │ ├── transformer_embeddings.py
│ │ │ ├── transformer_layer.py
│ │ │ ├── transformer_module.py
│ │ │ ├── transformer_pooler.py
│ │ │ ├── transformer_stack.py
│ │ │ └── util.py
│ │ ├── util.py
│ │ └── vision/
│ │ ├── __init__.py
│ │ ├── grid_embedder.py
│ │ ├── image2image.py
│ │ └── region_detector.py
│ ├── nn/
│ │ ├── __init__.py
│ │ ├── activations.py
│ │ ├── beam_search.py
│ │ ├── checkpoint/
│ │ │ ├── __init__.py
│ │ │ ├── checkpoint_wrapper.py
│ │ │ └── fairscale_checkpoint_wrapper.py
│ │ ├── chu_liu_edmonds.py
│ │ ├── initializers.py
│ │ ├── module.py
│ │ ├── parallel/
│ │ │ ├── __init__.py
│ │ │ ├── ddp_accelerator.py
│ │ │ ├── fairscale_fsdp_accelerator.py
│ │ │ └── sharded_module_mixin.py
│ │ ├── regularizers/
│ │ │ ├── __init__.py
│ │ │ ├── regularizer.py
│ │ │ ├── regularizer_applicator.py
│ │ │ └── regularizers.py
│ │ └── util.py
│ ├── predictors/
│ │ ├── __init__.py
│ │ ├── multitask.py
│ │ ├── predictor.py
│ │ ├── sentence_tagger.py
│ │ └── text_classifier.py
│ ├── py.typed
│ ├── sanity_checks/
│ │ ├── __init__.py
│ │ └── task_checklists/
│ │ └── __init__.py
│ ├── tools/
│ │ ├── EVALB/
│ │ │ ├── .gitignore
│ │ │ ├── COLLINS.prm
│ │ │ ├── LICENSE
│ │ │ ├── Makefile
│ │ │ ├── README
│ │ │ ├── bug/
│ │ │ │ ├── bug.gld
│ │ │ │ ├── bug.rsl-new
│ │ │ │ ├── bug.rsl-old
│ │ │ │ └── bug.tst
│ │ │ ├── evalb.c
│ │ │ ├── new.prm
│ │ │ ├── sample/
│ │ │ │ ├── sample.gld
│ │ │ │ ├── sample.prm
│ │ │ │ ├── sample.rsl
│ │ │ │ └── sample.tst
│ │ │ └── tgrep_proc.prl
│ │ ├── __init__.py
│ │ ├── archive_surgery.py
│ │ ├── create_elmo_embeddings_from_vocab.py
│ │ └── inspect_cache.py
│ ├── training/
│ │ ├── __init__.py
│ │ ├── callbacks/
│ │ │ ├── __init__.py
│ │ │ ├── backward.py
│ │ │ ├── callback.py
│ │ │ ├── confidence_checks.py
│ │ │ ├── console_logger.py
│ │ │ ├── log_writer.py
│ │ │ ├── should_validate.py
│ │ │ ├── tensorboard.py
│ │ │ ├── track_epoch.py
│ │ │ └── wandb.py
│ │ ├── checkpointer.py
│ │ ├── gradient_descent_trainer.py
│ │ ├── learning_rate_schedulers/
│ │ │ ├── __init__.py
│ │ │ ├── combined.py
│ │ │ ├── cosine.py
│ │ │ ├── learning_rate_scheduler.py
│ │ │ ├── linear_with_warmup.py
│ │ │ ├── noam.py
│ │ │ ├── polynomial_decay.py
│ │ │ ├── pytorch_lr_schedulers.py
│ │ │ └── slanted_triangular.py
│ │ ├── metric_tracker.py
│ │ ├── metrics/
│ │ │ ├── __init__.py
│ │ │ ├── attachment_scores.py
│ │ │ ├── auc.py
│ │ │ ├── average.py
│ │ │ ├── bleu.py
│ │ │ ├── boolean_accuracy.py
│ │ │ ├── categorical_accuracy.py
│ │ │ ├── covariance.py
│ │ │ ├── entropy.py
│ │ │ ├── evalb_bracketing_scorer.py
│ │ │ ├── f1_measure.py
│ │ │ ├── fbeta_measure.py
│ │ │ ├── fbeta_multi_label_measure.py
│ │ │ ├── fbeta_verbose_measure.py
│ │ │ ├── mean_absolute_error.py
│ │ │ ├── metric.py
│ │ │ ├── pearson_correlation.py
│ │ │ ├── perplexity.py
│ │ │ ├── rouge.py
│ │ │ ├── sequence_accuracy.py
│ │ │ ├── span_based_f1_measure.py
│ │ │ ├── spearman_correlation.py
│ │ │ └── unigram_recall.py
│ │ ├── momentum_schedulers/
│ │ │ ├── __init__.py
│ │ │ ├── inverted_triangular.py
│ │ │ └── momentum_scheduler.py
│ │ ├── moving_average.py
│ │ ├── no_op_trainer.py
│ │ ├── optimizers.py
│ │ ├── scheduler.py
│ │ ├── trainer.py
│ │ └── util.py
│ └── version.py
├── benchmarks/
│ ├── __init__.py
│ ├── data/
│ │ ├── __init__.py
│ │ └── tokenizers/
│ │ ├── __init__.py
│ │ └── character_tokenizer_bench.py
│ ├── nn/
│ │ └── util_bench.py
│ └── pytest.ini
├── codecov.yml
├── constraints.txt
├── dev-requirements.txt
├── docs/
│ └── css/
│ └── extra.css
├── mkdocs-skeleton.yml
├── mypy.ini
├── pyproject.toml
├── pytest.ini
├── requirements.in
├── requirements.txt
├── scripts/
│ ├── 24hr_diff.sh
│ ├── ai2_internal/
│ │ ├── resumable_train.sh
│ │ ├── resume_daemon.py
│ │ └── run_with_beaker.py
│ ├── build_docs.sh
│ ├── build_docs_config.py
│ ├── check_large_files.sh
│ ├── check_links.py
│ ├── check_torch_version.py
│ ├── close_stale_issues.py
│ ├── get_version.py
│ ├── ping_issue_assignees.py
│ ├── py2md.py
│ ├── release_notes.py
│ ├── tests/
│ │ ├── ai2_internal/
│ │ │ └── resume_daemon_test.py
│ │ └── py2md/
│ │ ├── basic_example.py
│ │ ├── basic_example_expected_output.md
│ │ └── py2md_test.py
│ └── train_fixtures.py
├── setup.py
├── test_fixtures/
│ ├── __init__.py
│ ├── basic_classifier/
│ │ ├── common.jsonnet
│ │ ├── experiment_from_archive.jsonnet
│ │ ├── experiment_seq2seq.jsonnet
│ │ ├── experiment_seq2vec.jsonnet
│ │ ├── parameters_inspection.json
│ │ └── serialization/
│ │ ├── best.th
│ │ └── vocabulary/
│ │ ├── labels.txt
│ │ ├── non_padded_namespaces.txt
│ │ └── tokens.txt
│ ├── common/
│ │ └── .gitignore
│ ├── data/
│ │ ├── babi.txt
│ │ ├── brown_corpus.txt
│ │ ├── conll2003.txt
│ │ ├── sequence_tagging.tsv
│ │ ├── shards/
│ │ │ ├── sequence_tagging_00.tsv
│ │ │ ├── sequence_tagging_01.tsv
│ │ │ └── sequence_tagging_02.tsv
│ │ └── text_classification_json/
│ │ ├── ag_news_corpus.jsonl
│ │ ├── ag_news_corpus_fake_sentiment_labels.jsonl
│ │ ├── imdb_corpus.jsonl
│ │ ├── imdb_corpus2.jsonl
│ │ └── integer_labels.jsonl
│ ├── elmo/
│ │ ├── config/
│ │ │ └── characters_token_embedder.json
│ │ ├── elmo_token_embeddings.hdf5
│ │ ├── lm_embeddings_0.hdf5
│ │ ├── lm_embeddings_1.hdf5
│ │ ├── lm_embeddings_2.hdf5
│ │ ├── lm_weights.hdf5
│ │ ├── options.json
│ │ ├── sentences.json
│ │ └── vocab_test.txt
│ ├── embeddings/
│ │ ├── fake_embeddings.5d.txt
│ │ ├── fake_embeddings.5d.txt.bz2
│ │ └── fake_embeddings.5d.txt.xz
│ ├── fairness/
│ │ ├── bias_embeddings.json
│ │ ├── definitional_pairs.json
│ │ ├── equalize_pairs.json
│ │ └── gender_specific_full.json
│ ├── plugins/
│ │ ├── .allennlp_plugins
│ │ └── d/
│ │ ├── __init__.py
│ │ └── d.py
│ ├── simple_tagger/
│ │ ├── experiment.json
│ │ ├── experiment_with_regularization.json
│ │ ├── model_test_case.jsonnet
│ │ ├── serialization/
│ │ │ ├── best.th
│ │ │ └── vocabulary/
│ │ │ ├── labels.txt
│ │ │ ├── non_padded_namespaces.txt
│ │ │ └── tokens.txt
│ │ └── serialization_full/
│ │ ├── best.th
│ │ ├── config.json
│ │ ├── meta.json
│ │ ├── metrics.json
│ │ ├── metrics_epoch_0.json
│ │ ├── model_state_e1_b0.th
│ │ ├── out.log
│ │ ├── training_state_e1_b0.th
│ │ └── vocabulary/
│ │ ├── labels.txt
│ │ ├── non_padded_namespaces.txt
│ │ └── tokens.txt
│ ├── simple_tagger_with_elmo/
│ │ ├── experiment.json
│ │ └── serialization/
│ │ ├── best.th
│ │ └── vocabulary/
│ │ ├── labels.txt
│ │ └── non_padded_namespaces.txt
│ ├── simple_tagger_with_span_f1/
│ │ ├── experiment.json
│ │ └── serialization/
│ │ ├── best.th
│ │ └── vocabulary/
│ │ ├── labels.txt
│ │ ├── non_padded_namespaces.txt
│ │ ├── test_tokens.txt
│ │ └── tokens.txt
│ └── utf-8_sample/
│ ├── archives/
│ │ ├── utf-8.tar.bz2
│ │ └── utf-8.tar.xz
│ └── utf-8_sample.txt
└── tests/
├── __init__.py
├── commands/
│ ├── __init__.py
│ ├── build_vocab_test.py
│ ├── cached_path_test.py
│ ├── checklist_test.py
│ ├── diff_test.py
│ ├── evaluate_test.py
│ ├── find_learning_rate_test.py
│ ├── main_test.py
│ ├── no_op_train_test.py
│ ├── predict_test.py
│ ├── print_results_test.py
│ ├── test_install_test.py
│ └── train_test.py
├── common/
│ ├── __init__.py
│ ├── cached_transformers_test.py
│ ├── file_utils_test.py
│ ├── from_params_test.py
│ ├── logging_test.py
│ ├── model_card_test.py
│ ├── params_test.py
│ ├── plugins_test.py
│ ├── push_to_hub_test.py
│ ├── registrable_test.py
│ ├── sequences_test.py
│ ├── task_card_test.py
│ ├── testing.py
│ └── util_test.py
├── confidence_checks/
│ ├── normalization_bias_verification_test.py
│ └── task_checklists/
│ ├── __init__.py
│ ├── sentiment_analysis_suite_test.py
│ ├── task_suite_test.py
│ └── utils_test.py
├── data/
│ ├── __init__.py
│ ├── data_loaders/
│ │ ├── __init__.py
│ │ ├── multiprocess_data_loader_test.py
│ │ ├── multitask_data_loader_test.py
│ │ └── multitask_scheduler_test.py
│ ├── dataset_readers/
│ │ ├── __init__.py
│ │ ├── babi_reader_test.py
│ │ ├── conll2003_test.py
│ │ ├── dataset_reader_test.py
│ │ ├── dataset_utils/
│ │ │ └── span_utils_test.py
│ │ ├── interleaving_dataset_reader_test.py
│ │ ├── sequence_tagging_test.py
│ │ ├── sharded_dataset_reader_test.py
│ │ └── text_classification_json_test.py
│ ├── dataset_test.py
│ ├── fields/
│ │ ├── __init__.py
│ │ ├── adjacency_field_test.py
│ │ ├── field_test.py
│ │ ├── flag_field_test.py
│ │ ├── index_field_test.py
│ │ ├── label_field_test.py
│ │ ├── list_field_test.py
│ │ ├── metadata_field_test.py
│ │ ├── multilabel_field_test.py
│ │ ├── sequence_label_field_test.py
│ │ ├── span_field_test.py
│ │ ├── tensor_field_test.py
│ │ ├── text_field_test.py
│ │ └── transformer_text_field_test.py
│ ├── image_loader_test.py
│ ├── instance_test.py
│ ├── samplers/
│ │ ├── __init__.py
│ │ ├── bucket_batch_sampler_test.py
│ │ ├── max_tokens_batch_sampler_test.py
│ │ └── sampler_test.py
│ ├── token_indexers/
│ │ ├── __init__.py
│ │ ├── character_token_indexer_test.py
│ │ ├── elmo_indexer_test.py
│ │ ├── pretrained_transformer_indexer_test.py
│ │ ├── pretrained_transformer_mismatched_indexer_test.py
│ │ ├── single_id_token_indexer_test.py
│ │ └── spacy_indexer_test.py
│ ├── tokenizers/
│ │ ├── __init__.py
│ │ ├── character_tokenizer_test.py
│ │ ├── letters_digits_tokenizer_test.py
│ │ ├── pretrained_transformer_tokenizer_test.py
│ │ ├── sentence_splitter_test.py
│ │ └── spacy_tokenizer_test.py
│ └── vocabulary_test.py
├── evaluation/
│ ├── __init__.py
│ ├── evaluator_tests.py
│ └── serializers/
│ ├── __init__.py
│ └── serializer_test.py
├── fairness/
│ ├── __init__.py
│ ├── bias_direction_test.py
│ ├── bias_metrics_test.py
│ ├── bias_mitigators_test.py
│ ├── bias_utils_test.py
│ └── fairness_metrics_test.py
├── interpret/
│ ├── __init__.py
│ ├── hotflip_test.py
│ ├── input_reduction_test.py
│ ├── integrated_gradient_test.py
│ ├── simple_gradient_test.py
│ ├── simple_influence_test.py
│ └── smooth_gradient_test.py
├── models/
│ ├── __init__.py
│ ├── archival_test.py
│ ├── basic_classifier_test.py
│ ├── model_test.py
│ ├── multitask_test.py
│ ├── simple_tagger_test.py
│ └── test_model_test_case.py
├── modules/
│ ├── attention/
│ │ ├── __init__.py
│ │ ├── additive_attention_test.py
│ │ ├── attention_test.py
│ │ ├── bilinear_attention_test.py
│ │ ├── cosine_attention_test.py
│ │ ├── dot_product_attention_test.py
│ │ ├── linear_attention_test.py
│ │ └── scaled_dot_product_attention_test.py
│ ├── augmented_lstm_test.py
│ ├── bimpm_matching_test.py
│ ├── conditional_random_field_test.py
│ ├── elmo_test.py
│ ├── encoder_base_test.py
│ ├── feedforward_test.py
│ ├── gated_sum_test.py
│ ├── highway_test.py
│ ├── lstm_cell_with_projection_test.py
│ ├── masked_layer_norm_test.py
│ ├── matrix_attention/
│ │ ├── __init__.py
│ │ ├── bilinear_matrix_attention_test.py
│ │ ├── cosine_matrix_attention_test.py
│ │ ├── dot_product_matrix_attention_test.py
│ │ ├── linear_matrix_attention_test.py
│ │ ├── matrix_attention_test.py
│ │ └── scaled_dot_product_matrix_attention_test.py
│ ├── maxout_test.py
│ ├── residual_with_layer_dropout_test.py
│ ├── sampled_softmax_loss_test.py
│ ├── scalar_mix_test.py
│ ├── seq2seq_encoder_test.py
│ ├── seq2seq_encoders/
│ │ ├── __init__.py
│ │ ├── compose_encoder_test.py
│ │ ├── feedforward_encoder_test.py
│ │ ├── gated_cnn_encoder_test.py
│ │ ├── pass_through_encoder_test.py
│ │ ├── pytorch_seq2seq_wrapper_test.py
│ │ └── pytorch_transformer_wrapper_test.py
│ ├── seq2vec_encoder_test.py
│ ├── seq2vec_encoders/
│ │ ├── __init__.py
│ │ ├── bert_pooler_test.py
│ │ ├── boe_encoder_test.py
│ │ ├── cls_pooler_test.py
│ │ ├── cnn_encoder_test.py
│ │ ├── cnn_highway_encoder_test.py
│ │ └── pytorch_seq2vec_wrapper_test.py
│ ├── span_extractors/
│ │ ├── __init__.py
│ │ ├── bidirectional_endpoint_span_extractor_test.py
│ │ ├── endpoint_span_extractor_test.py
│ │ ├── max_pooling_span_extractor_test.py
│ │ └── self_attentive_span_extractor_test.py
│ ├── stacked_alternating_lstm_test.py
│ ├── stacked_bidirectional_lstm_test.py
│ ├── stacked_elmo_lstm_test.py
│ ├── text_field_embedders/
│ │ ├── __init__.py
│ │ └── basic_text_field_embedder_test.py
│ ├── time_distributed_test.py
│ ├── token_embedders/
│ │ ├── __init__.py
│ │ ├── bag_of_word_counts_token_embedder_test.py
│ │ ├── elmo_token_embedder_test.py
│ │ ├── embedding_test.py
│ │ ├── pass_through_embedder_test.py
│ │ ├── pretrained_transformer_embedder_test.py
│ │ ├── pretrained_transformer_mismatched_embedder_test.py
│ │ └── token_characters_encoder_test.py
│ ├── transformer/
│ │ ├── activation_layer_test.py
│ │ ├── bimodal_attention_test.py
│ │ ├── bimodal_encoder_test.py
│ │ ├── output_layer_test.py
│ │ ├── positional_encoding_test.py
│ │ ├── self_attention_test.py
│ │ ├── t5_self_attention_test.py
│ │ ├── t5_test.py
│ │ ├── toolkit_test.py
│ │ ├── transformer_embeddings_test.py
│ │ ├── transformer_layer_test.py
│ │ ├── transformer_module_test.py
│ │ ├── transformer_pooler_test.py
│ │ └── transformer_stack_test.py
│ └── vision/
│ ├── __init__.py
│ ├── grid_embedder_test.py
│ └── region_detector_test.py
├── nn/
│ ├── __init__.py
│ ├── beam_search_test.py
│ ├── checkpoint/
│ │ ├── __init__.py
│ │ ├── checkpoint_wrapper_test.py
│ │ └── fairscale_checkpoint_wrapper_test.py
│ ├── chu_liu_edmonds_test.py
│ ├── initializers_test.py
│ ├── parallel/
│ │ ├── __init__.py
│ │ └── fairscale_fsdp_accelerator_test.py
│ ├── pretrained_model_initializer_test.py
│ ├── regularizers_test.py
│ └── util_test.py
├── predictors/
│ ├── __init__.py
│ ├── predictor_test.py
│ ├── sentence_tagger_test.py
│ └── text_classifier_test.py
├── training/
│ ├── __init__.py
│ ├── checkpointer_test.py
│ ├── learning_rate_schedulers/
│ │ ├── __init__.py
│ │ ├── combined_test.py
│ │ ├── cosine_test.py
│ │ ├── learning_rate_scheduler_test.py
│ │ └── slanted_triangular_test.py
│ ├── metrics/
│ │ ├── __init__.py
│ │ ├── attachment_scores_test.py
│ │ ├── auc_test.py
│ │ ├── average_test.py
│ │ ├── bleu_test.py
│ │ ├── boolean_accuracy_test.py
│ │ ├── categorical_accuracy_test.py
│ │ ├── covariance_test.py
│ │ ├── entropy_test.py
│ │ ├── evalb_bracketing_scorer_test.py
│ │ ├── f1_measure_test.py
│ │ ├── fbeta_measure_test.py
│ │ ├── fbeta_multi_label_measure_test.py
│ │ ├── fbeta_verbose_measure_test.py
│ │ ├── mean_absolute_error_test.py
│ │ ├── pearson_correlation_test.py
│ │ ├── rouge_test.py
│ │ ├── sequence_accuracy_test.py
│ │ ├── span_based_f1_measure_test.py
│ │ ├── spearman_correlation_test.py
│ │ └── unigram_recall_test.py
│ ├── momentum_schedulers/
│ │ ├── __init__.py
│ │ └── inverted_triangular_test.py
│ ├── moving_average_test.py
│ ├── no_op_trainer_test.py
│ ├── optimizer_test.py
│ ├── trainer_test.py
│ └── util_test.py
├── tutorials/
│ ├── __init__.py
│ └── tagger/
│ ├── __init__.py
│ └── basic_allennlp_test.py
└── version_test.py
Condensed preview — 711 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (4,172K chars).
[
{
"path": ".coveragerc",
"chars": 21,
"preview": "[run]\nomit = tests/*\n"
},
{
"path": ".dockerignore",
"chars": 86,
"preview": ".dockerignore\n**.pyc\n**/__pycache__\n.gitignore\n.git\n.coverage\n.benchmarks\n.mypy_cache\n"
},
{
"path": ".flake8",
"chars": 618,
"preview": "[flake8]\nmax-line-length = 115\n\nignore =\n # these rules don't play well with black\n E203 # whitespace before :..."
},
{
"path": ".github/ISSUE_TEMPLATE/bug_report.md",
"chars": 2753,
"preview": "---\nname: Bug report\nabout: Create a report to help us improve\ntitle: ''\nlabels: 'bug'\nassignees: ''\n\n---\n\n<!--\nPlease f..."
},
{
"path": ".github/ISSUE_TEMPLATE/feature_request.md",
"chars": 610,
"preview": "---\nname: Feature request\nabout: Suggest an idea for this project\ntitle: ''\nlabels: 'Feature request'\nassignees: ''\n\n---..."
},
{
"path": ".github/ISSUE_TEMPLATE/question.md",
"chars": 341,
"preview": "---\nname: Question\nabout: Ask a question\ntitle: ''\nlabels: 'question'\nassignees: ''\n\n---\n\nPlease ask questions on [Stack..."
},
{
"path": ".github/pull_request_template.md",
"chars": 1401,
"preview": "<!-- To ensure we can review your pull request promptly please complete this template entirely. -->\n\n<!-- Please referen..."
},
{
"path": ".github/workflows/cffconvert.yml",
"chars": 458,
"preview": "name: cffconvert\n\non:\n pull_request:\n paths:\n - CITATION.cff\n push:\n paths:\n - CITATION.cff\n\njobs:\n v..."
},
{
"path": ".github/workflows/ci.yml",
"chars": 20701,
"preview": "name: CI\n\nconcurrency:\n group: ${{ github.workflow }}-${{ github.ref }}\n cancel-in-progress: true\n\non:\n pull_request:..."
},
{
"path": ".github/workflows/issues.yml",
"chars": 1198,
"preview": "name: Issues Bot\n\non:\n # Uncomment this PR trigger for testing.\n # pull_request:\n # branches:\n # - main\n schedu..."
},
{
"path": ".gitignore",
"chars": 485,
"preview": "# build artifacts\n\n.eggs/\n.mypy_cache\nallennlp.egg-info/\nbuild/\ndist/\npip-wheel-metadata/\ncontext.tar.gz\n\n\n# dev tools..."
},
{
"path": "CHANGELOG.md",
"chars": 58505,
"preview": "# Changelog\n\nAll notable changes to this project will be documented in this file.\n\nThe format is based on [Keep a Change..."
},
{
"path": "CITATION.cff",
"chars": 1361,
"preview": "# YAML 1.2\n---\nauthors: \n -\n affiliation: \"Allen Institute for Artificial Intelligence\"\n family-names: Gardner..."
},
{
"path": "CONTRIBUTING.md",
"chars": 12068,
"preview": "# Contributing\n\nThanks for considering contributing! We want AllenNLP to be *the way* to do cutting-edge NLP research,..."
},
{
"path": "Dockerfile",
"chars": 1190,
"preview": "# This Dockerfile creates an environment suitable for downstream usage of AllenNLP.\n# It's built from a wheel installati..."
},
{
"path": "Dockerfile.test",
"chars": 952,
"preview": "# Used to build an image for running tests.\n\nARG TORCH=1.12.0-cuda11.3-python3.8\nFROM ghcr.io/allenai/pytorch:${TORCH}..."
},
{
"path": "LICENSE",
"chars": 11359,
"preview": " Apache License\n Version 2.0, January 2004..."
},
{
"path": "MANIFEST.in",
"chars": 168,
"preview": "include LICENSE\ninclude README.md\nrecursive-include allennlp *\nrecursive-include scripts *\nglobal-exclude .DS_Store *.py..."
},
{
"path": "Makefile",
"chars": 5221,
"preview": "SRC = allennlp\n\nMD_DOCS_ROOT = docs/\nMD_DOCS_API_ROOT = $(MD_DOCS_ROOT)api/\nMD_DOCS_SRC = $(filter-out $(SRC)/__main__.p..."
},
{
"path": "README.md",
"chars": 18737,
"preview": "<div align=\"center\">\n <br>\n <img src=\"https://raw.githubusercontent.com/allenai/allennlp/main/docs/img/allennlp-lo..."
},
{
"path": "RELEASE_PROCESS.md",
"chars": 2340,
"preview": "# AllenNLP GitHub and PyPI Release Process\n\nThis document describes the procedure for releasing new versions of the core..."
},
{
"path": "STYLE.md",
"chars": 4112,
"preview": "# AllenNLP Style Guide\n\nOur highest priority for code style is that our code be easily readable to someone new to the\nco..."
},
{
"path": "allennlp/__init__.py",
"chars": 764,
"preview": "# We get a lot of these spurious warnings,\n# see https://github.com/ContinuumIO/anaconda-issues/issues/6678\nimport warni..."
},
{
"path": "allennlp/__main__.py",
"chars": 1240,
"preview": "#!/usr/bin/env python\nimport logging\nimport os\nimport sys\n\n\nif os.environ.get(\"ALLENNLP_DEBUG\"):\n LEVEL = logging.DEB..."
},
{
"path": "allennlp/commands/__init__.py",
"chars": 4739,
"preview": "import argparse\nimport logging\nimport sys\nfrom typing import Any, Optional, Tuple, Set\nimport warnings\n\nfrom allennlp im..."
},
{
"path": "allennlp/commands/_checklist_internal.py",
"chars": 6023,
"preview": "\"\"\"\nThe `checklist` subcommand allows you to conduct behavioural\ntesting for your model's predictions using a trained mo..."
},
{
"path": "allennlp/commands/build_vocab.py",
"chars": 3253,
"preview": "\"\"\"\nSubcommand for building a vocabulary from a training config.\n\"\"\"\n\nimport argparse\nimport json\nimport logging\nimport..."
},
{
"path": "allennlp/commands/cached_path.py",
"chars": 3586,
"preview": "\"\"\"\nCLI to the the caching mechanism in `common.file_utils`.\n\"\"\"\n\nimport argparse\nimport logging\n\n\nfrom allennlp.command..."
},
{
"path": "allennlp/commands/checklist.py",
"chars": 1539,
"preview": "\"\"\"\nThe `checklist` subcommand allows you to conduct behavioural\ntesting for your model's predictions using a trained mo..."
},
{
"path": "allennlp/commands/count_instances.py",
"chars": 1679,
"preview": "\"\"\"\nSubcommand for counting the number of instances from a training config.\n\"\"\"\n\nimport argparse\nimport logging\n\n\nfrom a..."
},
{
"path": "allennlp/commands/diff.py",
"chars": 9172,
"preview": "\"\"\"\n# Examples\n\n```bash\nallennlp diff \\\n hf://roberta-large/pytorch_model.bin \\\n https://storage.googleapis.com/al..."
},
{
"path": "allennlp/commands/evaluate.py",
"chars": 15195,
"preview": "\"\"\"\nThe `evaluate` subcommand can be used to\nevaluate a trained model against a dataset\nand report any metrics calculate..."
},
{
"path": "allennlp/commands/find_learning_rate.py",
"chars": 11856,
"preview": "\"\"\"\nThe `find-lr` subcommand can be used to find a good learning rate for a model.\nIt requires a configuration file and..."
},
{
"path": "allennlp/commands/predict.py",
"chars": 10238,
"preview": "\"\"\"\nThe `predict` subcommand allows you to make bulk JSON-to-JSON\nor dataset to JSON predictions using a trained model a..."
},
{
"path": "allennlp/commands/print_results.py",
"chars": 2384,
"preview": "\"\"\"\nThe `print-results` subcommand allows you to print results from multiple\nallennlp serialization directories to the c..."
},
{
"path": "allennlp/commands/push_to_hf.py",
"chars": 2957,
"preview": "\"\"\"\nThe `push-to-hf` subcommand can be used to push a trained model to the\nHugging Face Hub ([hf.co](https://hf.co/)).\n\"..."
},
{
"path": "allennlp/commands/subcommand.py",
"chars": 1861,
"preview": "\"\"\"\nBase class for subcommands under `allennlp.run`.\n\"\"\"\n\nimport argparse\nfrom typing import Callable, Dict, Optional, T..."
},
{
"path": "allennlp/commands/test_install.py",
"chars": 1597,
"preview": "\"\"\"\nThe `test-install` subcommand provides a programmatic way to verify\nthat AllenNLP has been successfully installed.\n\"..."
},
{
"path": "allennlp/commands/train.py",
"chars": 33828,
"preview": "\"\"\"\nThe `train` subcommand can be used to train a model.\nIt requires a configuration file and a directory in\nwhich to wr..."
},
{
"path": "allennlp/common/__init__.py",
"chars": 351,
"preview": "from allennlp.common.from_params import FromParams\nfrom allennlp.common.lazy import Lazy\nfrom allennlp.common.params imp..."
},
{
"path": "allennlp/common/cached_transformers.py",
"chars": 9781,
"preview": "import logging\nimport re\nimport warnings\nfrom typing import Dict, NamedTuple, Optional, Tuple, Union, cast\n\nimport trans..."
},
{
"path": "allennlp/common/checks.py",
"chars": 3887,
"preview": "\"\"\"\nFunctions and exceptions for checking that\nAllenNLP and its models are configured correctly.\n\"\"\"\nimport logging\nimpo..."
},
{
"path": "allennlp/common/file_utils.py",
"chars": 21750,
"preview": "\"\"\"\nUtilities for working with the local dataset cache.\n\"\"\"\nimport bz2\nimport gzip\nimport lzma\nimport weakref\nfrom conte..."
},
{
"path": "allennlp/common/from_params.py",
"chars": 27647,
"preview": "import collections.abc\nfrom copy import deepcopy\nfrom pathlib import Path\nfrom typing import (\n Any,\n Callable,..."
},
{
"path": "allennlp/common/lazy.py",
"chars": 3271,
"preview": "import copy\nimport inspect\nfrom typing import Callable, Generic, TypeVar, Type, Union, Optional, Dict, Any\n\nfrom allennl..."
},
{
"path": "allennlp/common/logging.py",
"chars": 4241,
"preview": "import logging\nfrom logging import Filter\nimport os\nfrom os import PathLike\nfrom typing import Union\n\nimport sys\n\n\nclass..."
},
{
"path": "allennlp/common/meta.py",
"chars": 814,
"preview": "from os import PathLike\nfrom dataclasses import dataclass, asdict\nimport json\nimport logging\nfrom typing import Union\n\nf..."
},
{
"path": "allennlp/common/model_card.py",
"chars": 21820,
"preview": "\"\"\"\nA specification for defining model cards as described in\n[Model Cards for Model Reporting (Mitchell et al, 2019)]\n(h..."
},
{
"path": "allennlp/common/params.py",
"chars": 22683,
"preview": "import copy\nfrom itertools import chain\nimport json\nimport logging\nimport os\nimport zlib\nfrom collections import Ordered..."
},
{
"path": "allennlp/common/plugins.py",
"chars": 3515,
"preview": "\"\"\"\n# Plugin management.\n\nAllenNLP supports loading \"plugins\" dynamically. A plugin is just a Python package that\nprovid..."
},
{
"path": "allennlp/common/push_to_hf.py",
"chars": 6139,
"preview": "\"\"\"\nUtilities for pushing models to the Hugging Face Hub ([hf.co](https://hf.co/)).\n\"\"\"\n\nimport logging\nimport shutil\nim..."
},
{
"path": "allennlp/common/registrable.py",
"chars": 13396,
"preview": "\"\"\"\n`allennlp.common.registrable.Registrable` is a \"mixin\" for endowing\nany base class with a named registry for its sub..."
},
{
"path": "allennlp/common/sequences.py",
"chars": 2739,
"preview": "import bisect\nimport random\nfrom collections import abc\nfrom typing import Sequence, Optional, Union\n\n\nclass ShuffledSeq..."
},
{
"path": "allennlp/common/task_card.py",
"chars": 2776,
"preview": "\"\"\"\nA specification for defining task cards (derived from model cards).\nMotivation: A model's capabilities and limitatio..."
},
{
"path": "allennlp/common/testing/__init__.py",
"chars": 4455,
"preview": "\"\"\"\nUtilities and helpers for writing tests.\n\"\"\"\nfrom typing import Dict, Any, Optional, Union, Tuple, List\nimport torch..."
},
{
"path": "allennlp/common/testing/checklist_test.py",
"chars": 974,
"preview": "from typing import Optional\nfrom checklist.test_suite import TestSuite\nfrom checklist.test_types import MFT as MinimumFu..."
},
{
"path": "allennlp/common/testing/confidence_check_test.py",
"chars": 488,
"preview": "import torch\nfrom allennlp.data.vocabulary import Vocabulary\nfrom allennlp.models.model import Model\n\n\nclass FakeModelFo..."
},
{
"path": "allennlp/common/testing/distributed_test.py",
"chars": 2489,
"preview": "import datetime\nfrom typing import List, Dict, Any, Tuple, Callable\nimport torch\nimport torch.distributed as dist\nimport..."
},
{
"path": "allennlp/common/testing/interpret_test.py",
"chars": 1794,
"preview": "from allennlp.predictors import TextClassifierPredictor\nfrom allennlp.models.model import Model\nimport torch\n\n\nclass Fak..."
},
{
"path": "allennlp/common/testing/model_test_case.py",
"chars": 19584,
"preview": "import copy\nimport json\nfrom os import PathLike\nimport random\nfrom typing import Any, Dict, Iterable, Set, Union\n\nimport..."
},
{
"path": "allennlp/common/testing/test_case.py",
"chars": 1994,
"preview": "import logging\nimport os\nimport pathlib\nimport shutil\nimport tempfile\n\nfrom allennlp.common.checks import log_pytorch_ve..."
},
{
"path": "allennlp/common/tqdm.py",
"chars": 3097,
"preview": "\"\"\"\n`allennlp.common.tqdm.Tqdm` wraps tqdm so we can add configurable\nglobal defaults for certain tqdm parameters.\n\"\"\"\ni..."
},
{
"path": "allennlp/common/util.py",
"chars": 24218,
"preview": "\"\"\"\nVarious utilities that don't fit anywhere else.\n\"\"\"\nimport hashlib\nimport io\nfrom datetime import timedelta\nimport i..."
},
{
"path": "allennlp/confidence_checks/__init__.py",
"chars": 175,
"preview": "from allennlp.confidence_checks.verification_base import VerificationBase\nfrom allennlp.confidence_checks.normalization_..."
},
{
"path": "allennlp/confidence_checks/normalization_bias_verification.py",
"chars": 3601,
"preview": "\"\"\"\nCode based almost entirely from the [pytorch-lightning-snippets]\n(https://github.com/awaelchli/pytorch-lightning-sni..."
},
{
"path": "allennlp/confidence_checks/task_checklists/__init__.py",
"chars": 412,
"preview": "from allennlp.confidence_checks.task_checklists.task_suite import TaskSuite\nfrom allennlp.confidence_checks.task_checkli..."
},
{
"path": "allennlp/confidence_checks/task_checklists/question_answering_suite.py",
"chars": 6573,
"preview": "from typing import Optional, Iterable, Tuple, Union\nimport itertools\nimport numpy as np\n\nfrom checklist.editor import Mu..."
},
{
"path": "allennlp/confidence_checks/task_checklists/sentiment_analysis_suite.py",
"chars": 26259,
"preview": "from typing import Optional, Iterable, List, Union, Tuple\nimport numpy as np\n\nfrom checklist.test_suite import TestSuite..."
},
{
"path": "allennlp/confidence_checks/task_checklists/task_suite.py",
"chars": 16067,
"preview": "import sys\nimport logging\nfrom typing import Type, Optional, Dict, Any, Callable, List, Iterable, Union, TextIO, Tuple..."
},
{
"path": "allennlp/confidence_checks/task_checklists/textual_entailment_suite.py",
"chars": 23808,
"preview": "from typing import Optional, Tuple, Iterable, Callable, Union\nimport itertools\nimport numpy as np\n\nfrom checklist.test_s..."
},
{
"path": "allennlp/confidence_checks/task_checklists/utils.py",
"chars": 4615,
"preview": "import string\nfrom typing import Dict, Callable, List, Union\nimport numpy as np\nimport spacy\nfrom checklist.editor impor..."
},
{
"path": "allennlp/confidence_checks/verification_base.py",
"chars": 2459,
"preview": "\"\"\"\nCode based almost entirely on\nhttps://github.com/awaelchli/pytorch-lightning-snippets/commit/7db53f774715d635c59ef56..."
},
{
"path": "allennlp/data/__init__.py",
"chars": 692,
"preview": "from allennlp.data.data_loaders import (\n DataLoader,\n TensorDict,\n allennlp_collate,\n)\nfrom allennlp.data.data..."
},
{
"path": "allennlp/data/batch.py",
"chars": 9916,
"preview": "\"\"\"\nA :class:`Batch` represents a collection of `Instance` s to be fed\nthrough a model.\n\"\"\"\n\nimport logging\nfrom collect..."
},
{
"path": "allennlp/data/data_loaders/__init__.py",
"chars": 400,
"preview": "from allennlp.data.data_loaders.data_loader import DataLoader, TensorDict\nfrom allennlp.data.data_loaders.multiprocess_d..."
},
{
"path": "allennlp/data/data_loaders/data_collator.py",
"chars": 2764,
"preview": "from typing import List\n\nfrom transformers.data.data_collator import DataCollatorForLanguageModeling\nfrom allennlp.commo..."
},
{
"path": "allennlp/data/data_loaders/data_loader.py",
"chars": 1759,
"preview": "from typing import Dict, Union, Iterator\n\nimport torch\n\nfrom allennlp.common.registrable import Registrable\nfrom allennl..."
},
{
"path": "allennlp/data/data_loaders/multiprocess_data_loader.py",
"chars": 31151,
"preview": "from collections import deque\nimport logging\nfrom multiprocessing.process import BaseProcess\nfrom multiprocessing.connec..."
},
{
"path": "allennlp/data/data_loaders/multitask_data_loader.py",
"chars": 13896,
"preview": "from typing import Any, Dict, Iterable, Iterator, Union, Optional\nimport itertools\nimport math\n\nimport torch\n\n\nfrom alle..."
},
{
"path": "allennlp/data/data_loaders/multitask_epoch_sampler.py",
"chars": 3837,
"preview": "from typing import Any, Dict, Mapping\n\nfrom allennlp.common.registrable import Registrable\nfrom allennlp.data.data_loade..."
},
{
"path": "allennlp/data/data_loaders/multitask_scheduler.py",
"chars": 5581,
"preview": "from collections import defaultdict\nfrom typing import Any, Dict, Iterable, Union, List, Mapping\n\nimport more_itertools..."
},
{
"path": "allennlp/data/data_loaders/simple_data_loader.py",
"chars": 3499,
"preview": "import math\nimport random\nfrom typing import Optional, List, Iterator\n\n\nimport torch\n\nfrom allennlp.common.util import l..."
},
{
"path": "allennlp/data/dataset_readers/__init__.py",
"chars": 992,
"preview": "\"\"\"\nA :class:`~allennlp.data.dataset_readers.dataset_reader.DatasetReader`\nreads a file and converts it to a collection..."
},
{
"path": "allennlp/data/dataset_readers/babi.py",
"chars": 4098,
"preview": "import logging\n\nfrom typing import Dict, List\n\n\nfrom allennlp.common.file_utils import cached_path\nfrom allennlp.data.da..."
},
{
"path": "allennlp/data/dataset_readers/conll2003.py",
"chars": 9835,
"preview": "from typing import Dict, List, Optional, Sequence, Iterable\nimport itertools\nimport logging\nimport warnings\n\n\nfrom allen..."
},
{
"path": "allennlp/data/dataset_readers/dataset_reader.py",
"chars": 17479,
"preview": "from dataclasses import dataclass\nimport itertools\nfrom os import PathLike\nfrom typing import Iterable, Iterator, Option..."
},
{
"path": "allennlp/data/dataset_readers/dataset_utils/__init__.py",
"chars": 346,
"preview": "from allennlp.data.dataset_readers.dataset_utils.span_utils import enumerate_spans\nfrom allennlp.data.dataset_readers.da..."
},
{
"path": "allennlp/data/dataset_readers/dataset_utils/span_utils.py",
"chars": 17308,
"preview": "from typing import Callable, List, Set, Tuple, TypeVar, Optional\nimport warnings\n\nfrom allennlp.common.checks import Con..."
},
{
"path": "allennlp/data/dataset_readers/interleaving_dataset_reader.py",
"chars": 5024,
"preview": "from typing import Dict, Mapping, Iterable, Union, Optional\nimport json\n\n\nfrom allennlp.common.checks import Configurati..."
},
{
"path": "allennlp/data/dataset_readers/multitask.py",
"chars": 3063,
"preview": "from os import PathLike\nfrom typing import Dict, Iterator, Union, Optional\n\nfrom allennlp.data.instance import Instance..."
},
{
"path": "allennlp/data/dataset_readers/sequence_tagging.py",
"chars": 4015,
"preview": "from typing import Dict, List, Any\nimport logging\n\n\nfrom allennlp.common.file_utils import cached_path\nfrom allennlp.dat..."
},
{
"path": "allennlp/data/dataset_readers/sharded_dataset_reader.py",
"chars": 3483,
"preview": "import glob\nimport logging\nimport os\nfrom typing import Iterable\n\n\nfrom allennlp.common.checks import ConfigurationError..."
},
{
"path": "allennlp/data/dataset_readers/text_classification_json.py",
"chars": 6257,
"preview": "from typing import Dict, List, Union\nimport logging\nimport json\n\nfrom allennlp.common.file_utils import cached_path\nfrom..."
},
{
"path": "allennlp/data/fields/__init__.py",
"chars": 1094,
"preview": "\"\"\"\nA :class:`~allennlp.data.fields.field.Field` is some piece of data instance\nthat ends up as an array in a model.\n\"\"\"..."
},
{
"path": "allennlp/data/fields/adjacency_field.py",
"chars": 6504,
"preview": "from typing import Dict, List, Set, Tuple, Optional\nimport logging\nimport textwrap\n\n\nimport torch\n\nfrom allennlp.common...."
},
{
"path": "allennlp/data/fields/array_field.py",
"chars": 150,
"preview": "from allennlp.data.fields.tensor_field import TensorField\n\nArrayField = TensorField\n\"\"\"For backwards compatibility, we k..."
},
{
"path": "allennlp/data/fields/field.py",
"chars": 7321,
"preview": "from copy import deepcopy\nfrom typing import Dict, Generic, List, TypeVar, Any\n\nimport torch\n\nfrom allennlp.data.vocabul..."
},
{
"path": "allennlp/data/fields/flag_field.py",
"chars": 1374,
"preview": "from typing import Any, Dict, List\n\n\nfrom allennlp.data.fields.field import Field\n\n\nclass FlagField(Field[Any]):\n \"\"\"..."
},
{
"path": "allennlp/data/fields/index_field.py",
"chars": 2334,
"preview": "from typing import Dict\n\n\nimport torch\n\nfrom allennlp.data.fields.field import Field\nfrom allennlp.data.fields.sequence_..."
},
{
"path": "allennlp/data/fields/label_field.py",
"chars": 4853,
"preview": "from typing import Dict, Union, Set\nimport logging\n\n\nimport torch\n\nfrom allennlp.data.fields.field import Field\nfrom all..."
},
{
"path": "allennlp/data/fields/list_field.py",
"chars": 5211,
"preview": "from typing import Dict, List, Iterator, Sequence, Any\n\n\nfrom allennlp.data.fields.field import DataArray, Field\nfrom al..."
},
{
"path": "allennlp/data/fields/metadata_field.py",
"chars": 2353,
"preview": "from typing import Any, Dict, List, Mapping\n\n\nfrom allennlp.data.fields.field import DataArray, Field\n\n\nclass MetadataFi..."
},
{
"path": "allennlp/data/fields/multilabel_field.py",
"chars": 6287,
"preview": "from typing import Dict, Union, Sequence, Set, Optional, cast\nimport logging\n\n\nimport torch\n\nfrom allennlp.data.fields.f..."
},
{
"path": "allennlp/data/fields/namespace_swapping_field.py",
"chars": 2109,
"preview": "from typing import Dict, List, Any\n\n\nimport torch\n\nfrom allennlp.common.util import pad_sequence_to_length\nfrom allennlp..."
},
{
"path": "allennlp/data/fields/sequence_field.py",
"chars": 753,
"preview": "from allennlp.data.fields.field import DataArray, Field\n\n\nclass SequenceField(Field[DataArray]):\n \"\"\"\n A `Sequence..."
},
{
"path": "allennlp/data/fields/sequence_label_field.py",
"chars": 6328,
"preview": "from typing import Dict, List, Union, Set, Iterator\nimport logging\nimport textwrap\n\n\nimport torch\n\nfrom allennlp.common...."
},
{
"path": "allennlp/data/fields/span_field.py",
"chars": 2774,
"preview": "from typing import Dict, Tuple\n\n\nimport torch\n\nfrom allennlp.data.fields.field import Field\nfrom allennlp.data.fields.se..."
},
{
"path": "allennlp/data/fields/tensor_field.py",
"chars": 3092,
"preview": "from typing import Dict, Any, Union, Optional\n\nimport torch\nimport numpy as np\n\n\nfrom allennlp.data.fields.field import..."
},
{
"path": "allennlp/data/fields/text_field.py",
"chars": 8734,
"preview": "\"\"\"\nA `TextField` represents a string of text, the kind that you might want to represent with\nstandard word vectors, or..."
},
{
"path": "allennlp/data/fields/transformer_text_field.py",
"chars": 4410,
"preview": "from typing import Dict, Optional, List, Any, Union\n\n\nimport torch\nimport torch.nn.functional\n\nfrom allennlp.data.fields..."
},
{
"path": "allennlp/data/image_loader.py",
"chars": 7697,
"preview": "from os import PathLike\nfrom typing import Union, Sequence, Tuple, List, cast\n\n\nimport torch\nimport torchvision\nfrom tor..."
},
{
"path": "allennlp/data/instance.py",
"chars": 5050,
"preview": "from typing import Dict, MutableMapping, Mapping\n\nfrom allennlp.data.fields.field import DataArray, Field\nfrom allennlp...."
},
{
"path": "allennlp/data/samplers/__init__.py",
"chars": 219,
"preview": "from allennlp.data.samplers.batch_sampler import BatchSampler\nfrom allennlp.data.samplers.bucket_batch_sampler import Bu..."
},
{
"path": "allennlp/data/samplers/batch_sampler.py",
"chars": 624,
"preview": "from typing import List, Iterable, Sequence, Optional\n\nfrom allennlp.common.registrable import Registrable\nfrom allennlp..."
},
{
"path": "allennlp/data/samplers/bucket_batch_sampler.py",
"chars": 7426,
"preview": "import logging\nimport math\nfrom typing import List, Iterable, Tuple, Sequence, Optional\nimport random\n\nfrom allennlp.com..."
},
{
"path": "allennlp/data/samplers/max_tokens_batch_sampler.py",
"chars": 4649,
"preview": "import logging\nimport random\nfrom typing import List, Iterable, Iterator, TypeVar, Sequence\n\nfrom allennlp.data.instance..."
},
{
"path": "allennlp/data/token_indexers/__init__.py",
"chars": 733,
"preview": "\"\"\"\nA `TokenIndexer` determines how string tokens get represented as arrays of indices in a model.\n\"\"\"\n\nfrom allennlp.da..."
},
{
"path": "allennlp/data/token_indexers/elmo_indexer.py",
"chars": 5637,
"preview": "from typing import Dict, List\n\n\nimport torch\n\nfrom allennlp.common.util import pad_sequence_to_length\nfrom allennlp.data..."
},
{
"path": "allennlp/data/token_indexers/pretrained_transformer_indexer.py",
"chars": 10603,
"preview": "from typing import Dict, List, Optional, Tuple, Any\nimport logging\nimport torch\nfrom allennlp.common.util import pad_seq..."
},
{
"path": "allennlp/data/token_indexers/pretrained_transformer_mismatched_indexer.py",
"chars": 5715,
"preview": "from typing import Dict, List, Any, Optional\nimport logging\n\n\nimport torch\n\nfrom allennlp.common.util import pad_sequenc..."
},
{
"path": "allennlp/data/token_indexers/single_id_token_indexer.py",
"chars": 5266,
"preview": "from typing import Dict, List, Optional, Any\nimport itertools\n\n\nfrom allennlp.data.vocabulary import Vocabulary\nfrom all..."
},
{
"path": "allennlp/data/token_indexers/spacy_indexer.py",
"chars": 2326,
"preview": "from typing import Dict, List\n\n\nfrom spacy.tokens import Token as SpacyToken\nimport torch\nimport numpy\n\nfrom allennlp.co..."
},
{
"path": "allennlp/data/token_indexers/token_characters_indexer.py",
"chars": 6836,
"preview": "from typing import Dict, List\nimport itertools\nimport warnings\n\n\nimport torch\n\nfrom allennlp.common.checks import Config..."
},
{
"path": "allennlp/data/token_indexers/token_indexer.py",
"chars": 6275,
"preview": "from typing import Any, Dict, List\n\nimport torch\n\nfrom allennlp.common import Registrable\nfrom allennlp.common.util impo..."
},
{
"path": "allennlp/data/tokenizers/__init__.py",
"chars": 667,
"preview": "\"\"\"\nThis module contains various classes for performing\ntokenization.\n\"\"\"\n\nfrom allennlp.data.tokenizers.token_class imp..."
},
{
"path": "allennlp/data/tokenizers/character_tokenizer.py",
"chars": 3819,
"preview": "from typing import List, Union, Dict, Any\n\n\nfrom allennlp.data.tokenizers.token_class import Token\nfrom allennlp.data.to..."
},
{
"path": "allennlp/data/tokenizers/letters_digits_tokenizer.py",
"chars": 709,
"preview": "import re\nfrom typing import List\n\n\nfrom allennlp.data.tokenizers.token_class import Token\nfrom allennlp.data.tokenizers..."
},
{
"path": "allennlp/data/tokenizers/pretrained_transformer_tokenizer.py",
"chars": 20289,
"preview": "import copy\nimport dataclasses\nimport logging\nfrom typing import Any, Dict, List, Optional, Tuple, Iterable\n\n\nfrom trans..."
},
{
"path": "allennlp/data/tokenizers/sentence_splitter.py",
"chars": 3262,
"preview": "from typing import List, Dict, Any\n\n\nimport spacy\n\nfrom allennlp.common import Registrable\nfrom allennlp.common.util imp..."
},
{
"path": "allennlp/data/tokenizers/spacy_tokenizer.py",
"chars": 6690,
"preview": "from typing import List, Optional\n\n\nimport spacy\nfrom spacy.tokens import Doc\n\nfrom allennlp.common.util import get_spac..."
},
{
"path": "allennlp/data/tokenizers/token_class.py",
"chars": 4162,
"preview": "from dataclasses import dataclass\nfrom typing import Optional\n\n\n@dataclass(init=False, repr=False)\nclass Token:\n \"\"\"..."
},
{
"path": "allennlp/data/tokenizers/tokenizer.py",
"chars": 2774,
"preview": "from typing import List, Optional\nimport logging\n\nfrom allennlp.common import Registrable\nfrom allennlp.data.tokenizers...."
},
{
"path": "allennlp/data/tokenizers/whitespace_tokenizer.py",
"chars": 918,
"preview": "from typing import List, Dict, Any\n\n\nfrom allennlp.data.tokenizers.token_class import Token\nfrom allennlp.data.tokenizer..."
},
{
"path": "allennlp/data/vocabulary.py",
"chars": 43005,
"preview": "\"\"\"\nA Vocabulary maps strings to integers, allowing for strings to be mapped to an\nout-of-vocabulary token.\n\"\"\"\n\nimport..."
},
{
"path": "allennlp/evaluation/__init__.py",
"chars": 136,
"preview": "from allennlp.evaluation.evaluator import Evaluator, SimpleEvaluator\nfrom allennlp.evaluation.serializers.serializers im..."
},
{
"path": "allennlp/evaluation/evaluator.py",
"chars": 9022,
"preview": "\"\"\"\nEvaluator class for evaluating a model with a given dataset\n\"\"\"\nfrom typing import Union, Dict, Any, Optional\nfrom o..."
},
{
"path": "allennlp/evaluation/postprocessors/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "allennlp/evaluation/serializers/__init__.py",
"chars": 73,
"preview": "from allennlp.evaluation.serializers.serializers import SimpleSerializer\n"
},
{
"path": "allennlp/evaluation/serializers/serializers.py",
"chars": 3109,
"preview": "from typing import Optional, Dict, Any, Callable\nimport logging\nimport json\n\nfrom allennlp.common.util import sanitize\nf..."
},
{
"path": "allennlp/fairness/__init__.py",
"chars": 1523,
"preview": "\"\"\"\nThis module contains tools to:\n\n1. measure the fairness of models according to multiple definitions of fairness\n2. m..."
},
{
"path": "allennlp/fairness/adversarial_bias_mitigator.py",
"chars": 12019,
"preview": "\"\"\"\nA Model wrapper to adversarially mitigate biases in\npredictions produced by a pretrained model for a downstream task..."
},
{
"path": "allennlp/fairness/bias_direction.py",
"chars": 12946,
"preview": "\"\"\"\nA suite of differentiable methods to compute the bias direction\nor concept subspace representing binary protected va..."
},
{
"path": "allennlp/fairness/bias_direction_wrappers.py",
"chars": 9766,
"preview": "import torch\nfrom typing import Union, Optional\nfrom os import PathLike\n\nfrom allennlp.fairness.bias_direction import (..."
},
{
"path": "allennlp/fairness/bias_metrics.py",
"chars": 28905,
"preview": "\"\"\"\n\nA suite of metrics to quantify how much bias is encoded by word embeddings\nand determine the effectiveness of bias..."
},
{
"path": "allennlp/fairness/bias_mitigator_applicator.py",
"chars": 3751,
"preview": "\"\"\"\nA Model wrapper to mitigate biases in\ncontextual embeddings during finetuning\non a downstream task and test time.\n\nB..."
},
{
"path": "allennlp/fairness/bias_mitigator_wrappers.py",
"chars": 9578,
"preview": "import torch\nfrom typing import Union, Optional\nfrom os import PathLike\n\nfrom allennlp.fairness.bias_mitigators import (..."
},
{
"path": "allennlp/fairness/bias_mitigators.py",
"chars": 24434,
"preview": "\"\"\"\nA suite of differentiable methods to mitigate\nbiases for binary concepts in embeddings.\n\"\"\"\n\nimport torch\nimport num..."
},
{
"path": "allennlp/fairness/bias_utils.py",
"chars": 3798,
"preview": "import torch\nimport json\nfrom os import PathLike\nfrom typing import List, Tuple, Union, Optional\n\nfrom allennlp.common.f..."
},
{
"path": "allennlp/fairness/fairness_metrics.py",
"chars": 31786,
"preview": "\"\"\"\nFairness metrics are based on:\n\n1. Barocas, S.; Hardt, M.; and Narayanan, A. 2019. [Fairness and machine learning](h..."
},
{
"path": "allennlp/interpret/__init__.py",
"chars": 250,
"preview": "from allennlp.interpret.attackers.attacker import Attacker\nfrom allennlp.interpret.saliency_interpreters.saliency_interp..."
},
{
"path": "allennlp/interpret/attackers/__init__.py",
"chars": 188,
"preview": "from allennlp.interpret.attackers.attacker import Attacker\nfrom allennlp.interpret.attackers.input_reduction import Inpu..."
},
{
"path": "allennlp/interpret/attackers/attacker.py",
"chars": 2270,
"preview": "from typing import List\n\nfrom allennlp.common import Registrable\nfrom allennlp.common.util import JsonDict\nfrom allennlp..."
},
{
"path": "allennlp/interpret/attackers/hotflip.py",
"chars": 20112,
"preview": "from copy import deepcopy\nfrom typing import Dict, List, Tuple\n\nimport numpy\nimport torch\n\nfrom allennlp.common.util imp..."
},
{
"path": "allennlp/interpret/attackers/input_reduction.py",
"chars": 9505,
"preview": "from copy import deepcopy\nfrom typing import List, Tuple\nimport heapq\n\nimport numpy as np\nimport torch\n\nfrom allennlp.co..."
},
{
"path": "allennlp/interpret/attackers/utils.py",
"chars": 1930,
"preview": "from allennlp.common.util import JsonDict\nfrom allennlp.data import Instance\n\n\ndef get_fields_to_compare(\n inputs: Js..."
},
{
"path": "allennlp/interpret/influence_interpreters/__init__.py",
"chars": 184,
"preview": "from allennlp.interpret.influence_interpreters.influence_interpreter import InfluenceInterpreter\nfrom allennlp.interpret..."
},
{
"path": "allennlp/interpret/influence_interpreters/influence_interpreter.py",
"chars": 17224,
"preview": "import logging\nfrom os import PathLike\nimport re\nfrom typing import List, Optional, NamedTuple, Sequence, Union, Dict, A..."
},
{
"path": "allennlp/interpret/influence_interpreters/simple_influence.py",
"chars": 10197,
"preview": "import logging\nfrom typing import List, Optional, Tuple, Union, Sequence\n\nimport numpy as np\n\nimport torch\nimport torch...."
},
{
"path": "allennlp/interpret/saliency_interpreters/__init__.py",
"chars": 354,
"preview": "from allennlp.interpret.saliency_interpreters.saliency_interpreter import SaliencyInterpreter\nfrom allennlp.interpret.sa..."
},
{
"path": "allennlp/interpret/saliency_interpreters/integrated_gradient.py",
"chars": 4611,
"preview": "import math\nfrom typing import List, Dict, Any\n\nimport numpy\nimport torch\n\nfrom allennlp.common.util import JsonDict, sa..."
},
{
"path": "allennlp/interpret/saliency_interpreters/saliency_interpreter.py",
"chars": 2806,
"preview": "from typing import List\n\nimport numpy\nimport torch\n\nfrom allennlp.common import Registrable\nfrom allennlp.common.util im..."
},
{
"path": "allennlp/interpret/saliency_interpreters/simple_gradient.py",
"chars": 3636,
"preview": "import math\n\nfrom typing import List\nimport numpy\nimport torch\n\nfrom allennlp.common.util import JsonDict, sanitize\nfrom..."
},
{
"path": "allennlp/interpret/saliency_interpreters/smooth_gradient.py",
"chars": 3268,
"preview": "import math\nfrom typing import Dict, Any\n\nimport numpy\nimport torch\n\nfrom allennlp.common.util import JsonDict, sanitize..."
},
{
"path": "allennlp/models/__init__.py",
"chars": 390,
"preview": "\"\"\"\nThese submodules contain the classes for AllenNLP models,\nall of which are subclasses of `Model`.\n\"\"\"\n\nfrom allennlp..."
},
{
"path": "allennlp/models/archival.py",
"chars": 12838,
"preview": "\"\"\"\nHelper functions for archiving models and restoring archived models.\n\"\"\"\nfrom os import PathLike\nfrom pathlib import..."
},
{
"path": "allennlp/models/basic_classifier.py",
"chars": 7287,
"preview": "from typing import Dict, Optional\n\n\nimport torch\n\nfrom allennlp.data import TextFieldTensors, Vocabulary\nfrom allennlp.d..."
},
{
"path": "allennlp/models/heads/__init__.py",
"chars": 109,
"preview": "from allennlp.models.heads.head import Head\nfrom allennlp.models.heads.classifier_head import ClassifierHead\n"
},
{
"path": "allennlp/models/heads/classifier_head.py",
"chars": 5334,
"preview": "from typing import Dict, Optional\n\n\nimport torch\n\nfrom allennlp.data import Vocabulary\nfrom allennlp.models.heads.head i..."
},
{
"path": "allennlp/models/heads/head.py",
"chars": 1021,
"preview": "from allennlp.models.model import Model\n\n\nclass Head(Model):\n \"\"\"\n A `Head` is a `Model` that takes _already encod..."
},
{
"path": "allennlp/models/model.py",
"chars": 23026,
"preview": "\"\"\"\n`Model` is an abstract class representing\nan AllenNLP model.\n\"\"\"\n\nimport logging\nimport os\nfrom os import PathLike\ni..."
},
{
"path": "allennlp/models/multitask.py",
"chars": 10104,
"preview": "from collections import defaultdict\nimport inspect\nfrom typing import Any, Dict, List, Set, Union, Mapping\n\n\nimport torc..."
},
{
"path": "allennlp/models/simple_tagger.py",
"chars": 9558,
"preview": "from typing import Dict, Optional, List, Any\n\nimport numpy\n\nimport torch\nfrom torch.nn.modules.linear import Linear\nimpo..."
},
{
"path": "allennlp/modules/__init__.py",
"chars": 1304,
"preview": "\"\"\"\nCustom PyTorch\n`Module <https://pytorch.org/docs/master/nn.html#torch.nn.Module>`_ s\nthat are used as components in..."
},
{
"path": "allennlp/modules/attention/__init__.py",
"chars": 530,
"preview": "from allennlp.modules.attention.attention import Attention\nfrom allennlp.modules.attention.bilinear_attention import Bil..."
},
{
"path": "allennlp/modules/attention/additive_attention.py",
"chars": 2409,
"preview": "import torch\nfrom torch.nn.parameter import Parameter\n\nfrom allennlp.modules.attention.attention import Attention\n\n\n@Att..."
},
{
"path": "allennlp/modules/attention/attention.py",
"chars": 1694,
"preview": "\"\"\"\nAn *attention* module that computes the similarity between\nan input vector and the rows of a matrix.\n\"\"\"\n\nimport tor..."
},
{
"path": "allennlp/modules/attention/bilinear_attention.py",
"chars": 2329,
"preview": "import torch\nfrom torch.nn.parameter import Parameter\n\nfrom allennlp.modules.attention.attention import Attention\nfrom a..."
},
{
"path": "allennlp/modules/attention/cosine_attention.py",
"chars": 757,
"preview": "import torch\n\nfrom allennlp.modules.attention.attention import Attention\nfrom allennlp.nn import util\n\n\n@Attention.regis..."
},
{
"path": "allennlp/modules/attention/dot_product_attention.py",
"chars": 571,
"preview": "import torch\n\nfrom allennlp.modules.attention.attention import Attention\n\n\n@Attention.register(\"dot_product\")\nclass DotP..."
},
{
"path": "allennlp/modules/attention/linear_attention.py",
"chars": 3307,
"preview": "import math\n\nimport torch\nfrom torch.nn import Parameter\n\nfrom allennlp.modules.attention.attention import Attention\nfro..."
},
{
"path": "allennlp/modules/attention/scaled_dot_product_attention.py",
"chars": 1403,
"preview": "import math\nfrom typing import Optional\n\nimport torch\n\n\nfrom allennlp.modules.attention.dot_product_attention import Dot..."
},
{
"path": "allennlp/modules/augmented_lstm.py",
"chars": 21575,
"preview": "\"\"\"\nAn LSTM with Recurrent Dropout and the option to use highway\nconnections between layers.\nBased on PyText version (th..."
},
{
"path": "allennlp/modules/backbones/__init__.py",
"chars": 230,
"preview": "from allennlp.modules.backbones.backbone import Backbone\nfrom allennlp.modules.backbones.pretrained_transformer_backbone..."
},
{
"path": "allennlp/modules/backbones/backbone.py",
"chars": 2188,
"preview": "from typing import Dict\n\nimport torch\n\nfrom allennlp.common import Registrable\n\n\nclass Backbone(torch.nn.Module, Registr..."
},
{
"path": "allennlp/modules/backbones/pretrained_transformer_backbone.py",
"chars": 6077,
"preview": "from typing import Dict, Optional, Any\n\n\nimport torch\n\nfrom allennlp.data.fields.text_field import TextFieldTensors\nfrom..."
},
{
"path": "allennlp/modules/backbones/vilbert_backbone.py",
"chars": 8088,
"preview": "import logging\nfrom typing import Dict, List\n\nimport torch\n\n\nfrom allennlp.data.fields.text_field import TextFieldTensor..."
},
{
"path": "allennlp/modules/bimpm_matching.py",
"chars": 15794,
"preview": "\"\"\"\nMulti-perspective matching layer\n\"\"\"\n\nfrom typing import Tuple, List\n\nimport torch\nimport torch.nn as nn\nimport torc..."
},
{
"path": "allennlp/modules/conditional_random_field/__init__.py",
"chars": 531,
"preview": "from allennlp.modules.conditional_random_field.conditional_random_field import (\n ConditionalRandomField,\n allowed..."
},
{
"path": "allennlp/modules/conditional_random_field/conditional_random_field.py",
"chars": 19649,
"preview": "\"\"\"\nConditional random field\n\"\"\"\nfrom typing import List, Tuple, Dict, Union\n\nimport torch\n\nfrom allennlp.common.checks..."
},
{
"path": "allennlp/modules/conditional_random_field/conditional_random_field_wemission.py",
"chars": 3913,
"preview": "\"\"\"\nConditional random field with emission-based weighting\n\"\"\"\nfrom typing import List, Tuple\n\nimport torch\n\nfrom allenn..."
},
{
"path": "allennlp/modules/conditional_random_field/conditional_random_field_wlannoy.py",
"chars": 10687,
"preview": "\"\"\"\nConditional random field with weighting based on Lannoy et al. (2019) approach\n\"\"\"\nfrom typing import List, Tuple\n\ni..."
},
{
"path": "allennlp/modules/conditional_random_field/conditional_random_field_wtrans.py",
"chars": 4214,
"preview": "\"\"\"\nConditional random field with emission- and transition-based weighting\n\"\"\"\nfrom typing import List, Tuple\n\nimport to..."
},
{
"path": "allennlp/modules/elmo.py",
"chars": 29090,
"preview": "import json\nimport logging\nimport warnings\nfrom typing import Any, Dict, List, Union\n\nimport numpy\nimport torch\n\nfrom to..."
},
{
"path": "allennlp/modules/elmo_lstm.py",
"chars": 15170,
"preview": "\"\"\"\nA stacked bidirectional LSTM with skip connections between layers.\n\"\"\"\nimport warnings\nfrom typing import List, Opti..."
},
{
"path": "allennlp/modules/encoder_base.py",
"chars": 16887,
"preview": "from typing import Tuple, Union, Optional, Callable, Any\nimport torch\nfrom torch.nn.utils.rnn import pack_padded_sequenc..."
},
{
"path": "allennlp/modules/feedforward.py",
"chars": 4165,
"preview": "\"\"\"\nA feed-forward neural network.\n\"\"\"\nfrom typing import List, Union\n\nimport torch\n\nfrom allennlp.common import FromPar..."
},
{
"path": "allennlp/modules/gated_sum.py",
"chars": 1346,
"preview": "import torch\n\nfrom allennlp.nn import Activation\n\n\nclass GatedSum(torch.nn.Module):\n \"\"\"\n This `Module` represents..."
},
{
"path": "allennlp/modules/highway.py",
"chars": 2635,
"preview": "\"\"\"\nA [Highway layer](https://arxiv.org/abs/1505.00387) that does a gated combination of a linear\ntransformation and a n..."
},
{
"path": "allennlp/modules/input_variational_dropout.py",
"chars": 1245,
"preview": "import torch\n\n\nclass InputVariationalDropout(torch.nn.Dropout):\n \"\"\"\n Apply the dropout technique in Gal and Ghahr..."
},
{
"path": "allennlp/modules/layer_norm.py",
"chars": 1115,
"preview": "import torch\n\nfrom allennlp.nn import util\n\n\nclass LayerNorm(torch.nn.Module):\n\n \"\"\"\n An implementation of [Layer..."
}
]
// ... and 511 more files (download for full content)
About this extraction
This page contains the full source code of the allenai/allennlp GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 727 files (3.8 MB), approximately 1.0M tokens. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.