[ { "path": ".gitignore", "content": ".*.swp\n*.o\n*.bin\n*.vec\n*.bc\n.DS_Store\ndata\nfasttext\nresult\nwebsite/node_modules/\npackage-lock.json\nnode_modules/\n" }, { "path": "CMakeLists.txt", "content": "#\n# Copyright (c) 2016-present, Facebook, Inc.\n# All rights reserved.\n#\n# This source code is licensed under the MIT license found in the\n# LICENSE file in the root directory of this source tree.\n#\n\ncmake_minimum_required(VERSION 2.8.9)\nproject(fasttext)\n\nset(CMAKE_CXX_STANDARD 17)\n\n# The version number.\nset (fasttext_VERSION_MAJOR 0)\nset (fasttext_VERSION_MINOR 1)\n\ninclude_directories(fasttext)\n\nset(CMAKE_CXX_FLAGS \" -pthread -std=c++17 -funroll-loops -O3 -march=native\")\n\nset(HEADER_FILES\n src/args.h\n src/autotune.h\n src/densematrix.h\n src/dictionary.h\n src/fasttext.h\n src/loss.h\n src/matrix.h\n src/meter.h\n src/model.h\n src/productquantizer.h\n src/quantmatrix.h\n src/real.h\n src/utils.h\n src/vector.h)\n\nset(SOURCE_FILES\n src/args.cc\n src/autotune.cc\n src/densematrix.cc\n src/dictionary.cc\n src/fasttext.cc\n src/loss.cc\n src/main.cc\n src/matrix.cc\n src/meter.cc\n src/model.cc\n src/productquantizer.cc\n src/quantmatrix.cc\n src/utils.cc\n src/vector.cc)\n\n\nif (NOT MSVC)\n include(GNUInstallDirs)\n configure_file(\"fasttext.pc.in\" \"fasttext.pc\" @ONLY)\n install(FILES \"${CMAKE_BINARY_DIR}/fasttext.pc\" DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig)\nendif()\n\nadd_library(fasttext-shared SHARED ${SOURCE_FILES} ${HEADER_FILES})\nadd_library(fasttext-static STATIC ${SOURCE_FILES} ${HEADER_FILES})\nadd_library(fasttext-static_pic STATIC ${SOURCE_FILES} ${HEADER_FILES})\nset_target_properties(fasttext-shared PROPERTIES OUTPUT_NAME fasttext\n SOVERSION \"${fasttext_VERSION_MAJOR}\")\nset_target_properties(fasttext-static PROPERTIES OUTPUT_NAME fasttext)\nset_target_properties(fasttext-static_pic PROPERTIES OUTPUT_NAME fasttext_pic\n POSITION_INDEPENDENT_CODE True)\nadd_executable(fasttext-bin src/main.cc)\ntarget_link_libraries(fasttext-bin pthread fasttext-static)\nset_target_properties(fasttext-bin PROPERTIES PUBLIC_HEADER \"${HEADER_FILES}\" OUTPUT_NAME fasttext)\ninstall (TARGETS fasttext-shared\n LIBRARY DESTINATION lib)\ninstall (TARGETS fasttext-static\n ARCHIVE DESTINATION lib)\ninstall (TARGETS fasttext-static_pic\n ARCHIVE DESTINATION lib)\ninstall (TARGETS fasttext-bin\n RUNTIME DESTINATION bin\n PUBLIC_HEADER DESTINATION include/fasttext)\n" }, { "path": "CODE_OF_CONDUCT.md", "content": "# Code of Conduct\n\n## Our Pledge\n\nIn the interest of fostering an open and welcoming environment, we as\ncontributors and maintainers pledge to make participation in our project and\nour community a harassment-free experience for everyone, regardless of age, body\nsize, disability, ethnicity, sex characteristics, gender identity and expression,\nlevel of experience, education, socio-economic status, nationality, personal\nappearance, race, religion, or sexual identity and orientation.\n\n## Our Standards\n\nExamples of behavior that contributes to creating a positive environment\ninclude:\n\n* Using welcoming and inclusive language\n* Being respectful of differing viewpoints and experiences\n* Gracefully accepting constructive criticism\n* Focusing on what is best for the community\n* Showing empathy towards other community members\n\nExamples of unacceptable behavior by participants include:\n\n* The use of sexualized language or imagery and unwelcome sexual attention or\n advances\n* Trolling, insulting/derogatory comments, and personal or political attacks\n* Public or private harassment\n* Publishing others' private information, such as a physical or electronic\n address, without explicit permission\n* Other conduct which could reasonably be considered inappropriate in a\n professional setting\n\n## Our Responsibilities\n\nProject maintainers are responsible for clarifying the standards of acceptable\nbehavior and are expected to take appropriate and fair corrective action in\nresponse to any instances of unacceptable behavior.\n\nProject maintainers have the right and responsibility to remove, edit, or\nreject comments, commits, code, wiki edits, issues, and other contributions\nthat are not aligned to this Code of Conduct, or to ban temporarily or\npermanently any contributor for other behaviors that they deem inappropriate,\nthreatening, offensive, or harmful.\n\n## Scope\n\nThis Code of Conduct applies within all project spaces, and it also applies when\nan individual is representing the project or its community in public spaces.\nExamples of representing a project or community include using an official\nproject e-mail address, posting via an official social media account, or acting\nas an appointed representative at an online or offline event. Representation of\na project may be further defined and clarified by project maintainers.\n\n## Enforcement\n\nInstances of abusive, harassing, or otherwise unacceptable behavior may be\nreported by contacting the project team at . All\ncomplaints will be reviewed and investigated and will result in a response that\nis deemed necessary and appropriate to the circumstances. The project team is\nobligated to maintain confidentiality with regard to the reporter of an incident.\nFurther details of specific enforcement policies may be posted separately.\n\nProject maintainers who do not follow or enforce the Code of Conduct in good\nfaith may face temporary or permanent repercussions as determined by other\nmembers of the project's leadership.\n\n## Attribution\n\nThis Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,\navailable at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html\n\n[homepage]: https://www.contributor-covenant.org\n\nFor answers to common questions about this code of conduct, see\nhttps://www.contributor-covenant.org/faq\n\n" }, { "path": "CONTRIBUTING.md", "content": "# Contributing to fastText\nWe want to make contributing to this project as easy and transparent as possible.\n\n## Issues\nWe use GitHub issues to track public bugs. Please ensure your description is clear and has sufficient instructions to be able to reproduce the issue.\n\n### Reproducing issues\nPlease make sure that the issue you mention is not a result of one of the existing third-party libraries. For example, please do not post an issue if you encountered an error within a third-party Python library. We can only help you with errors which can be directly reproduced either with our C++ code or the corresponding Python bindings. If you do find an error, please post detailed steps to reproduce it. If we can't reproduce your error, we can't help you fix it.\n\n## Pull Requests\nPlease post an Issue before submitting a pull request. This might save you some time as it is possible we can't support your contribution, albeit we try our best to accomodate your (planned) work and highly appreciate your time. Generally, it is best to have a pull request emerge from an issue rather than the other way around.\n\nTo create a pull request:\n\n1. Fork the repo and create your branch from `master`.\n2. If you've added code that should be tested, add tests.\n3. If you've changed APIs, update the documentation.\n4. Ensure the test suite passes.\n5. Make sure your code lints.\n6. If you haven't already, complete the Contributor License Agreement (\"CLA\").\n\n## Tests\nFirst, you will need to make sure you have the required data. For that, please have a look at the fetch_test_data.sh script under tests. Next run the tests using the runtests.py script passing a path to the directory containing the datasets.\n\n## Contributor License Agreement (\"CLA\")\nIn order to accept your pull request, we need you to submit a CLA. You only need\nto do this once to work on any of Facebook's open source projects.\n\nComplete your CLA here: \n\n## License\nBy contributing to fastText, you agree that your contributions will be licensed under its MIT license.\n" }, { "path": "LICENSE", "content": "MIT License\n\nCopyright (c) 2016-present, Facebook, Inc.\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n" }, { "path": "MANIFEST.in", "content": "include LICENSE\ninclude PATENTS\n\nrecursive-include python *.md *.rst\nrecursive-include src *.h\n" }, { "path": "Makefile", "content": "#\n# Copyright (c) 2016-present, Facebook, Inc.\n# All rights reserved.\n#\n# This source code is licensed under the MIT license found in the\n# LICENSE file in the root directory of this source tree.\n#\n\nCXX = c++\nCXXFLAGS = -pthread -std=c++17 -march=native\nOBJS = args.o autotune.o matrix.o dictionary.o loss.o productquantizer.o densematrix.o quantmatrix.o vector.o model.o utils.o meter.o fasttext.o\nINCLUDES = -I.\n\nopt: CXXFLAGS += -O3 -funroll-loops -DNDEBUG\nopt: fasttext\n\ncoverage: CXXFLAGS += -O0 -fno-inline -fprofile-arcs --coverage\ncoverage: fasttext\n\ndebug: CXXFLAGS += -g -O0 -fno-inline\ndebug: fasttext\n\nwasm: webassembly/fasttext_wasm.js\n\nwasmdebug: export EMCC_DEBUG=1\nwasmdebug: webassembly/fasttext_wasm.js\n\n\nargs.o: src/args.cc src/args.h\n\t$(CXX) $(CXXFLAGS) -c src/args.cc\n\nautotune.o: src/autotune.cc src/autotune.h\n\t$(CXX) $(CXXFLAGS) -c src/autotune.cc\n\nmatrix.o: src/matrix.cc src/matrix.h\n\t$(CXX) $(CXXFLAGS) -c src/matrix.cc\n\ndictionary.o: src/dictionary.cc src/dictionary.h src/args.h\n\t$(CXX) $(CXXFLAGS) -c src/dictionary.cc\n\nloss.o: src/loss.cc src/loss.h src/matrix.h src/real.h\n\t$(CXX) $(CXXFLAGS) -c src/loss.cc\n\nproductquantizer.o: src/productquantizer.cc src/productquantizer.h src/utils.h\n\t$(CXX) $(CXXFLAGS) -c src/productquantizer.cc\n\ndensematrix.o: src/densematrix.cc src/densematrix.h src/utils.h src/matrix.h\n\t$(CXX) $(CXXFLAGS) -c src/densematrix.cc\n\nquantmatrix.o: src/quantmatrix.cc src/quantmatrix.h src/utils.h src/matrix.h\n\t$(CXX) $(CXXFLAGS) -c src/quantmatrix.cc\n\nvector.o: src/vector.cc src/vector.h src/utils.h\n\t$(CXX) $(CXXFLAGS) -c src/vector.cc\n\nmodel.o: src/model.cc src/model.h src/args.h\n\t$(CXX) $(CXXFLAGS) -c src/model.cc\n\nutils.o: src/utils.cc src/utils.h\n\t$(CXX) $(CXXFLAGS) -c src/utils.cc\n\nmeter.o: src/meter.cc src/meter.h\n\t$(CXX) $(CXXFLAGS) -c src/meter.cc\n\nfasttext.o: src/fasttext.cc src/*.h\n\t$(CXX) $(CXXFLAGS) -c src/fasttext.cc\n\nfasttext: $(OBJS) src/fasttext.cc src/main.cc\n\t$(CXX) $(CXXFLAGS) $(OBJS) src/main.cc -o fasttext\n\nclean:\n\trm -rf *.o *.gcno *.gcda fasttext *.bc webassembly/fasttext_wasm.js webassembly/fasttext_wasm.wasm\n\n\nEMCXX = em++\nEMCXXFLAGS = --bind --std=c++11 -s WASM=1 -s ALLOW_MEMORY_GROWTH=1 -s \"EXTRA_EXPORTED_RUNTIME_METHODS=['addOnPostRun', 'FS']\" -s \"DISABLE_EXCEPTION_CATCHING=0\" -s \"EXCEPTION_DEBUG=1\" -s \"FORCE_FILESYSTEM=1\" -s \"MODULARIZE=1\" -s \"EXPORT_ES6=1\" -s 'EXPORT_NAME=\"FastTextModule\"' -Isrc/\nEMOBJS = args.bc autotune.bc matrix.bc dictionary.bc loss.bc productquantizer.bc densematrix.bc quantmatrix.bc vector.bc model.bc utils.bc meter.bc fasttext.bc main.bc\n\n\nmain.bc: webassembly/fasttext_wasm.cc\n\t$(EMCXX) $(EMCXXFLAGS) webassembly/fasttext_wasm.cc -o main.bc\n\nargs.bc: src/args.cc src/args.h\n\t$(EMCXX) $(EMCXXFLAGS) src/args.cc -o args.bc\n\nautotune.bc: src/autotune.cc src/autotune.h\n\t$(EMCXX) $(EMCXXFLAGS) src/autotune.cc -o autotune.bc\n\nmatrix.bc: src/matrix.cc src/matrix.h\n\t$(EMCXX) $(EMCXXFLAGS) src/matrix.cc -o matrix.bc\n\ndictionary.bc: src/dictionary.cc src/dictionary.h src/args.h\n\t$(EMCXX) $(EMCXXFLAGS) src/dictionary.cc -o dictionary.bc\n\nloss.bc: src/loss.cc src/loss.h src/matrix.h src/real.h\n\t$(EMCXX) $(EMCXXFLAGS) src/loss.cc -o loss.bc\n\nproductquantizer.bc: src/productquantizer.cc src/productquantizer.h src/utils.h\n\t$(EMCXX) $(EMCXXFLAGS) src/productquantizer.cc -o productquantizer.bc\n\ndensematrix.bc: src/densematrix.cc src/densematrix.h src/utils.h src/matrix.h\n\t$(EMCXX) $(EMCXXFLAGS) src/densematrix.cc -o densematrix.bc\n\nquantmatrix.bc: src/quantmatrix.cc src/quantmatrix.h src/utils.h src/matrix.h\n\t$(EMCXX) $(EMCXXFLAGS) src/quantmatrix.cc -o quantmatrix.bc\n\nvector.bc: src/vector.cc src/vector.h src/utils.h\n\t$(EMCXX) $(EMCXXFLAGS) src/vector.cc -o vector.bc\n\nmodel.bc: src/model.cc src/model.h src/args.h\n\t$(EMCXX) $(EMCXXFLAGS) src/model.cc -o model.bc\n\nutils.bc: src/utils.cc src/utils.h\n\t$(EMCXX) $(EMCXXFLAGS) src/utils.cc -o utils.bc\n\nmeter.bc: src/meter.cc src/meter.h\n\t$(EMCXX) $(EMCXXFLAGS) src/meter.cc -o meter.bc\n\nfasttext.bc: src/fasttext.cc src/*.h\n\t$(EMCXX) $(EMCXXFLAGS) src/fasttext.cc -o fasttext.bc\n\nwebassembly/fasttext_wasm.js: $(EMOBJS) webassembly/fasttext_wasm.cc Makefile\n\t$(EMCXX) $(EMCXXFLAGS) $(EMOBJS) -o webassembly/fasttext_wasm.js\n\n\n" }, { "path": "PACKAGE", "content": "load(\"@fbcode_macros//build_defs:package_local_utils.bzl\", \"package_local_utils\")\n\npackage_local_utils.set_clang_version(15, True)\n" }, { "path": "README.md", "content": "# fastText\n[fastText](https://fasttext.cc/) is a library for efficient learning of word representations and sentence classification.\n\n[![CircleCI](https://circleci.com/gh/facebookresearch/fastText/tree/master.svg?style=svg)](https://circleci.com/gh/facebookresearch/fastText/tree/master)\n\n## Table of contents\n\n* [Resources](#resources)\n * [Models](#models)\n * [Supplementary data](#supplementary-data)\n * [FAQ](#faq)\n * [Cheatsheet](#cheatsheet)\n* [Requirements](#requirements)\n* [Building fastText](#building-fasttext)\n * [Getting the source code](#getting-the-source-code)\n * [Building fastText using make (preferred)](#building-fasttext-using-make-preferred)\n * [Building fastText using cmake](#building-fasttext-using-cmake)\n * [Building fastText for Python](#building-fasttext-for-python)\n* [Example use cases](#example-use-cases)\n * [Word representation learning](#word-representation-learning)\n * [Obtaining word vectors for out-of-vocabulary words](#obtaining-word-vectors-for-out-of-vocabulary-words)\n * [Text classification](#text-classification)\n* [Full documentation](#full-documentation)\n* [References](#references)\n * [Enriching Word Vectors with Subword Information](#enriching-word-vectors-with-subword-information)\n * [Bag of Tricks for Efficient Text Classification](#bag-of-tricks-for-efficient-text-classification)\n * [FastText.zip: Compressing text classification models](#fasttextzip-compressing-text-classification-models)\n* [Join the fastText community](#join-the-fasttext-community)\n* [License](#license)\n\n## Resources\n\n### Models\n- Recent state-of-the-art [English word vectors](https://fasttext.cc/docs/en/english-vectors.html).\n- Word vectors for [157 languages trained on Wikipedia and Crawl](https://github.com/facebookresearch/fastText/blob/master/docs/crawl-vectors.md).\n- Models for [language identification](https://fasttext.cc/docs/en/language-identification.html#content) and [various supervised tasks](https://fasttext.cc/docs/en/supervised-models.html#content).\n\n### Supplementary data\n- The preprocessed [YFCC100M data](https://fasttext.cc/docs/en/dataset.html#content) used in [2].\n\n### FAQ\n\nYou can find [answers to frequently asked questions](https://fasttext.cc/docs/en/faqs.html#content) on our [website](https://fasttext.cc/).\n\n### Cheatsheet\n\nWe also provide a [cheatsheet](https://fasttext.cc/docs/en/cheatsheet.html#content) full of useful one-liners.\n\n## Requirements\n\nWe are continuously building and testing our library, CLI and Python bindings under various docker images using [circleci](https://circleci.com/).\n\nGenerally, **fastText** builds on modern Mac OS and Linux distributions.\nSince it uses some C++11 features, it requires a compiler with good C++11 support.\nThese include :\n\n* (g++-4.7.2 or newer) or (clang-3.3 or newer)\n\nCompilation is carried out using a Makefile, so you will need to have a working **make**.\nIf you want to use **cmake** you need at least version 2.8.9.\n\nOne of the oldest distributions we successfully built and tested the CLI under is [Debian jessie](https://www.debian.org/releases/jessie/).\n\nFor the word-similarity evaluation script you will need:\n\n* Python 2.6 or newer\n* NumPy & SciPy\n\nFor the python bindings (see the subdirectory python) you will need:\n\n* Python version 2.7 or >=3.4\n* NumPy & SciPy\n* [pybind11](https://github.com/pybind/pybind11)\n\nOne of the oldest distributions we successfully built and tested the Python bindings under is [Debian jessie](https://www.debian.org/releases/jessie/).\n\nIf these requirements make it impossible for you to use fastText, please open an issue and we will try to accommodate you.\n\n## Building fastText\n\nWe discuss building the latest stable version of fastText.\n\n### Getting the source code\n\nYou can find our [latest stable release](https://github.com/facebookresearch/fastText/releases/latest) in the usual place.\n\nThere is also the master branch that contains all of our most recent work, but comes along with all the usual caveats of an unstable branch. You might want to use this if you are a developer or power-user.\n\n### Building fastText using make (preferred)\n\n```\n$ wget https://github.com/facebookresearch/fastText/archive/v0.9.2.zip\n$ unzip v0.9.2.zip\n$ cd fastText-0.9.2\n$ make\n```\n\nThis will produce object files for all the classes as well as the main binary `fasttext`.\nIf you do not plan on using the default system-wide compiler, update the two macros defined at the beginning of the Makefile (CC and INCLUDES).\n\n### Building fastText using cmake\n\nFor now this is not part of a release, so you will need to clone the master branch.\n\n```\n$ git clone https://github.com/facebookresearch/fastText.git\n$ cd fastText\n$ mkdir build && cd build && cmake ..\n$ make && make install\n```\n\nThis will create the fasttext binary and also all relevant libraries (shared, static, PIC).\n\n### Building fastText for Python\n\nFor now this is not part of a release, so you will need to clone the master branch.\n\n```\n$ git clone https://github.com/facebookresearch/fastText.git\n$ cd fastText\n$ pip install .\n```\n\nFor further information and introduction see python/README.md\n\n## Example use cases\n\nThis library has two main use cases: word representation learning and text classification.\nThese were described in the two papers [1](#enriching-word-vectors-with-subword-information) and [2](#bag-of-tricks-for-efficient-text-classification).\n\n### Word representation learning\n\nIn order to learn word vectors, as described in [1](#enriching-word-vectors-with-subword-information), do:\n\n```\n$ ./fasttext skipgram -input data.txt -output model\n```\n\nwhere `data.txt` is a training file containing `UTF-8` encoded text.\nBy default the word vectors will take into account character n-grams from 3 to 6 characters.\nAt the end of optimization the program will save two files: `model.bin` and `model.vec`.\n`model.vec` is a text file containing the word vectors, one per line.\n`model.bin` is a binary file containing the parameters of the model along with the dictionary and all hyper parameters.\nThe binary file can be used later to compute word vectors or to restart the optimization.\n\n### Obtaining word vectors for out-of-vocabulary words\n\nThe previously trained model can be used to compute word vectors for out-of-vocabulary words.\nProvided you have a text file `queries.txt` containing words for which you want to compute vectors, use the following command:\n\n```\n$ ./fasttext print-word-vectors model.bin < queries.txt\n```\n\nThis will output word vectors to the standard output, one vector per line.\nThis can also be used with pipes:\n\n```\n$ cat queries.txt | ./fasttext print-word-vectors model.bin\n```\n\nSee the provided scripts for an example. For instance, running:\n\n```\n$ ./word-vector-example.sh\n```\n\nwill compile the code, download data, compute word vectors and evaluate them on the rare words similarity dataset RW [Thang et al. 2013].\n\n### Text classification\n\nThis library can also be used to train supervised text classifiers, for instance for sentiment analysis.\nIn order to train a text classifier using the method described in [2](#bag-of-tricks-for-efficient-text-classification), use:\n\n```\n$ ./fasttext supervised -input train.txt -output model\n```\n\nwhere `train.txt` is a text file containing a training sentence per line along with the labels.\nBy default, we assume that labels are words that are prefixed by the string `__label__`.\nThis will output two files: `model.bin` and `model.vec`.\nOnce the model was trained, you can evaluate it by computing the precision and recall at k (P@k and R@k) on a test set using:\n\n```\n$ ./fasttext test model.bin test.txt k\n```\n\nThe argument `k` is optional, and is equal to `1` by default.\n\nIn order to obtain the k most likely labels for a piece of text, use:\n\n```\n$ ./fasttext predict model.bin test.txt k\n```\n\nor use `predict-prob` to also get the probability for each label\n\n```\n$ ./fasttext predict-prob model.bin test.txt k\n```\n\nwhere `test.txt` contains a piece of text to classify per line.\nDoing so will print to the standard output the k most likely labels for each line.\nThe argument `k` is optional, and equal to `1` by default.\nSee `classification-example.sh` for an example use case.\nIn order to reproduce results from the paper [2](#bag-of-tricks-for-efficient-text-classification), run `classification-results.sh`, this will download all the datasets and reproduce the results from Table 1.\n\nIf you want to compute vector representations of sentences or paragraphs, please use:\n\n```\n$ ./fasttext print-sentence-vectors model.bin < text.txt\n```\n\nThis assumes that the `text.txt` file contains the paragraphs that you want to get vectors for.\nThe program will output one vector representation per line in the file.\n\nYou can also quantize a supervised model to reduce its memory usage with the following command:\n\n```\n$ ./fasttext quantize -output model\n```\nThis will create a `.ftz` file with a smaller memory footprint. All the standard functionality, like `test` or `predict` work the same way on the quantized models:\n```\n$ ./fasttext test model.ftz test.txt\n```\nThe quantization procedure follows the steps described in [3](#fasttextzip-compressing-text-classification-models). You can\nrun the script `quantization-example.sh` for an example.\n\n\n## Full documentation\n\nInvoke a command without arguments to list available arguments and their default values:\n\n```\n$ ./fasttext supervised\nEmpty input or output path.\n\nThe following arguments are mandatory:\n -input training file path\n -output output file path\n\nThe following arguments are optional:\n -verbose verbosity level [2]\n\nThe following arguments for the dictionary are optional:\n -minCount minimal number of word occurrences [1]\n -minCountLabel minimal number of label occurrences [0]\n -wordNgrams max length of word ngram [1]\n -bucket number of buckets [2000000]\n -minn min length of char ngram [0]\n -maxn max length of char ngram [0]\n -t sampling threshold [0.0001]\n -label labels prefix [__label__]\n\nThe following arguments for training are optional:\n -lr learning rate [0.1]\n -lrUpdateRate change the rate of updates for the learning rate [100]\n -dim size of word vectors [100]\n -ws size of the context window [5]\n -epoch number of epochs [5]\n -neg number of negatives sampled [5]\n -loss loss function {ns, hs, softmax} [softmax]\n -thread number of threads [12]\n -pretrainedVectors pretrained word vectors for supervised learning []\n -saveOutput whether output params should be saved [0]\n\nThe following arguments for quantization are optional:\n -cutoff number of words and ngrams to retain [0]\n -retrain finetune embeddings if a cutoff is applied [0]\n -qnorm quantizing the norm separately [0]\n -qout quantizing the classifier [0]\n -dsub size of each sub-vector [2]\n```\n\nDefaults may vary by mode. (Word-representation modes `skipgram` and `cbow` use a default `-minCount` of 5.)\n\n## References\n\nPlease cite [1](#enriching-word-vectors-with-subword-information) if using this code for learning word representations or [2](#bag-of-tricks-for-efficient-text-classification) if using for text classification.\n\n### Enriching Word Vectors with Subword Information\n\n[1] P. Bojanowski\\*, E. Grave\\*, A. Joulin, T. Mikolov, [*Enriching Word Vectors with Subword Information*](https://arxiv.org/abs/1607.04606)\n\n```\n@article{bojanowski2017enriching,\n title={Enriching Word Vectors with Subword Information},\n author={Bojanowski, Piotr and Grave, Edouard and Joulin, Armand and Mikolov, Tomas},\n journal={Transactions of the Association for Computational Linguistics},\n volume={5},\n year={2017},\n issn={2307-387X},\n pages={135--146}\n}\n```\n\n### Bag of Tricks for Efficient Text Classification\n\n[2] A. Joulin, E. Grave, P. Bojanowski, T. Mikolov, [*Bag of Tricks for Efficient Text Classification*](https://arxiv.org/abs/1607.01759)\n\n```\n@InProceedings{joulin2017bag,\n title={Bag of Tricks for Efficient Text Classification},\n author={Joulin, Armand and Grave, Edouard and Bojanowski, Piotr and Mikolov, Tomas},\n booktitle={Proceedings of the 15th Conference of the European Chapter of the Association for Computational Linguistics: Volume 2, Short Papers},\n month={April},\n year={2017},\n publisher={Association for Computational Linguistics},\n pages={427--431},\n}\n```\n\n### FastText.zip: Compressing text classification models\n\n[3] A. Joulin, E. Grave, P. Bojanowski, M. Douze, H. Jégou, T. Mikolov, [*FastText.zip: Compressing text classification models*](https://arxiv.org/abs/1612.03651)\n\n```\n@article{joulin2016fasttext,\n title={FastText.zip: Compressing text classification models},\n author={Joulin, Armand and Grave, Edouard and Bojanowski, Piotr and Douze, Matthijs and J{\\'e}gou, H{\\'e}rve and Mikolov, Tomas},\n journal={arXiv preprint arXiv:1612.03651},\n year={2016}\n}\n```\n\n(\\* These authors contributed equally.)\n\n\n## Join the fastText community\n\n* Facebook page: https://www.facebook.com/groups/1174547215919768\n* Google group: https://groups.google.com/forum/#!forum/fasttext-library\n* Contact: [egrave@fb.com](mailto:egrave@fb.com), [bojanowski@fb.com](mailto:bojanowski@fb.com), [ajoulin@fb.com](mailto:ajoulin@fb.com), [tmikolov@fb.com](mailto:tmikolov@fb.com)\n\nSee the CONTRIBUTING file for information about how to help out.\n\n## License\n\nfastText is MIT-licensed.\n" }, { "path": "alignment/README.md", "content": "## Alignment of Word Embeddings\n\nThis directory provides code for learning alignments between word embeddings in different languages.\n\nThe code is in Python 3 and requires [NumPy](http://www.numpy.org/).\n\nThe script `example.sh` shows how to use this code to learn and evaluate a bilingual alignment of word embeddings.\n\nThe word embeddings used in [1] can be found on the [fastText project page](https://fasttext.cc) and the supervised bilingual lexicons on the [MUSE project page](https://github.com/facebookresearch/MUSE).\n\n### Supervised alignment\n\nThe script `align.py` aligns word embeddings from two languages using a bilingual lexicon as supervision.\nThe details of this approach can be found in [1].\n\n### Unsupervised alignment\n\nThe script `unsup_align.py` aligns word embeddings from two languages without requiring any supervision.\nAdditionally, the script `unsup_multialign.py` aligns multiple languages to a common space with no supervision.\nThe details of these approaches can be found in [2] and [3] respectively.\n\nIn addition to NumPy, the unsupervised methods require the [Python Optimal Transport](https://pot.readthedocs.io/en/stable/) toolbox.\n\n### Download\n\nWikipedia fastText embeddings aligned with our method can be found [here](https://fasttext.cc/docs/en/aligned-vectors.html).\n\n### References\n\nIf you use the supervised alignment method, please cite:\n\n[1] A. Joulin, P. Bojanowski, T. Mikolov, H. Jegou, E. Grave, [*Loss in Translation: Learning Bilingual Word Mapping with a Retrieval Criterion*](https://arxiv.org/abs/1804.07745)\n\n```\n@InProceedings{joulin2018loss,\n title={Loss in Translation: Learning Bilingual Word Mapping with a Retrieval Criterion},\n author={Joulin, Armand and Bojanowski, Piotr and Mikolov, Tomas and J\\'egou, Herv\\'e and Grave, Edouard},\n year={2018},\n booktitle={Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing},\n}\n```\n\nIf you use the unsupervised bilingual alignment method, please cite:\n\n[2] E. Grave, A. Joulin, Q. Berthet, [*Unsupervised Alignment of Embeddings with Wasserstein Procrustes*](https://arxiv.org/abs/1805.11222)\n\n```\n@article{grave2018unsupervised,\n title={Unsupervised Alignment of Embeddings with Wasserstein Procrustes},\n author={Grave, Edouard and Joulin, Armand and Berthet, Quentin},\n journal={arXiv preprint arXiv:1805.11222},\n year={2018}\n}\n```\n\nIf you use the unsupervised alignment script `unsup_multialign.py`, please cite:\n\n[3] J. Alaux, E. Grave, M. Cuturi, A. Joulin, [*Unsupervised Hyperalignment for Multilingual Word Embeddings*](https://arxiv.org/abs/1811.01124)\n\n```\n@article{alaux2018unsupervised,\n title={Unsupervised hyperalignment for multilingual word embeddings},\n author={Alaux, Jean and Grave, Edouard and Cuturi, Marco and Joulin, Armand},\n journal={arXiv preprint arXiv:1811.01124},\n year={2018}\n}\n```\n" }, { "path": "alignment/align.py", "content": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n#\n# Copyright (c) 2018-present, Facebook, Inc.\n# All rights reserved.\n#\n# This source code is licensed under the license found in the\n# LICENSE file in the root directory of this source tree.\n\nimport numpy as np\nimport argparse\nfrom utils import *\nimport sys\n\nparser = argparse.ArgumentParser(description='RCSLS for supervised word alignment')\n\nparser.add_argument(\"--src_emb\", type=str, default='', help=\"Load source embeddings\")\nparser.add_argument(\"--tgt_emb\", type=str, default='', help=\"Load target embeddings\")\nparser.add_argument('--center', action='store_true', help='whether to center embeddings or not')\n\nparser.add_argument(\"--dico_train\", type=str, default='', help=\"train dictionary\")\nparser.add_argument(\"--dico_test\", type=str, default='', help=\"validation dictionary\")\n\nparser.add_argument(\"--output\", type=str, default='', help=\"where to save aligned embeddings\")\n\nparser.add_argument(\"--knn\", type=int, default=10, help=\"number of nearest neighbors in RCSL/CSLS\")\nparser.add_argument(\"--maxneg\", type=int, default=200000, help=\"Maximum number of negatives for the Extended RCSLS\")\nparser.add_argument(\"--maxsup\", type=int, default=-1, help=\"Maximum number of training examples\")\nparser.add_argument(\"--maxload\", type=int, default=200000, help=\"Maximum number of loaded vectors\")\n\nparser.add_argument(\"--model\", type=str, default=\"none\", help=\"Set of constraints: spectral or none\")\nparser.add_argument(\"--reg\", type=float, default=0.0 , help='regularization parameters')\n\nparser.add_argument(\"--lr\", type=float, default=1.0, help='learning rate')\nparser.add_argument(\"--niter\", type=int, default=10, help='number of iterations')\nparser.add_argument('--sgd', action='store_true', help='use sgd')\nparser.add_argument(\"--batchsize\", type=int, default=10000, help=\"batch size for sgd\")\n\nparams = parser.parse_args()\n\n###### SPECIFIC FUNCTIONS ######\n# functions specific to RCSLS\n# the rest of the functions are in utils.py\n\ndef getknn(sc, x, y, k=10):\n sidx = np.argpartition(sc, -k, axis=1)[:, -k:]\n ytopk = y[sidx.flatten(), :]\n ytopk = ytopk.reshape(sidx.shape[0], sidx.shape[1], y.shape[1])\n f = np.sum(sc[np.arange(sc.shape[0])[:, None], sidx])\n df = np.dot(ytopk.sum(1).T, x)\n return f / k, df / k\n\n\ndef rcsls(X_src, Y_tgt, Z_src, Z_tgt, R, knn=10):\n X_trans = np.dot(X_src, R.T)\n f = 2 * np.sum(X_trans * Y_tgt)\n df = 2 * np.dot(Y_tgt.T, X_src)\n fk0, dfk0 = getknn(np.dot(X_trans, Z_tgt.T), X_src, Z_tgt, knn)\n fk1, dfk1 = getknn(np.dot(np.dot(Z_src, R.T), Y_tgt.T).T, Y_tgt, Z_src, knn)\n f = f - fk0 -fk1\n df = df - dfk0 - dfk1.T\n return -f / X_src.shape[0], -df / X_src.shape[0]\n\n\ndef proj_spectral(R):\n U, s, V = np.linalg.svd(R)\n s[s > 1] = 1\n s[s < 0] = 0\n return np.dot(U, np.dot(np.diag(s), V))\n\n\n###### MAIN ######\n\n# load word embeddings\nwords_tgt, x_tgt = load_vectors(params.tgt_emb, maxload=params.maxload, center=params.center)\nwords_src, x_src = load_vectors(params.src_emb, maxload=params.maxload, center=params.center)\n\n# load validation bilingual lexicon\nsrc2tgt, lexicon_size = load_lexicon(params.dico_test, words_src, words_tgt)\n\n# word --> vector indices\nidx_src = idx(words_src)\nidx_tgt = idx(words_tgt)\n\n# load train bilingual lexicon\npairs = load_pairs(params.dico_train, idx_src, idx_tgt)\nif params.maxsup > 0 and params.maxsup < len(pairs):\n pairs = pairs[:params.maxsup]\n\n# selecting training vector pairs\nX_src, Y_tgt = select_vectors_from_pairs(x_src, x_tgt, pairs)\n\n# adding negatives for RCSLS\nZ_src = x_src[:params.maxneg, :]\nZ_tgt = x_tgt[:params.maxneg, :]\n\n# initialization:\nR = procrustes(X_src, Y_tgt)\nnnacc = compute_nn_accuracy(np.dot(x_src, R.T), x_tgt, src2tgt, lexicon_size=lexicon_size)\nprint(\"[init -- Procrustes] NN: %.4f\"%(nnacc))\nsys.stdout.flush()\n\n# optimization\nfold, Rold = 0, []\nniter, lr = params.niter, params.lr\n\nfor it in range(0, niter + 1):\n if lr < 1e-4:\n break\n\n if params.sgd:\n indices = np.random.choice(X_src.shape[0], size=params.batchsize, replace=False)\n f, df = rcsls(X_src[indices, :], Y_tgt[indices, :], Z_src, Z_tgt, R, params.knn)\n else:\n f, df = rcsls(X_src, Y_tgt, Z_src, Z_tgt, R, params.knn)\n\n if params.reg > 0:\n R *= (1 - lr * params.reg)\n R -= lr * df\n if params.model == \"spectral\":\n R = proj_spectral(R)\n\n print(\"[it=%d] f = %.4f\" % (it, f))\n sys.stdout.flush()\n\n if f > fold and it > 0 and not params.sgd:\n lr /= 2\n f, R = fold, Rold\n\n fold, Rold = f, R\n\n if (it > 0 and it % 10 == 0) or it == niter:\n nnacc = compute_nn_accuracy(np.dot(x_src, R.T), x_tgt, src2tgt, lexicon_size=lexicon_size)\n print(\"[it=%d] NN = %.4f - Coverage = %.4f\" % (it, nnacc, len(src2tgt) / lexicon_size))\n\nnnacc = compute_nn_accuracy(np.dot(x_src, R.T), x_tgt, src2tgt, lexicon_size=lexicon_size)\nprint(\"[final] NN = %.4f - Coverage = %.4f\" % (nnacc, len(src2tgt) / lexicon_size))\n\nif params.output != \"\":\n print(\"Saving all aligned vectors at %s\" % params.output)\n words_full, x_full = load_vectors(params.src_emb, maxload=-1, center=params.center, verbose=False)\n x = np.dot(x_full, R.T)\n x /= np.linalg.norm(x, axis=1)[:, np.newaxis] + 1e-8\n save_vectors(params.output, x, words_full)\n save_matrix(params.output + \"-mat\", R)\n" }, { "path": "alignment/eval.py", "content": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n#\n# Copyright (c) 2018-present, Facebook, Inc.\n# All rights reserved.\n#\n# This source code is licensed under the license found in the\n# LICENSE file in the root directory of this source tree.\n\nimport io\nimport numpy as np\nimport argparse\nfrom utils import *\n\nparser = argparse.ArgumentParser(description='Evaluation of word alignment')\nparser.add_argument(\"--src_emb\", type=str, default='', help=\"Load source embeddings\")\nparser.add_argument(\"--tgt_emb\", type=str, default='', help=\"Load target embeddings\")\nparser.add_argument('--center', action='store_true', help='whether to center embeddings or not')\nparser.add_argument(\"--src_mat\", type=str, default='', help=\"Load source alignment matrix. If none given, the aligment matrix is the identity.\")\nparser.add_argument(\"--tgt_mat\", type=str, default='', help=\"Load target alignment matrix. If none given, the aligment matrix is the identity.\")\nparser.add_argument(\"--dico_test\", type=str, default='', help=\"test dictionary\")\nparser.add_argument(\"--maxload\", type=int, default=200000)\nparser.add_argument(\"--nomatch\", action='store_true', help=\"no exact match in lexicon\")\nparams = parser.parse_args()\n\n\n###### SPECIFIC FUNCTIONS ######\n# function specific to evaluation\n# the rest of the functions are in utils.py\n\ndef load_transform(fname, d1=300, d2=300):\n fin = io.open(fname, 'r', encoding='utf-8', newline='\\n', errors='ignore')\n R = np.zeros([d1, d2])\n for i, line in enumerate(fin):\n tokens = line.split(' ')\n R[i, :] = np.array(tokens[0:d2], dtype=float)\n return R\n\n\n###### MAIN ######\n\nprint(\"Evaluation of alignment on %s\" % params.dico_test)\nif params.nomatch:\n print(\"running without exact string matches\")\n\nwords_tgt, x_tgt = load_vectors(params.tgt_emb, maxload=params.maxload, center=params.center)\nwords_src, x_src = load_vectors(params.src_emb, maxload=params.maxload, center=params.center)\n\nif params.tgt_mat != \"\":\n R_tgt = load_transform(params.tgt_mat)\n x_tgt = np.dot(x_tgt, R_tgt)\nif params.src_mat != \"\":\n R_src = load_transform(params.src_mat)\n x_src = np.dot(x_src, R_src)\n\nsrc2tgt, lexicon_size = load_lexicon(params.dico_test, words_src, words_tgt)\n\nnnacc = compute_nn_accuracy(x_src, x_tgt, src2tgt, lexicon_size=lexicon_size)\ncslsproc = compute_csls_accuracy(x_src, x_tgt, src2tgt, lexicon_size=lexicon_size)\nprint(\"NN = %.4f - CSLS = %.4f - Coverage = %.4f\" % (nnacc, cslsproc, len(src2tgt) / lexicon_size))\n" }, { "path": "alignment/example.sh", "content": "#!/bin/usr/env sh\n# Copyright (c) 2018-present, Facebook, Inc.\n# All rights reserved.\n#\n# This source code is licensed under the license found in the\n# LICENSE file in the root directory of this source tree.\n\nset -e\ns=${1:-en}\nt=${2:-es}\necho \"Example based on the ${s}->${t} alignment\"\n\nif [ ! -d data/ ]; then\n mkdir -p data;\nfi\n\nif [ ! -d res/ ]; then\n mkdir -p res;\nfi\n\ndico_train=data/${s}-${t}.0-5000.txt\nif [ ! -f \"${dico_train}\" ]; then\n DICO=$(basename -- \"${dico_train}\")\n wget -c \"https://dl.fbaipublicfiles.com/arrival/dictionaries/${DICO}\" -P data/\nfi\n\ndico_test=data/${s}-${t}.5000-6500.txt\nif [ ! -f \"${dico_test}\" ]; then\n DICO=$(basename -- \"${dico_test}\")\n wget -c \"https://dl.fbaipublicfiles.com/arrival/dictionaries/${DICO}\" -P data/\nfi\n\nsrc_emb=data/wiki.${s}.vec\nif [ ! -f \"${src_emb}\" ]; then\n EMB=$(basename -- \"${src_emb}\")\n wget -c \"https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/${EMB}\" -P data/\nfi\n\ntgt_emb=data/wiki.${t}.vec\nif [ ! -f \"${tgt_emb}\" ]; then\n EMB=$(basename -- \"${tgt_emb}\")\n wget -c \"https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/${EMB}\" -P data/\nfi\n\noutput=res/wiki.${s}-${t}.vec\n\npython3 align.py --src_emb \"${src_emb}\" --tgt_emb \"${tgt_emb}\" \\\n --dico_train \"${dico_train}\" --dico_test \"${dico_test}\" --output \"${output}\" \\\n --lr 25 --niter 10\npython3 eval.py --src_emb \"${output}\" --tgt_emb \"${tgt_emb}\" \\\n --dico_test \"${dico_test}\"\n" }, { "path": "alignment/unsup_align.py", "content": "#!/usr/bin/env python3\n# Copyright (c) 2018-present, Facebook, Inc.\n# All rights reserved.\n#\n# This source code is licensed under the MIT license found in the\n# LICENSE file in the root directory of this source tree.\n\nimport codecs, sys, time, math, argparse, ot\nimport numpy as np\nfrom utils import *\n\nparser = argparse.ArgumentParser(description='Wasserstein Procrustes for Embedding Alignment')\nparser.add_argument('--model_src', type=str, help='Path to source word embeddings')\nparser.add_argument('--model_tgt', type=str, help='Path to target word embeddings')\nparser.add_argument('--lexicon', type=str, help='Path to the evaluation lexicon')\nparser.add_argument('--output_src', default='', type=str, help='Path to save the aligned source embeddings')\nparser.add_argument('--output_tgt', default='', type=str, help='Path to save the aligned target embeddings')\nparser.add_argument('--seed', default=1111, type=int, help='Random number generator seed')\nparser.add_argument('--nepoch', default=5, type=int, help='Number of epochs')\nparser.add_argument('--niter', default=5000, type=int, help='Initial number of iterations')\nparser.add_argument('--bsz', default=500, type=int, help='Initial batch size')\nparser.add_argument('--lr', default=500., type=float, help='Learning rate')\nparser.add_argument('--nmax', default=20000, type=int, help='Vocabulary size for learning the alignment')\nparser.add_argument('--reg', default=0.05, type=float, help='Regularization parameter for sinkhorn')\nargs = parser.parse_args()\n\n\ndef objective(X, Y, R, n=5000):\n Xn, Yn = X[:n], Y[:n]\n C = -np.dot(np.dot(Xn, R), Yn.T)\n P = ot.sinkhorn(np.ones(n), np.ones(n), C, 0.025, stopThr=1e-3)\n return 1000 * np.linalg.norm(np.dot(Xn, R) - np.dot(P, Yn)) / n\n\n\ndef sqrt_eig(x):\n U, s, VT = np.linalg.svd(x, full_matrices=False)\n return np.dot(U, np.dot(np.diag(np.sqrt(s)), VT))\n\n\ndef align(X, Y, R, lr=10., bsz=200, nepoch=5, niter=1000,\n nmax=10000, reg=0.05, verbose=True):\n for epoch in range(1, nepoch + 1):\n for _it in range(1, niter + 1):\n # sample mini-batch\n xt = X[np.random.permutation(nmax)[:bsz], :]\n yt = Y[np.random.permutation(nmax)[:bsz], :]\n # compute OT on minibatch\n C = -np.dot(np.dot(xt, R), yt.T)\n P = ot.sinkhorn(np.ones(bsz), np.ones(bsz), C, reg, stopThr=1e-3)\n # compute gradient\n G = - np.dot(xt.T, np.dot(P, yt))\n R -= lr / bsz * G\n # project on orthogonal matrices\n U, s, VT = np.linalg.svd(R)\n R = np.dot(U, VT)\n bsz *= 2\n niter //= 4\n if verbose:\n print(\"epoch: %d obj: %.3f\" % (epoch, objective(X, Y, R)))\n return R\n\n\ndef convex_init(X, Y, niter=100, reg=0.05, apply_sqrt=False):\n n, d = X.shape\n if apply_sqrt:\n X, Y = sqrt_eig(X), sqrt_eig(Y)\n K_X, K_Y = np.dot(X, X.T), np.dot(Y, Y.T)\n K_Y *= np.linalg.norm(K_X) / np.linalg.norm(K_Y)\n K2_X, K2_Y = np.dot(K_X, K_X), np.dot(K_Y, K_Y)\n P = np.ones([n, n]) / float(n)\n for it in range(1, niter + 1):\n G = np.dot(P, K2_X) + np.dot(K2_Y, P) - 2 * np.dot(K_Y, np.dot(P, K_X))\n q = ot.sinkhorn(np.ones(n), np.ones(n), G, reg, stopThr=1e-3)\n alpha = 2.0 / float(2.0 + it)\n P = alpha * q + (1.0 - alpha) * P\n obj = np.linalg.norm(np.dot(P, K_X) - np.dot(K_Y, P))\n print(obj)\n return procrustes(np.dot(P, X), Y).T\n\n\nprint(\"\\n*** Wasserstein Procrustes ***\\n\")\n\nnp.random.seed(args.seed)\n\nmaxload = 200000\nw_src, x_src = load_vectors(args.model_src, maxload, norm=True, center=True)\nw_tgt, x_tgt = load_vectors(args.model_tgt, maxload, norm=True, center=True)\nsrc2trg, _ = load_lexicon(args.lexicon, w_src, w_tgt)\n\nprint(\"\\nComputing initial mapping with convex relaxation...\")\nt0 = time.time()\nR0 = convex_init(x_src[:2500], x_tgt[:2500], reg=args.reg, apply_sqrt=True)\nprint(\"Done [%03d sec]\" % math.floor(time.time() - t0))\n\nprint(\"\\nComputing mapping with Wasserstein Procrustes...\")\nt0 = time.time()\nR = align(x_src, x_tgt, R0.copy(), bsz=args.bsz, lr=args.lr, niter=args.niter,\n nepoch=args.nepoch, reg=args.reg, nmax=args.nmax)\nprint(\"Done [%03d sec]\" % math.floor(time.time() - t0))\n\nacc = compute_nn_accuracy(x_src, np.dot(x_tgt, R.T), src2trg)\nprint(\"\\nPrecision@1: %.3f\\n\" % acc)\n\nif args.output_src != '':\n x_src = x_src / np.linalg.norm(x_src, 2, 1).reshape([-1, 1])\n save_vectors(args.output_src, x_src, w_src)\nif args.output_tgt != '':\n x_tgt = x_tgt / np.linalg.norm(x_tgt, 2, 1).reshape([-1, 1])\n save_vectors(args.output_tgt, np.dot(x_tgt, R.T), w_tgt)\n" }, { "path": "alignment/unsup_multialign.py", "content": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n#\n# Copyright (c) 2019-present, Facebook, Inc.\n# All rights reserved.\n#\n# This source code is licensed under the license found in the\n# LICENSE file in the root directory of this source tree.\n\nimport io, os, ot, argparse, random\nimport numpy as np\nfrom utils import *\n\nparser = argparse.ArgumentParser(description=' ')\n\nparser.add_argument('--embdir', default='data/', type=str)\nparser.add_argument('--outdir', default='output/', type=str)\nparser.add_argument('--lglist', default='en-fr-es-it-pt-de-pl-ru-da-nl-cs', type=str,\n help='list of languages. The first element is the pivot. Example: en-fr-es to align English, French and Spanish with English as the pivot.')\n\nparser.add_argument('--maxload', default=20000, type=int, help='Max number of loaded vectors')\nparser.add_argument('--uniform', action='store_true', help='switch to uniform probability of picking language pairs')\n\n# optimization parameters for the square loss\nparser.add_argument('--epoch', default=2, type=int, help='nb of epochs for square loss')\nparser.add_argument('--niter', default=500, type=int, help='max number of iteration per epoch for square loss')\nparser.add_argument('--lr', default=0.1, type=float, help='learning rate for square loss')\nparser.add_argument('--bsz', default=500, type=int, help='batch size for square loss')\n\n# optimization parameters for the RCSLS loss\nparser.add_argument('--altepoch', default=100, type=int, help='nb of epochs for RCSLS loss')\nparser.add_argument('--altlr', default=25, type=float, help='learning rate for RCSLS loss')\nparser.add_argument(\"--altbsz\", type=int, default=1000, help=\"batch size for RCSLS\")\n\nargs = parser.parse_args()\n\n###### SPECIFIC FUNCTIONS ######\n\ndef getknn(sc, x, y, k=10):\n sidx = np.argpartition(sc, -k, axis=1)[:, -k:]\n ytopk = y[sidx.flatten(), :]\n ytopk = ytopk.reshape(sidx.shape[0], sidx.shape[1], y.shape[1])\n f = np.sum(sc[np.arange(sc.shape[0])[:, None], sidx])\n df = np.dot(ytopk.sum(1).T, x)\n return f / k, df / k\n\n\ndef rcsls(Xi, Xj, Zi, Zj, R, knn=10):\n X_trans = np.dot(Xi, R.T)\n f = 2 * np.sum(X_trans * Xj)\n df = 2 * np.dot(Xj.T, Xi)\n fk0, dfk0 = getknn(np.dot(X_trans, Zj.T), Xi, Zj, knn)\n fk1, dfk1 = getknn(np.dot(np.dot(Zi, R.T), Xj.T).T, Xj, Zi, knn)\n f = f - fk0 -fk1\n df = df - dfk0 - dfk1.T\n return -f / Xi.shape[0], -df.T / Xi.shape[0]\n\n\ndef GWmatrix(emb0):\n N = np.shape(emb0)[0]\n N2 = .5* np.linalg.norm(emb0, axis=1).reshape(1, N)\n C2 = np.tile(N2.transpose(), (1, N)) + np.tile(N2, (N, 1))\n C2 -= np.dot(emb0,emb0.T)\n return C2\n\n\ndef gromov_wasserstein(x_src, x_tgt, C2):\n N = x_src.shape[0]\n C1 = GWmatrix(x_src)\n M = ot.gromov_wasserstein(C1,C2,np.ones(N),np.ones(N),'square_loss',epsilon=0.55,max_iter=100,tol=1e-4)\n return procrustes(np.dot(M,x_tgt), x_src)\n\n\ndef align(EMB, TRANS, lglist, args):\n nmax, l = args.maxload, len(lglist)\n # create a list of language pairs to sample from\n # (default == higher probability to pick a language pair contianing the pivot)\n # if --uniform: uniform probability of picking a language pair\n samples = []\n for i in range(l):\n for j in range(l):\n if j == i :\n continue\n if j > 0 and args.uniform == False:\n samples.append((0,j))\n if i > 0 and args.uniform == False:\n samples.append((i,0))\n samples.append((i,j))\n\n # optimization of the l2 loss\n print('start optimizing L2 loss')\n lr0, bsz, nepoch, niter = args.lr, args.bsz, args.epoch, args.niter\n for epoch in range(nepoch):\n print(\"start epoch %d / %d\"%(epoch+1, nepoch))\n ones = np.ones(bsz)\n f, fold, nb, lr = 0.0, 0.0, 0.0, lr0\n for it in range(niter):\n if it > 1 and f > fold + 1e-3:\n lr /= 2\n if lr < .05:\n break\n fold = f\n f, nb = 0.0, 0.0\n for k in range(100 * (l-1)):\n (i,j) = random.choice(samples)\n embi = EMB[i][np.random.permutation(nmax)[:bsz], :]\n embj = EMB[j][np.random.permutation(nmax)[:bsz], :]\n perm = ot.sinkhorn(ones, ones, np.linalg.multi_dot([embi, -TRANS[i], TRANS[j].T,embj.T]), reg = 0.025, stopThr = 1e-3)\n grad = np.linalg.multi_dot([embi.T, perm, embj])\n f -= np.trace(np.linalg.multi_dot([TRANS[i].T, grad, TRANS[j]])) / embi.shape[0]\n nb += 1\n if i > 0:\n TRANS[i] = proj_ortho(TRANS[i] + lr * np.dot(grad, TRANS[j]))\n if j > 0:\n TRANS[j] = proj_ortho(TRANS[j] + lr * np.dot(grad.transpose(), TRANS[i]))\n print(\"iter %d / %d - epoch %d - loss: %.5f lr: %.4f\" % (it, niter, epoch+1, f / nb , lr))\n print(\"end of epoch %d - loss: %.5f - lr: %.4f\" % (epoch+1, f / max(nb,1), lr))\n niter, bsz = max(int(niter/2),2), min(1000, bsz * 2)\n #end for epoch in range(nepoch):\n\n # optimization of the RCSLS loss\n print('start optimizing RCSLS loss')\n f, fold, nb, lr = 0.0, 0.0, 0.0, args.altlr\n for epoch in range(args.altepoch):\n if epoch > 1 and f-fold > -1e-4 * abs(fold):\n lr/= 2\n if lr < 1e-1:\n break\n fold = f\n f, nb = 0.0, 0.0\n for k in range(round(nmax / args.altbsz) * 10 * (l-1)):\n (i,j) = random.choice(samples)\n sgdidx = np.random.choice(nmax, size=args.altbsz, replace=False)\n embi = EMB[i][sgdidx, :]\n embj = EMB[j][:nmax, :]\n # crude alignment approximation:\n T = np.dot(TRANS[i], TRANS[j].T)\n scores = np.linalg.multi_dot([embi, T, embj.T])\n perm = np.zeros_like(scores)\n perm[np.arange(len(scores)), scores.argmax(1)] = 1\n embj = np.dot(perm, embj)\n # normalization over a subset of embeddings for speed up\n fi, grad = rcsls(embi, embj, embi, embj, T.T)\n f += fi\n nb += 1\n if i > 0:\n TRANS[i] = proj_ortho(TRANS[i] - lr * np.dot(grad, TRANS[j]))\n if j > 0:\n TRANS[j] = proj_ortho(TRANS[j] - lr * np.dot(grad.transpose(), TRANS[i]))\n print(\"epoch %d - loss: %.5f - lr: %.4f\" % (epoch+1, f / max(nb,1), lr))\n #end for epoch in range(args.altepoch):\n return TRANS\n\ndef convex_init(X, Y, niter=100, reg=0.05, apply_sqrt=False):\n n, d = X.shape\n K_X, K_Y = np.dot(X, X.T), np.dot(Y, Y.T)\n K_Y *= np.linalg.norm(K_X) / np.linalg.norm(K_Y)\n K2_X, K2_Y = np.dot(K_X, K_X), np.dot(K_Y, K_Y)\n P = np.ones([n, n]) / float(n)\n for it in range(1, niter + 1):\n G = np.dot(P, K2_X) + np.dot(K2_Y, P) - 2 * np.dot(K_Y, np.dot(P, K_X))\n q = ot.sinkhorn(np.ones(n), np.ones(n), G, reg, stopThr=1e-3)\n alpha = 2.0 / float(2.0 + it)\n P = alpha * q + (1.0 - alpha) * P\n return procrustes(np.dot(P, X), Y).T\n\n\n###### MAIN ######\n\nlglist = args.lglist.split('-')\nl = len(lglist)\n\n# embs:\nEMB = {}\nfor i in range(l):\n fn = args.embdir + '/wiki.' + lglist[i] + '.vec'\n _, vecs = load_vectors(fn, maxload=args.maxload)\n EMB[i] = vecs\n\n#init\nprint(\"Computing initial bilingual apping with Gromov-Wasserstein...\")\nTRANS={}\nmaxinit = 2000\nemb0 = EMB[0][:maxinit,:]\nC0 = GWmatrix(emb0)\nTRANS[0] = np.eye(300)\nfor i in range(1, l):\n print(\"init \"+lglist[i])\n embi = EMB[i][:maxinit,:]\n TRANS[i] = gromov_wasserstein(embi, emb0, C0)\n\n# align\nalign(EMB, TRANS, lglist, args)\n\nprint('saving matrices in ' + args.outdir)\nlanguages=''.join(lglist)\nfor i in range(l):\n save_matrix(args.outdir + '/W-' + languages + '-' + lglist[i], TRANS[i])\n" }, { "path": "alignment/utils.py", "content": "#!/usr/bin/env python3\n# Copyright (c) 2018-present, Facebook, Inc.\n# All rights reserved.\n#\n# This source code is licensed under the license found in the\n# LICENSE file in the root directory of this source tree.\n\nimport io\nimport numpy as np\nimport collections\n\n\ndef load_vectors(fname, maxload=200000, norm=True, center=False, verbose=True):\n if verbose:\n print(\"Loading vectors from %s\" % fname)\n fin = io.open(fname, 'r', encoding='utf-8', newline='\\n', errors='ignore')\n n, d = map(int, fin.readline().split())\n if maxload > 0:\n n = min(n, maxload)\n x = np.zeros([n, d])\n words = []\n for i, line in enumerate(fin):\n if i >= n:\n break\n tokens = line.rstrip().split(' ')\n words.append(tokens[0])\n v = np.array(tokens[1:], dtype=float)\n x[i, :] = v\n if norm:\n x /= np.linalg.norm(x, axis=1)[:, np.newaxis] + 1e-8\n if center:\n x -= x.mean(axis=0)[np.newaxis, :]\n x /= np.linalg.norm(x, axis=1)[:, np.newaxis] + 1e-8\n if verbose:\n print(\"%d word vectors loaded\" % (len(words)))\n return words, x\n\n\ndef idx(words):\n w2i = {}\n for i, w in enumerate(words):\n if w not in w2i:\n w2i[w] = i\n return w2i\n\n\ndef save_vectors(fname, x, words):\n n, d = x.shape\n fout = io.open(fname, 'w', encoding='utf-8')\n fout.write(u\"%d %d\\n\" % (n, d))\n for i in range(n):\n fout.write(words[i] + \" \" + \" \".join(map(lambda a: \"%.4f\" % a, x[i, :])) + \"\\n\")\n fout.close()\n\n\ndef save_matrix(fname, x):\n n, d = x.shape\n fout = io.open(fname, 'w', encoding='utf-8')\n fout.write(u\"%d %d\\n\" % (n, d))\n for i in range(n):\n fout.write(\" \".join(map(lambda a: \"%.4f\" % a, x[i, :])) + \"\\n\")\n fout.close()\n\n\ndef procrustes(X_src, Y_tgt):\n U, s, V = np.linalg.svd(np.dot(Y_tgt.T, X_src))\n return np.dot(U, V)\n\n\ndef select_vectors_from_pairs(x_src, y_tgt, pairs):\n n = len(pairs)\n d = x_src.shape[1]\n x = np.zeros([n, d])\n y = np.zeros([n, d])\n for k, ij in enumerate(pairs):\n i, j = ij\n x[k, :] = x_src[i, :]\n y[k, :] = y_tgt[j, :]\n return x, y\n\n\ndef load_lexicon(filename, words_src, words_tgt, verbose=True):\n f = io.open(filename, 'r', encoding='utf-8')\n lexicon = collections.defaultdict(set)\n idx_src , idx_tgt = idx(words_src), idx(words_tgt)\n vocab = set()\n for line in f:\n word_src, word_tgt = line.split()\n if word_src in idx_src and word_tgt in idx_tgt:\n lexicon[idx_src[word_src]].add(idx_tgt[word_tgt])\n vocab.add(word_src)\n if verbose:\n coverage = len(lexicon) / float(len(vocab))\n print(\"Coverage of source vocab: %.4f\" % (coverage))\n return lexicon, float(len(vocab))\n\n\ndef load_pairs(filename, idx_src, idx_tgt, verbose=True):\n f = io.open(filename, 'r', encoding='utf-8')\n pairs = []\n tot = 0\n for line in f:\n a, b = line.rstrip().split(' ')\n tot += 1\n if a in idx_src and b in idx_tgt:\n pairs.append((idx_src[a], idx_tgt[b]))\n if verbose:\n coverage = (1.0 * len(pairs)) / tot\n print(\"Found pairs for training: %d - Total pairs in file: %d - Coverage of pairs: %.4f\" % (len(pairs), tot, coverage))\n return pairs\n\n\ndef compute_nn_accuracy(x_src, x_tgt, lexicon, bsz=100, lexicon_size=-1):\n if lexicon_size < 0:\n lexicon_size = len(lexicon)\n idx_src = list(lexicon.keys())\n acc = 0.0\n x_src /= np.linalg.norm(x_src, axis=1)[:, np.newaxis] + 1e-8\n x_tgt /= np.linalg.norm(x_tgt, axis=1)[:, np.newaxis] + 1e-8\n for i in range(0, len(idx_src), bsz):\n e = min(i + bsz, len(idx_src))\n scores = np.dot(x_tgt, x_src[idx_src[i:e]].T)\n pred = scores.argmax(axis=0)\n for j in range(i, e):\n if pred[j - i] in lexicon[idx_src[j]]:\n acc += 1.0\n return acc / lexicon_size\n\n\ndef compute_csls_accuracy(x_src, x_tgt, lexicon, lexicon_size=-1, k=10, bsz=1024):\n if lexicon_size < 0:\n lexicon_size = len(lexicon)\n idx_src = list(lexicon.keys())\n\n x_src /= np.linalg.norm(x_src, axis=1)[:, np.newaxis] + 1e-8\n x_tgt /= np.linalg.norm(x_tgt, axis=1)[:, np.newaxis] + 1e-8\n\n sr = x_src[list(idx_src)]\n sc = np.dot(sr, x_tgt.T)\n similarities = 2 * sc\n sc2 = np.zeros(x_tgt.shape[0])\n for i in range(0, x_tgt.shape[0], bsz):\n j = min(i + bsz, x_tgt.shape[0])\n sc_batch = np.dot(x_tgt[i:j, :], x_src.T)\n dotprod = np.partition(sc_batch, -k, axis=1)[:, -k:]\n sc2[i:j] = np.mean(dotprod, axis=1)\n similarities -= sc2[np.newaxis, :]\n\n nn = np.argmax(similarities, axis=1).tolist()\n correct = 0.0\n for k in range(0, len(lexicon)):\n if nn[k] in lexicon[idx_src[k]]:\n correct += 1.0\n return correct / lexicon_size\n" }, { "path": "classification-example.sh", "content": "#!/usr/bin/env bash\n#\n# Copyright (c) 2016-present, Facebook, Inc.\n# All rights reserved.\n#\n# This source code is licensed under the MIT license found in the\n# LICENSE file in the root directory of this source tree.\n#\n\nmyshuf() {\n perl -MList::Util=shuffle -e 'print shuffle(<>);' \"$@\";\n}\n\nnormalize_text() {\n tr '[:upper:]' '[:lower:]' | sed -e 's/^/__label__/g' | \\\n sed -e \"s/'/ ' /g\" -e 's/\"//g' -e 's/\\./ \\. /g' -e 's/
/ /g' \\\n -e 's/,/ , /g' -e 's/(/ ( /g' -e 's/)/ ) /g' -e 's/\\!/ \\! /g' \\\n -e 's/\\?/ \\? /g' -e 's/\\;/ /g' -e 's/\\:/ /g' | tr -s \" \" | myshuf\n}\n\nRESULTDIR=result\nDATADIR=data\n\nmkdir -p \"${RESULTDIR}\"\nmkdir -p \"${DATADIR}\"\n\nif [ ! -f \"${DATADIR}/dbpedia.train\" ]\nthen\n wget -c \"https://drive.google.com/uc?export=download&id=0Bz8a_Dbh9QhbQ2Vic1kxMmZZQ1k\" -O \"${DATADIR}/dbpedia_csv.tar.gz\"\n tar -xzvf \"${DATADIR}/dbpedia_csv.tar.gz\" -C \"${DATADIR}\"\n cat \"${DATADIR}/dbpedia_csv/train.csv\" | normalize_text > \"${DATADIR}/dbpedia.train\"\n cat \"${DATADIR}/dbpedia_csv/test.csv\" | normalize_text > \"${DATADIR}/dbpedia.test\"\nfi\n\nmake\n\n./fasttext supervised -input \"${DATADIR}/dbpedia.train\" -output \"${RESULTDIR}/dbpedia\" -dim 10 -lr 0.1 -wordNgrams 2 -minCount 1 -bucket 10000000 -epoch 5 -thread 4\n\n./fasttext test \"${RESULTDIR}/dbpedia.bin\" \"${DATADIR}/dbpedia.test\"\n\n./fasttext predict \"${RESULTDIR}/dbpedia.bin\" \"${DATADIR}/dbpedia.test\" > \"${RESULTDIR}/dbpedia.test.predict\"\n" }, { "path": "classification-results.sh", "content": "#!/usr/bin/env bash\n#\n# Copyright (c) 2016-present, Facebook, Inc.\n# All rights reserved.\n#\n# This source code is licensed under the MIT license found in the\n# LICENSE file in the root directory of this source tree.\n#\n\n# This script produces the results from Table 1 in the following paper:\n# Bag of Tricks for Efficient Text Classification, arXiv 1607.01759, 2016\n\nmyshuf() {\n perl -MList::Util=shuffle -e 'print shuffle(<>);' \"$@\";\n}\n\nnormalize_text() {\n tr '[:upper:]' '[:lower:]' | sed -e 's/^/__label__/g' | \\\n sed -e \"s/'/ ' /g\" -e 's/\"//g' -e 's/\\./ \\. /g' -e 's/
/ /g' \\\n -e 's/,/ , /g' -e 's/(/ ( /g' -e 's/)/ ) /g' -e 's/\\!/ \\! /g' \\\n -e 's/\\?/ \\? /g' -e 's/\\;/ /g' -e 's/\\:/ /g' | tr -s \" \" | myshuf\n}\n\nDATASET=(\n ag_news\n sogou_news\n dbpedia\n yelp_review_polarity\n yelp_review_full\n yahoo_answers\n amazon_review_full\n amazon_review_polarity\n)\n\nID=(\n 0Bz8a_Dbh9QhbUDNpeUdjb0wxRms # ag_news\n 0Bz8a_Dbh9QhbUkVqNEszd0pHaFE # sogou_news\n 0Bz8a_Dbh9QhbQ2Vic1kxMmZZQ1k # dbpedia\n 0Bz8a_Dbh9QhbNUpYQ2N3SGlFaDg # yelp_review_polarity\n 0Bz8a_Dbh9QhbZlU4dXhHTFhZQU0 # yelp_review_full\n 0Bz8a_Dbh9Qhbd2JNdDBsQUdocVU # yahoo_answers\n 0Bz8a_Dbh9QhbZVhsUnRWRDhETzA # amazon_review_full\n 0Bz8a_Dbh9QhbaW12WVVZS2drcnM # amazon_review_polarity\n)\n\n# These learning rates were chosen by validation on a subset of the training set.\nLR=( 0.25 0.5 0.5 0.1 0.1 0.1 0.05 0.05 )\n\nRESULTDIR=result\nDATADIR=data\n\nmkdir -p \"${RESULTDIR}\"\nmkdir -p \"${DATADIR}\"\n\n# Small datasets first\n\nfor i in {0..0}\ndo\n echo \"Downloading dataset ${DATASET[i]}\"\n if [ ! -f \"${DATADIR}/${DATASET[i]}.train\" ]\n then\n wget -c \"https://drive.google.com/uc?export=download&id=${ID[i]}\" -O \"${DATADIR}/${DATASET[i]}_csv.tar.gz\"\n tar -xzvf \"${DATADIR}/${DATASET[i]}_csv.tar.gz\" -C \"${DATADIR}\"\n cat \"${DATADIR}/${DATASET[i]}_csv/train.csv\" | normalize_text > \"${DATADIR}/${DATASET[i]}.train\"\n cat \"${DATADIR}/${DATASET[i]}_csv/test.csv\" | normalize_text > \"${DATADIR}/${DATASET[i]}.test\"\n fi\ndone\n\n# Large datasets require a bit more work due to the extra request page\n\nfor i in {1..7}\ndo\n echo \"Downloading dataset ${DATASET[i]}\"\n if [ ! -f \"${DATADIR}/${DATASET[i]}.train\" ]\n then\n curl -c /tmp/cookies \"https://drive.google.com/uc?export=download&id=${ID[i]}\" > /tmp/intermezzo.html\n curl -L -b /tmp/cookies \"https://drive.google.com$(cat /tmp/intermezzo.html | grep -Po 'uc-download-link\" [^>]* href=\"\\K[^\"]*' | sed 's/\\&/\\&/g')\" > \"${DATADIR}/${DATASET[i]}_csv.tar.gz\"\n tar -xzvf \"${DATADIR}/${DATASET[i]}_csv.tar.gz\" -C \"${DATADIR}\"\n cat \"${DATADIR}/${DATASET[i]}_csv/train.csv\" | normalize_text > \"${DATADIR}/${DATASET[i]}.train\"\n cat \"${DATADIR}/${DATASET[i]}_csv/test.csv\" | normalize_text > \"${DATADIR}/${DATASET[i]}.test\"\n fi\ndone\n\nmake\n\nfor i in {0..7}\ndo\n echo \"Working on dataset ${DATASET[i]}\"\n ./fasttext supervised -input \"${DATADIR}/${DATASET[i]}.train\" \\\n -output \"${RESULTDIR}/${DATASET[i]}\" -dim 10 -lr \"${LR[i]}\" -wordNgrams 2 \\\n -minCount 1 -bucket 10000000 -epoch 5 -thread 4 > /dev/null\n ./fasttext test \"${RESULTDIR}/${DATASET[i]}.bin\" \\\n \"${DATADIR}/${DATASET[i]}.test\"\ndone\n" }, { "path": "crawl/README.md", "content": "## Preprocessing Common Crawl\n\nThis code downloads, preprocesses and splits per language the data from [Common Crawl](http://commoncrawl.org/).\n\nThis script uses the scripts and language identifier of [1].\n\nThis code inherits its requirements form [fastText](https://github.com/facebookresearch/fastText).\n\nSet the variable WET_PATHS_URL to the crawl you want to process.\nPlease also set the variables NUM_LANGID and NUM_DEDUP in `download_crawl.sh` according to the capacity of your machine.\nLangid processes are mostly limited by CPU usage, while dedup processes are likely to be limited by RAM usage (each use 2GB of RAM).\n\n### Reference\n\nIf you use this code, please cite:\n\n[1] E. Grave*, P. Bojanowski*, P. Gupta, A. Joulin, T. Mikolov, [*Learning Word Vectors for 157 Languages*](https://arxiv.org/abs/1802.06893)\n\n```\n@inproceedings{grave2018learning,\n title={Learning Word Vectors for 157 Languages},\n author={Grave, Edouard and Bojanowski, Piotr and Gupta, Prakhar and Joulin, Armand and Mikolov, Tomas},\n booktitle={Proceedings of the International Conference on Language Resources and Evaluation (LREC 2018)},\n year={2018}\n}\n```\n" }, { "path": "crawl/dedup.cc", "content": "// Copyright (c) 2018-present, Facebook, Inc.\n// All rights reserved.\n//\n// This source code is licensed under the MIT license found in the\n// LICENSE file in the root directory of this source tree.\n\n#include \n#include \n#include \n#include \n#include \n\nuint64_t fnv1a_64(uint8_t *data, size_t sz, uint64_t h=14695981039346656037ull)\n{\n for (size_t i = 0; i < sz; i++, data++) {\n h ^= uint64_t(*data);\n h *= 1099511628211ull;\n }\n return h;\n}\n\nint main(int argc, char** argv)\n{\n uint64_t init_values[] = {\n 14695981039346656037ull,\n 9425296925403859339ull,\n 13716263814064014149ull,\n 3525492407291847033ull,\n 8607404175481815707ull,\n 9818874561736458749ull,\n 10026508429719773353ull,\n 3560712257386009938ull\n };\n size_t n = 1ull<<34, num_hashes = 2;\n std::vector seen(n);\n\n std::ios_base::sync_with_stdio(false);\n\n for (std::string line; std::getline(std::cin, line);) {\n bool b = true;\n for (size_t i = 0; i < num_hashes; i++) {\n uint64_t h = fnv1a_64((uint8_t*) line.data(), line.length(), init_values[i]) % n;\n b = b && seen[h];\n seen[h] = true;\n }\n if (!b) {\n std::cout << line << std::endl;\n }\n }\n return 0;\n}\n" }, { "path": "crawl/download_crawl.sh", "content": "#!/bin/usr/env sh\n# Copyright (c) 2018-present, Facebook, Inc.\n# All rights reserved.\n#\n# This source code is licensed under the MIT license found in the\n# LICENSE file in the root directory of this source tree.\n\nset -e\n\n# Set this variable to the crawl you want to process.\nWET_PATHS_URL=\"https://commoncrawl.s3.amazonaws.com/crawl-data/CC-MAIN-2018-34/wet.paths.gz\"\n\n# Set NUM_LANGID and NUM_DEDUP according to the capacity of your machine.\n# Please note that each dedup process uses 2GB of RAM, while langid is\n# mostly limited by cpu usage.\nNUM_LANGID=12\nNUM_DEDUP=8\nURL=\"https://commoncrawl.s3.amazonaws.com/\"\n\nif [ ! -d fastText ]; then\n git clone https://github.com/facebookresearch/fastText.git\nfi\n\nif [ ! -f fastText/fasttext ]; then\n cd fastText\n make\n cd ..\nfi\n\nif [ ! -f lid.176.bin ]; then\n wget https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.bin\nfi\n\nif [ ! -d tmp ]; then\n mkdir tmp\nfi\n\nif [ ! -d shard ]; then\n mkdir shard\nfi\n\nif [ ! -f wet.paths ]; then\n wget \"${WET_PATHS_URL}\"\n gunzip wet.paths.gz\nfi\n\n## Language identification\ncat wet.paths | xargs -n 1 -P \"${NUM_LANGID}\" -I '{}' sh process_wet_file.sh \"${URL}{}\"\n\n## Deduplication\ng++ -std=c++11 -O3 -o dedup dedup.cc\ng++ -std=c++11 -O3 -o filter_utf8 filter_utf8.cc\nfind shard -name '*.txt' | xargs -n 1 -P \"${NUM_DEDUP}\" -I '{}' sh filter_dedup.sh \"{}\"\n\n## Example of data filtering + tokenization\ngit clone https://github.com/moses-smt/mosesdecoder.git\nperl mosesdecoder/scripts/tokenizer/tokenizer.perl -l es < shard/es.dedup > shard/es.tok\n" }, { "path": "crawl/filter_dedup.sh", "content": "#!/bin/usr/env sh\n# Copyright (c) 2018-present, Facebook, Inc.\n# All rights reserved.\n#\n# This source code is licensed under the MIT license found in the\n# LICENSE file in the root directory of this source tree.\n\nset -e\n\nLG=$(basename --suffix=\".txt\" \"${1}\")\n\n./filter_utf8 < \"shard/${LG}.txt\" \\\n | ./dedup > \"shard/${LG}.dedup\"\n" }, { "path": "crawl/filter_utf8.cc", "content": "// Copyright (c) 2018-present, Facebook, Inc.\n// All rights reserved.\n//\n// This source code is licensed under the MIT license found in the\n// LICENSE file in the root directory of this source tree.\n\n#include \n#include \n#include \n\n// Check that the next n bytes are continuation bytes.\nbool continuation(uint8_t* str, int n)\n{\n for (int i = 0; i < n; i++) {\n if ((str[i] & 0xc0) != 0x80) return false;\n }\n return true;\n}\n\n// Invalid UTF8 correspond to codepoints which are larger than U+10FFFF.\n// This value is encoded in UTF8 as:\n// * 11110.100 10.001111 10.111111 10.111111\n// We thus check if the first byte is larger than 0xf4, or if it is equal\n// to 0xf4 and the second byte is larger than 0x8f.\nbool invalid(uint8_t* str)\n{\n return str[0] > 0xf4 || (str[0] == 0xf4 && str[1] > 0x8f);\n}\n\n// Surrogate halves corresponds to the range U+D800 through U+DFFF,\n// which are encoded in UTF8 as:\n// * 1110.1101 10.100000 10.000000\n// * 1110.1101 10.111111 10.111111\n// We thus check is the first byte is equal to 0xed and if the\n// sixth bit of the second byte is set.\nbool surrogate(uint8_t* str)\n{\n return str[0] == 0xed && str[1] & 0x20;\n}\n\n// Sequences of length 2 are overlong if the leading 4 bits (noted as y)\n// are equal to 0: 110.yyyyx 10xxxxxx\nbool overlong_2(uint8_t* str)\n{\n return (str[0] & 0x1e) == 0;\n}\n\n// Sequences of lenth 3 are overlong if the leading 5 bits (noted as y)\n// are equal to 0: 1110.yyyy 10.yxxxxx 10.xxxxxx\nbool overlong_3(uint8_t* str)\n{\n return (str[0] & 0x0f) == 0 && (str[1] & 0x20) == 0;\n}\n\n// Sequences of length 4 are overlong if the leading 5 bits (noted as y)\n// are equal to 0: 11110.yyy 10.yyxxxx 10.xxxxxx 10.xxxxxx\nbool overlong_4(uint8_t* str)\n{\n return (str[0] & 0x07) == 0 && (str[1] & 0x30) == 0;\n}\n\nbool valid_utf8(uint8_t* str, size_t length)\n{\n uint8_t* end = str + length;\n while (str < end) {\n if (str[0] < 0x80) {\n // 0.xxxxxxx\n str += 1;\n } else if ((str[0] & 0xe0) == 0xc0) {\n // 110.xxxxx 10.xxxxxx\n if (str + 1 >= end) return false;\n if (!continuation(str + 1, 1)) return false;\n if (overlong_2(str)) return false;\n str += 2;\n } else if ((str[0] & 0xf0) == 0xe0) {\n // 1110.xxxx 10.xxxxxx 10.xxxxxx\n if (str + 2 >= end) return false;\n if (!continuation(str + 1, 2)) return false;\n if (overlong_3(str)) return false;\n if (surrogate(str)) return false;\n str += 3;\n } else if ((str[0] & 0xf8) == 0xf0) {\n // 11110.xxx 10.xxxxxx 10.xxxxxx 10.xxxxxx\n if (str + 3 >= end) return false;\n if (!continuation(str + 1, 3)) return false;\n if (overlong_4(str)) return false;\n if (invalid(str)) return false;\n str += 4;\n } else {\n return false;\n }\n }\n return true;\n}\n\nint main(int argc, char** argv)\n{\n std::ios_base::sync_with_stdio(false);\n for (std::string line; std::getline(std::cin, line);) {\n if (valid_utf8((uint8_t*) line.data(), line.length())) {\n std::cout << line << std::endl;\n }\n }\n return 0;\n}\n" }, { "path": "crawl/process_wet_file.sh", "content": "#!/bin/usr/env sh\n# Copyright (c) 2018-present, Facebook, Inc.\n# All rights reserved.\n#\n# This source code is licensed under the MIT license found in the\n# LICENSE file in the root directory of this source tree.\n\nset -e\n\nURL=$1\n\nFILENAME=$(basename --suffix=\".warc.wet.gz\" \"${URL}\")\n\necho \"Processing ${FILENAME}.\"\n\nwget -q -P tmp \"${URL}\"\n\n#echo \"Extracting ${FILENAME}.warc.wet.gz\"\ngunzip \"tmp/${FILENAME}.warc.wet.gz\"\n\n#echo \"Language identification for ${FILENAME}.warc.wet\"\nfastText/fasttext predict-prob lid.176.bin \"tmp/${FILENAME}.warc.wet\" > \"tmp/${FILENAME}.lid\"\n\n#echo \"Splitting ${FILENAME}.warc.wet per language\"\npaste \"tmp/${FILENAME}.lid\" \"tmp/${FILENAME}.warc.wet\" | \\\n awk '($2 > 0.8 || ($1==\"__label__hr\" && $2 > 0.4)) && length() > 100 {lang = substr($1, 10); $1=\"\"; $2=\"\"; print $0 >> \"shard/\"lang\".txt\"}'\n\n#echo \"Removing tmp files\"\nrm \"tmp/${FILENAME}.lid\"\nrm \"tmp/${FILENAME}.warc.wet\"\n" }, { "path": "docs/aligned-vectors.md", "content": "---\nid: aligned-vectors\ntitle: Aligned word vectors\n---\n\nWe are publishing aligned word vectors for 44 languages based on the pre-trained vectors computed on [*Wikipedia*](https://www.wikipedia.org) using fastText.\nThe alignments are performed with the RCSLS method described in [*Joulin et al (2018)*](https://arxiv.org/abs/1804.07745).\n\n### Vectors\n\nThe aligned vectors can be downloaded from:\n\n|||||\n|-|-|-|-|\n| Afrikaans: [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-aligned/wiki.af.align.vec) | Arabic: [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-aligned/wiki.ar.align.vec) | Bulgarian: [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-aligned/wiki.bg.align.vec) | Bengali: [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-aligned/wiki.bn.align.vec) |\n| Bosnian: [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-aligned/wiki.bs.align.vec) | Catalan: [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-aligned/wiki.ca.align.vec) | Czech: [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-aligned/wiki.cs.align.vec) | Danish: [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-aligned/wiki.da.align.vec) |\n| German: [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-aligned/wiki.de.align.vec) | Greek: [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-aligned/wiki.el.align.vec) | English: [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-aligned/wiki.en.align.vec) | Spanish: [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-aligned/wiki.es.align.vec) |\n| Estonian: [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-aligned/wiki.et.align.vec) | Persian: [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-aligned/wiki.fa.align.vec) | Finnish: [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-aligned/wiki.fi.align.vec) | French: [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-aligned/wiki.fr.align.vec) |\n| Hebrew: [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-aligned/wiki.he.align.vec) | Hindi: [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-aligned/wiki.hi.align.vec) | Croatian: [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-aligned/wiki.hr.align.vec) | Hungarian: [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-aligned/wiki.hu.align.vec) |\n| Indonesian: [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-aligned/wiki.id.align.vec) | Italian: [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-aligned/wiki.it.align.vec) | Korean: [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-aligned/wiki.ko.align.vec) | Lithuanian: [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-aligned/wiki.lt.align.vec) |\n| Latvian: [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-aligned/wiki.lv.align.vec) | Macedonian: [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-aligned/wiki.mk.align.vec) | Malay: [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-aligned/wiki.ms.align.vec) | Dutch: [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-aligned/wiki.nl.align.vec) |\n| Norwegian: [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-aligned/wiki.no.align.vec) | Polish: [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-aligned/wiki.pl.align.vec) | Portuguese: [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-aligned/wiki.pt.align.vec) | Romanian: [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-aligned/wiki.ro.align.vec) |\n| Russian: [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-aligned/wiki.ru.align.vec) | Slovak: [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-aligned/wiki.sk.align.vec) | Slovenian: [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-aligned/wiki.sl.align.vec) | Albanian: [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-aligned/wiki.sq.align.vec) |\n| Swedish: [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-aligned/wiki.sv.align.vec) | Tamil: [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-aligned/wiki.ta.align.vec) | Thai: [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-aligned/wiki.th.align.vec) | Tagalog: [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-aligned/wiki.tl.align.vec) |\n| Turkish: [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-aligned/wiki.tr.align.vec) | Ukrainian: [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-aligned/wiki.uk.align.vec) | Vietnamese: [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-aligned/wiki.vi.align.vec) | Chinese: [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-aligned/wiki.zh.align.vec) |\n\n### Format\n\nThe word vectors come in the default text format of fastText.\nThe first line gives the number of vectors and their dimension.\nThe other lines contain a word followed by its vector. Each value is space separated.\n\n### License\n\nThe word vectors are distributed under the [*Creative Commons Attribution-Share-Alike License 3.0*](https://creativecommons.org/licenses/by-sa/3.0/).\n\n### References\n\nIf you use these word vectors, please cite the following papers:\n\n[1] A. Joulin, P. Bojanowski, T. Mikolov, H. Jegou, E. Grave, [*Loss in Translation: Learning Bilingual Word Mapping with a Retrieval Criterion*](https://arxiv.org/abs/1804.07745)\n\n```markup\n@InProceedings{joulin2018loss,\n title={Loss in Translation: Learning Bilingual Word Mapping with a Retrieval Criterion},\n author={Joulin, Armand and Bojanowski, Piotr and Mikolov, Tomas and J\\'egou, Herv\\'e and Grave, Edouard},\n year={2018},\n booktitle={Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing},\n}\n```\n\n[2] P. Bojanowski\\*, E. Grave\\*, A. Joulin, T. Mikolov, [*Enriching Word Vectors with Subword Information*](https://arxiv.org/abs/1607.04606)\n\n```markup\n@article{bojanowski2017enriching,\n title={Enriching Word Vectors with Subword Information},\n author={Bojanowski, Piotr and Grave, Edouard and Joulin, Armand and Mikolov, Tomas},\n journal={Transactions of the Association for Computational Linguistics},\n volume={5},\n year={2017},\n issn={2307-387X},\n pages={135--146}\n}\n```\n" }, { "path": "docs/api.md", "content": "---\nid: api\ntitle:API\n---\n\nWe automatically generate our [API documentation](/docs/en/html/index.html) with doxygen.\n" }, { "path": "docs/autotune.md", "content": "---\nid: autotune\ntitle: Automatic hyperparameter optimization\n---\n\nAs we saw in [the tutorial](/docs/en/supervised-tutorial.html#more-epochs-and-larger-learning-rate), finding the best hyperparameters is crucial for building efficient models. However, searching the best hyperparameters manually is difficult. Parameters are dependent and the effect of each parameter vary from one dataset to another.\n\nFastText's autotune feature allows you to find automatically the best hyperparameters for your dataset.\n\n# How to use it\n\nIn order to activate hyperparameter optimization, we must provide a validation file with the `-autotune-validation` argument.\n\nFor example, using the same data as our [tutorial example](/docs/en/supervised-tutorial.html#our-first-classifier), the autotune can be used in the following way:\n\n\n\n```sh\n>> ./fasttext supervised -input cooking.train -output model_cooking -autotune-validation cooking.valid\n```\n\n```py\n>>> import fasttext\n>>> model = fasttext.train_supervised(input='cooking.train', autotuneValidationFile='cooking.valid')\n```\n\n\n\nThen, fastText will search the hyperparameters that gives the best f1-score on `cooking.valid` file:\n```sh\nProgress: 100.0% Trials: 27 Best score: 0.406763 ETA: 0h 0m 0s\n```\n\nNow we can test the obtained model with:\n\n\n```sh\n>> ./fasttext test model_cooking.bin cooking.valid\nN 3000\nP@1 0.666\nR@1 0.288\n```\n\n```py\n>>> model.test(\"cooking.valid\")\n(3000L, 0.666, 0.288)\n```\n\n\n\nBy default, the search will take 5 minutes. You can set the timeout in seconds with the `-autotune-duration` argument. For example, if you want to set the limit to 10 minutes:\n\n\n\n```sh\n>> ./fasttext supervised -input cooking.train -output model_cooking -autotune-validation cooking.valid -autotune-duration 600\n```\n\n```py\n>>> import fasttext\n>>> model = fasttext.train_supervised(input='cooking.train', autotuneValidationFile='cooking.valid', autotuneDuration=600)\n```\n\n\n\nWhile autotuning, fastText displays the best f1-score found so far. If we decide to stop the tuning before the time limit, we can send one `SIGINT` signal (via `CTLR-C` for example). FastText will then finish the current training, and retrain with the best parameters found so far.\n\n\n\n# Constrain model size\n\nAs you may know, fastText can compress the model with [quantization](/docs/en/cheatsheet.html#quantization). However, this compression task comes with its own [hyperparameters](/docs/en/options.html) (`-cutoff`, `-retrain`, `-qnorm`, `-qout`, `-dsub`) that have a consequence on the accuracy and the size of the final model.\n\nFortunately, autotune can also find the hyperparameters for this compression task while targeting the desired model size. To this end, we can set the `-autotune-modelsize` argument:\n\n\n\n```sh\n>> ./fasttext supervised -input cooking.train -output model_cooking -autotune-validation cooking.valid -autotune-modelsize 2M\n```\nThis will produce a `.ftz` file with the best accuracy having the desired size:\n```sh\n>> ls -la model_cooking.ftz\n-rw-r--r--. 1 celebio users 1990862 Aug 25 05:39 model_cooking.ftz\n>> ./fasttext test model_cooking.ftz cooking.valid\nN 3000\nP@1 0.57\nR@1 0.246\n```\n\n```py\n>>> import fasttext\n>>> model = fasttext.train_supervised(input='cooking.train', autotuneValidationFile='cooking.valid', autotuneModelSize=\"2M\")\n```\nIf you save the model, you will obtain a model file with the desired size:\n```py\n>>> model.save_model(\"model_cooking.ftz\")\n>>> import os\n>>> os.stat(\"model_cooking.ftz\").st_size\n1990862\n>>> model.test(\"cooking.valid\")\n(3000L, 0.57, 0.246)\n```\n\n\n\n# How to set the optimization metric?\n\n\n\n
\nBy default, autotune will test the validation file you provide, exactly the same way as `./fasttext test model_cooking.bin cooking.valid` and try to optimize to get the highest [f1-score](https://en.wikipedia.org/wiki/F1_score).\n\nBut, if we want to optimize the score of a specific label, say `__label__baking`, we can set the `-autotune-metric` argument:\n\n```sh\n>> ./fasttext supervised -input cooking.train -output model_cooking -autotune-validation cooking.valid -autotune-metric f1:__label__baking\n```\n\nThis is equivalent to manually optimize the f1-score we get when we test with `./fasttext test-label model_cooking.bin cooking.valid | grep __label__baking` in command line.\n\nSometimes, you may be interested in predicting more than one label. For example, if you were optimizing the hyperparameters manually to get the best score to predict two labels, you would test with `./fasttext test model_cooking.bin cooking.valid 2`. You can also tell autotune to optimize the parameters by testing two labels with the `-autotune-predictions` argument.\n\n
\nBy default, autotune will test the validation file you provide, exactly the same way as `model.test(\"cooking.valid\")` and try to optimize to get the highest [f1-score](https://en.wikipedia.org/wiki/F1_score).\n\nBut, if we want to optimize the score of a specific label, say `__label__baking`, we can set the `autotuneMetric` argument:\n\n```py\n>>> import fasttext\n>>> model = fasttext.train_supervised(input='cooking.train', autotuneValidationFile='cooking.valid', autotuneMetric=\"f1:__label__baking\")\n```\n\nThis is equivalent to manually optimize the f1-score we get when we test with `model.test_label('cooking.valid')['__label__baking']`.\n\nSometimes, you may be interested in predicting more than one label. For example, if you were optimizing the hyperparameters manually to get the best score to predict two labels, you would test with `model.test(\"cooking.valid\", k=2)`. You can also tell autotune to optimize the parameters by testing two labels with the `autotunePredictions` argument.\n\n\nYou can also force autotune to optimize for the best precision for a given recall, or the best recall for a given precision, for all labels, or for a specific label:\n\nFor example, in order to get the best precision at recall = `30%`:\n```sh\n>> ./fasttext supervised [...] -autotune-metric precisionAtRecall:30\n```\nAnd to get the best precision at recall = `30%` for the label `__label__baking`:\n```sh\n>> ./fasttext supervised [...] -autotune-metric precisionAtRecall:30:__label__baking\n```\n\nSimilarly, you can use `recallAtPrecision`:\n```sh\n>> ./fasttext supervised [...] -autotune-metric recallAtPrecision:30\n>> ./fasttext supervised [...] -autotune-metric recallAtPrecision:30:__label__baking\n```\n\n\n" }, { "path": "docs/cheatsheet.md", "content": "---\nid: cheatsheet\ntitle: Cheatsheet\n---\n\n## Word representation learning\n\nIn order to learn word vectors do:\n\n```bash\n$ ./fasttext skipgram -input data.txt -output model\n```\n\n## Obtaining word vectors\n\nPrint word vectors for a text file `queries.txt` containing words.\n\n```bash\n$ ./fasttext print-word-vectors model.bin < queries.txt\n```\n\n## Text classification\n\nIn order to train a text classifier do:\n\n```bash\n$ ./fasttext supervised -input train.txt -output model\n```\n\nOnce the model was trained, you can evaluate it by computing the precision and recall at k (P@k and R@k) on a test set using:\n\n```bash\n$ ./fasttext test model.bin test.txt 1\n```\n\nIn order to obtain the k most likely labels for a piece of text, use:\n\n```bash\n$ ./fasttext predict model.bin test.txt k\n```\n\nIn order to obtain the k most likely labels and their associated probabilities for a piece of text, use:\n\n```bash\n$ ./fasttext predict-prob model.bin test.txt k\n```\n\nIf you want to compute vector representations of sentences or paragraphs, please use:\n\n```bash\n$ ./fasttext print-sentence-vectors model.bin < text.txt\n```\n\n## Quantization\n\nIn order to create a `.ftz` file with a smaller memory footprint do:\n\n```bash\n$ ./fasttext quantize -output model\n```\n\nAll other commands such as test also work with this model\n\n```bash\n$ ./fasttext test model.ftz test.txt\n```\n\n## Autotune\n\nActivate hyperparameter optimization with `-autotune-validation` argument:\n\n```bash\n$ ./fasttext supervised -input train.txt -output model -autotune-validation valid.txt\n```\n\nSet timeout (in seconds):\n```bash\n$ ./fasttext supervised -input train.txt -output model -autotune-validation valid.txt -autotune-duration 600\n```\n\nConstrain the final model size:\n```bash\n$ ./fasttext supervised -input train.txt -output model -autotune-validation valid.txt -autotune-modelsize 2M\n```\n\n\n\n\n\n" }, { "path": "docs/crawl-vectors.md", "content": "---\nid: crawl-vectors\ntitle: Word vectors for 157 languages\n---\n\nWe distribute pre-trained word vectors for 157 languages, trained on [*Common Crawl*](http://commoncrawl.org/) and [*Wikipedia*](https://www.wikipedia.org) using fastText.\nThese models were trained using CBOW with position-weights, in dimension 300, with character n-grams of length 5, a window of size 5 and 10 negatives.\nWe also distribute three new word analogy datasets, for French, Hindi and Polish.\n\n### Download directly with command line or from python\n\nIn order to download with command line or from python code, you must have installed the python package as [described here](/docs/en/support.html#building-fasttext-python-module).\n\n\n\n```bash\n$ ./download_model.py en # English\nDownloading https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.en.300.bin.gz\n (19.78%) [=========> ]\n```\nOnce the download is finished, use the model as usual:\n```bash\n$ ./fasttext nn cc.en.300.bin 10\nQuery word?\n```\n\n```py\n>>> import fasttext.util\n>>> fasttext.util.download_model('en', if_exists='ignore') # English\n>>> ft = fasttext.load_model('cc.en.300.bin')\n```\n\n\n#### 🤗 HuggingFace Integration\nWord vectors for 157 languages available on the Hugging Face Hub under the [`fasttext`](https://huggingface.co/models?library=fasttext) tag and more documentation is available [here](https://huggingface.co/facebook/fasttext-en-vectors/blob/main/README.md).\n\n```python\n>>> import fasttext\n>>> from huggingface_hub import hf_hub_download\n\n>>> model_path = hf_hub_download(repo_id=\"facebook/fasttext-en-vectors\", filename=\"model.bin\")\n>>> model = fasttext.load_model(model_path)\n```\n\n### Adapt the dimension\n\nThe pre-trained word vectors we distribute have dimension 300. If you need a smaller size, you can use our dimension reducer.\nIn order to use that feature, you must have installed the python package as [described here](/docs/en/support.html#building-fasttext-python-module).\n\nFor example, in order to get vectors of dimension 100:\n\n\n\n```bash\n$ ./reduce_model.py cc.en.300.bin 100\nLoading model\nReducing matrix dimensions\nSaving model\ncc.en.100.bin saved\n```\nThen you can use the `cc.en.100.bin` model file as usual.\n\n\n```py\n>>> import fasttext\n>>> import fasttext.util\n>>> ft = fasttext.load_model('cc.en.300.bin')\n>>> ft.get_dimension()\n300\n>>> fasttext.util.reduce_model(ft, 100)\n>>> ft.get_dimension()\n100\n```\nThen you can use `ft` model object as usual:\n```py\n>>> ft.get_word_vector('hello').shape\n(100,)\n>>> ft.get_nearest_neighbors('hello')\n[(0.775576114654541, u'heyyyy'), (0.7686290144920349, u'hellow'), (0.7663413286209106, u'hello-'), (0.7579624056816101, u'heyyyyy'), (0.7495524287223816, u'hullo'), (0.7473770380020142, u'.hello'), (0.7407292127609253, u'Hiiiii'), (0.7402616739273071, u'hellooo'), (0.7399682402610779, u'hello.'), (0.7396857738494873, u'Heyyyyy')]\n```\nor save it for later use:\n```py\n>>> ft.save_model('cc.en.100.bin')\n```\n\n\n\n### Format\n\nThe word vectors are available in both binary and text formats.\n\nUsing the binary models, vectors for out-of-vocabulary words can be obtained with\n```\n$ ./fasttext print-word-vectors wiki.it.300.bin < oov_words.txt\n```\nwhere the file oov_words.txt contains out-of-vocabulary words.\n\nIn the text format, each line contain a word followed by its vector.\nEach value is space separated, and words are sorted by frequency in descending order.\nThese text models can easily be loaded in Python using the following code:\n```python\nimport io\n\ndef load_vectors(fname):\n fin = io.open(fname, 'r', encoding='utf-8', newline='\\n', errors='ignore')\n n, d = map(int, fin.readline().split())\n data = {}\n for line in fin:\n tokens = line.rstrip().split(' ')\n data[tokens[0]] = map(float, tokens[1:])\n return data\n```\n\n### Tokenization\n\nWe used the [*Stanford word segmenter*](https://nlp.stanford.edu/software/segmenter.html) for Chinese, [*Mecab*](http://taku910.github.io/mecab/) for Japanese and [*UETsegmenter*](https://github.com/phongnt570/UETsegmenter) for Vietnamese.\nFor languages using the Latin, Cyrillic, Hebrew or Greek scripts, we used the tokenizer from the [*Europarl*](http://www.statmt.org/europarl/) preprocessing tools.\nFor the remaining languages, we used the ICU tokenizer.\n\nMore information about the training of these models can be found in the article [*Learning Word Vectors for 157 Languages*](https://arxiv.org/abs/1802.06893).\n\n### License\n\nThe word vectors are distributed under the [*Creative Commons Attribution-Share-Alike License 3.0*](https://creativecommons.org/licenses/by-sa/3.0/).\n\n### References\n\nIf you use these word vectors, please cite the following paper:\n\nE. Grave\\*, P. Bojanowski\\*, P. Gupta, A. Joulin, T. Mikolov, [*Learning Word Vectors for 157 Languages*](https://arxiv.org/abs/1802.06893)\n\n```markup\n@inproceedings{grave2018learning,\n title={Learning Word Vectors for 157 Languages},\n author={Grave, Edouard and Bojanowski, Piotr and Gupta, Prakhar and Joulin, Armand and Mikolov, Tomas},\n booktitle={Proceedings of the International Conference on Language Resources and Evaluation (LREC 2018)},\n year={2018}\n}\n```\n\n### Evaluation datasets\n\nThe analogy evaluation datasets described in the paper are available here: [French](https://dl.fbaipublicfiles.com/fasttext/word-analogies/questions-words-fr.txt), [Hindi](https://dl.fbaipublicfiles.com/fasttext/word-analogies/questions-words-hi.txt), [Polish](https://dl.fbaipublicfiles.com/fasttext/word-analogies/questions-words-pl.txt).\n\n### Models\n\nThe models can be downloaded from:\n\n||||\n|-|-|-|\n| Afrikaans: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.af.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.af.300.vec.gz) | Albanian: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.sq.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.sq.300.vec.gz) | Alemannic: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.als.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.als.300.vec.gz) |\n| Amharic: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.am.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.am.300.vec.gz) | Arabic: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.ar.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.ar.300.vec.gz) | Aragonese: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.an.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.an.300.vec.gz) |\n| Armenian: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.hy.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.hy.300.vec.gz) | Assamese: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.as.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.as.300.vec.gz) | Asturian: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.ast.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.ast.300.vec.gz) |\n| Azerbaijani: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.az.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.az.300.vec.gz) | Bashkir: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.ba.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.ba.300.vec.gz) | Basque: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.eu.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.eu.300.vec.gz) |\n| Bavarian: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.bar.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.bar.300.vec.gz) | Belarusian: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.be.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.be.300.vec.gz) | Bengali: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.bn.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.bn.300.vec.gz) |\n| Bihari: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.bh.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.bh.300.vec.gz) | Bishnupriya Manipuri: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.bpy.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.bpy.300.vec.gz) | Bosnian: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.bs.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.bs.300.vec.gz) |\n| Breton: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.br.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.br.300.vec.gz) | Bulgarian: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.bg.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.bg.300.vec.gz) | Burmese: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.my.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.my.300.vec.gz) |\n| Catalan: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.ca.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.ca.300.vec.gz) | Cebuano: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.ceb.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.ceb.300.vec.gz) | Central Bicolano: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.bcl.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.bcl.300.vec.gz) |\n| Chechen: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.ce.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.ce.300.vec.gz) | Chinese: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.zh.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.zh.300.vec.gz) | Chuvash: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.cv.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.cv.300.vec.gz) |\n| Corsican: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.co.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.co.300.vec.gz) | Croatian: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.hr.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.hr.300.vec.gz) | Czech: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.cs.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.cs.300.vec.gz) |\n| Danish: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.da.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.da.300.vec.gz) | Divehi: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.dv.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.dv.300.vec.gz) | Dutch: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.nl.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.nl.300.vec.gz) |\n| Eastern Punjabi: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.pa.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.pa.300.vec.gz) | Egyptian Arabic: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.arz.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.arz.300.vec.gz) | Emilian-Romagnol: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.eml.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.eml.300.vec.gz) |\n| English: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.en.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.en.300.vec.gz) | Erzya: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.myv.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.myv.300.vec.gz) | Esperanto: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.eo.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.eo.300.vec.gz) |\n| Estonian: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.et.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.et.300.vec.gz) | Fiji Hindi: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.hif.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.hif.300.vec.gz) | Finnish: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.fi.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.fi.300.vec.gz) |\n| French: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.fr.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.fr.300.vec.gz) | Galician: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.gl.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.gl.300.vec.gz) | Georgian: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.ka.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.ka.300.vec.gz) |\n| German: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.de.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.de.300.vec.gz) | Goan Konkani: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.gom.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.gom.300.vec.gz) | Greek: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.el.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.el.300.vec.gz) |\n| Gujarati: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.gu.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.gu.300.vec.gz) | Haitian: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.ht.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.ht.300.vec.gz) | Hebrew: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.he.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.he.300.vec.gz) |\n| Hill Mari: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.mrj.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.mrj.300.vec.gz) | Hindi: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.hi.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.hi.300.vec.gz) | Hungarian: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.hu.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.hu.300.vec.gz) |\n| Icelandic: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.is.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.is.300.vec.gz) | Ido: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.io.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.io.300.vec.gz) | Ilokano: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.ilo.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.ilo.300.vec.gz) |\n| Indonesian: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.id.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.id.300.vec.gz) | Interlingua: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.ia.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.ia.300.vec.gz) | Irish: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.ga.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.ga.300.vec.gz) |\n| Italian: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.it.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.it.300.vec.gz) | Japanese: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.ja.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.ja.300.vec.gz) | Javanese: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.jv.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.jv.300.vec.gz) |\n| Kannada: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.kn.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.kn.300.vec.gz) | Kapampangan: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.pam.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.pam.300.vec.gz) | Kazakh: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.kk.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.kk.300.vec.gz) |\n| Khmer: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.km.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.km.300.vec.gz) | Kirghiz: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.ky.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.ky.300.vec.gz) | Korean: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.ko.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.ko.300.vec.gz) |\n| Kurdish (Kurmanji): [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.ku.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.ku.300.vec.gz) | Kurdish (Sorani): [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.ckb.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.ckb.300.vec.gz) | Latin: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.la.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.la.300.vec.gz) |\n| Latvian: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.lv.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.lv.300.vec.gz) | Limburgish: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.li.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.li.300.vec.gz) | Lithuanian: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.lt.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.lt.300.vec.gz) |\n| Lombard: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.lmo.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.lmo.300.vec.gz) | Low Saxon: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.nds.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.nds.300.vec.gz) | Luxembourgish: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.lb.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.lb.300.vec.gz) |\n| Macedonian: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.mk.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.mk.300.vec.gz) | Maithili: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.mai.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.mai.300.vec.gz) | Malagasy: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.mg.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.mg.300.vec.gz) |\n| Malay: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.ms.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.ms.300.vec.gz) | Malayalam: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.ml.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.ml.300.vec.gz) | Maltese: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.mt.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.mt.300.vec.gz) |\n| Manx: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.gv.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.gv.300.vec.gz) | Marathi: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.mr.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.mr.300.vec.gz) | Mazandarani: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.mzn.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.mzn.300.vec.gz) |\n| Meadow Mari: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.mhr.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.mhr.300.vec.gz) | Minangkabau: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.min.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.min.300.vec.gz) | Mingrelian: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.xmf.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.xmf.300.vec.gz) |\n| Mirandese: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.mwl.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.mwl.300.vec.gz) | Mongolian: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.mn.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.mn.300.vec.gz) | Nahuatl: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.nah.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.nah.300.vec.gz) |\n| Neapolitan: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.nap.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.nap.300.vec.gz) | Nepali: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.ne.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.ne.300.vec.gz) | Newar: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.new.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.new.300.vec.gz) |\n| North Frisian: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.frr.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.frr.300.vec.gz) | Northern Sotho: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.nso.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.nso.300.vec.gz) | Norwegian (Bokmål): [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.no.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.no.300.vec.gz) |\n| Norwegian (Nynorsk): [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.nn.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.nn.300.vec.gz) | Occitan: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.oc.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.oc.300.vec.gz) | Oriya: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.or.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.or.300.vec.gz) |\n| Ossetian: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.os.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.os.300.vec.gz) | Palatinate German: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.pfl.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.pfl.300.vec.gz) | Pashto: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.ps.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.ps.300.vec.gz) |\n| Persian: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.fa.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.fa.300.vec.gz) | Piedmontese: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.pms.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.pms.300.vec.gz) | Polish: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.pl.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.pl.300.vec.gz) |\n| Portuguese: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.pt.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.pt.300.vec.gz) | Quechua: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.qu.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.qu.300.vec.gz) | Romanian: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.ro.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.ro.300.vec.gz) |\n| Romansh: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.rm.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.rm.300.vec.gz) | Russian: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.ru.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.ru.300.vec.gz) | Sakha: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.sah.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.sah.300.vec.gz) |\n| Sanskrit: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.sa.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.sa.300.vec.gz) | Sardinian: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.sc.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.sc.300.vec.gz) | Scots: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.sco.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.sco.300.vec.gz) |\n| Scottish Gaelic: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.gd.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.gd.300.vec.gz) | Serbian: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.sr.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.sr.300.vec.gz) | Serbo-Croatian: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.sh.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.sh.300.vec.gz) |\n| Sicilian: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.scn.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.scn.300.vec.gz) | Sindhi: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.sd.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.sd.300.vec.gz) | Sinhalese: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.si.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.si.300.vec.gz) |\n| Slovak: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.sk.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.sk.300.vec.gz) | Slovenian: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.sl.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.sl.300.vec.gz) | Somali: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.so.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.so.300.vec.gz) |\n| Southern Azerbaijani: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.azb.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.azb.300.vec.gz) | Spanish: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.es.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.es.300.vec.gz) | Sundanese: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.su.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.su.300.vec.gz) |\n| Swahili: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.sw.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.sw.300.vec.gz) | Swedish: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.sv.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.sv.300.vec.gz) | Tagalog: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.tl.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.tl.300.vec.gz) |\n| Tajik: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.tg.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.tg.300.vec.gz) | Tamil: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.ta.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.ta.300.vec.gz) | Tatar: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.tt.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.tt.300.vec.gz) |\n| Telugu: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.te.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.te.300.vec.gz) | Thai: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.th.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.th.300.vec.gz) | Tibetan: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.bo.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.bo.300.vec.gz) |\n| Turkish: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.tr.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.tr.300.vec.gz) | Turkmen: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.tk.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.tk.300.vec.gz) | Ukrainian: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.uk.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.uk.300.vec.gz) |\n| Upper Sorbian: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.hsb.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.hsb.300.vec.gz) | Urdu: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.ur.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.ur.300.vec.gz) | Uyghur: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.ug.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.ug.300.vec.gz) |\n| Uzbek: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.uz.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.uz.300.vec.gz) | Venetian: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.vec.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.vec.300.vec.gz) | Vietnamese: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.vi.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.vi.300.vec.gz) |\n| Volapük: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.vo.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.vo.300.vec.gz) | Walloon: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.wa.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.wa.300.vec.gz) | Waray: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.war.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.war.300.vec.gz) |\n| Welsh: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.cy.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.cy.300.vec.gz) | West Flemish: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.vls.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.vls.300.vec.gz) | West Frisian: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.fy.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.fy.300.vec.gz) |\n| Western Punjabi: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.pnb.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.pnb.300.vec.gz) | Yiddish: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.yi.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.yi.300.vec.gz) | Yoruba: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.yo.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.yo.300.vec.gz) |\n| Zazaki: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.diq.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.diq.300.vec.gz) | Zeelandic: [bin](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.zea.300.bin.gz), [text](https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.zea.300.vec.gz) |\n" }, { "path": "docs/dataset.md", "content": "---\nid: dataset\ntitle: Datasets\n---\n\n[Download YFCC100M Dataset](https://fb-public.box.com/s/htfdbrvycvroebv9ecaezaztocbcnsdn)\n" }, { "path": "docs/english-vectors.md", "content": "---\nid: english-vectors\ntitle: English word vectors\n---\n\nThis page gathers several pre-trained word vectors trained using fastText.\n\n### Download pre-trained word vectors\n\nPre-trained word vectors learned on different sources can be downloaded below:\n\n1. [wiki-news-300d-1M.vec.zip](https://dl.fbaipublicfiles.com/fasttext/vectors-english/wiki-news-300d-1M.vec.zip): 1 million word vectors trained on Wikipedia 2017, UMBC webbase corpus and statmt.org news dataset (16B tokens).\n2. [wiki-news-300d-1M-subword.vec.zip](https://dl.fbaipublicfiles.com/fasttext/vectors-english/wiki-news-300d-1M-subword.vec.zip): 1 million word vectors trained with subword infomation on Wikipedia 2017, UMBC webbase corpus and statmt.org news dataset (16B tokens).\n3. [crawl-300d-2M.vec.zip](https://dl.fbaipublicfiles.com/fasttext/vectors-english/crawl-300d-2M.vec.zip): 2 million word vectors trained on Common Crawl (600B tokens).\n4. [crawl-300d-2M-subword.zip](https://dl.fbaipublicfiles.com/fasttext/vectors-english/crawl-300d-2M-subword.zip): 2 million word vectors trained with subword information on Common Crawl (600B tokens).\n\n### Format\n\nThe first line of the file contains the number of words in the vocabulary and the size of the vectors.\nEach line contains a word followed by its vectors, like in the default fastText text format.\nEach value is space separated. Words are ordered by descending frequency.\nThese text models can easily be loaded in Python using the following code:\n```python\nimport io\n\ndef load_vectors(fname):\n fin = io.open(fname, 'r', encoding='utf-8', newline='\\n', errors='ignore')\n n, d = map(int, fin.readline().split())\n data = {}\n for line in fin:\n tokens = line.rstrip().split(' ')\n data[tokens[0]] = map(float, tokens[1:])\n return data\n```\n\n### License\n\nThese word vectors are distributed under the [*Creative Commons Attribution-Share-Alike License 3.0*](https://creativecommons.org/licenses/by-sa/3.0/).\n\n### References\n\nIf you use these word vectors, please cite the following paper:\n\nT. Mikolov, E. Grave, P. Bojanowski, C. Puhrsch, A. Joulin. [*Advances in Pre-Training Distributed Word Representations*](https://arxiv.org/abs/1712.09405)\n\n```markup\n@inproceedings{mikolov2018advances,\n title={Advances in Pre-Training Distributed Word Representations},\n author={Mikolov, Tomas and Grave, Edouard and Bojanowski, Piotr and Puhrsch, Christian and Joulin, Armand},\n booktitle={Proceedings of the International Conference on Language Resources and Evaluation (LREC 2018)},\n year={2018}\n}\n```\n" }, { "path": "docs/faqs.md", "content": "---\nid: faqs\ntitle:FAQ\n---\n\n## What is fastText? Are there tutorials?\n\nFastText is a library for text classification and representation. It transforms text into continuous vectors that can later be used on any language related task. A few tutorials are available.\n\n## How can I reduce the size of my fastText models?\n\nfastText uses a hashtable for either word or character ngrams. The size of the hashtable directly impacts the size of a model. To reduce the size of the model, it is possible to reduce the size of this table with the option '-hash'. For example a good value is 20000. Another option that greatly impacts the size of a model is the size of the vectors (-dim). This dimension can be reduced to save space but this can significantly impact performance. If that still produce a model that is too big, one can further reduce the size of a trained model with the quantization option.\n```bash\n./fasttext quantize -output model\n```\n\n## What would be the best way to represent word phrases rather than words?\n\nCurrently the best approach to represent word phrases or sentence is to take a bag of words of word vectors. Additionally, for phrases like “New York”, preprocessing the data so that it becomes a single token “New_York” can greatly help.\n\n## Why does fastText produce vectors even for unknown words?\n\nOne of the key features of fastText word representation is its ability to produce vectors for any words, even made-up ones.\nIndeed, fastText word vectors are built from vectors of substrings of characters contained in it.\nThis allows to build vectors even for misspelled words or concatenation of words.\n\n## Why is the hierarchical softmax slightly worse in performance than the full softmax?\n\nThe hierarchical softmax is an approximation of the full softmax loss that allows to train on large number of class efficiently. This is often at the cost of a few percent of accuracy.\nNote also that this loss is thought for classes that are unbalanced, that is some classes are more frequent than others. If your dataset has a balanced number of examples per class, it is worth trying the negative sampling loss (-loss ns -neg 100).\nHowever, negative sampling will still be very slow at test time, since the full softmax will be computed.\n\n## Can we run fastText program on a GPU?\n\nAs of now, fastText only works on CPU.\nPlease note that one of the goal of fastText is to be an efficient CPU tool, allowing to train models without requiring a GPU.\n\n## Can I use fastText with python? Or other languages?\n\n[Python is officially supported](/docs/en/support.html#building-fasttext-python-module).\nThere are few unofficial wrappers for javascript, lua and other languages available on github.\n\n## Can I use fastText with continuous data?\n\nFastText works on discrete tokens and thus cannot be directly used on continuous tokens. However, one can discretize continuous tokens to use fastText on them, for example by rounding values to a specific digit (\"12.3\" becomes \"12\").\n\n## There are misspellings in the dictionary. Should we improve text normalization?\n\nIf the words are infrequent, there is no need to worry.\n\n## I'm encountering a NaN, why could this be?\n\nYou'll likely see this behavior because your learning rate is too high. Try reducing it until you don't see this error anymore.\n\n## My compiler / architecture can't build fastText. What should I do?\nTry a newer version of your compiler. We try to maintain compatibility with older versions of gcc and many platforms, however sometimes maintaining backwards compatibility becomes very hard. In general, compilers and tool chains that ship with LTS versions of major linux distributions should be fair game. In any case, create an issue with your compiler version and architecture and we'll try to implement compatibility.\n\n## How do I run fastText in a fully reproducible way? Each time I run it I get different results.\nIf you run fastText multiple times you'll obtain slightly different results each time due to the optimization algorithm (asynchronous stochastic gradient descent, or Hogwild). If you need to get the same results (e.g. to confront different input params set) you have to set the 'thread' parameter to 1. In this way you'll get exactly the same performances at each run (with the same input params).\n\n\n## Why do I get a probability of 1.00001?\nThis is a known rounding issue. You can consider it as 1.0.\n\n## How can I change the dimension of word vectors of a model file?\nIf you already trained a model, or downloaded a pre-trained word vectors model, you can adapt the dimension of the word vectors with the `reduce_model.py` script or by calling `fasttext.util.reduce_model` from python, as [described here](/docs/en/crawl-vectors.html#adapt-the-dimension)\n" }, { "path": "docs/language-identification.md", "content": "---\nid: language-identification\ntitle: Language identification\n---\n\n### Description\n\nWe distribute two models for language identification, which can recognize 176 languages (see the list of ISO codes below). These models were trained on data from [Wikipedia](https://www.wikipedia.org/), [Tatoeba](https://tatoeba.org/eng/) and [SETimes](http://nlp.ffzg.hr/resources/corpora/setimes/), used under [CC-BY-SA](http://creativecommons.org/licenses/by-sa/3.0/).\n\nWe distribute two versions of the models:\n\n* [lid.176.bin](https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.bin), which is faster and slightly more accurate, but has a file size of 126MB ;\n* [lid.176.ftz](https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.ftz), which is the compressed version of the model, with a file size of 917kB.\n\nThese models were trained on UTF-8 data, and therefore expect UTF-8 as input.\n\n#### Updated model (NLLB project)\nA newer LID (**L**anguage **ID**entification) model was [released as part of the NLLB project](https://github.com/facebookresearch/fairseq/tree/nllb#lid-model) under [CC-BY-NC 4.0](LICENSE.model.md) license. \n\n* [lid218e.bin](https://tinyurl.com/nllblid218e) uses different language codes from the original models—the ISO 639-3 code (e.g. \"eng\", \"fra\", \"rus\") plus an additional code describing the script (e.g., \"eng_Latn\", \"ukr_Cyrl\")—and has a file size of 1.2GB.\n\nYou can read more about the data the model was trained on [here](https://github.com/facebookresearch/fairseq/blob/nllb/README.md#datasets).\n\n#### 🤗 HuggingFace Integration\nThis model is [available](https://huggingface.co/facebook/fasttext-language-identification) on the Hugging Face Hub. \n\n```python\n>>> import fasttext\n>>> from huggingface_hub import hf_hub_download\n\n>>> model_path = hf_hub_download(repo_id=\"facebook/fasttext-language-identification\", filename=\"model.bin\")\n>>> model = fasttext.load_model(model_path)\n>>> model.predict(\"Hello, world!\")\n\n(('__label__eng_Latn',), array([0.81148803]))\n\n>>> model.predict(\"Hello, world!\", k=5)\n\n(('__label__eng_Latn', '__label__vie_Latn', '__label__nld_Latn', '__label__pol_Latn', '__label__deu_Latn'), \n array([0.61224753, 0.21323682, 0.09696738, 0.01359863, 0.01319415]))\n```\n\n\n### License\n\nThe models are distributed under the [*Creative Commons Attribution-Share-Alike License 3.0*](https://creativecommons.org/licenses/by-sa/3.0/).\n\n### List of supported languages\n```\naf als am an ar arz as ast av az azb ba bar bcl be bg bh bn bo bpy br bs bxr ca cbk ce ceb ckb co cs cv cy da de diq dsb dty dv el eml en eo es et eu fa fi fr frr fy ga gd gl gn gom gu gv he hi hif hr hsb ht hu hy ia id ie ilo io is it ja jbo jv ka kk km kn ko krc ku kv kw ky la lb lez li lmo lo lrc lt lv mai mg mhr min mk ml mn mr mrj ms mt mwl my myv mzn nah nap nds ne new nl nn no oc or os pa pam pfl pl pms pnb ps pt qu rm ro ru rue sa sah sc scn sco sd sh si sk sl so sq sr su sv sw ta te tg th tk tl tr tt tyv ug uk ur uz vec vep vi vls vo wa war wuu xal xmf yi yo yue zh\n```\n\n### References\n\nIf you use these models, please cite the following papers:\n\n[1] A. Joulin, E. Grave, P. Bojanowski, T. Mikolov, [*Bag of Tricks for Efficient Text Classification*](https://arxiv.org/abs/1607.01759)\n```\n@article{joulin2016bag,\n title={Bag of Tricks for Efficient Text Classification},\n author={Joulin, Armand and Grave, Edouard and Bojanowski, Piotr and Mikolov, Tomas},\n journal={arXiv preprint arXiv:1607.01759},\n year={2016}\n}\n```\n[2] A. Joulin, E. Grave, P. Bojanowski, M. Douze, H. Jégou, T. Mikolov, [*FastText.zip: Compressing text classification models* ](https://arxiv.org/abs/1612.03651)\n```\n@article{joulin2016fasttext,\n title={FastText.zip: Compressing text classification models},\n author={Joulin, Armand and Grave, Edouard and Bojanowski, Piotr and Douze, Matthijs and J{\\'e}gou, H{\\'e}rve and Mikolov, Tomas},\n journal={arXiv preprint arXiv:1612.03651},\n year={2016}\n}\n```\n" }, { "path": "docs/options.md", "content": "---\nid: options\ntitle: List of options\n---\n\nInvoke a command without arguments to list available arguments and their default values:\n\n```bash\n$ ./fasttext supervised\nEmpty input or output path.\n\nThe following arguments are mandatory:\n -input training file path\n -output output file path\n\n The following arguments are optional:\n -verbose verbosity level [2]\n\n The following arguments for the dictionary are optional:\n -minCount minimal number of word occurrences [1]\n -minCountLabel minimal number of label occurrences [0]\n -wordNgrams max length of word ngram [1]\n -bucket number of buckets [2000000]\n -minn min length of char ngram [0]\n -maxn max length of char ngram [0]\n -t sampling threshold [0.0001]\n -label labels prefix [__label__]\n\n The following arguments for training are optional:\n -lr learning rate [0.1]\n -lrUpdateRate change the rate of updates for the learning rate [100]\n -dim size of word vectors [100]\n -ws size of the context window [5]\n -epoch number of epochs [5]\n -neg number of negatives sampled [5]\n -loss loss function {ns, hs, softmax} [softmax]\n -thread number of threads [12]\n -pretrainedVectors pretrained word vectors for supervised learning []\n -saveOutput whether output params should be saved [0]\n\n The following arguments for quantization are optional:\n -cutoff number of words and ngrams to retain [0]\n -retrain finetune embeddings if a cutoff is applied [0]\n -qnorm quantizing the norm separately [0]\n -qout quantizing the classifier [0]\n -dsub size of each sub-vector [2]\n```\n\nDefaults may vary by mode. (Word-representation modes `skipgram` and `cbow` use a default `-minCount` of 5.)\n\n\nHyperparameter optimization (autotune) is activated when you provide a validation file with `-autotune-validation` argument.\n```text\nThe following arguments are for autotune:\n -autotune-validation validation file to be used for evaluation\n -autotune-metric metric objective {f1, f1:labelname} [f1]\n -autotune-predictions number of predictions used for evaluation [1]\n -autotune-duration maximum duration in seconds [300]\n -autotune-modelsize constraint model file size [] (empty = do not quantize)\n```\n" }, { "path": "docs/pretrained-vectors.md", "content": "---\nid: pretrained-vectors\ntitle: Wiki word vectors\n---\n\nWe are publishing pre-trained word vectors for 294 languages, trained on [*Wikipedia*](https://www.wikipedia.org) using fastText.\nThese vectors in dimension 300 were obtained using the skip-gram model described in [*Bojanowski et al. (2016)*](https://arxiv.org/abs/1607.04606) with default parameters.\n\nPlease note that a newer version of multi-lingual word vectors are available at: [Word vectors for 157 languages](https://fasttext.cc/docs/en/crawl-vectors.html).\n\n### Models\n\nThe models can be downloaded from:\n\n||||\n|-|-|-|\n| Abkhazian: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ab.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ab.vec) | Acehnese: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ace.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ace.vec) | Adyghe: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ady.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ady.vec) |\n| Afar: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.aa.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.aa.vec) | Afrikaans: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.af.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.af.vec) | Akan: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ak.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ak.vec) |\n| Albanian: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.sq.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.sq.vec) | Alemannic: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.als.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.als.vec) | Amharic: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.am.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.am.vec) |\n| Anglo_Saxon: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ang.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ang.vec) | Arabic: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ar.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ar.vec) | Aragonese: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.an.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.an.vec) |\n| Aramaic: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.arc.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.arc.vec) | Armenian: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.hy.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.hy.vec) | Aromanian: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.roa_rup.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.roa_rup.vec) |\n| Assamese: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.as.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.as.vec) | Asturian: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ast.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ast.vec) | Avar: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.av.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.av.vec) |\n| Aymara: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ay.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ay.vec) | Azerbaijani: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.az.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.az.vec) | Bambara: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.bm.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.bm.vec) |\n| Banjar: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.bjn.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.bjn.vec) | Banyumasan: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.map_bms.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.map_bms.vec) | Bashkir: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ba.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ba.vec) |\n| Basque: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.eu.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.eu.vec) | Bavarian: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.bar.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.bar.vec) | Belarusian: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.be.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.be.vec) |\n| Bengali: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.bn.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.bn.vec) | Bihari: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.bh.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.bh.vec) | Bishnupriya Manipuri: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.bpy.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.bpy.vec) |\n| Bislama: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.bi.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.bi.vec) | Bosnian: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.bs.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.bs.vec) | Breton: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.br.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.br.vec) |\n| Buginese: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.bug.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.bug.vec) | Bulgarian: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.bg.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.bg.vec) | Burmese: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.my.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.my.vec) |\n| Buryat: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.bxr.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.bxr.vec) | Cantonese: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.zh_yue.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.zh_yue.vec) | Catalan: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ca.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ca.vec) |\n| Cebuano: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ceb.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ceb.vec) | Central Bicolano: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.bcl.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.bcl.vec) | Chamorro: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ch.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ch.vec) |\n| Chavacano: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.cbk_zam.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.cbk_zam.vec) | Chechen: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ce.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ce.vec) | Cherokee: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.chr.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.chr.vec) |\n| Cheyenne: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.chy.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.chy.vec) | Chichewa: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ny.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ny.vec) | Chinese: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.zh.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.zh.vec) |\n| Choctaw: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.cho.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.cho.vec) | Chuvash: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.cv.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.cv.vec) | Classical Chinese: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.zh_classical.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.zh_classical.vec) |\n| Cornish: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.kw.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.kw.vec) | Corsican: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.co.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.co.vec) | Cree: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.cr.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.cr.vec) |\n| Crimean Tatar: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.crh.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.crh.vec) | Croatian: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.hr.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.hr.vec) | Czech: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.cs.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.cs.vec) |\n| Danish: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.da.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.da.vec) | Divehi: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.dv.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.dv.vec) | Dutch: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.nl.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.nl.vec) |\n| Dutch Low Saxon: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.nds_nl.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.nds_nl.vec) | Dzongkha: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.dz.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.dz.vec) | Eastern Punjabi: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.pa.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.pa.vec) |\n| Egyptian Arabic: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.arz.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.arz.vec) | Emilian_Romagnol: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.eml.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.eml.vec) | English: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.en.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.en.vec) |\n| Erzya: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.myv.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.myv.vec) | Esperanto: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.eo.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.eo.vec) | Estonian: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.et.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.et.vec) |\n| Ewe: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ee.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ee.vec) | Extremaduran: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ext.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ext.vec) | Faroese: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.fo.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.fo.vec) |\n| Fiji Hindi: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.hif.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.hif.vec) | Fijian: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.fj.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.fj.vec) | Finnish: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.fi.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.fi.vec) |\n| Franco_Provençal: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.frp.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.frp.vec) | French: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.fr.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.fr.vec) | Friulian: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.fur.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.fur.vec) |\n| Fula: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ff.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ff.vec) | Gagauz: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.gag.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.gag.vec) | Galician: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.gl.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.gl.vec) |\n| Gan: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.gan.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.gan.vec) | Georgian: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ka.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ka.vec) | German: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.de.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.de.vec) |\n| Gilaki: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.glk.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.glk.vec) | Goan Konkani: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.gom.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.gom.vec) | Gothic: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.got.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.got.vec) |\n| Greek: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.el.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.el.vec) | Greenlandic: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.kl.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.kl.vec) | Guarani: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.gn.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.gn.vec) |\n| Gujarati: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.gu.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.gu.vec) | Haitian: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ht.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ht.vec) | Hakka: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.hak.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.hak.vec) |\n| Hausa: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ha.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ha.vec) | Hawaiian: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.haw.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.haw.vec) | Hebrew: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.he.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.he.vec) |\n| Herero: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.hz.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.hz.vec) | Hill Mari: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.mrj.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.mrj.vec) | Hindi: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.hi.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.hi.vec) |\n| Hiri Motu: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ho.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ho.vec) | Hungarian: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.hu.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.hu.vec) | Icelandic: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.is.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.is.vec) |\n| Ido: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.io.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.io.vec) | Igbo: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ig.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ig.vec) | Ilokano: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ilo.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ilo.vec) |\n| Indonesian: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.id.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.id.vec) | Interlingua: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ia.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ia.vec) | Interlingue: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ie.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ie.vec) |\n| Inuktitut: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.iu.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.iu.vec) | Inupiak: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ik.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ik.vec) | Irish: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ga.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ga.vec) |\n| Italian: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.it.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.it.vec) | Jamaican Patois: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.jam.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.jam.vec) | Japanese: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ja.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ja.vec) |\n| Javanese: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.jv.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.jv.vec) | Kabardian: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.kbd.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.kbd.vec) | Kabyle: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.kab.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.kab.vec) |\n| Kalmyk: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.xal.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.xal.vec) | Kannada: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.kn.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.kn.vec) | Kanuri: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.kr.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.kr.vec) |\n| Kapampangan: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.pam.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.pam.vec) | Karachay_Balkar: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.krc.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.krc.vec) | Karakalpak: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.kaa.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.kaa.vec) |\n| Kashmiri: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ks.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ks.vec) | Kashubian: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.csb.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.csb.vec) | Kazakh: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.kk.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.kk.vec) |\n| Khmer: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.km.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.km.vec) | Kikuyu: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ki.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ki.vec) | Kinyarwanda: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.rw.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.rw.vec) |\n| Kirghiz: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ky.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ky.vec) | Kirundi: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.rn.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.rn.vec) | Komi: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.kv.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.kv.vec) |\n| Komi_Permyak: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.koi.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.koi.vec) | Kongo: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.kg.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.kg.vec) | Korean: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ko.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ko.vec) |\n| Kuanyama: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.kj.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.kj.vec) | Kurdish (Kurmanji): [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ku.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ku.vec) | Kurdish (Sorani): [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ckb.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ckb.vec) |\n| Ladino: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.lad.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.lad.vec) | Lak: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.lbe.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.lbe.vec) | Lao: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.lo.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.lo.vec) |\n| Latgalian: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ltg.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ltg.vec) | Latin: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.la.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.la.vec) | Latvian: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.lv.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.lv.vec) |\n| Lezgian: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.lez.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.lez.vec) | Ligurian: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.lij.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.lij.vec) | Limburgish: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.li.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.li.vec) |\n| Lingala: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ln.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ln.vec) | Lithuanian: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.lt.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.lt.vec) | Livvi_Karelian: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.olo.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.olo.vec) |\n| Lojban: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.jbo.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.jbo.vec) | Lombard: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.lmo.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.lmo.vec) | Low Saxon: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.nds.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.nds.vec) |\n| Lower Sorbian: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.dsb.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.dsb.vec) | Luganda: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.lg.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.lg.vec) | Luxembourgish: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.lb.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.lb.vec) |\n| Macedonian: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.mk.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.mk.vec) | Maithili: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.mai.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.mai.vec) | Malagasy: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.mg.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.mg.vec) |\n| Malay: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ms.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ms.vec) | Malayalam: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ml.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ml.vec) | Maltese: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.mt.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.mt.vec) |\n| Manx: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.gv.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.gv.vec) | Maori: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.mi.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.mi.vec) | Marathi: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.mr.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.mr.vec) |\n| Marshallese: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.mh.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.mh.vec) | Mazandarani: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.mzn.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.mzn.vec) | Meadow Mari: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.mhr.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.mhr.vec) |\n| Min Dong: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.cdo.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.cdo.vec) | Min Nan: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.zh_min_nan.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.zh_min_nan.vec) | Minangkabau: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.min.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.min.vec) |\n| Mingrelian: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.xmf.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.xmf.vec) | Mirandese: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.mwl.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.mwl.vec) | Moksha: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.mdf.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.mdf.vec) |\n| Moldovan: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.mo.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.mo.vec) | Mongolian: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.mn.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.mn.vec) | Muscogee: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.mus.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.mus.vec) |\n| Nahuatl: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.nah.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.nah.vec) | Nauruan: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.na.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.na.vec) | Navajo: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.nv.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.nv.vec) |\n| Ndonga: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ng.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ng.vec) | Neapolitan: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.nap.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.nap.vec) | Nepali: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ne.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ne.vec) |\n| Newar: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.new.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.new.vec) | Norfolk: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.pih.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.pih.vec) | Norman: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.nrm.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.nrm.vec) |\n| North Frisian: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.frr.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.frr.vec) | Northern Luri: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.lrc.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.lrc.vec) | Northern Sami: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.se.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.se.vec) |\n| Northern Sotho: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.nso.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.nso.vec) | Norwegian (Bokmål): [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.no.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.no.vec) | Norwegian (Nynorsk): [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.nn.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.nn.vec) |\n| Novial: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.nov.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.nov.vec) | Nuosu: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ii.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ii.vec) | Occitan: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.oc.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.oc.vec) |\n| Old Church Slavonic: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.cu.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.cu.vec) | Oriya: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.or.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.or.vec) | Oromo: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.om.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.om.vec) |\n| Ossetian: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.os.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.os.vec) | Palatinate German: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.pfl.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.pfl.vec) | Pali: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.pi.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.pi.vec) |\n| Pangasinan: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.pag.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.pag.vec) | Papiamentu: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.pap.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.pap.vec) | Pashto: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ps.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ps.vec) |\n| Pennsylvania German: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.pdc.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.pdc.vec) | Persian: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.fa.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.fa.vec) | Picard: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.pcd.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.pcd.vec) |\n| Piedmontese: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.pms.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.pms.vec) | Polish: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.pl.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.pl.vec) | Pontic: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.pnt.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.pnt.vec) |\n| Portuguese: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.pt.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.pt.vec) | Quechua: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.qu.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.qu.vec) | Ripuarian: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ksh.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ksh.vec) |\n| Romani: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.rmy.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.rmy.vec) | Romanian: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ro.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ro.vec) | Romansh: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.rm.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.rm.vec) |\n| Russian: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ru.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ru.vec) | Rusyn: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.rue.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.rue.vec) | Sakha: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.sah.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.sah.vec) |\n| Samoan: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.sm.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.sm.vec) | Samogitian: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.bat_smg.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.bat_smg.vec) | Sango: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.sg.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.sg.vec) |\n| Sanskrit: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.sa.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.sa.vec) | Sardinian: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.sc.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.sc.vec) | Saterland Frisian: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.stq.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.stq.vec) |\n| Scots: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.sco.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.sco.vec) | Scottish Gaelic: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.gd.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.gd.vec) | Serbian: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.sr.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.sr.vec) |\n| Serbo_Croatian: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.sh.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.sh.vec) | Sesotho: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.st.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.st.vec) | Shona: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.sn.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.sn.vec) |\n| Sicilian: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.scn.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.scn.vec) | Silesian: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.szl.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.szl.vec) | Simple English: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.simple.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.simple.vec) |\n| Sindhi: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.sd.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.sd.vec) | Sinhalese: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.si.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.si.vec) | Slovak: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.sk.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.sk.vec) |\n| Slovenian: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.sl.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.sl.vec) | Somali: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.so.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.so.vec) | Southern Azerbaijani: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.azb.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.azb.vec) |\n| Spanish: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.es.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.es.vec) | Sranan: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.srn.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.srn.vec) | Sundanese: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.su.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.su.vec) |\n| Swahili: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.sw.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.sw.vec) | Swati: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ss.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ss.vec) | Swedish: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.sv.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.sv.vec) |\n| Tagalog: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.tl.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.tl.vec) | Tahitian: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ty.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ty.vec) | Tajik: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.tg.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.tg.vec) |\n| Tamil: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ta.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ta.vec) | Tarantino: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.roa_tara.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.roa_tara.vec) | Tatar: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.tt.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.tt.vec) |\n| Telugu: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.te.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.te.vec) | Tetum: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.tet.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.tet.vec) | Thai: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.th.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.th.vec) |\n| Tibetan: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.bo.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.bo.vec) | Tigrinya: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ti.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ti.vec) | Tok Pisin: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.tpi.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.tpi.vec) |\n| Tongan: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.to.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.to.vec) | Tsonga: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ts.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ts.vec) | Tswana: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.tn.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.tn.vec) |\n| Tulu: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.tcy.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.tcy.vec) | Tumbuka: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.tum.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.tum.vec) | Turkish: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.tr.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.tr.vec) |\n| Turkmen: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.tk.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.tk.vec) | Tuvan: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.tyv.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.tyv.vec) | Twi: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.tw.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.tw.vec) |\n| Udmurt: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.udm.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.udm.vec) | Ukrainian: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.uk.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.uk.vec) | Upper Sorbian: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.hsb.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.hsb.vec) |\n| Urdu: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ur.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ur.vec) | Uyghur: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ug.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ug.vec) | Uzbek: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.uz.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.uz.vec) |\n| Venda: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ve.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.ve.vec) | Venetian: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.vec.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.vec.vec) | Vepsian: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.vep.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.vep.vec) |\n| Vietnamese: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.vi.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.vi.vec) | Volapük: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.vo.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.vo.vec) | Võro: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.fiu_vro.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.fiu_vro.vec) |\n| Walloon: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.wa.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.wa.vec) | Waray: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.war.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.war.vec) | Welsh: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.cy.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.cy.vec) |\n| West Flemish: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.vls.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.vls.vec) | West Frisian: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.fy.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.fy.vec) | Western Punjabi: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.pnb.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.pnb.vec) |\n| Wolof: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.wo.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.wo.vec) | Wu: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.wuu.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.wuu.vec) | Xhosa: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.xh.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.xh.vec) |\n| Yiddish: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.yi.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.yi.vec) | Yoruba: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.yo.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.yo.vec) | Zazaki: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.diq.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.diq.vec) |\n| Zeelandic: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.zea.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.zea.vec) | Zhuang: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.za.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.za.vec) | Zulu: [*bin+text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.zu.zip), [*text*](https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.zu.vec) |\n\n### Format\n\nThe word vectors come in both the binary and text default formats of fastText.\nIn the text format, each line contains a word followed by its vector. Each value is space separated.\nWords are ordered by their frequency in a descending order.\n\n### License\n\nThe word vectors are distributed under the [*Creative Commons Attribution-Share-Alike License 3.0*](https://creativecommons.org/licenses/by-sa/3.0/).\n\n### References\n\nIf you use these word vectors, please cite the following paper:\n\nP. Bojanowski\\*, E. Grave\\*, A. Joulin, T. Mikolov, [*Enriching Word Vectors with Subword Information*](https://arxiv.org/abs/1607.04606)\n\n```markup\n@article{bojanowski2017enriching,\n title={Enriching Word Vectors with Subword Information},\n author={Bojanowski, Piotr and Grave, Edouard and Joulin, Armand and Mikolov, Tomas},\n journal={Transactions of the Association for Computational Linguistics},\n volume={5},\n year={2017},\n issn={2307-387X},\n pages={135--146}\n}\n```\n" }, { "path": "docs/python-module.md", "content": "---\nid: python-module\ntitle: Python module\n---\n\nIn this document we present how to use fastText in python.\n\n## Table of contents\n\n* [Requirements](#requirements)\n* [Installation](#installation)\n* [Usage overview](#usage-overview)\n * [Word representation model](#word-representation-model)\n * [Text classification model](#text-classification-model)\n * [IMPORTANT: Preprocessing data / encoding conventions](#important-preprocessing-data-encoding-conventions)\n * [More examples](#more-examples)\n* [API](#api)\n * [`train_unsupervised` parameters](#train_unsupervised-parameters)\n * [`train_supervised` parameters](#train_supervised-parameters)\n * [`model` object](#model-object)\n\n\n# Requirements\n\n[fastText](https://fasttext.cc/) builds on modern Mac OS and Linux distributions.\nSince it uses C\\++11 features, it requires a compiler with good C++11 support. You will need [Python](https://www.python.org/) (version 2.7 or ≥ 3.4), [NumPy](http://www.numpy.org/) & [SciPy](https://www.scipy.org/) and [pybind11](https://github.com/pybind/pybind11).\n\n\n# Installation\n\nTo install the latest release, you can do :\n```bash\n$ pip install fasttext\n```\n\nor, to get the latest development version of fasttext, you can install from our github repository :\n```bash\n$ git clone https://github.com/facebookresearch/fastText.git\n$ cd fastText\n$ sudo pip install .\n$ # or :\n$ sudo python setup.py install\n```\n\n# Usage overview\n\n\n## Word representation model\n\nIn order to learn word vectors, as [described here](/docs/en/references.html#enriching-word-vectors-with-subword-information), we can use `fasttext.train_unsupervised` function like this:\n\n\n```py\nimport fasttext\n\n# Skipgram model :\nmodel = fasttext.train_unsupervised('data.txt', model='skipgram')\n\n# or, cbow model :\nmodel = fasttext.train_unsupervised('data.txt', model='cbow')\n\n```\n\nwhere `data.txt` is a training file containing utf-8 encoded text.\n\n\nThe returned `model` object represents your learned model, and you can use it to retrieve information.\n\n```py\nprint(model.words) # list of words in dictionary\nprint(model['king']) # get the vector of the word 'king'\n```\n\n\n### Saving and loading a model object\n\nYou can save your trained model object by calling the function `save_model`.\n```py\nmodel.save_model(\"model_filename.bin\")\n```\n\nand retrieve it later thanks to the function `load_model` :\n```py\nmodel = fasttext.load_model(\"model_filename.bin\")\n```\n\nFor more information about word representation usage of fasttext, you can refer to our [word representations tutorial](/docs/en/unsupervised-tutorial.html).\n\n\n## Text classification model\n\nIn order to train a text classifier using the method [described here](/docs/en/references.html#bag-of-tricks-for-efficient-text-classification), we can use `fasttext.train_supervised` function like this:\n\n\n```py\nimport fasttext\n\nmodel = fasttext.train_supervised('data.train.txt')\n```\n\nwhere `data.train.txt` is a text file containing a training sentence per line along with the labels. By default, we assume that labels are words that are prefixed by the string `__label__`\n\nOnce the model is trained, we can retrieve the list of words and labels:\n\n```py\nprint(model.words)\nprint(model.labels)\n```\n\nTo evaluate our model by computing the precision at 1 (P@1) and the recall on a test set, we use the `test` function:\n\n```py\ndef print_results(N, p, r):\n print(\"N\\t\" + str(N))\n print(\"P@{}\\t{:.3f}\".format(1, p))\n print(\"R@{}\\t{:.3f}\".format(1, r))\n\nprint_results(*model.test('test.txt'))\n```\n\nWe can also predict labels for a specific text :\n\n```py\nmodel.predict(\"Which baking dish is best to bake a banana bread ?\")\n```\n\nBy default, `predict` returns only one label : the one with the highest probability. You can also predict more than one label by specifying the parameter `k`:\n```py\nmodel.predict(\"Which baking dish is best to bake a banana bread ?\", k=3)\n```\n\nIf you want to predict more than one sentence you can pass an array of strings :\n\n```py\nmodel.predict([\"Which baking dish is best to bake a banana bread ?\", \"Why not put knives in the dishwasher?\"], k=3)\n```\n\n\nOf course, you can also save and load a model to/from a file as [in the word representation usage](#saving-and-loading-a-model-object).\n\nFor more information about text classification usage of fasttext, you can refer to our [text classification tutorial](/docs/en/supervised-tutorial.html).\n\n\n\n\n### Compress model files with quantization\n\nWhen you want to save a supervised model file, fastText can compress it in order to have a much smaller model file by sacrificing only a little bit performance.\n\n```py\n# with the previously trained `model` object, call :\nmodel.quantize(input='data.train.txt', retrain=True)\n\n# then display results and save the new model :\nprint_results(*model.test(valid_data))\nmodel.save_model(\"model_filename.ftz\")\n```\n\n`model_filename.ftz` will have a much smaller size than `model_filename.bin`.\n\nFor further reading on quantization, you can refer to [this paragraph from our blog post](/blog/2017/10/02/blog-post.html#model-compression).\n\n\n## IMPORTANT: Preprocessing data / encoding conventions\n\nIn general it is important to properly preprocess your data. In particular our example scripts in the [root folder](https://github.com/facebookresearch/fastText) do this.\n\nfastText assumes UTF-8 encoded text. All text must be [unicode for Python2](https://docs.python.org/2/library/functions.html#unicode) and [str for Python3](https://docs.python.org/3.5/library/stdtypes.html#textseq). The passed text will be [encoded as UTF-8 by pybind11](https://pybind11.readthedocs.io/en/master/advanced/cast/strings.html?highlight=utf-8#strings-bytes-and-unicode-conversions) before passed to the fastText C++ library. This means it is important to use UTF-8 encoded text when building a model. On Unix-like systems you can convert text using [iconv](https://en.wikipedia.org/wiki/Iconv).\n\nfastText will tokenize (split text into pieces) based on the following ASCII characters (bytes). In particular, it is not aware of UTF-8 whitespace. We advice the user to convert UTF-8 whitespace / word boundaries into one of the following symbols as appropiate.\n\n* space\n* tab\n* vertical tab\n* carriage return\n* formfeed\n* the null character\n\nThe newline character is used to delimit lines of text. In particular, the EOS token is appended to a line of text if a newline character is encountered. The only exception is if the number of tokens exceeds the MAX\\_LINE\\_SIZE constant as defined in the [Dictionary header](https://github.com/facebookresearch/fastText/blob/master/src/dictionary.h). This means if you have text that is not separate by newlines, such as the [fil9 dataset](http://mattmahoney.net/dc/textdata), it will be broken into chunks with MAX\\_LINE\\_SIZE of tokens and the EOS token is not appended.\n\nThe length of a token is the number of UTF-8 characters by considering the [leading two bits of a byte](https://en.wikipedia.org/wiki/UTF-8#Description) to identify [subsequent bytes of a multi-byte sequence](https://github.com/facebookresearch/fastText/blob/master/src/dictionary.cc). Knowing this is especially important when choosing the minimum and maximum length of subwords. Further, the EOS token (as specified in the [Dictionary header](https://github.com/facebookresearch/fastText/blob/master/src/dictionary.h)) is considered a character and will not be broken into subwords.\n\n## More examples\n\nIn order to have a better knowledge of fastText models, please consider the main [README](https://github.com/facebookresearch/fastText/blob/master/README.md) and in particular [the tutorials on our website](https://fasttext.cc/docs/en/supervised-tutorial.html).\n\nYou can find further python examples in [the doc folder](https://github.com/facebookresearch/fastText/tree/master/python/doc/examples).\n\nAs with any package you can get help on any Python function using the help function.\n\nFor example\n\n```\n+>>> import fasttext\n+>>> help(fasttext.FastText)\n\nHelp on module fasttext.FastText in fasttext:\n\nNAME\n fasttext.FastText\n\nDESCRIPTION\n # Copyright (c) 2017-present, Facebook, Inc.\n # All rights reserved.\n #\n # This source code is licensed under the MIT license found in the\n # LICENSE file in the root directory of this source tree.\n\nFUNCTIONS\n load_model(path)\n Load a model given a filepath and return a model object.\n\n tokenize(text)\n Given a string of text, tokenize it and return a list of tokens\n[...]\n```\n\n\n# API\n\n\n## `train_unsupervised` parameters\n\n```python\n input # training file path (required)\n model # unsupervised fasttext model {cbow, skipgram} [skipgram]\n lr # learning rate [0.05]\n dim # size of word vectors [100]\n ws # size of the context window [5]\n epoch # number of epochs [5]\n minCount # minimal number of word occurences [5]\n minn # min length of char ngram [3]\n maxn # max length of char ngram [6]\n neg # number of negatives sampled [5]\n wordNgrams # max length of word ngram [1]\n loss # loss function {ns, hs, softmax, ova} [ns]\n bucket # number of buckets [2000000]\n thread # number of threads [number of cpus]\n lrUpdateRate # change the rate of updates for the learning rate [100]\n t # sampling threshold [0.0001]\n verbose # verbose [2]\n```\n\n## `train_supervised` parameters\n\n```python\n input # training file path (required)\n lr # learning rate [0.1]\n dim # size of word vectors [100]\n ws # size of the context window [5]\n epoch # number of epochs [5]\n minCount # minimal number of word occurences [1]\n minCountLabel # minimal number of label occurences [1]\n minn # min length of char ngram [0]\n maxn # max length of char ngram [0]\n neg # number of negatives sampled [5]\n wordNgrams # max length of word ngram [1]\n loss # loss function {ns, hs, softmax, ova} [softmax]\n bucket # number of buckets [2000000]\n thread # number of threads [number of cpus]\n lrUpdateRate # change the rate of updates for the learning rate [100]\n t # sampling threshold [0.0001]\n label # label prefix ['__label__']\n verbose # verbose [2]\n pretrainedVectors # pretrained word vectors (.vec file) for supervised learning []\n```\n\n## `model` object\n\n`train_supervised`, `train_unsupervised` and `load_model` functions return an instance of `_FastText` class, that we generaly name `model` object.\n\nThis object exposes those training arguments as properties : `lr`, `dim`, `ws`, `epoch`, `minCount`, `minCountLabel`, `minn`, `maxn`, `neg`, `wordNgrams`, `loss`, `bucket`, `thread`, `lrUpdateRate`, `t`, `label`, `verbose`, `pretrainedVectors`. So `model.wordNgrams` will give you the max length of word ngram used for training this model.\n\nIn addition, the object exposes several functions :\n\n```python\n get_dimension # Get the dimension (size) of a lookup vector (hidden layer).\n # This is equivalent to `dim` property.\n get_input_vector # Given an index, get the corresponding vector of the Input Matrix.\n get_input_matrix # Get a copy of the full input matrix of a Model.\n get_labels # Get the entire list of labels of the dictionary\n # This is equivalent to `labels` property.\n get_line # Split a line of text into words and labels.\n get_output_matrix # Get a copy of the full output matrix of a Model.\n get_sentence_vector # Given a string, get a single vector represenation. This function\n # assumes to be given a single line of text. We split words on\n # whitespace (space, newline, tab, vertical tab) and the control\n # characters carriage return, formfeed and the null character.\n get_subword_id # Given a subword, return the index (within input matrix) it hashes to.\n get_subwords # Given a word, get the subwords and their indicies.\n get_word_id # Given a word, get the word id within the dictionary.\n get_word_vector # Get the vector representation of word.\n get_words # Get the entire list of words of the dictionary\n # This is equivalent to `words` property.\n is_quantized # whether the model has been quantized\n predict # Given a string, get a list of labels and a list of corresponding probabilities.\n quantize # Quantize the model reducing the size of the model and it's memory footprint.\n save_model # Save the model to the given path\n test # Evaluate supervised model using file given by path\n test_label # Return the precision and recall score for each label. \n```\n\nThe properties `words`, `labels` return the words and labels from the dictionary :\n```py\nmodel.words # equivalent to model.get_words()\nmodel.labels # equivalent to model.get_labels()\n```\n\nThe object overrides `__getitem__` and `__contains__` functions in order to return the representation of a word and to check if a word is in the vocabulary.\n\n```py\nmodel['king'] # equivalent to model.get_word_vector('king')\n'king' in model # equivalent to `'king' in model.get_words()`\n```\n" }, { "path": "docs/references.md", "content": "---\nid: references\ntitle: References\n---\n\nPlease cite [1](#enriching-word-vectors-with-subword-information) if using this code for learning word representations or [2](#bag-of-tricks-for-efficient-text-classification) if using for text classification.\n\n[1] P. Bojanowski\\*, E. Grave\\*, A. Joulin, T. Mikolov, [*Enriching Word Vectors with Subword Information*](https://arxiv.org/abs/1607.04606)\n\n```markup\n@article{bojanowski2016enriching,\n title={Enriching Word Vectors with Subword Information},\n author={Bojanowski, Piotr and Grave, Edouard and Joulin, Armand and Mikolov, Tomas},\n journal={arXiv preprint arXiv:1607.04606},\n year={2016}\n}\n```\n\n[2] A. Joulin, E. Grave, P. Bojanowski, T. Mikolov, [*Bag of Tricks for Efficient Text Classification*](https://arxiv.org/abs/1607.01759)\n\n```markup\n@article{joulin2016bag,\n title={Bag of Tricks for Efficient Text Classification},\n author={Joulin, Armand and Grave, Edouard and Bojanowski, Piotr and Mikolov, Tomas},\n journal={arXiv preprint arXiv:1607.01759},\n year={2016}\n}\n```\n\n[3] A. Joulin, E. Grave, P. Bojanowski, M. Douze, H. Jégou, T. Mikolov, [*FastText.zip: Compressing text classification models*](https://arxiv.org/abs/1612.03651)\n\n```markup\n@article{joulin2016fasttext,\n title={FastText.zip: Compressing text classification models},\n author={Joulin, Armand and Grave, Edouard and Bojanowski, Piotr and Douze, Matthijs and J{\\'e}gou, H{\\'e}rve and Mikolov, Tomas},\n journal={arXiv preprint arXiv:1612.03651},\n year={2016}\n}\n```\n\n(\\* These authors contributed equally.)\n" }, { "path": "docs/supervised-models.md", "content": "---\nid: supervised-models\ntitle: Supervised models\n---\n\nThis page gathers several pre-trained supervised models on several datasets.\n\n### Description\n\nThe regular models are trained using the procedure described in [1]. They can be reproduced using the classification-results.sh script within our github repository. The quantized models are build by using the respective supervised settings and adding the following flags to the quantize subcommand.\n\n```bash\n-qnorm -retrain -cutoff 100000\n```\n\n### Table of models\n\nEach entry describes the test accuracy and size of the model. You can click on a table cell to download the corresponding model.\n\n| dataset | ag news | amazon review full | amazon review polarity | dbpedia |\n|-----------|-----------------------|-----------------------|------------------------|------------------------|\n| regular | [0.924 / 387MB](https://dl.fbaipublicfiles.com/fasttext/supervised-models/ag_news.bin) | [0.603 / 462MB](https://dl.fbaipublicfiles.com/fasttext/supervised-models/amazon_review_full.bin) | [0.946 / 471MB](https://dl.fbaipublicfiles.com/fasttext/supervised-models/amazon_review_polarity.bin) | [0.986 / 427MB](https://dl.fbaipublicfiles.com/fasttext/supervised-models/dbpedia.bin) |\n| compressed | [0.92 / 1.6MB](https://dl.fbaipublicfiles.com/fasttext/supervised-models/ag_news.ftz) | [0.599 / 1.6MB]( https://dl.fbaipublicfiles.com/fasttext/supervised-models/amazon_review_full.ftz) | [0.93 / 1.6MB]( https://dl.fbaipublicfiles.com/fasttext/supervised-models/amazon_review_polarity.ftz) | [0.984 / 1.7MB]( https://dl.fbaipublicfiles.com/fasttext/supervised-models/dbpedia.ftz) |\n\n| dataset | sogou news | yahoo answers | yelp review polarity | yelp review full |\n|-----------|----------------------|------------------------|----------------------|------------------------|\n| regular | [0.969 / 402MB](https://dl.fbaipublicfiles.com/fasttext/supervised-models/sogou_news.bin) | [0.724 / 494MB](https://dl.fbaipublicfiles.com/fasttext/supervised-models/yahoo_answers.bin)| [0.957 / 409MB](https://dl.fbaipublicfiles.com/fasttext/supervised-models/yelp_review_polarity.bin)| [0.639 / 412MB](https://dl.fbaipublicfiles.com/fasttext/supervised-models/yelp_review_full.bin)|\n| compressed | [0.968 / 1.4MB](https://dl.fbaipublicfiles.com/fasttext/supervised-models/sogou_news.ftz) | [0.717 / 1.6MB](https://dl.fbaipublicfiles.com/fasttext/supervised-models/yahoo_answers.ftz) | [0.957 / 1.5MB](https://dl.fbaipublicfiles.com/fasttext/supervised-models/yelp_review_polarity.ftz) | [0.636 / 1.5MB](https://dl.fbaipublicfiles.com/fasttext/supervised-models/yelp_review_full.ftz) |\n\n### References\n\nIf you use these models, please cite the following paper:\n\n[1] A. Joulin, E. Grave, P. Bojanowski, T. Mikolov, [*Bag of Tricks for Efficient Text Classification*](https://arxiv.org/abs/1607.01759)\n\n```markup\n@article{joulin2016bag,\n title={Bag of Tricks for Efficient Text Classification},\n author={Joulin, Armand and Grave, Edouard and Bojanowski, Piotr and Mikolov, Tomas},\n journal={arXiv preprint arXiv:1607.01759},\n year={2016}\n}\n```\n\n[2] A. Joulin, E. Grave, P. Bojanowski, M. Douze, H. Jégou, T. Mikolov, [*FastText.zip: Compressing text classification models*](https://arxiv.org/abs/1612.03651)\n\n```markup\n@article{joulin2016fasttext,\n title={FastText.zip: Compressing text classification models},\n author={Joulin, Armand and Grave, Edouard and Bojanowski, Piotr and Douze, Matthijs and J{\\'e}gou, H{\\'e}rve and Mikolov, Tomas},\n journal={arXiv preprint arXiv:1612.03651},\n year={2016}\n}\n```\n" }, { "path": "docs/supervised-tutorial.md", "content": "---\nid: supervised-tutorial\ntitle: Text classification\n---\n\nText classification is a core problem to many applications, like spam detection, sentiment analysis or smart replies. In this tutorial, we describe how to build a text classifier with the fastText tool.\n\n## What is text classification?\n\nThe goal of text classification is to assign documents (such as emails, posts, text messages, product reviews, etc...) to one or multiple categories. Such categories can be review scores, spam v.s. non-spam, or the language in which the document was typed. Nowadays, the dominant approach to build such classifiers is machine learning, that is learning classification rules from examples. In order to build such classifiers, we need labeled data, which consists of documents and their corresponding categories (or tags, or labels).\n\nAs an example, we build a classifier which automatically classifies stackexchange questions about cooking into one of several possible tags, such as `pot`, `bowl` or `baking`.\n\n## Installing fastText\n\nThe first step of this tutorial is to install and build fastText. It only requires a c++ compiler with good support of c++11.\n\nLet us start by downloading the [most recent release](https://github.com/facebookresearch/fastText/releases):\n\n```bash\n$ wget https://github.com/facebookresearch/fastText/archive/v0.9.2.zip\n$ unzip v0.9.2.zip\n```\n\nMove to the fastText directory and build it:\n\n```bash\n$ cd fastText-0.9.2\n# for command line tool :\n$ make\n# for python bindings :\n$ pip install .\n```\n\n\n\n
\nRunning the binary without any argument will print the high level documentation, showing the different use cases supported by fastText:\n\n```bash\n>> ./fasttext\nusage: fasttext \n\nThe commands supported by fasttext are:\n\n supervised train a supervised classifier\n quantize quantize a model to reduce the memory usage\n test evaluate a supervised classifier\n predict predict most likely labels\n predict-prob predict most likely labels with probabilities\n skipgram train a skipgram model\n cbow train a cbow model\n print-word-vectors print word vectors given a trained model\n print-sentence-vectors print sentence vectors given a trained model\n nn query for nearest neighbors\n analogies query for analogies\n\n```\n\nIn this tutorial, we mainly use the `supervised`, `test` and `predict` subcommands, which corresponds to learning (and using) text classifier. For an introduction to the other functionalities of fastText, please see the [tutorial about learning word vectors](https://fasttext.cc/docs/en/unsupervised-tutorial.html).\n\n\n
\nCalling the help function will show high level documentation of the library:\n```py\n>>> import fasttext\n>>> help(fasttext.FastText)\nHelp on module fasttext.FastText in fasttext:\n\nNAME\n fasttext.FastText\n\nDESCRIPTION\n # Copyright (c) 2017-present, Facebook, Inc.\n # All rights reserved.\n #\n # This source code is licensed under the MIT license found in the\n # LICENSE file in the root directory of this source tree.\n\nFUNCTIONS\n load_model(path)\n Load a model given a filepath and return a model object.\n\n read_args(arg_list, arg_dict, arg_names, default_values)\n\n tokenize(text)\n Given a string of text, tokenize it and return a list of tokens\n\n train_supervised(*kargs, **kwargs)\n Train a supervised model and return a model object.\n\n input must be a filepath. The input text does not need to be tokenized\n as per the tokenize function, but it must be preprocessed and encoded\n as UTF-8. You might want to consult standard preprocessing scripts such\n as tokenizer.perl mentioned here: http://www.statmt.org/wmt07/baseline.html\n\n The input file must must contain at least one label per line. For an\n example consult the example datasets which are part of the fastText\n repository such as the dataset pulled by classification-example.sh.\n\n train_unsupervised(*kargs, **kwargs)\n Train an unsupervised model and return a model object.\n\n input must be a filepath. The input text does not need to be tokenized\n as per the tokenize function, but it must be preprocessed and encoded\n as UTF-8. You might want to consult standard preprocessing scripts such\n as tokenizer.perl mentioned here: http://www.statmt.org/wmt07/baseline.html\n\n The input field must not contain any labels or use the specified label prefix\n unless it is ok for those words to be ignored. For an example consult the\n dataset pulled by the example script word-vector-example.sh, which is\n part of the fastText repository.\n```\n\nIn this tutorial, we mainly use the `train_supervised`, which returns a model object, and call `test` and `predict` on this object. That corresponds to learning (and using) text classifier. For an introduction to the other functionalities of fastText, please see the [tutorial about learning word vectors](https://fasttext.cc/docs/en/unsupervised-tutorial.html).\n\n\n## Getting and preparing the data\n\nAs mentioned in the introduction, we need labeled data to train our supervised classifier. In this tutorial, we are interested in building a classifier to automatically recognize the topic of a stackexchange question about cooking. Let's download examples of questions from [the cooking section of Stackexchange](http://cooking.stackexchange.com/), and their associated tags:\n\n```bash\n>> wget https://dl.fbaipublicfiles.com/fasttext/data/cooking.stackexchange.tar.gz && tar xvzf cooking.stackexchange.tar.gz\n>> head cooking.stackexchange.txt\n```\n\nEach line of the text file contains a list of labels, followed by the corresponding document. All the labels start by the `__label__` prefix, which is how fastText recognize what is a label or what is a word. The model is then trained to predict the labels given the word in the document.\n\nBefore training our first classifier, we need to split the data into train and validation. We will use the validation set to evaluate how good the learned classifier is on new data.\n\n```bash\n>> wc cooking.stackexchange.txt\n 15404 169582 1401900 cooking.stackexchange.txt\n```\n\nOur full dataset contains 15404 examples. Let's split it into a training set of 12404 examples and a validation set of 3000 examples:\n\n```bash\n>> head -n 12404 cooking.stackexchange.txt > cooking.train\n>> tail -n 3000 cooking.stackexchange.txt > cooking.valid\n```\n\n## Our first classifier\n\nWe are now ready to train our first classifier:\n\n\n\n```bash\n>> ./fasttext supervised -input cooking.train -output model_cooking\nRead 0M words\nNumber of words: 14598\nNumber of labels: 734\nProgress: 100.0% words/sec/thread: 75109 lr: 0.000000 loss: 5.708354 eta: 0h0m\n```\n\nThe `-input` command line option indicates the file containing the training examples, while the `-output` option indicates where to save the model. At the end of training, a file `model_cooking.bin`, containing the trained classifier, is created in the current directory.\n\n\n```py\n>>> import fasttext\n>>> model = fasttext.train_supervised(input=\"cooking.train\")\nRead 0M words\nNumber of words: 14598\nNumber of labels: 734\nProgress: 100.0% words/sec/thread: 75109 lr: 0.000000 loss: 5.708354 eta: 0h0m\n```\nThe `input` argument indicates the file containing the training examples. We can now use the `model` variable to access information on the trained model.\n\nWe can also call `save_model` to save it as a file and load it later with `load_model` function.\n```py\n>>> model.save_model(\"model_cooking.bin\")\n```\n\n\n\nNow, we can test our classifier, by :\n\n\n```bash\n>> ./fasttext predict model_cooking.bin -\n```\n\nand then typing a sentence. Let's first try the sentence:\n\n*Which baking dish is best to bake a banana bread ?*\n\nThe predicted tag is `baking` which fits well to this question. Let us now try a second example:\n\n*Why not put knives in the dishwasher?*\n\n\n```py\n>>> model.predict(\"Which baking dish is best to bake a banana bread ?\")\n((u'__label__baking',), array([0.15613931]))\n```\nThe predicted tag is `baking` which fits well to this question. Let us now try a second example:\n\n```py\n>>> model.predict(\"Why not put knives in the dishwasher?\")\n((u'__label__food-safety',), array([0.08686075]))\n```\n\n\n\n\nThe label predicted by the model is `food-safety`, which is not relevant. Somehow, the model seems to fail on simple examples.\n\nTo get a better sense of its quality, let's test it on the validation data by running:\n\n\n\n```bash\n>> ./fasttext test model_cooking.bin cooking.valid\nN 3000\nP@1 0.124\nR@1 0.0541\nNumber of examples: 3000\n```\nThe output of fastText are the precision at one (`P@1`) and the recall at one (`R@1`).\n\n\n```py\n>>> model.test(\"cooking.valid\")\n(3000L, 0.124, 0.0541)\n```\nThe output are the number of samples (here `3000`), the precision at one (`0.124`) and the recall at one (`0.0541`).\n\n\nWe can also compute the precision at five and recall at five with:\n\n\n\n```bash\n>> ./fasttext test model_cooking.bin cooking.valid 5\nN 3000\nP@5 0.0668\nR@5 0.146\nNumber of examples: 3000\n```\n\n```py\n>>> model.test(\"cooking.valid\", k=5)\n(3000L, 0.0668, 0.146)\n```\n\n\n\n## Advanced readers: precision and recall\n\nThe precision is the number of correct labels among the labels predicted by fastText. The recall is the number of labels that successfully were predicted, among all the real labels. Let's take an example to make this more clear:\n\n*Why not put knives in the dishwasher?*\n\nOn Stack Exchange, this sentence is labeled with three tags: `equipment`, `cleaning` and `knives`. The top five labels predicted by the model can be obtained with:\n\n\n\n```bash\n>> ./fasttext predict model_cooking.bin - 5\n```\n\n```py\n>>> model.predict(\"Why not put knives in the dishwasher?\", k=5)\n((u'__label__food-safety', u'__label__baking', u'__label__equipment', u'__label__substitutions', u'__label__bread'), array([0.0857 , 0.0657, 0.0454, 0.0333, 0.0333]))\n```\n\n\nare `food-safety`, `baking`, `equipment`, `substitutions` and `bread`.\n\nThus, one out of five labels predicted by the model is correct, giving a precision of 0.20. Out of the three real labels, only one is predicted by the model, giving a recall of 0.33.\n\nFor more details, see [the related Wikipedia page](https://en.wikipedia.org/wiki/Precision_and_recall).\n\n## Making the model better\n\nThe model obtained by running fastText with the default arguments is pretty bad at classifying new questions. Let's try to improve the performance, by changing the default parameters.\n\n### preprocessing the data\n\nLooking at the data, we observe that some words contain uppercase letter or punctuation. One of the first step to improve the performance of our model is to apply some simple pre-processing. A crude normalization can be obtained using command line tools such as `sed` and `tr`:\n\n```bash\n>> cat cooking.stackexchange.txt | sed -e \"s/\\([.\\!?,'/()]\\)/ \\1 /g\" | tr \"[:upper:]\" \"[:lower:]\" > cooking.preprocessed.txt\n>> head -n 12404 cooking.preprocessed.txt > cooking.train\n>> tail -n 3000 cooking.preprocessed.txt > cooking.valid\n```\n\nLet's train a new model on the pre-processed data:\n\n\n\n```bash\n>> ./fasttext supervised -input cooking.train -output model_cooking\nRead 0M words\nNumber of words: 9012\nNumber of labels: 734\nProgress: 100.0% words/sec/thread: 82041 lr: 0.000000 loss: 5.671649 eta: 0h0m\n\n>> ./fasttext test model_cooking.bin cooking.valid\nN 3000\nP@1 0.164\nR@1 0.0717\nNumber of examples: 3000\n```\n\n```py\n>>> import fasttext\n>>> model = fasttext.train_supervised(input=\"cooking.train\")\nRead 0M words\nNumber of words: 9012\nNumber of labels: 734\nProgress: 100.0% words/sec/thread: 82041 lr: 0.000000 loss: 5.671649 eta: 0h0m\n\n>>> model.test(\"cooking.valid\")\n(3000L, 0.164, 0.0717)\n```\n\n\nWe observe that thanks to the pre-processing, the vocabulary is smaller (from 14k words to 9k). The precision is also starting to go up by 4%!\n\n### more epochs and larger learning rate\n\nBy default, fastText sees each training example only five times during training, which is pretty small, given that our training set only have 12k training examples. The number of times each examples is seen (also known as the number of epochs), can be increased using the `-epoch` option:\n\n\n\n```bash\n>> ./fasttext supervised -input cooking.train -output model_cooking -epoch 25\nRead 0M words\nNumber of words: 9012\nNumber of labels: 734\nProgress: 100.0% words/sec/thread: 77633 lr: 0.000000 loss: 7.147976 eta: 0h0m\n```\n\n```py\n>>> import fasttext\n>>> model = fasttext.train_supervised(input=\"cooking.train\", epoch=25)\nRead 0M words\nNumber of words: 9012\nNumber of labels: 734\nProgress: 100.0% words/sec/thread: 77633 lr: 0.000000 loss: 7.147976 eta: 0h0m\n```\n\n\nLet's test the new model:\n\n\n\n```bash\n>> ./fasttext test model_cooking.bin cooking.valid\nN 3000\nP@1 0.501\nR@1 0.218\nNumber of examples: 3000\n```\n\n```py\n>>> model.test(\"cooking.valid\")\n(3000L, 0.501, 0.218)\n```\n\n\nThis is much better! Another way to change the learning speed of our model is to increase (or decrease) the learning rate of the algorithm. This corresponds to how much the model changes after processing each example. A learning rate of 0 would mean that the model does not change at all, and thus, does not learn anything. Good values of the learning rate are in the range `0.1 - 1.0`.\n\n\n\n```bash\n>> ./fasttext supervised -input cooking.train -output model_cooking -lr 1.0\nRead 0M words\nNumber of words: 9012\nNumber of labels: 734\nProgress: 100.0% words/sec/thread: 81469 lr: 0.000000 loss: 6.405640 eta: 0h0m\n\n>> ./fasttext test model_cooking.bin cooking.valid\nN 3000\nP@1 0.563\nR@1 0.245\nNumber of examples: 3000\n```\n\n```py\n>>> model = fasttext.train_supervised(input=\"cooking.train\", lr=1.0)\nRead 0M words\nNumber of words: 9012\nNumber of labels: 734\nProgress: 100.0% words/sec/thread: 81469 lr: 0.000000 loss: 6.405640 eta: 0h0m\n\n>>> model.test(\"cooking.valid\")\n(3000L, 0.563, 0.245)\n```\n\n\nEven better! Let's try both together:\n\n\n\n```bash\n>> ./fasttext supervised -input cooking.train -output model_cooking -lr 1.0 -epoch 25\nRead 0M words\nNumber of words: 9012\nNumber of labels: 734\nProgress: 100.0% words/sec/thread: 76394 lr: 0.000000 loss: 4.350277 eta: 0h0m\n\n>> ./fasttext test model_cooking.bin cooking.valid\nN 3000\nP@1 0.585\nR@1 0.255\nNumber of examples: 3000\n```\n\n```py\n>>> model = fasttext.train_supervised(input=\"cooking.train\", lr=1.0, epoch=25)\nRead 0M words\nNumber of words: 9012\nNumber of labels: 734\nProgress: 100.0% words/sec/thread: 76394 lr: 0.000000 loss: 4.350277 eta: 0h0m\n\n>>> model.test(\"cooking.valid\")\n(3000L, 0.585, 0.255)\n```\n\n\nLet us now add a few more features to improve even further our performance!\n\n### word n-grams\n\nFinally, we can improve the performance of a model by using word bigrams, instead of just unigrams. This is especially important for classification problems where word order is important, such as sentiment analysis.\n\n\n\n```bash\n>> ./fasttext supervised -input cooking.train -output model_cooking -lr 1.0 -epoch 25 -wordNgrams 2\nRead 0M words\nNumber of words: 9012\nNumber of labels: 734\nProgress: 100.0% words/sec/thread: 75366 lr: 0.000000 loss: 3.226064 eta: 0h0m\n\n>> ./fasttext test model_cooking.bin cooking.valid\nN 3000\nP@1 0.599\nR@1 0.261\nNumber of examples: 3000\n```\n\n```py\n>>> model = fasttext.train_supervised(input=\"cooking.train\", lr=1.0, epoch=25, wordNgrams=2)\nRead 0M words\nNumber of words: 9012\nNumber of labels: 734\nProgress: 100.0% words/sec/thread: 75366 lr: 0.000000 loss: 3.226064 eta: 0h0m\n\n>>> model.test(\"cooking.valid\")\n(3000L, 0.599, 0.261)\n```\n\n\nWith a few steps, we were able to go from a precision at one of 12.4% to 59.9%. Important steps included:\n\n* preprocessing the data ;\n* changing the number of epochs (using the option `-epoch`, standard range `[5 - 50]`) ;\n* changing the learning rate (using the option `-lr`, standard range `[0.1 - 1.0]`) ;\n* using word n-grams (using the option `-wordNgrams`, standard range `[1 - 5]`).\n\n## Advanced readers: What is a Bigram?\n\nA 'unigram' refers to a single undividing unit, or token, usually used as an input to a model. For example a unigram can be a word or a letter depending on the model. In fastText, we work at the word level and thus unigrams are words.\n\nSimilarly we denote by 'bigram' the concatenation of 2 consecutive tokens or words. Similarly we often talk about n-gram to refer to the concatenation any n consecutive tokens.\n\nFor example, in the sentence, 'Last donut of the night', the unigrams are 'last', 'donut', 'of', 'the' and 'night'. The bigrams are: 'Last donut', 'donut of', 'of the' and 'the night'.\n\nBigrams are particularly interesting because, for most sentences, you can reconstruct the order of the words just by looking at a bag of n-grams.\n\nLet us illustrate this by a simple exercise, given the following bigrams, try to reconstruct the original sentence: 'all out', 'I am', 'of bubblegum', 'out of' and 'am all'.\nIt is common to refer to a word as a unigram.\n\n## Scaling things up\n\nSince we are training our model on a few thousands of examples, the training only takes a few seconds. But training models on larger datasets, with more labels can start to be too slow. A potential solution to make the training faster is to use the [hierarchical softmax](#advanced-readers-hierarchical-softmax), instead of the regular softmax. This can be done with the option `-loss hs`:\n\n\n\n```bash\n>> ./fasttext supervised -input cooking.train -output model_cooking -lr 1.0 -epoch 25 -wordNgrams 2 -bucket 200000 -dim 50 -loss hs\nRead 0M words\nNumber of words: 9012\nNumber of labels: 734\nProgress: 100.0% words/sec/thread: 2199406 lr: 0.000000 loss: 1.718807 eta: 0h0m\n```\n\n```py\n>>> model = fasttext.train_supervised(input=\"cooking.train\", lr=1.0, epoch=25, wordNgrams=2, bucket=200000, dim=50, loss='hs')\nRead 0M words\nNumber of words: 9012\nNumber of labels: 734\nProgress: 100.0% words/sec/thread: 2199406 lr: 0.000000 loss: 1.718807 eta: 0h0m\n```\n\n\nTraining should now take less than a second.\n\n\n## Advanced readers: hierarchical softmax\n\nThe hierarchical softmax is a loss function that approximates the softmax with a much faster computation.\n\nThe idea is to build a binary tree whose leaves correspond to the labels. Each intermediate node has a binary decision activation (e.g. sigmoid) that is trained, and predicts if we should go to the left or to the right. The probability of the output unit is then given by the product of the probabilities of intermediate nodes along the path from the root to the output unit leave.\n\nFor a detailed explanation, you can have a look on [this video](https://www.youtube.com/watch?v=B95LTf2rVWM).\n\nIn fastText, we use a Huffman tree, so that the lookup time is faster for more frequent outputs and thus the average lookup time for the output is optimal.\n\n## Multi-label classification\n\nWhen we want to assign a document to multiple labels, we can still use the softmax loss and play with the parameters for prediction, namely the number of labels to predict and the threshold for the predicted probability. However playing with these arguments can be tricky and unintuitive since the probabilities must sum to 1.\n\nA convenient way to handle multiple labels is to use independent binary classifiers for each label. This can be done with `-loss one-vs-all` or `-loss ova`.\n\n\n\n```bash\n>> ./fasttext supervised -input cooking.train -output model_cooking -lr 0.5 -epoch 25 -wordNgrams 2 -bucket 200000 -dim 50 -loss one-vs-all\nRead 0M words\nNumber of words: 14543\nNumber of labels: 735\nProgress: 100.0% words/sec/thread: 72104 lr: 0.000000 loss: 4.340807 ETA: 0h 0m\n```\n\n```py\n>>> import fasttext\n>>> model = fasttext.train_supervised(input=\"cooking.train\", lr=0.5, epoch=25, wordNgrams=2, bucket=200000, dim=50, loss='ova')\nRead 0M words\nNumber of words: 14543\nNumber of labels: 735\nProgress: 100.0% words/sec/thread: 72104 lr: 0.000000 loss: 4.340807 ETA: 0h 0m\n```\n\n\nIt is a good idea to decrease the learning rate compared to other loss functions.\n\nNow let's have a look on our predictions, we want as many prediction as possible (argument `-1`) and we want only labels with probability higher or equal to `0.5` :\n\n\n```bash\n>> ./fasttext predict-prob model_cooking.bin - -1 0.5\n```\nand then type the sentence:\n\n*Which baking dish is best to bake a banana bread ?*\n\nwe get:\n```\n__label__baking 1.00000 __label__bananas 0.939923 __label__bread 0.592677\n```\n\n```py\n>>> model.predict(\"Which baking dish is best to bake a banana bread ?\", k=-1, threshold=0.5)\n((u''__label__baking, u'__label__bananas', u'__label__bread'), array([1.00000, 0.939923, 0.592677]))\n```\n\n\n\n\n\n\n
\nWe can also evaluate our results with the `test` command :\n```bash\n>> ./fasttext test model_cooking.bin cooking.valid -1 0.5\nN 3000\nP@-1 0.702\nR@-1 0.2\nNumber of examples: 3000\n```\nand play with the threshold to obtain desired precision/recall metrics :\n\n```bash\n>> ./fasttext test model_cooking.bin cooking.valid -1 0.1\nN 3000\nP@-1 0.591\nR@-1 0.272\nNumber of examples: 3000\n```\n\n
\nWe can also evaluate our results with the `test` function:\n```py\n>>> model.test(\"cooking.valid\", k=-1)\n(3000L, 0.702, 0.2)\n```\n\n\n\n## Conclusion\n\nIn this tutorial, we gave a brief overview of how to use fastText to train powerful text classifiers. We had a light overview of some of the most important options to tune.\n" }, { "path": "docs/support.md", "content": "---\nid: support\ntitle: Get started\n---\n\n## What is fastText?\n\nfastText is a library for efficient learning of word representations and sentence classification.\n\n## Requirements\n\nfastText builds on modern Mac OS and Linux distributions.\nSince it uses C++11 features, it requires a compiler with good C++11 support.\nThese include :\n\n* (gcc-4.6.3 or newer) or (clang-3.3 or newer)\n\nCompilation is carried out using a Makefile, so you will need to have a working **make**.\nFor the word-similarity evaluation script you will need:\n\n* python 2.6 or newer\n* numpy & scipy\n\n## Building fastText as a command line tool\n\nIn order to build `fastText`, use the following:\n\n```bash\n$ git clone https://github.com/facebookresearch/fastText.git\n$ cd fastText\n$ make\n```\n\nThis will produce object files for all the classes as well as the main binary `fasttext`.\nIf you do not plan on using the default system-wide compiler, update the two macros defined at the beginning of the Makefile (CC and INCLUDES).\n\n\n## Building `fasttext` python module\n\nIn order to build `fasttext` module for python, use the following:\n\n```bash\n$ git clone https://github.com/facebookresearch/fastText.git\n$ cd fastText\n$ sudo pip install .\n$ # or :\n$ sudo python setup.py install\n```\n\nThen verify the installation went well :\n```bash\n$ python\nPython 2.7.15 |(default, May 1 2018, 18:37:05)\nType \"help\", \"copyright\", \"credits\" or \"license\" for more information.\n>>> import fasttext\n>>>\n```\nIf you don't see any error message, the installation was successful.\n" }, { "path": "docs/unsupervised-tutorials.md", "content": "---\nid: unsupervised-tutorial\ntitle: Word representations\n---\nA popular idea in modern machine learning is to represent words by vectors. These vectors capture hidden information about a language, like word analogies or semantic. It is also used to improve performance of text classifiers.\n\nIn this tutorial, we show how to build these word vectors with the fastText tool. To download and install fastText, follow the first steps of [the tutorial on text classification](https://fasttext.cc/docs/en/supervised-tutorial.html).\n\n## Getting the data\n\nIn order to compute word vectors, you need a large text corpus. Depending on the corpus, the word vectors will capture different information. In this tutorial, we focus on Wikipedia's articles but other sources could be considered, like news or Webcrawl (more examples [here](http://statmt.org/)). To download a raw dump of Wikipedia, run the following command:\n\n```bash\nwget https://dumps.wikimedia.org/enwiki/latest/enwiki-latest-pages-articles.xml.bz2\n```\n\nDownloading the Wikipedia corpus takes some time. Instead, lets restrict our study to the first 1 billion bytes of English Wikipedia. They can be found on Matt Mahoney's [website](http://mattmahoney.net/):\n\n```bash\n$ mkdir data\n$ wget -c http://mattmahoney.net/dc/enwik9.zip -P data\n$ unzip data/enwik9.zip -d data\n```\n\nA raw Wikipedia dump contains a lot of HTML / XML data. We pre-process it with the wikifil.pl script bundled with fastText (this script was originally developed by Matt Mahoney, and can be found on his [website](http://mattmahoney.net/)).\n\n```bash\n$ perl wikifil.pl data/enwik9 > data/fil9\n```\n\nWe can check the file by running the following command:\n\n```bash\n$ head -c 80 data/fil9\nanarchism originated as a term of abuse first used against early working class\n```\n\nThe text is nicely pre-processed and can be used to learn our word vectors.\n\n## Training word vectors\n\n\n\nLearning word vectors on this data can now be achieved with a single command:\n\n```bash\n$ mkdir result\n$ ./fasttext skipgram -input data/fil9 -output result/fil9\n```\n\nTo decompose this command line: ./fastext calls the binary fastText executable (see how to install fastText [here](https://fasttext.cc/docs/en/support.html)) with the 'skipgram' model (it can also be 'cbow'). We then specify the requires options '-input' for the location of the data and '-output' for the location where the word representations will be saved.\n\nWhile fastText is running, the progress and estimated time to completion is shown on your screen. Once the program finishes, there should be two files in the result directory:\n\n```bash\n$ ls -l result\n-rw-r-r-- 1 bojanowski 1876110778 978480850 Dec 20 11:01 fil9.bin\n-rw-r-r-- 1 bojanowski 1876110778 190004182 Dec 20 11:01 fil9.vec\n```\n\nThe `fil9.bin` file is a binary file that stores the whole fastText model and can be subsequently loaded. The `fil9.vec` file is a text file that contains the word vectors, one per line for each word in the vocabulary:\n\n```bash\n$ head -n 4 result/fil9.vec\n218316 100\nthe -0.10363 -0.063669 0.032436 -0.040798 0.53749 0.00097867 0.10083 0.24829 ...\nof -0.0083724 0.0059414 -0.046618 -0.072735 0.83007 0.038895 -0.13634 0.60063 ...\none 0.32731 0.044409 -0.46484 0.14716 0.7431 0.24684 -0.11301 0.51721 0.73262 ...\n```\n\nThe first line is a header containing the number of words and the dimensionality of the vectors. The subsequent lines are the word vectors for all words in the vocabulary, sorted by decreasing frequency.\n\nLearning word vectors on this data can now be achieved with a single command:\n```py\n>>> import fasttext\n>>> model = fasttext.train_unsupervised('data/fil9')\n```\nWhile fastText is running, the progress and estimated time to completion is shown on your screen. Once the training finishes, `model` variable contains information on the trained model, and can be used for querying:\n\n```py\n>>> model.words\n[u'the', u'of', u'one', u'zero', u'and', u'in', u'two', u'a', u'nine', u'to', u'is', ...\n```\nIt returns all words in the vocabulary, sorted by decreasing frequency. We can get the word vector by:\n```py\n>>> model.get_word_vector(\"the\")\narray([-0.03087516, 0.09221972, 0.17660329, 0.17308897, 0.12863874,\n 0.13912526, -0.09851588, 0.00739991, 0.37038437, -0.00845221,\n ...\n -0.21184735, -0.05048715, -0.34571868, 0.23765688, 0.23726143],\n dtype=float32)\n```\n\nWe can save this model on disk as a binary file:\n```py\n>>> model.save_model(\"result/fil9.bin\")\n```\nand reload it later instead of training again:\n```py\n$ python\n>>> import fasttext\n>>> model = fasttext.load_model(\"result/fil9.bin\")\n```\n\n\n\n## Advanced readers: skipgram versus cbow\n\nfastText provides two models for computing word representations: skipgram and cbow ('**c**ontinuous-**b**ag-**o**f-**w**ords').\n\nThe skipgram model learns to predict a target word thanks to a nearby word. On the other hand, the cbow model predicts the target word according to its context. The context is represented as a bag of the words contained in a fixed size window around the target word.\n\nLet us illustrate this difference with an example: given the sentence *'Poets have been mysteriously silent on the subject of cheese'* and the target word '*silent*', a skipgram model tries to predict the target using a random close-by word, like '*subject' *or* '*mysteriously*'**. *The cbow model takes all the words in a surrounding window, like {*been, *mysteriously*, on, the*}, and uses the sum of their vectors to predict the target. The figure below summarizes this difference with another example.\n\n![cbow vs skipgram](https://fasttext.cc/img/cbo_vs_skipgram.png)\nTo train a cbow model with fastText, you run the following command:\n\n\n\n```bash\n./fasttext cbow -input data/fil9 -output result/fil9\n```\n\n```py\n>>> import fasttext\n>>> model = fasttext.train_unsupervised('data/fil9', \"cbow\")\n```\n\n\nIn practice, we observe that skipgram models works better with subword information than cbow.\n\n## Advanced readers: playing with the parameters\n\nSo far, we run fastText with the default parameters, but depending on the data, these parameters may not be optimal. Let us give an introduction to some of the key parameters for word vectors.\n\nThe most important parameters of the model are its dimension and the range of size for the subwords. The dimension (*dim*) controls the size of the vectors, the larger they are the more information they can capture but requires more data to be learned. But, if they are too large, they are harder and slower to train. By default, we use 100 dimensions, but any value in the 100-300 range is as popular. The subwords are all the substrings contained in a word between the minimum size (*minn*) and the maximal size (*maxn*). By default, we take all the subword between 3 and 6 characters, but other range could be more appropriate to different languages:\n\n\n\n```bash\n$ ./fasttext skipgram -input data/fil9 -output result/fil9 -minn 2 -maxn 5 -dim 300\n```\n\n```py\n>>> import fasttext\n>>> model = fasttext.train_unsupervised('data/fil9', minn=2, maxn=5, dim=300)\n```\n\n\nDepending on the quantity of data you have, you may want to change the parameters of the training. The *epoch* parameter controls how many times the model will loop over your data. By default, we loop over the dataset 5 times. If you dataset is extremely massive, you may want to loop over it less often. Another important parameter is the learning rate -*lr*. The higher the learning rate is, the faster the model converge to a solution but at the risk of overfitting to the dataset. The default value is 0.05 which is a good compromise. If you want to play with it we suggest to stay in the range of [0.01, 1]:\n\n\n\n```bash\n$ ./fasttext skipgram -input data/fil9 -output result/fil9 -epoch 1 -lr 0.5\n```\n\n```py\n>>> import fasttext\n>>> model = fasttext.train_unsupervised('data/fil9', epoch=1, lr=0.5)\n```\n\n\nFinally , fastText is multi-threaded and uses 12 threads by default. If you have less CPU cores (say 4), you can easily set the number of threads using the *thread* flag:\n\n\n\n```bash\n$ ./fasttext skipgram -input data/fil9 -output result/fil9 -thread 4\n```\n\n```py\n>>> import fasttext\n>>> model = fasttext.train_unsupervised('data/fil9', thread=4)\n```\n\n\n\n## Printing word vectors\n\nSearching and printing word vectors directly from the `fil9.vec` file is cumbersome. Fortunately, there is a `print-word-vectors` functionality in fastText.\n\nFor example, we can print the word vectors of words *asparagus,* *pidgey* and *yellow* with the following command:\n\n\n```bash\n$ echo \"asparagus pidgey yellow\" | ./fasttext print-word-vectors result/fil9.bin\nasparagus 0.46826 -0.20187 -0.29122 -0.17918 0.31289 -0.31679 0.17828 -0.04418 ...\npidgey -0.16065 -0.45867 0.10565 0.036952 -0.11482 0.030053 0.12115 0.39725 ...\nyellow -0.39965 -0.41068 0.067086 -0.034611 0.15246 -0.12208 -0.040719 -0.30155 ...\n```\n\n```py\n>>> [model.get_word_vector(x) for x in [\"asparagus\", \"pidgey\", \"yellow\"]]\n[array([-0.25751096, -0.18716481, 0.06921121, 0.06455903, 0.29168844,\n 0.15426874, -0.33448914, -0.427215 , 0.7813013 , -0.10600132,\n ...\n 0.37090245, 0.39266172, -0.4555302 , 0.27452755, 0.00467369],\n dtype=float32),\n array([-0.20613593, -0.25325796, -0.2422259 , -0.21067499, 0.32879013,\n 0.7269511 , 0.3782259 , 0.11274897, 0.246764 , -0.6423613 ,\n ...\n 0.46302193, 0.2530962 , -0.35795924, 0.5755718 , 0.09843876],\n dtype=float32),\n array([-0.304823 , 0.2543754 , -0.2198013 , -0.25421786, 0.11219151,\n 0.38286993, -0.22636674, -0.54023844, 0.41095474, -0.3505803 ,\n ...\n 0.54788435, 0.36740595, -0.5678512 , 0.07523401, -0.08701935],\n dtype=float32)]\n\n```\n\n\nA nice feature is that you can also query for words that did not appear in your data! Indeed words are represented by the sum of its substrings. As long as the unknown word is made of known substrings, there is a representation of it!\n\nAs an example let's try with a misspelled word:\n\n\n\n```bash\n$ echo \"enviroment\" | ./fasttext print-word-vectors result/fil9.bin\n```\n\n```py\n>>> model.get_word_vector(\"enviroment\")\n```\n\n\nYou still get a word vector for it! But how good it is? Let's find out in the next sections!\n\n\n## Nearest neighbor queries\n\nA simple way to check the quality of a word vector is to look at its nearest neighbors. This give an intuition of the type of semantic information the vectors are able to capture.\n\nThis can be achieved with the nearest neighbor (*nn*) functionality. For example, we can query the 10 nearest neighbors of a word by running the following command:\n\n\n\n```bash\n$ ./fasttext nn result/fil9.bin\nPre-computing word vectors... done.\n```\n\nThen we are prompted to type our query word, let us try *asparagus* :\n\n```bash\nQuery word? asparagus\nbeetroot 0.812384\ntomato 0.806688\nhorseradish 0.805928\nspinach 0.801483\nlicorice 0.791697\nlingonberries 0.781507\nasparagales 0.780756\nlingonberry 0.778534\ncelery 0.774529\nbeets 0.773984\n```\n\n```py\n>>> model.get_nearest_neighbors('asparagus')\n[(0.812384, u'beetroot'), (0.806688, u'tomato'), (0.805928, u'horseradish'), (0.801483, u'spinach'), (0.791697, u'licorice'), (0.781507, u'lingonberries'), (0.780756, u'asparagales'), (0.778534, u'lingonberry'), (0.774529, u'celery'), (0.773984, u'beets')]\n```\n\n\nNice! It seems that vegetable vectors are similar. Note that the nearest neighbor is the word *asparagus* itself, this means that this word appeared in the dataset. What about pokemons?\n\n\n\n```bash\nQuery word? pidgey\npidgeot 0.891801\npidgeotto 0.885109\npidge 0.884739\npidgeon 0.787351\npok 0.781068\npikachu 0.758688\ncharizard 0.749403\nsquirtle 0.742582\nbeedrill 0.741579\ncharmeleon 0.733625\n```\n\n```py\n>>> model.get_nearest_neighbors('pidgey')\n[(0.891801, u'pidgeot'), (0.885109, u'pidgeotto'), (0.884739, u'pidge'), (0.787351, u'pidgeon'), (0.781068, u'pok'), (0.758688, u'pikachu'), (0.749403, u'charizard'), (0.742582, u'squirtle'), (0.741579, u'beedrill'), (0.733625, u'charmeleon')]\n```\n\n\n\nDifferent evolution of the same Pokemon have close-by vectors! But what about our misspelled word, is its vector close to anything reasonable? Let s find out:\n\n\n\n```bash\nQuery word? enviroment\nenviromental 0.907951\nenviron 0.87146\nenviro 0.855381\nenvirons 0.803349\nenvironnement 0.772682\nenviromission 0.761168\nrealclimate 0.716746\nenvironment 0.702706\nacclimatation 0.697196\necotourism 0.697081\n```\n\n```py\n>>> model.get_nearest_neighbors('enviroment')\n[(0.907951, u'enviromental'), (0.87146, u'environ'), (0.855381, u'enviro'), (0.803349, u'environs'), (0.772682, u'environnement'), (0.761168, u'enviromission'), (0.716746, u'realclimate'), (0.702706, u'environment'), (0.697196, u'acclimatation'), (0.697081, u'ecotourism')]\n```\n\n\n\nThanks to the information contained within the word, the vector of our misspelled word matches to reasonable words! It is not perfect but the main information has been captured.\n\n## Advanced reader: measure of similarity\n\nIn order to find nearest neighbors, we need to compute a similarity score between words. Our words are represented by continuous word vectors and we can thus apply simple similarities to them. In particular we use the cosine of the angles between two vectors. This similarity is computed for all words in the vocabulary, and the 10 most similar words are shown. Of course, if the word appears in the vocabulary, it will appear on top, with a similarity of 1.\n\n## Word analogies\n\nIn a similar spirit, one can play around with word analogies. For example, we can see if our model can guess what is to France, and what Berlin is to Germany.\n\n\n\n\n\nThis can be done with the *analogies* functionality. It takes a word triplet (like *Germany Berlin France*) and outputs the analogy:\n\n\n\n```bash\n$ ./fasttext analogies result/fil9.bin\nPre-computing word vectors... done.\nQuery triplet (A - B + C)? berlin germany france\nparis 0.896462\nbourges 0.768954\nlouveciennes 0.765569\ntoulouse 0.761916\nvalenciennes 0.760251\nmontpellier 0.752747\nstrasbourg 0.744487\nmeudon 0.74143\nbordeaux 0.740635\npigneaux 0.736122\n```\n\n```py\n>>> model.get_analogies(\"berlin\", \"germany\", \"france\")\n[(0.896462, u'paris'), (0.768954, u'bourges'), (0.765569, u'louveciennes'), (0.761916, u'toulouse'), (0.760251, u'valenciennes'), (0.752747, u'montpellier'), (0.744487, u'strasbourg'), (0.74143, u'meudon'), (0.740635, u'bordeaux'), (0.736122, u'pigneaux')]\n```\n\n\nThe answer provided by our model is *Paris*, which is correct. Let's have a look at a less obvious example:\n\n\n\n```bash\nQuery triplet (A - B + C)? psx sony nintendo\ngamecube 0.803352\nnintendogs 0.792646\nplaystation 0.77344\nsega 0.772165\ngameboy 0.767959\narcade 0.754774\nplaystationjapan 0.753473\ngba 0.752909\ndreamcast 0.74907\nfamicom 0.745298\n```\n\n```py\n>>> model.get_analogies(\"psx\", \"sony\", \"nintendo\")\n[(0.803352, u'gamecube'), (0.792646, u'nintendogs'), (0.77344, u'playstation'), (0.772165, u'sega'), (0.767959, u'gameboy'), (0.754774, u'arcade'), (0.753473, u'playstationjapan'), (0.752909, u'gba'), (0.74907, u'dreamcast'), (0.745298, u'famicom')]\n```\n\n\n\nOur model considers that the *nintendo* analogy of a *psx* is the *gamecube*, which seems reasonable. Of course the quality of the analogies depend on the dataset used to train the model and one can only hope to cover fields only in the dataset.\n\n\n## Importance of character n-grams\n\nUsing subword-level information is particularly interesting to build vectors for unknown words. For example, the word *gearshift* does not exist on Wikipedia but we can still query its closest existing words:\n\n\n\n```bash\nQuery word? gearshift\ngearing 0.790762\nflywheels 0.779804\nflywheel 0.777859\ngears 0.776133\ndriveshafts 0.756345\ndriveshaft 0.755679\ndaisywheel 0.749998\nwheelsets 0.748578\nepicycles 0.744268\ngearboxes 0.73986\n```\n\n```py\n>>> model.get_nearest_neighbors('gearshift')\n[(0.790762, u'gearing'), (0.779804, u'flywheels'), (0.777859, u'flywheel'), (0.776133, u'gears'), (0.756345, u'driveshafts'), (0.755679, u'driveshaft'), (0.749998, u'daisywheel'), (0.748578, u'wheelsets'), (0.744268, u'epicycles'), (0.73986, u'gearboxes')]\n```\n\n\n\n\nMost of the retrieved words share substantial substrings but a few are actually quite different, like *cogwheel*. You can try other words like *sunbathe* or *grandnieces*.\n\nNow that we have seen the interest of subword information for unknown words, let's check how it compares to a model that does not use subword information. To train a model without subwords, just run the following command:\n\n\n\n```bash\n$ ./fasttext skipgram -input data/fil9 -output result/fil9-none -maxn 0\n```\nThe results are saved in result/fil9-non.vec and result/fil9-non.bin.\n\n```py\n>>> model_without_subwords = fasttext.train_unsupervised('data/fil9', maxn=0)\n```\n\n\n\nTo illustrate the difference, let us take an uncommon word in Wikipedia, like *accomodation* which is a misspelling of *accommodation**.* Here is the nearest neighbors obtained without subwords:\n\n\n\n```bash\n$ ./fasttext nn result/fil9-none.bin\nQuery word? accomodation\nsunnhordland 0.775057\naccomodations 0.769206\nadministrational 0.753011\nlaponian 0.752274\nammenities 0.750805\ndachas 0.75026\nvuosaari 0.74172\nhostelling 0.739995\ngreenbelts 0.733975\nasserbo 0.732465\n```\n\n```py\n>>> model_without_subwords.get_nearest_neighbors('accomodation')\n[(0.775057, u'sunnhordland'), (0.769206, u'accomodations'), (0.753011, u'administrational'), (0.752274, u'laponian'), (0.750805, u'ammenities'), (0.75026, u'dachas'), (0.74172, u'vuosaari'), (0.739995, u'hostelling'), (0.733975, u'greenbelts'), (0.732465, u'asserbo')]\n```\n\n\nThe result does not make much sense, most of these words are unrelated. On the other hand, using subword information gives the following list of nearest neighbors:\n\n\n\n```bash\nQuery word? accomodation\naccomodations 0.96342\naccommodation 0.942124\naccommodations 0.915427\naccommodative 0.847751\naccommodating 0.794353\naccomodated 0.740381\namenities 0.729746\ncatering 0.725975\naccomodate 0.703177\nhospitality 0.701426\n```\n\n```py\n>>> model.get_nearest_neighbors('accomodation')\n[(0.96342, u'accomodations'), (0.942124, u'accommodation'), (0.915427, u'accommodations'), (0.847751, u'accommodative'), (0.794353, u'accommodating'), (0.740381, u'accomodated'), (0.729746, u'amenities'), (0.725975, u'catering'), (0.703177, u'accomodate'), (0.701426, u'hospitality')]\n```\n\n\nThe nearest neighbors capture different variation around the word *accommodation*. We also get semantically related words such as *amenities* or *catering*.\n\n## Conclusion\n\nIn this tutorial, we show how to obtain word vectors from Wikipedia. This can be done for any language and we provide [pre-trained models](https://fasttext.cc/docs/en/pretrained-vectors.html) with the default setting for 294 of them.\n" }, { "path": "docs/webassembly-module.md", "content": "---\nid: webassembly-module\ntitle: WebAssembly module\n---\n\nIn this document we present how to use fastText in javascript with WebAssembly.\n\n## Table of contents\n\n* [Requirements](#requirements)\n* [Building WebAssembly binaries](#building-webassembly-binaries)\n* [Build a webpage that uses fastText](#build-a-webpage-that-uses-fasttext)\n* [Load a model](#load-a-model)\n* [Train a model](#train-a-model)\n * [Disclaimer](#disclaimer)\n * [Text classification](#text-classification)\n * [Word representations](#word-representations)\n* [Quantized models](#quantized-models)\n* [API](#api)\n * [`model` object](#model-object)\n * [`loadModel`](#loadmodel)\n * [`trainSupervised`](#trainsupervised)\n * [`trainUnsupervised`](#trainunsupervised)\n\n# Requirements\n\nFor building [fastText](https://fasttext.cc/) with WebAssembly bindings, we will need:\n - a compiler with good C++11 support, since it uses C\\++11 features,\n - [emscripten](https://emscripten.org/),\n - a [browser that supports WebAssembly](https://caniuse.com/#feat=wasm).\n\n\n# Building WebAssembly binaries\n\nFirst, download and install emscripten sdk as [described here](https://emscripten.org/docs/getting_started/downloads.html#installation-instructions).\n\n\nWe need to make sure we activated the PATH for emscripten:\n```bash\n$ source /path/to/emsdk/emsdk_env.sh\n```\n\nClone [fastText repository](https://github.com/facebookresearch/fastText/):\n\n```bash\n$ git clone git@github.com:facebookresearch/fastText.git\n```\n\nBuild WebAssembly binaries:\n```bash\n$ cd fastText\n$ make wasm\n```\n\nThis will create `fasttext_wasm.wasm` and `fasttext_wasm.js` in the `webassembly` folder.\n\n- `fasttext_wasm.wasm` is the binary file that will be loaded in the webassembly's virtual machine.\n- `fasttext_wasm.js` is a javascript file built by emscripten, that helps to load `fasttext_wasm.wasm` file in the virtual machine and provides some helper functions.\n- `fasttext.js` is the wrapper that provides a nice API for fastText. \n\nAs the user of the library, we will interact with classes and methods defined in `fasttext.js`. We won't deal with `fasttext_wasm.*` files, but they are necessary to run fastText in the javascript's VM.\n\n# Build a webpage that uses fastText\n\nIn this section we are going to build a minimal HTML page that loads fastText WebAssembly module.\n\nAt the root of the repository, create a folder `webassembly-test`, and copy the files mentioned in the previous section:\n\n```bash\n$ mkdir webassembly-test\n$ cp webassembly/fasttext_wasm.wasm webassembly-test/\n$ cp webassembly/fasttext_wasm.js webassembly-test/\n$ cp webassembly/fasttext.js webassembly-test/\n```\n\nInside that folder, create `test.html` file containing:\n```html\n\n\n\n \n \n\n\n \n\n\n```\n\nIt is important to add the attribute `type=\"module\"` to the script tag, because we use ES6 style imports. `addOnPostRun` is a function that helps to provide a handler that is called when fastText is successfully loaded in the virtual machine. Once we are called inside that function, we can create an instance of `FastText`, that we will use to access the api.\n\n\nLet's test it.\n\nOpening `test.html` directly in the browser won't work since we are dynamically loading webassembly resources. The `test.html` file must be served from a webserver. The easiest way to achieve this is to use python's simple http server module:\n\n```bash\n$ cd webassembly-test\n$ python -m SimpleHTTPServer\n```\n\nThen browse `http://localhost:8000/test.html` in your browser. If everything worked as expected, you should see `FastText {f: FastText}` in the javascript console.\n\n\n# Load a model\n\nIn order to load a fastText model that was already trained, we can use `loadModel` function. In the example below we use `lid.176.ftz` that you can download from [here](/docs/en/language-identification.html).\n\nPlace the model file you want to load inside the same directory than the HTML file, and inside the script part:\n```javascript\nimport {FastText, addOnPostRun} from \"./fasttext.js\";\n\nconst printVector = function(predictions) {\n for (let i=0; i {\n let ft = new FastText();\n\n const url = \"lid.176.ftz\";\n ft.loadModel(url).then(model => {\n \n console.log(\"Model loaded.\")\n\n let text = \"Bonjour à tous. Ceci est du français\";\n console.log(text);\n printVector(model.predict(text, 5, 0.0));\n\n text = \"Hello, world. This is english\";\n console.log(text);\n printVector(model.predict(text, 5, 0.0));\n\n text = \"Merhaba dünya. Bu da türkçe\"\n console.log(text);\n printVector(model.predict(text, 5, 0.0));\n });\n});\n```\n\n`loadModel` function returns a [Promise](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise) that resolves to a `model` object. We can then use [`model` object](#model-object) to call various methods, such as `predict`.\n\nWe define `printVector` function that loops through a representation of `std::vector` in javascript, and displays the items. Here, we use it to display prediction results.\n\nYou can also refer to `webassembly/doc/examples/predict.html` in the source code.\n\n# Calling other methods\n\nOnce the model is loaded, you can call any method like `model.getDimension()` or `model.getSubwords(word)`. You can refer to [this](#api) section of the document for a complete API. You can also have a look to `webassembly/doc/examples/misc.html` file in the source code for further examples.\n\n\n# Train a model\n\n### Disclaimer\n\nIt is also possible to train a model inside the browser with fastText's WebAssembly API. The training can be slow because at the time of writing, it is not possible to use multithreading in WebAssembly (along with dynamic memory growth). So most of the time, we would train a model with the python or command line tool, eventually quantize it, and load it in the WebAssembly module. However, training a model inside the browser can be useful for creating animations or educational tools.\n \n### Text classification\n\nPlace the `cooking.train` file (as described [here](/docs/en/supervised-tutorial.html)) inside the same directory:\n\n```javascript\nimport {FastText, addOnPostRun} from \"./fasttext.js\";\n\nconst trainCallback = (progress, loss, wst, lr, eta) => {\n console.log([progress, loss, wst, lr, eta]);\n};\n\naddOnPostRun(() => {\n let ft = new FastText();\n\n ft.trainSupervised(\"cooking.train\", {\n 'lr':1.0,\n 'epoch':10,\n 'loss':'hs',\n 'wordNgrams':2,\n 'dim':50,\n 'bucket':200000\n }, trainCallback).then(model => {\n console.log('Trained.');\n });\n});\n```\n\n`trainCallback` function is called by the module to show progress, average training cost, number of words per second (per thread, but there is only one thread), learning rate, estimated remaining time.\n\n\n### Word representations\n\nPlace the `fil9` file (as described [here](/docs/en/unsupervised-tutorial.html)) inside the same directory:\n\n```javascript\nimport {FastText, addOnPostRun} from \"./fasttext.js\";\n\nconst trainCallback = (progress, loss, wst, lr, eta) => {\n console.log([progress, loss, wst, lr, eta]);\n};\n\naddOnPostRun(() => {\n let ft = new FastText();\n\n ft.trainUnsupervised(\"fil9\", 'skipgram', {\n 'lr':0.1,\n 'epoch':1,\n 'loss':'ns',\n 'wordNgrams':2,\n 'dim':50,\n 'bucket':200000\n }, trainCallback).then(model => {\n console.log('Trained.');\n });\n});\n```\n\n# Quantized models\n\nQuantization is a technique that reduces the size of your models. You can quantize your model as [described here](/docs/en/faqs.html#how-can-i-reduce-the-size-of-my-fasttext-models).\n\nYou can load a quantized model in fastText's WebAssembly module, as we did in [\"Load a model\" section](#load-a-model).\n\n\nIn the context of web, it is particularly useful to have smaller models since they can be downloaded much faster. You can use our autotune feature as [described here](/docs/en/autotune.html#constrain-model-size) in order to find the best trade-off between accuracy and model size that fits your needs.\n\n\n# API\n\n## `model` object\n\n`trainSupervised`, `trainUnsupervised` and `loadModel` functions return a Promise that resolves to an instance of `FastTextModel` class, that we generaly name `model` object.\n\nThis object exposes several functions:\n\n```javascript\nisQuant // true if the model is quantized.\ngetDimension // the dimension (size) of a lookup vector (hidden layer).\ngetWordVector(word) // the vector representation of `word`.\ngetSentenceVector(text) // the vector representation of `text`.\ngetNearestNeighbors(word, k=10) // nearest `k` neighbors of `word`.\ngetAnalogies(wordA, wordB, wordC, k) // nearest `k` neighbors of the operation `wordA - wordB + wordC`.\ngetWordId(word) // get the word id within the dictionary.\ngetSubwordId(subword) // the index (within input matrix) a subword hashes to.\ngetSubwords(word) // the subwords and their indicies.\ngetInputVector(ind) // given an index, get the corresponding vector of the Input Matrix.\npredict(text, k = 1, threshold = 0.0) // Given a string, get a list of labels and a list of corresponding\n // probabilities. k controls the number of returned labels.\ngetInputMatrix() // get a reference to the full input matrix of a (non-quantized) Model.\ngetOutputMatrix() // get a reference to the full output matrix of a (non-quantized) Model.\ngetWords() // get the entire list of words of the dictionary including the frequency\n // of the individual words. This does not include any subwords. For that\n // please consult the function get_subwords.\ngetLabels() // get the entire list of labels of the dictionary including the frequency\ngetLine(text) // split a line of text into words and labels.\nsaveModel() // saves the model file in WebAssembly's in-memory FS and returns a blob\ntest(url, k, threshold) // downloads the test file from the specified url, evaluates the supervised model with it.\n```\n\nYou can also have a look to `webassembly/doc/examples/misc.html` file in the source code for further examples.\n\n## `loadModel`\n\nYou can load a model as follows:\n\n`ft.loadModel(url);`\n\n`loadModel` returns a [Promise](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise) that resolves to a [`model` object](#model-object).\n\n\n## `trainSupervised` \n\nYou can train a text classification model with fastText's WebAssembly API as follows:\n\n`ft.trainSupervised(trainFile, args, trainCallback);`\n\n- `trainFile`: the url of the input file\n- `args`: a dictionary with following keys:\n```javascript\n lr # learning rate [0.1]\n dim # size of word vectors [100]\n ws # size of the context window [5]\n epoch # number of epochs [5]\n minCount # minimal number of word occurences [1]\n minCountLabel # minimal number of label occurences [1]\n minn # min length of char ngram [0]\n maxn # max length of char ngram [0]\n neg # number of negatives sampled [5]\n wordNgrams # max length of word ngram [1]\n loss # loss function {ns, hs, softmax, ova} [softmax]\n bucket # number of buckets [2000000]\n thread # number of threads [number of cpus]\n lrUpdateRate # change the rate of updates for the learning rate [100]\n t # sampling threshold [0.0001]\n label # label prefix ['__label__']\n```\n- `trainCallback` is the name of the function that will be called during training to provide various information. Set this argument to `null` if you don't need a callback, or provide a function that has the following signature: `function myCallback(progress, loss, wst, lr, eta){ ... }`\n\n`trainSupervised` returns a [Promise](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise) that resolves to a [`model` object](#model-object).\n\n## `trainUnsupervised` \n\nYou can train a word representation model with fastText's WebAssembly API as follows:\n\n`ft.trainUnsupervised(trainFile, modelname, args, trainCallback);`\n\n- `trainFile`: the url of the input file\n- `modelName`: must be `\"cbow\"` or `\"skipgram\"`\n- `args`: a dictionary with following keys:\n```javascript\n lr # learning rate [0.05]\n dim # size of word vectors [100]\n ws # size of the context window [5]\n epoch # number of epochs [5]\n minCount # minimal number of word occurences [5]\n minn # min length of char ngram [3]\n maxn # max length of char ngram [6]\n neg # number of negatives sampled [5]\n wordNgrams # max length of word ngram [1]\n loss # loss function {ns, hs, softmax, ova} [ns]\n bucket # number of buckets [2000000]\n thread # number of threads [number of cpus]\n lrUpdateRate # change the rate of updates for the learning rate [100]\n t # sampling threshold [0.0001]\n```\n- `trainCallback` is the name of the function that will be called during training to provide various information. Set this argument to `null` if you don't need a callback, or provide a function that has the following signature: `function myCallback(progress, loss, wst, lr, eta){ ... }`\n\n`trainUnsupervised` returns a [Promise](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise) that resolves to a [`model` object](#model-object).\n\n\n\n" }, { "path": "download_model.py", "content": "#!/usr/bin/env python\n# -*- coding: utf-8 -*-\n#\n# Copyright (c) 2017-present, Facebook, Inc.\n# All rights reserved.\n#\n# This source code is licensed under the MIT license found in the\n# LICENSE file in the root directory of this source tree.\n\nfrom __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\nfrom __future__ import unicode_literals\n\nimport argparse\n\nimport fasttext.util\n\n\nargs = None\n\n\ndef command_download(lang_id, if_exists):\n \"\"\"\n Download pre-trained common-crawl vectors from fastText's website\n https://fasttext.cc/docs/en/crawl-vectors.html\n \"\"\"\n fasttext.util.download_model(lang_id, if_exists)\n\n\ndef main():\n global args\n\n parser = argparse.ArgumentParser(\n description='fastText helper tool to reduce model dimensions.')\n parser.add_argument(\"language\", type=str, default=\"en\",\n help=\"language identifier of the pre-trained vectors. For example `en` or `fr`.\")\n parser.add_argument(\"--overwrite\", action=\"store_true\",\n help=\"overwrite if file exists.\")\n\n args = parser.parse_args()\n\n command_download(args.language, if_exists=(\n 'overwrite' if args.overwrite else 'strict'))\n\n\nif __name__ == '__main__':\n main()\n" }, { "path": "eval.py", "content": "#!/usr/bin/env python\n# -*- coding: utf-8 -*-\n#\n# Copyright (c) 2016-present, Facebook, Inc.\n# All rights reserved.\n#\n# This source code is licensed under the MIT license found in the\n# LICENSE file in the root directory of this source tree.\n#\n\nfrom __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\nfrom __future__ import unicode_literals\nimport numpy as np\nfrom scipy import stats\nimport os\nimport math\nimport argparse\n\n\ndef compat_splitting(line):\n return line.decode('utf8').split()\n\n\ndef similarity(v1, v2):\n n1 = np.linalg.norm(v1)\n n2 = np.linalg.norm(v2)\n return np.dot(v1, v2) / n1 / n2\n\n\nparser = argparse.ArgumentParser(description='Process some integers.')\nparser.add_argument(\n '--model',\n '-m',\n dest='modelPath',\n action='store',\n required=True,\n help='path to model'\n)\nparser.add_argument(\n '--data',\n '-d',\n dest='dataPath',\n action='store',\n required=True,\n help='path to data'\n)\nargs = parser.parse_args()\n\nvectors = {}\nfin = open(args.modelPath, 'rb')\nfor _, line in enumerate(fin):\n try:\n tab = compat_splitting(line)\n vec = np.array(tab[1:], dtype=float)\n word = tab[0]\n if np.linalg.norm(vec) == 0:\n continue\n if not word in vectors:\n vectors[word] = vec\n except ValueError:\n continue\n except UnicodeDecodeError:\n continue\nfin.close()\n\nmysim = []\ngold = []\ndrop = 0.0\nnwords = 0.0\n\nfin = open(args.dataPath, 'rb')\nfor line in fin:\n tline = compat_splitting(line)\n word1 = tline[0].lower()\n word2 = tline[1].lower()\n nwords = nwords + 1.0\n\n if (word1 in vectors) and (word2 in vectors):\n v1 = vectors[word1]\n v2 = vectors[word2]\n d = similarity(v1, v2)\n mysim.append(d)\n gold.append(float(tline[2]))\n else:\n drop = drop + 1.0\nfin.close()\n\ncorr = stats.spearmanr(mysim, gold)\ndataset = os.path.basename(args.dataPath)\nprint(\n \"{0:20s}: {1:2.0f} (OOV: {2:2.0f}%)\"\n .format(dataset, corr[0] * 100, math.ceil(drop / nwords * 100.0))\n)\n" }, { "path": "fasttext.pc.in", "content": "prefix=@CMAKE_INSTALL_PREFIX@\nexec_prefix=@CMAKE_INSTALL_FULL_LIBEXECDIR@\nlibdir=@CMAKE_INSTALL_FULL_LIBDIR@\nincludedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@\n\nName: @PROJECT_NAME@\nDescription: Efficient learning of word representations and sentence classification\nVersion: @PROJECT_VERSION@\nLibs: -L${libdir} -lfasttext\nCflags: -I${includedir}\n" }, { "path": "get-wikimedia.sh", "content": "#!/usr/bin/env bash\n#\n# Copyright (c) 2016-present, Facebook, Inc.\n# All rights reserved.\n#\n# This source code is licensed under the MIT license found in the\n# LICENSE file in the root directory of this source tree.\n#\n\nset -e\n\nnormalize_text() {\n sed -e \"s/’/'/g\" -e \"s/′/'/g\" -e \"s/''/ /g\" -e \"s/'/ ' /g\" -e \"s/“/\\\"/g\" -e \"s/”/\\\"/g\" \\\n -e 's/\"/ \" /g' -e 's/\\./ \\. /g' -e 's/
/ /g' -e 's/, / , /g' -e 's/(/ ( /g' -e 's/)/ ) /g' -e 's/\\!/ \\! /g' \\\n -e 's/\\?/ \\? /g' -e 's/\\;/ /g' -e 's/\\:/ /g' -e 's/-/ - /g' -e 's/=/ /g' -e 's/=/ /g' -e 's/*/ /g' -e 's/|/ /g' \\\n -e 's/«/ /g' | tr 0-9 \" \"\n}\n\nexport LANGUAGE=en_US.UTF-8\nexport LC_ALL=en_US.UTF-8\nexport LANG=en_US.UTF-8\n\nNOW=$(date +\"%Y%m%d\")\n\nROOT=\"data/wikimedia/${NOW}\"\nmkdir -p \"${ROOT}\"\necho \"Saving data in \"\"$ROOT\"\nread -r -p \"Choose a language (e.g. en, bh, fr, etc.): \" choice\nLANG=\"$choice\"\necho \"Chosen language: \"\"$LANG\"\nread -r -p \"Continue to download (WARNING: This might be big and can take a long time!)(y/n)? \" choice\ncase \"$choice\" in\n y|Y ) echo \"Starting download...\";;\n n|N ) echo \"Exiting\";exit 1;;\n * ) echo \"Invalid answer\";exit 1;;\nesac\nwget -c \"https://dumps.wikimedia.org/\"\"$LANG\"\"wiki/latest/\"\"${LANG}\"\"wiki-latest-pages-articles.xml.bz2\" -P \"${ROOT}\"\necho \"Processing \"\"$ROOT\"/\"$LANG\"\"wiki-latest-pages-articles.xml.bz2\"\nbzip2 -c -d \"$ROOT\"/\"$LANG\"\"wiki-latest-pages-articles.xml.bz2\" | awk '{print tolower($0);}' | perl -e '\n# Program to filter Wikipedia XML dumps to \"clean\" text consisting only of lowercase\n# letters (a-z, converted from A-Z), and spaces (never consecutive)...\n# All other characters are converted to spaces. Only text which normally appears.\n# in the web browser is displayed. Tables are removed. Image captions are.\n# preserved. Links are converted to normal text. Digits are spelled out.\n# *** Modified to not spell digits or throw away non-ASCII characters ***\n# Written by Matt Mahoney, June 10, 2006. This program is released to the public domain.\n$/=\">\"; # input record separator\nwhile (<>) {\n if (/ ... \n if (/#redirect/i) {$text=0;} # remove #REDIRECT\n if ($text) {\n # Remove any text not normally visible\n if (/<\\/text>/) {$text=0;}\n s/<.*>//; # remove xml tags\n s/&/&/g; # decode URL encoded chars\n s/<//g;\n s///g; # remove references ... \n s/<[^>]*>//g; # remove xhtml tags\n s/\\[http:[^] ]*/[/g; # remove normal url, preserve visible text\n s/\\|thumb//ig; # remove images links, preserve caption\n s/\\|left//ig;\n s/\\|right//ig;\n s/\\|\\d+px//ig;\n s/\\[\\[image:[^\\[\\]]*\\|//ig;\n s/\\[\\[category:([^|\\]]*)[^]]*\\]\\]/[[$1]]/ig; # show categories without markup\n s/\\[\\[[a-z\\-]*:[^\\]]*\\]\\]//g; # remove links to other languages\n s/\\[\\[[^\\|\\]]*\\|/[[/g; # remove wiki url, preserve visible text\n s/{{[^}]*}}//g; # remove {{icons}} and {tables}\n s/{[^}]*}//g;\n s/\\[//g; # remove [ and ]\n s/\\]//g;\n s/&[^;]*;/ /g; # remove URL encoded chars\n $_=\" $_ \";\n chop;\n print $_;\n }\n}\n' | normalize_text | awk '{if (NF>1) print;}' | tr -s \" \" | shuf > \"${ROOT}\"/wiki.\"${LANG}\".txt\n" }, { "path": "pyproject.toml", "content": "[build-system]\nrequires = [\"setuptools\", \"wheel\", \"pybind11\"]\n" }, { "path": "python/README.md", "content": "# fastText [![CircleCI](https://circleci.com/gh/facebookresearch/fastText/tree/master.svg?style=svg)](https://circleci.com/gh/facebookresearch/fastText/tree/master)\n\n[fastText](https://fasttext.cc/) is a library for efficient learning of word representations and sentence classification.\n\nIn this document we present how to use fastText in python.\n\n## Table of contents\n\n* [Requirements](#requirements)\n* [Installation](#installation)\n* [Usage overview](#usage-overview)\n * [Word representation model](#word-representation-model)\n * [Text classification model](#text-classification-model)\n * [IMPORTANT: Preprocessing data / encoding conventions](#important-preprocessing-data-encoding-conventions)\n * [More examples](#more-examples)\n* [API](#api)\n * [`train_unsupervised` parameters](#train_unsupervised-parameters)\n * [`train_supervised` parameters](#train_supervised-parameters)\n * [`model` object](#model-object)\n\n\n# Requirements\n\n[fastText](https://fasttext.cc/) builds on modern Mac OS and Linux distributions.\nSince it uses C\\++11 features, it requires a compiler with good C++11 support. You will need [Python](https://www.python.org/) (version 2.7 or ≥ 3.4), [NumPy](http://www.numpy.org/) & [SciPy](https://www.scipy.org/) and [pybind11](https://github.com/pybind/pybind11).\n\n\n# Installation\n\nTo install the latest release, you can do :\n```bash\n$ pip install fasttext\n```\n\nor, to get the latest development version of fasttext, you can install from our github repository :\n```bash\n$ git clone https://github.com/facebookresearch/fastText.git\n$ cd fastText\n$ sudo pip install .\n$ # or :\n$ sudo python setup.py install\n```\n\n# Usage overview\n\n\n## Word representation model\n\nIn order to learn word vectors, as [described here](https://fasttext.cc/docs/en/references.html#enriching-word-vectors-with-subword-information), we can use `fasttext.train_unsupervised` function like this:\n\n\n```py\nimport fasttext\n\n# Skipgram model :\nmodel = fasttext.train_unsupervised('data.txt', model='skipgram')\n\n# or, cbow model :\nmodel = fasttext.train_unsupervised('data.txt', model='cbow')\n\n```\n\nwhere `data.txt` is a training file containing utf-8 encoded text.\n\n\nThe returned `model` object represents your learned model, and you can use it to retrieve information.\n\n```py\nprint(model.words) # list of words in dictionary\nprint(model['king']) # get the vector of the word 'king'\n```\n\n\n### Saving and loading a model object\n\nYou can save your trained model object by calling the function `save_model`.\n```py\nmodel.save_model(\"model_filename.bin\")\n```\n\nand retrieve it later thanks to the function `load_model` :\n```py\nmodel = fasttext.load_model(\"model_filename.bin\")\n```\n\nFor more information about word representation usage of fasttext, you can refer to our [word representations tutorial](https://fasttext.cc/docs/en/unsupervised-tutorial.html).\n\n\n## Text classification model\n\nIn order to train a text classifier using the method [described here](https://fasttext.cc/docs/en/references.html#bag-of-tricks-for-efficient-text-classification), we can use `fasttext.train_supervised` function like this:\n\n\n```py\nimport fasttext\n\nmodel = fasttext.train_supervised('data.train.txt')\n```\n\nwhere `data.train.txt` is a text file containing a training sentence per line along with the labels. By default, we assume that labels are words that are prefixed by the string `__label__`\n\nOnce the model is trained, we can retrieve the list of words and labels:\n\n```py\nprint(model.words)\nprint(model.labels)\n```\n\nTo evaluate our model by computing the precision at 1 (P@1) and the recall on a test set, we use the `test` function:\n\n```py\ndef print_results(N, p, r):\n print(\"N\\t\" + str(N))\n print(\"P@{}\\t{:.3f}\".format(1, p))\n print(\"R@{}\\t{:.3f}\".format(1, r))\n\nprint_results(*model.test('test.txt'))\n```\n\nWe can also predict labels for a specific text :\n\n```py\nmodel.predict(\"Which baking dish is best to bake a banana bread ?\")\n```\n\nBy default, `predict` returns only one label : the one with the highest probability. You can also predict more than one label by specifying the parameter `k`:\n```py\nmodel.predict(\"Which baking dish is best to bake a banana bread ?\", k=3)\n```\n\nIf you want to predict more than one sentence you can pass an array of strings :\n\n```py\nmodel.predict([\"Which baking dish is best to bake a banana bread ?\", \"Why not put knives in the dishwasher?\"], k=3)\n```\n\n\nOf course, you can also save and load a model to/from a file as [in the word representation usage](#saving-and-loading-a-model-object).\n\nFor more information about text classification usage of fasttext, you can refer to our [text classification tutorial](https://fasttext.cc/docs/en/supervised-tutorial.html).\n\n\n\n\n### Compress model files with quantization\n\nWhen you want to save a supervised model file, fastText can compress it in order to have a much smaller model file by sacrificing only a little bit performance.\n\n```py\n# with the previously trained `model` object, call :\nmodel.quantize(input='data.train.txt', retrain=True)\n\n# then display results and save the new model :\nprint_results(*model.test(valid_data))\nmodel.save_model(\"model_filename.ftz\")\n```\n\n`model_filename.ftz` will have a much smaller size than `model_filename.bin`.\n\nFor further reading on quantization, you can refer to [this paragraph from our blog post](https://fasttext.cc/blog/2017/10/02/blog-post.html#model-compression).\n\n\n## IMPORTANT: Preprocessing data / encoding conventions\n\nIn general it is important to properly preprocess your data. In particular our example scripts in the [root folder](https://github.com/facebookresearch/fastText) do this.\n\nfastText assumes UTF-8 encoded text. All text must be [unicode for Python2](https://docs.python.org/2/library/functions.html#unicode) and [str for Python3](https://docs.python.org/3.5/library/stdtypes.html#textseq). The passed text will be [encoded as UTF-8 by pybind11](https://pybind11.readthedocs.io/en/master/advanced/cast/strings.html?highlight=utf-8#strings-bytes-and-unicode-conversions) before passed to the fastText C++ library. This means it is important to use UTF-8 encoded text when building a model. On Unix-like systems you can convert text using [iconv](https://en.wikipedia.org/wiki/Iconv).\n\nfastText will tokenize (split text into pieces) based on the following ASCII characters (bytes). In particular, it is not aware of UTF-8 whitespace. We advice the user to convert UTF-8 whitespace / word boundaries into one of the following symbols as appropiate.\n\n* space\n* tab\n* vertical tab\n* carriage return\n* formfeed\n* the null character\n\nThe newline character is used to delimit lines of text. In particular, the EOS token is appended to a line of text if a newline character is encountered. The only exception is if the number of tokens exceeds the MAX\\_LINE\\_SIZE constant as defined in the [Dictionary header](https://github.com/facebookresearch/fastText/blob/master/src/dictionary.h). This means if you have text that is not separate by newlines, such as the [fil9 dataset](http://mattmahoney.net/dc/textdata), it will be broken into chunks with MAX\\_LINE\\_SIZE of tokens and the EOS token is not appended.\n\nThe length of a token is the number of UTF-8 characters by considering the [leading two bits of a byte](https://en.wikipedia.org/wiki/UTF-8#Description) to identify [subsequent bytes of a multi-byte sequence](https://github.com/facebookresearch/fastText/blob/master/src/dictionary.cc). Knowing this is especially important when choosing the minimum and maximum length of subwords. Further, the EOS token (as specified in the [Dictionary header](https://github.com/facebookresearch/fastText/blob/master/src/dictionary.h)) is considered a character and will not be broken into subwords.\n\n## More examples\n\nIn order to have a better knowledge of fastText models, please consider the main [README](https://github.com/facebookresearch/fastText/blob/master/README.md) and in particular [the tutorials on our website](https://fasttext.cc/docs/en/supervised-tutorial.html).\n\nYou can find further python examples in [the doc folder](https://github.com/facebookresearch/fastText/tree/master/python/doc/examples).\n\nAs with any package you can get help on any Python function using the help function.\n\nFor example\n\n```\n+>>> import fasttext\n+>>> help(fasttext.FastText)\n\nHelp on module fasttext.FastText in fasttext:\n\nNAME\n fasttext.FastText\n\nDESCRIPTION\n # Copyright (c) 2017-present, Facebook, Inc.\n # All rights reserved.\n #\n # This source code is licensed under the MIT license found in the\n # LICENSE file in the root directory of this source tree.\n\nFUNCTIONS\n load_model(path)\n Load a model given a filepath and return a model object.\n\n tokenize(text)\n Given a string of text, tokenize it and return a list of tokens\n[...]\n```\n\n\n# API\n\n\n## `train_unsupervised` parameters\n\n```python\n input # training file path (required)\n model # unsupervised fasttext model {cbow, skipgram} [skipgram]\n lr # learning rate [0.05]\n dim # size of word vectors [100]\n ws # size of the context window [5]\n epoch # number of epochs [5]\n minCount # minimal number of word occurences [5]\n minn # min length of char ngram [3]\n maxn # max length of char ngram [6]\n neg # number of negatives sampled [5]\n wordNgrams # max length of word ngram [1]\n loss # loss function {ns, hs, softmax, ova} [ns]\n bucket # number of buckets [2000000]\n thread # number of threads [number of cpus]\n lrUpdateRate # change the rate of updates for the learning rate [100]\n t # sampling threshold [0.0001]\n verbose # verbose [2]\n```\n\n## `train_supervised` parameters\n\n```python\n input # training file path (required)\n lr # learning rate [0.1]\n dim # size of word vectors [100]\n ws # size of the context window [5]\n epoch # number of epochs [5]\n minCount # minimal number of word occurences [1]\n minCountLabel # minimal number of label occurences [1]\n minn # min length of char ngram [0]\n maxn # max length of char ngram [0]\n neg # number of negatives sampled [5]\n wordNgrams # max length of word ngram [1]\n loss # loss function {ns, hs, softmax, ova} [softmax]\n bucket # number of buckets [2000000]\n thread # number of threads [number of cpus]\n lrUpdateRate # change the rate of updates for the learning rate [100]\n t # sampling threshold [0.0001]\n label # label prefix ['__label__']\n verbose # verbose [2]\n pretrainedVectors # pretrained word vectors (.vec file) for supervised learning []\n```\n\n## `model` object\n\n`train_supervised`, `train_unsupervised` and `load_model` functions return an instance of `_FastText` class, that we generaly name `model` object.\n\nThis object exposes those training arguments as properties : `lr`, `dim`, `ws`, `epoch`, `minCount`, `minCountLabel`, `minn`, `maxn`, `neg`, `wordNgrams`, `loss`, `bucket`, `thread`, `lrUpdateRate`, `t`, `label`, `verbose`, `pretrainedVectors`. So `model.wordNgrams` will give you the max length of word ngram used for training this model.\n\nIn addition, the object exposes several functions :\n\n```python\n get_dimension # Get the dimension (size) of a lookup vector (hidden layer).\n # This is equivalent to `dim` property.\n get_input_vector # Given an index, get the corresponding vector of the Input Matrix.\n get_input_matrix # Get a copy of the full input matrix of a Model.\n get_labels # Get the entire list of labels of the dictionary\n # This is equivalent to `labels` property.\n get_line # Split a line of text into words and labels.\n get_output_matrix # Get a copy of the full output matrix of a Model.\n get_sentence_vector # Given a string, get a single vector represenation. This function\n # assumes to be given a single line of text. We split words on\n # whitespace (space, newline, tab, vertical tab) and the control\n # characters carriage return, formfeed and the null character.\n get_subword_id # Given a subword, return the index (within input matrix) it hashes to.\n get_subwords # Given a word, get the subwords and their indicies.\n get_word_id # Given a word, get the word id within the dictionary.\n get_word_vector # Get the vector representation of word.\n get_words # Get the entire list of words of the dictionary\n # This is equivalent to `words` property.\n is_quantized # whether the model has been quantized\n predict # Given a string, get a list of labels and a list of corresponding probabilities.\n quantize # Quantize the model reducing the size of the model and it's memory footprint.\n save_model # Save the model to the given path\n test # Evaluate supervised model using file given by path\n test_label # Return the precision and recall score for each label. \n```\n\nThe properties `words`, `labels` return the words and labels from the dictionary :\n```py\nmodel.words # equivalent to model.get_words()\nmodel.labels # equivalent to model.get_labels()\n```\n\nThe object overrides `__getitem__` and `__contains__` functions in order to return the representation of a word and to check if a word is in the vocabulary.\n\n```py\nmodel['king'] # equivalent to model.get_word_vector('king')\n'king' in model # equivalent to `'king' in model.get_words()`\n```\n\n\nJoin the fastText community\n---------------------------\n\n- [Facebook page](https://www.facebook.com/groups/1174547215919768)\n- [Stack overflow](https://stackoverflow.com/questions/tagged/fasttext)\n- [Google group](https://groups.google.com/forum/#!forum/fasttext-library)\n- [GitHub](https://github.com/facebookresearch/fastText)\n" }, { "path": "python/README.rst", "content": "fastText |CircleCI|\n===================\n\n`fastText `__ is a library for efficient learning\nof word representations and sentence classification.\n\nIn this document we present how to use fastText in python.\n\nTable of contents\n-----------------\n\n- `Requirements <#requirements>`__\n- `Installation <#installation>`__\n- `Usage overview <#usage-overview>`__\n- `Word representation model <#word-representation-model>`__\n- `Text classification model <#text-classification-model>`__\n- `IMPORTANT: Preprocessing data / encoding\n conventions <#important-preprocessing-data-encoding-conventions>`__\n- `More examples <#more-examples>`__\n- `API <#api>`__\n- `train_unsupervised parameters <#train_unsupervised-parameters>`__\n- `train_supervised parameters <#train_supervised-parameters>`__\n- `model object <#model-object>`__\n\nRequirements\n============\n\n`fastText `__ builds on modern Mac OS and Linux\ndistributions. Since it uses C++11 features, it requires a compiler with\ngood C++11 support. You will need `Python `__\n(version 2.7 or ≥ 3.4), `NumPy `__ &\n`SciPy `__ and\n`pybind11 `__.\n\nInstallation\n============\n\nTo install the latest release, you can do :\n\n.. code:: bash\n\n $ pip install fasttext\n\nor, to get the latest development version of fasttext, you can install\nfrom our github repository :\n\n.. code:: bash\n\n $ git clone https://github.com/facebookresearch/fastText.git\n $ cd fastText\n $ sudo pip install .\n $ # or :\n $ sudo python setup.py install\n\nUsage overview\n==============\n\nWord representation model\n-------------------------\n\nIn order to learn word vectors, as `described\nhere `__,\nwe can use ``fasttext.train_unsupervised`` function like this:\n\n.. code:: py\n\n import fasttext\n\n # Skipgram model :\n model = fasttext.train_unsupervised('data.txt', model='skipgram')\n\n # or, cbow model :\n model = fasttext.train_unsupervised('data.txt', model='cbow')\n\nwhere ``data.txt`` is a training file containing utf-8 encoded text.\n\nThe returned ``model`` object represents your learned model, and you can\nuse it to retrieve information.\n\n.. code:: py\n\n print(model.words) # list of words in dictionary\n print(model['king']) # get the vector of the word 'king'\n\nSaving and loading a model object\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\nYou can save your trained model object by calling the function\n``save_model``.\n\n.. code:: py\n\n model.save_model(\"model_filename.bin\")\n\nand retrieve it later thanks to the function ``load_model`` :\n\n.. code:: py\n\n model = fasttext.load_model(\"model_filename.bin\")\n\nFor more information about word representation usage of fasttext, you\ncan refer to our `word representations\ntutorial `__.\n\nText classification model\n-------------------------\n\nIn order to train a text classifier using the method `described\nhere `__,\nwe can use ``fasttext.train_supervised`` function like this:\n\n.. code:: py\n\n import fasttext\n\n model = fasttext.train_supervised('data.train.txt')\n\nwhere ``data.train.txt`` is a text file containing a training sentence\nper line along with the labels. By default, we assume that labels are\nwords that are prefixed by the string ``__label__``\n\nOnce the model is trained, we can retrieve the list of words and labels:\n\n.. code:: py\n\n print(model.words)\n print(model.labels)\n\nTo evaluate our model by computing the precision at 1 (P@1) and the\nrecall on a test set, we use the ``test`` function:\n\n.. code:: py\n\n def print_results(N, p, r):\n print(\"N\\t\" + str(N))\n print(\"P@{}\\t{:.3f}\".format(1, p))\n print(\"R@{}\\t{:.3f}\".format(1, r))\n\n print_results(*model.test('test.txt'))\n\nWe can also predict labels for a specific text :\n\n.. code:: py\n\n model.predict(\"Which baking dish is best to bake a banana bread ?\")\n\nBy default, ``predict`` returns only one label : the one with the\nhighest probability. You can also predict more than one label by\nspecifying the parameter ``k``:\n\n.. code:: py\n\n model.predict(\"Which baking dish is best to bake a banana bread ?\", k=3)\n\nIf you want to predict more than one sentence you can pass an array of\nstrings :\n\n.. code:: py\n\n model.predict([\"Which baking dish is best to bake a banana bread ?\", \"Why not put knives in the dishwasher?\"], k=3)\n\nOf course, you can also save and load a model to/from a file as `in the\nword representation usage <#saving-and-loading-a-model-object>`__.\n\nFor more information about text classification usage of fasttext, you\ncan refer to our `text classification\ntutorial `__.\n\nCompress model files with quantization\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\nWhen you want to save a supervised model file, fastText can compress it\nin order to have a much smaller model file by sacrificing only a little\nbit performance.\n\n.. code:: py\n\n # with the previously trained `model` object, call :\n model.quantize(input='data.train.txt', retrain=True)\n\n # then display results and save the new model :\n print_results(*model.test(valid_data))\n model.save_model(\"model_filename.ftz\")\n\n``model_filename.ftz`` will have a much smaller size than\n``model_filename.bin``.\n\nFor further reading on quantization, you can refer to `this paragraph\nfrom our blog\npost `__.\n\nIMPORTANT: Preprocessing data / encoding conventions\n----------------------------------------------------\n\nIn general it is important to properly preprocess your data. In\nparticular our example scripts in the `root\nfolder `__ do this.\n\nfastText assumes UTF-8 encoded text. All text must be `unicode for\nPython2 `__\nand `str for\nPython3 `__.\nThe passed text will be `encoded as UTF-8 by\npybind11 `__\nbefore passed to the fastText C++ library. This means it is important to\nuse UTF-8 encoded text when building a model. On Unix-like systems you\ncan convert text using `iconv `__.\n\nfastText will tokenize (split text into pieces) based on the following\nASCII characters (bytes). In particular, it is not aware of UTF-8\nwhitespace. We advice the user to convert UTF-8 whitespace / word\nboundaries into one of the following symbols as appropiate.\n\n- space\n- tab\n- vertical tab\n- carriage return\n- formfeed\n- the null character\n\nThe newline character is used to delimit lines of text. In particular,\nthe EOS token is appended to a line of text if a newline character is\nencountered. The only exception is if the number of tokens exceeds the\nMAX\\_LINE\\_SIZE constant as defined in the `Dictionary\nheader `__.\nThis means if you have text that is not separate by newlines, such as\nthe `fil9 dataset `__, it will be\nbroken into chunks with MAX\\_LINE\\_SIZE of tokens and the EOS token is\nnot appended.\n\nThe length of a token is the number of UTF-8 characters by considering\nthe `leading two bits of a\nbyte `__ to identify\n`subsequent bytes of a multi-byte\nsequence `__.\nKnowing this is especially important when choosing the minimum and\nmaximum length of subwords. Further, the EOS token (as specified in the\n`Dictionary\nheader `__)\nis considered a character and will not be broken into subwords.\n\nMore examples\n-------------\n\nIn order to have a better knowledge of fastText models, please consider\nthe main\n`README `__\nand in particular `the tutorials on our\nwebsite `__.\n\nYou can find further python examples in `the doc\nfolder `__.\n\nAs with any package you can get help on any Python function using the\nhelp function.\n\nFor example\n\n::\n\n +>>> import fasttext\n +>>> help(fasttext.FastText)\n\n Help on module fasttext.FastText in fasttext:\n\n NAME\n fasttext.FastText\n\n DESCRIPTION\n # Copyright (c) 2017-present, Facebook, Inc.\n # All rights reserved.\n #\n # This source code is licensed under the MIT license found in the\n # LICENSE file in the root directory of this source tree.\n\n FUNCTIONS\n load_model(path)\n Load a model given a filepath and return a model object.\n\n tokenize(text)\n Given a string of text, tokenize it and return a list of tokens\n [...]\n\nAPI\n===\n\n``train_unsupervised`` parameters\n---------------------------------\n\n.. code:: python\n\n input # training file path (required)\n model # unsupervised fasttext model {cbow, skipgram} [skipgram]\n lr # learning rate [0.05]\n dim # size of word vectors [100]\n ws # size of the context window [5]\n epoch # number of epochs [5]\n minCount # minimal number of word occurences [5]\n minn # min length of char ngram [3]\n maxn # max length of char ngram [6]\n neg # number of negatives sampled [5]\n wordNgrams # max length of word ngram [1]\n loss # loss function {ns, hs, softmax, ova} [ns]\n bucket # number of buckets [2000000]\n thread # number of threads [number of cpus]\n lrUpdateRate # change the rate of updates for the learning rate [100]\n t # sampling threshold [0.0001]\n verbose # verbose [2]\n\n``train_supervised`` parameters\n-------------------------------\n\n.. code:: python\n\n input # training file path (required)\n lr # learning rate [0.1]\n dim # size of word vectors [100]\n ws # size of the context window [5]\n epoch # number of epochs [5]\n minCount # minimal number of word occurences [1]\n minCountLabel # minimal number of label occurences [1]\n minn # min length of char ngram [0]\n maxn # max length of char ngram [0]\n neg # number of negatives sampled [5]\n wordNgrams # max length of word ngram [1]\n loss # loss function {ns, hs, softmax, ova} [softmax]\n bucket # number of buckets [2000000]\n thread # number of threads [number of cpus]\n lrUpdateRate # change the rate of updates for the learning rate [100]\n t # sampling threshold [0.0001]\n label # label prefix ['__label__']\n verbose # verbose [2]\n pretrainedVectors # pretrained word vectors (.vec file) for supervised learning []\n\n``model`` object\n----------------\n\n``train_supervised``, ``train_unsupervised`` and ``load_model``\nfunctions return an instance of ``_FastText`` class, that we generaly\nname ``model`` object.\n\nThis object exposes those training arguments as properties : ``lr``,\n``dim``, ``ws``, ``epoch``, ``minCount``, ``minCountLabel``, ``minn``,\n``maxn``, ``neg``, ``wordNgrams``, ``loss``, ``bucket``, ``thread``,\n``lrUpdateRate``, ``t``, ``label``, ``verbose``, ``pretrainedVectors``.\nSo ``model.wordNgrams`` will give you the max length of word ngram used\nfor training this model.\n\nIn addition, the object exposes several functions :\n\n.. code:: python\n\n get_dimension # Get the dimension (size) of a lookup vector (hidden layer).\n # This is equivalent to `dim` property.\n get_input_vector # Given an index, get the corresponding vector of the Input Matrix.\n get_input_matrix # Get a copy of the full input matrix of a Model.\n get_labels # Get the entire list of labels of the dictionary\n # This is equivalent to `labels` property.\n get_line # Split a line of text into words and labels.\n get_output_matrix # Get a copy of the full output matrix of a Model.\n get_sentence_vector # Given a string, get a single vector represenation. This function\n # assumes to be given a single line of text. We split words on\n # whitespace (space, newline, tab, vertical tab) and the control\n # characters carriage return, formfeed and the null character.\n get_subword_id # Given a subword, return the index (within input matrix) it hashes to.\n get_subwords # Given a word, get the subwords and their indicies.\n get_word_id # Given a word, get the word id within the dictionary.\n get_word_vector # Get the vector representation of word.\n get_words # Get the entire list of words of the dictionary\n # This is equivalent to `words` property.\n is_quantized # whether the model has been quantized\n predict # Given a string, get a list of labels and a list of corresponding probabilities.\n quantize # Quantize the model reducing the size of the model and it's memory footprint.\n save_model # Save the model to the given path\n test # Evaluate supervised model using file given by path\n test_label # Return the precision and recall score for each label.\n\nThe properties ``words``, ``labels`` return the words and labels from\nthe dictionary :\n\n.. code:: py\n\n model.words # equivalent to model.get_words()\n model.labels # equivalent to model.get_labels()\n\nThe object overrides ``__getitem__`` and ``__contains__`` functions in\norder to return the representation of a word and to check if a word is\nin the vocabulary.\n\n.. code:: py\n\n model['king'] # equivalent to model.get_word_vector('king')\n 'king' in model # equivalent to `'king' in model.get_words()`\n\nJoin the fastText community\n---------------------------\n\n- `Facebook page `__\n- `Stack\n overflow `__\n- `Google\n group `__\n- `GitHub `__\n\n.. |CircleCI| image:: https://circleci.com/gh/facebookresearch/fastText/tree/master.svg?style=svg\n :target: https://circleci.com/gh/facebookresearch/fastText/tree/master\n" }, { "path": "python/benchmarks/README.rst", "content": "These programs allow us to compare the performance of a few key operations when consindering changes. \n\nIt is important to run these to make sure a change doesn't introduce a regression.\n" }, { "path": "python/benchmarks/get_word_vector.py", "content": "# Copyright (c) 2017-present, Facebook, Inc.\n# All rights reserved.\n#\n# This source code is licensed under the MIT license found in the\n# LICENSE file in the root directory of this source tree.\n\nfrom __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\nfrom __future__ import unicode_literals\n\nfrom fasttext import load_model\nfrom fasttext import tokenize\nimport sys\nimport time\nimport argparse\n\n\ndef get_word_vector(data, model):\n t1 = time.time()\n print(\"Reading\")\n with open(data, \"r\") as f:\n tokens = tokenize(f.read())\n t2 = time.time()\n print(\"Read TIME: \" + str(t2 - t1))\n print(\"Read NUM : \" + str(len(tokens)))\n f = load_model(model)\n # This is not equivalent to piping the data into\n # print-word-vector, because the data is tokenized\n # first.\n t3 = time.time()\n i = 0\n for t in tokens:\n f.get_word_vector(t)\n i += 1\n if i % 10000 == 0:\n sys.stderr.write(\"\\ri: \" + str(float(i / len(tokens))))\n sys.stderr.flush()\n t4 = time.time()\n print(\"\\nVectoring: \" + str(t4 - t3))\n\n\nif __name__ == \"__main__\":\n parser = argparse.ArgumentParser(\n description=\"Simple benchmark for get_word_vector.\"\n )\n parser.add_argument(\"model\", help=\"A model file to use for benchmarking.\")\n parser.add_argument(\"data\", help=\"A data file to use for benchmarking.\")\n args = parser.parse_args()\n get_word_vector(args.data, args.model)\n" }, { "path": "python/doc/examples/FastTextEmbeddingBag.py", "content": "#!/usr/bin/env python\n\n# Copyright (c) 2017-present, Facebook, Inc.\n# All rights reserved.\n#\n# This source code is licensed under the MIT license found in the\n# LICENSE file in the root directory of this source tree.\n\n# NOTE: This requires PyTorch! We do not provide installation scripts to install PyTorch.\n# It is up to you to install this dependency if you want to execute this example.\n# PyTorch's website should give you clear instructions on this: http://pytorch.org/\n\nfrom __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\nfrom __future__ import unicode_literals\nfrom torch.nn.modules.sparse import EmbeddingBag\nimport numpy as np\nimport torch\nimport random\nimport string\nimport time\nfrom fasttext import load_model\nfrom torch.autograd import Variable\n\n\nclass FastTextEmbeddingBag(EmbeddingBag):\n def __init__(self, model_path):\n self.model = load_model(model_path)\n input_matrix = self.model.get_input_matrix()\n input_matrix_shape = input_matrix.shape\n super().__init__(input_matrix_shape[0], input_matrix_shape[1])\n self.weight.data.copy_(torch.FloatTensor(input_matrix))\n\n def forward(self, words):\n word_subinds = np.empty([0], dtype=np.int64)\n word_offsets = [0]\n for word in words:\n _, subinds = self.model.get_subwords(word)\n word_subinds = np.concatenate((word_subinds, subinds))\n word_offsets.append(word_offsets[-1] + len(subinds))\n word_offsets = word_offsets[:-1]\n ind = Variable(torch.LongTensor(word_subinds))\n offsets = Variable(torch.LongTensor(word_offsets))\n return super().forward(ind, offsets)\n\n\ndef random_word(N):\n return ''.join(\n random.choices(\n string.ascii_uppercase + string.ascii_lowercase + string.digits,\n k=N\n )\n )\n\n\nif __name__ == \"__main__\":\n ft_emb = FastTextEmbeddingBag(\"fil9.bin\")\n model = load_model(\"fil9.bin\")\n num_lines = 200\n total_seconds = 0.0\n total_words = 0\n for _ in range(num_lines):\n words = [\n random_word(random.randint(1, 10))\n for _ in range(random.randint(15, 25))\n ]\n total_words += len(words)\n words_average_length = sum([len(word) for word in words]) / len(words)\n start = time.clock()\n words_emb = ft_emb(words)\n total_seconds += (time.clock() - start)\n for i in range(len(words)):\n word = words[i]\n ft_word_emb = model.get_word_vector(word)\n py_emb = np.array(words_emb[i].data)\n assert (np.isclose(ft_word_emb, py_emb).all())\n print(\n \"Avg. {:2.5f} seconds to build embeddings for {} lines with a total of {} words.\".\n format(total_seconds, num_lines, total_words)\n )\n" }, { "path": "python/doc/examples/bin_to_vec.py", "content": "#!/usr/bin/env python\n\n# Copyright (c) 2017-present, Facebook, Inc.\n# All rights reserved.\n#\n# This source code is licensed under the MIT license found in the\n# LICENSE file in the root directory of this source tree.\n\nfrom __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\nfrom __future__ import unicode_literals\nfrom __future__ import division, absolute_import, print_function\n\nfrom fasttext import load_model\nimport argparse\nimport errno\n\nif __name__ == \"__main__\":\n parser = argparse.ArgumentParser(\n description=(\"Print fasttext .vec file to stdout from .bin file\")\n )\n parser.add_argument(\n \"model\",\n help=\"Model to use\",\n )\n args = parser.parse_args()\n\n f = load_model(args.model)\n words = f.get_words()\n print(str(len(words)) + \" \" + str(f.get_dimension()))\n for w in words:\n v = f.get_word_vector(w)\n vstr = \"\"\n for vi in v:\n vstr += \" \" + str(vi)\n try:\n print(w + vstr)\n except IOError as e:\n if e.errno == errno.EPIPE:\n pass\n" }, { "path": "python/doc/examples/compute_accuracy.py", "content": "#!/usr/bin/env python\n\n# Copyright (c) 2017-present, Facebook, Inc.\n# All rights reserved.\n#\n# This source code is licensed under the MIT license found in the\n# LICENSE file in the root directory of this source tree.\n\nfrom __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\nfrom __future__ import unicode_literals\nfrom __future__ import division, absolute_import, print_function\n\nfrom fasttext import load_model\nfrom fasttext import util\nimport argparse\nimport numpy as np\n\n\ndef process_question(question, cossims, model, words, vectors):\n correct = 0\n num_qs = 0\n num_lines = 0\n for line in question:\n num_lines += 1\n qwords = line.split()\n # We lowercase all words to correspond to the preprocessing\n # we applied to our data.\n qwords = [x.lower().strip() for x in qwords]\n # If one of the words is not in the vocabulary we skip this question\n found = True\n for w in qwords:\n if w not in words:\n found = False\n break\n if not found:\n continue\n # The first three words form the query\n # We retrieve their word vectors and normalize them\n query = qwords[:3]\n query = [model.get_word_vector(x) for x in query]\n query = [x / np.linalg.norm(x) for x in query]\n # Get the query vector. Example:\n # Germany - Berlin + France\n query = query[1] - query[0] + query[2]\n # We don't need to rank all the words, only until we found\n # the first word not equal to our set of query words.\n ban_set = list(map(lambda x: words.index(x), qwords[:3]))\n if words[util.find_nearest_neighbor(\n query, vectors, ban_set, cossims=cossims\n )] == qwords[3]:\n correct += 1\n num_qs += 1\n return correct, num_qs, num_lines\n\n\n# We use the same conventions as within compute-accuracy\ndef print_compute_accuracy_score(\n question, correct, num_qs, total_accuracy, semantic_accuracy,\n syntactic_accuracy\n):\n print(\n (\n \"{0:>30}: ACCURACY TOP1: {3:.2f} % ({1} / {2})\\t Total accuracy: {4:.2f} % Semantic accuracy: {5:.2f} % Syntactic accuracy: {6:.2f} %\"\n ).format(\n question,\n correct,\n num_qs,\n correct / float(num_qs) * 100 if num_qs > 0 else 0,\n total_accuracy * 100,\n semantic_accuracy * 100,\n syntactic_accuracy * 100,\n )\n )\n\n\nif __name__ == \"__main__\":\n parser = argparse.ArgumentParser(\n description=(\n \"compute_accuracy equivalent in Python. \"\n \"See https://github.com/tmikolov/word2vec/blob/master/demo-word-accuracy.sh\"\n )\n )\n parser.add_argument(\n \"model\",\n help=\"Model to use\",\n )\n parser.add_argument(\n \"question_words\",\n help=\"word questions similar to tmikolov's file (see help for link)\",\n )\n parser.add_argument(\n \"threshold\",\n help=\"threshold used to limit number of words used\",\n )\n args = parser.parse_args()\n args.threshold = int(args.threshold)\n\n # Retrieve list of normalized word vectors for the first words up\n # until the threshold count.\n f = load_model(args.model)\n # Gets words with associated frequeny sorted by default by descending order\n words, freq = f.get_words(include_freq=True)\n words = words[:args.threshold]\n vectors = np.zeros((len(words), f.get_dimension()), dtype=float)\n for i in range(len(words)):\n wv = f.get_word_vector(words[i])\n wv = wv / np.linalg.norm(wv)\n vectors[i] = wv\n\n total_correct = 0\n total_qs = 0\n total_num_lines = 0\n\n total_se_correct = 0\n total_se_qs = 0\n\n total_sy_correct = 0\n total_sy_qs = 0\n\n qid = 0\n questions = []\n with open(args.question_words, 'r') as fqw:\n questions = fqw.read().split(':')[1:]\n # For efficiency preallocate the memory to calculate cosine similarities\n cossims = np.zeros(len(words), dtype=float)\n for question in questions:\n quads = question.split('\\n')\n question = quads[0].strip()\n quads = quads[1:-1]\n correct, num_qs, num_lines = process_question(\n quads, cossims, f, words, vectors\n )\n total_qs += num_qs\n total_correct += correct\n total_num_lines += num_lines\n\n if (qid < 5):\n total_se_correct += correct\n total_se_qs += num_qs\n else:\n total_sy_correct += correct\n total_sy_qs += num_qs\n\n print_compute_accuracy_score(\n question,\n correct,\n num_qs,\n total_correct / float(total_qs) if total_qs > 0 else 0,\n total_se_correct / float(total_se_qs) if total_se_qs > 0 else 0,\n total_sy_correct / float(total_sy_qs) if total_sy_qs > 0 else 0,\n )\n qid += 1\n\n print(\n \"Questions seen / total: {0} {1} {2:.2f} %\".\n format(\n total_qs,\n total_num_lines,\n total_qs / total_num_lines * 100 if total_num_lines > 0 else 0,\n )\n )\n" }, { "path": "python/doc/examples/get_vocab.py", "content": "#!/usr/bin/env python\n\n# Copyright (c) 2017-present, Facebook, Inc.\n# All rights reserved.\n#\n# This source code is licensed under the MIT license found in the\n# LICENSE file in the root directory of this source tree.\n\nfrom __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\nfrom __future__ import unicode_literals\nfrom __future__ import division, absolute_import, print_function\n\nfrom fasttext import load_model\nimport argparse\nimport errno\n\nif __name__ == \"__main__\":\n parser = argparse.ArgumentParser(\n description=(\n \"Print words or labels and frequency of a model's dictionary\"\n )\n )\n parser.add_argument(\n \"model\",\n help=\"Model to use\",\n )\n parser.add_argument(\n \"-l\",\n \"--labels\",\n help=\"Print labels instead of words\",\n action='store_true',\n default=False,\n )\n args = parser.parse_args()\n\n f = load_model(args.model)\n if args.labels:\n words, freq = f.get_labels(include_freq=True)\n else:\n words, freq = f.get_words(include_freq=True)\n for w, f in zip(words, freq):\n try:\n print(w + \"\\t\" + str(f))\n except IOError as e:\n if e.errno == errno.EPIPE:\n pass\n" }, { "path": "python/doc/examples/train_supervised.py", "content": "#!/usr/bin/env python\n\n# Copyright (c) 2017-present, Facebook, Inc.\n# All rights reserved.\n#\n# This source code is licensed under the MIT license found in the\n# LICENSE file in the root directory of this source tree.\n\nfrom __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\nfrom __future__ import unicode_literals\n\nimport os\nfrom fasttext import train_supervised\n\n\ndef print_results(N, p, r):\n print(\"N\\t\" + str(N))\n print(\"P@{}\\t{:.3f}\".format(1, p))\n print(\"R@{}\\t{:.3f}\".format(1, r))\n\n\nif __name__ == \"__main__\":\n train_data = os.path.join(os.getenv(\"DATADIR\", ''), 'cooking.train')\n valid_data = os.path.join(os.getenv(\"DATADIR\", ''), 'cooking.valid')\n\n # train_supervised uses the same arguments and defaults as the fastText cli\n model = train_supervised(\n input=train_data, epoch=25, lr=1.0, wordNgrams=2, verbose=2, minCount=1\n )\n print_results(*model.test(valid_data))\n\n model = train_supervised(\n input=train_data, epoch=25, lr=1.0, wordNgrams=2, verbose=2, minCount=1,\n loss=\"hs\"\n )\n print_results(*model.test(valid_data))\n model.save_model(\"cooking.bin\")\n\n model.quantize(input=train_data, qnorm=True, retrain=True, cutoff=100000)\n print_results(*model.test(valid_data))\n model.save_model(\"cooking.ftz\")\n" }, { "path": "python/doc/examples/train_unsupervised.py", "content": "#!/usr/bin/env python\n\n# Copyright (c) 2017-present, Facebook, Inc.\n# All rights reserved.\n#\n# This source code is licensed under the MIT license found in the\n# LICENSE file in the root directory of this source tree.\n\nfrom __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\nfrom __future__ import unicode_literals\nfrom __future__ import division, absolute_import, print_function\n\nfrom fasttext import train_unsupervised\nimport numpy as np\nimport os\nfrom scipy import stats\n\n\n# Because of fasttext we don't need to account for OOV\ndef compute_similarity(data_path):\n def similarity(v1, v2):\n n1 = np.linalg.norm(v1)\n n2 = np.linalg.norm(v2)\n return np.dot(v1, v2) / n1 / n2\n\n mysim = []\n gold = []\n\n with open(data_path, 'rb') as fin:\n for line in fin:\n tline = line.split()\n word1 = tline[0].lower()\n word2 = tline[1].lower()\n\n v1 = model.get_word_vector(word1)\n v2 = model.get_word_vector(word2)\n d = similarity(v1, v2)\n mysim.append(d)\n gold.append(float(tline[2]))\n\n corr = stats.spearmanr(mysim, gold)\n dataset = os.path.basename(data_path)\n correlation = corr[0] * 100\n return dataset, correlation, 0\n\n\nif __name__ == \"__main__\":\n model = train_unsupervised(\n input=os.path.join(os.getenv(\"DATADIR\", ''), 'fil9'),\n model='skipgram',\n )\n model.save_model(\"fil9.bin\")\n dataset, corr, oov = compute_similarity('rw.txt')\n print(\"{0:20s}: {1:2.0f} (OOV: {2:2.0f}%)\".format(dataset, corr, 0))\n" }, { "path": "quantization-example.sh", "content": "myshuf() {\n perl -MList::Util=shuffle -e 'print shuffle(<>);' \"$@\";\n}\n\nnormalize_text() {\n tr '[:upper:]' '[:lower:]' | sed -e 's/^/__label__/g' | \\\n sed -e \"s/'/ ' /g\" -e 's/\"//g' -e 's/\\./ \\. /g' -e 's/
/ /g' \\\n -e 's/,/ , /g' -e 's/(/ ( /g' -e 's/)/ ) /g' -e 's/\\!/ \\! /g' \\\n -e 's/\\?/ \\? /g' -e 's/\\;/ /g' -e 's/\\:/ /g' | tr -s \" \" | myshuf\n}\n\nRESULTDIR=result\nDATADIR=data\n\nmkdir -p \"${RESULTDIR}\"\nmkdir -p \"${DATADIR}\"\n\nif [ ! -f \"${DATADIR}/dbpedia.train\" ]\nthen\n wget -c \"https://drive.google.com/uc?export=download&id=0Bz8a_Dbh9QhbQ2Vic1kxMmZZQ1k\" -O \"${DATADIR}/dbpedia_csv.tar.gz\"\n tar -xzvf \"${DATADIR}/dbpedia_csv.tar.gz\" -C \"${DATADIR}\"\n cat \"${DATADIR}/dbpedia_csv/train.csv\" | normalize_text > \"${DATADIR}/dbpedia.train\"\n cat \"${DATADIR}/dbpedia_csv/test.csv\" | normalize_text > \"${DATADIR}/dbpedia.test\"\nfi\n\nmake\n\necho \"Training...\"\n./fasttext supervised -input \"${DATADIR}/dbpedia.train\" -output \"${RESULTDIR}/dbpedia\" -dim 10 -lr 0.1 -wordNgrams 2 -minCount 1 -bucket 10000000 -epoch 5 -thread 4\n\necho \"Quantizing...\"\n./fasttext quantize -output \"${RESULTDIR}/dbpedia\" -input \"${DATADIR}/dbpedia.train\" -qnorm -retrain -epoch 1 -cutoff 100000\n\necho \"Testing original model...\"\n./fasttext test \"${RESULTDIR}/dbpedia.bin\" \"${DATADIR}/dbpedia.test\"\necho \"Testing quantized model...\"\n./fasttext test \"${RESULTDIR}/dbpedia.ftz\" \"${DATADIR}/dbpedia.test\"\n\nwc -c < \"${RESULTDIR}/dbpedia.bin\" | awk '{print \"Size of the original model:\\t\",$1;}'\nwc -c < \"${RESULTDIR}/dbpedia.ftz\" | awk '{print \"Size of the quantized model:\\t\",$1;}'\n" }, { "path": "reduce_model.py", "content": "#!/usr/bin/env python\n# -*- coding: utf-8 -*-\n#\n# Copyright (c) 2017-present, Facebook, Inc.\n# All rights reserved.\n#\n# This source code is licensed under the MIT license found in the\n# LICENSE file in the root directory of this source tree.\n\nfrom __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\nfrom __future__ import unicode_literals\n\nimport argparse\nimport os\nimport re\nimport sys\n\nimport fasttext\nimport fasttext.util\n\nargs = None\n\n\ndef eprint(*args, **kwargs):\n print(*args, file=sys.stderr, **kwargs)\n\n\ndef guess_target_name(model_file, initial_dim, target_dim):\n \"\"\"\n Given a model name with the convention a..b, this function\n returns the model's name with `target_dim` value.\n For example model_file name `cc.en.300.bin` with initial dim 300 becomes\n `cc.en.100.bin` when the `target_dim` is 100.\n \"\"\"\n prg = re.compile(\"(.*).%s.(.*)\" % initial_dim)\n m = prg.match(model_file)\n if m:\n return \"%s.%d.%s\" % (m.group(1), target_dim, m.group(2))\n\n sp_ext = os.path.splitext(model_file)\n return \"%s.%d%s\" % (sp_ext[0], target_dim, sp_ext[1])\n\n\ndef command_reduce(model_file, target_dim, if_exists):\n \"\"\"\n Given a `model_file`, this function reduces its dimension to `target_dim`\n by applying a PCA.\n \"\"\"\n eprint(\"Loading model\")\n\n ft = fasttext.load_model(model_file)\n initial_dim = ft.get_dimension()\n if target_dim >= initial_dim:\n raise Exception(\"Target dimension (%d) should be less than initial dimension (%d).\" % (\n target_dim, initial_dim))\n\n result_filename = guess_target_name(model_file, initial_dim, target_dim)\n if os.path.isfile(result_filename):\n if if_exists == 'overwrite':\n pass\n elif if_exists == 'strict':\n raise Exception(\n \"File already exists. Use --overwrite to overwrite.\")\n elif if_exists == 'ignore':\n return result_filename\n\n eprint(\"Reducing matrix dimensions\")\n fasttext.util.reduce_model(ft, target_dim)\n\n eprint(\"Saving model\")\n ft.save_model(result_filename)\n eprint(\"%s saved\" % result_filename)\n\n return result_filename\n\n\ndef main():\n global args\n\n parser = argparse.ArgumentParser(\n description='fastText helper tool to reduce model dimensions.')\n parser.add_argument(\"model\", type=str,\n help=\"model file to reduce. model.bin\")\n parser.add_argument(\"dim\", type=int,\n help=\"targeted dimension of word vectors.\")\n parser.add_argument(\"--overwrite\", action=\"store_true\",\n help=\"overwrite if file exists.\")\n\n args = parser.parse_args()\n\n command_reduce(args.model, args.dim, if_exists=(\n 'overwrite' if args.overwrite else 'strict'))\n\n\nif __name__ == '__main__':\n main()\n" }, { "path": "runtests.py", "content": "#!/usr/bin/env python\n# -*- coding: utf-8 -*-\n#\n# Copyright (c) 2016-present, Facebook, Inc.\n# All rights reserved.\n#\n# This source code is licensed under the MIT license found in the\n# LICENSE file in the root directory of this source tree.\n#\n\n# To run the integration tests you must first fetch all the required test data.\n# Have a look at tests/fetch_test_data.sh\n# You will then need to point this script to the corresponding folder\n\nfrom __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\nfrom __future__ import unicode_literals\n\nimport unittest\nimport argparse\nfrom fasttext.tests import gen_tests\nfrom fasttext.tests import gen_unit_tests\n\n\ndef run_tests(tests):\n suite = unittest.TestLoader().loadTestsFromTestCase(tests)\n unittest.TextTestRunner(verbosity=3).run(suite)\n\n\nif __name__ == \"__main__\":\n parser = argparse.ArgumentParser()\n parser.add_argument(\n \"-u\", \"--unit-tests\", help=\"run unit tests\", action=\"store_true\"\n )\n parser.add_argument(\n \"-i\",\n \"--integration-tests\",\n help=\"run integration tests\",\n action=\"store_true\"\n )\n parser.add_argument(\n \"-v\",\n \"--verbose\",\n default=1,\n help=\"verbosity level (default 1)\",\n type=int,\n )\n parser.add_argument(\"--data-dir\", help=\"Full path to data directory\")\n args = parser.parse_args()\n if args.unit_tests:\n run_tests(gen_unit_tests(verbose=args.verbose))\n if args.integration_tests:\n if args.data_dir is None:\n raise ValueError(\n \"Need data directory! Consult tests/fetch_test_data.sh\"\n )\n run_tests(gen_tests(args.data_dir, verbose=args.verbose))\n if not args.unit_tests and not args.integration_tests:\n print(\"Ran no tests\")\n" }, { "path": "scripts/kbcompletion/README.md", "content": "# Fast Linear Model for Knowledge Graph Embeddings\n\n## Knowledge base completion\n\nThese scripts require the [fastText library](https://github.com/facebookresearch/fastText).\n\nRun the data.sh script to download and format the datasets. Then run any of the scripts to train and test on a given dataset.\n\n## Reference\n\nIf you use this code please cite:\n\n@article{joulin2017fast,\n title={Fast Linear Model for Knowledge Graph Embeddings},\n author={Joulin, Armand and Grave, Edouard and Bojanowski, Piotr and Nickel, Maximilian and Mikolov, Tomas},\n journal={arXiv preprint arXiv:1710.10881},\n year={2017}\n}\n\n" }, { "path": "scripts/kbcompletion/data.sh", "content": "#!/usr/bin/env bash\n#\n# Copyright (c) 2017-present, Facebook, Inc.\n# All rights reserved.\n#\n# This source code is licensed under the MIT license found in the\n# LICENSE file in the root directory of this source tree.\n#\nset -e\nDATADIR=data/\n\nif [ ! -d \"$DATADIR\" ]; then\n mkdir $DATADIR\nfi\n\ncd $DATADIR\necho \"preparing WN18\"\n#wget -P . https://everest.hds.utc.fr/lib/exe/fetch.php?media=en:wordnet-mlj12.tar.gz\n#mv fetch.php\\?media\\=en\\:wordnet-mlj12.tar.gz wordnet-mlj12.tar.gz\nwget -P . https://github.com/mana-ysh/knowledge-graph-embeddings/raw/master/dat/wordnet-mlj12.tar.gz\ntar -xzvf wordnet-mlj12.tar.gz\nDIR=wordnet-mlj12\nfor f in ${DIR}/wordnet-ml*.txt;\ndo\n fn=${DIR}/ft_$(basename $f)\n awk '{print \"__label__\"$1,\"0_\"$2, $3;print $1,\"1_\"$2,\" __label__\"$3}' < ${f} > ${fn};\ndone\ncat ${DIR}/ft_* > ${DIR}/ft_wordnet-mlj12-full.txt\ncat ${DIR}/ft_*train.txt ${DIR}/ft_*valid.txt > ${DIR}/ft_wordnet-mlj12-valid+train.txt\n\necho \"preparing FB15K\"\n#wget https://everest.hds.utc.fr/lib/exe/fetch.php?media=en:fb15k.tgz\n#mv fetch.php\\?media\\=en\\:fb15k.tgz fb15k.tgz\nwget https://github.com/mana-ysh/knowledge-graph-embeddings/raw/master/dat/fb15k.tgz\ntar -xzvf fb15k.tgz\nDIR=FB15k/\nfor f in ${DIR}/freebase*.txt;\ndo\n fn=${DIR}/ft_$(basename $f)\n echo $f \" --> \" $fn\n awk '{print \"__label__\"$1,\"0_\"$2, $3;print $1,\"1_\"$2,\" __label__\"$3}' < ${f} > ${fn};\ndone\ncat ${DIR}/ft_* > ${DIR}/ft_freebase_mtr100_mte100-full.txt\ncat ${DIR}/ft_*train.txt ${DIR}/ft_*valid.txt > ${DIR}/ft_freebase_mtr100_mte100-valid+train.txt\n\necho \"preparing FB15K-237\"\nwget https://download.microsoft.com/download/8/7/0/8700516A-AB3D-4850-B4BB-805C515AECE1/FB15K-237.2.zip\nunzip FB15K-237.2.zip\nDIR=Release/\nfor f in train.txt test.txt valid.txt\ndo\n fn=${DIR}/ft_$(basename $f)\n echo $f \" --> \" $fn\n awk -F \"\\t\" '{print \"__label__\"$1,\"0_\"$2, $3;print $1,\"1_\"$2,\" __label__\"$3}' < ${DIR}/${f} > ${fn};\ndone\ncat ${DIR}/ft_*.txt > ${DIR}/ft_full.txt\ncat ${DIR}/ft_train.txt ${DIR}/ft_valid.txt > ${DIR}/ft_valid+train.txt\n\necho \"preparing SVO\"\nwget . https://everest.hds.utc.fr/lib/exe/fetch.php?media=en:svo-tensor-dataset.tar.gz\nmv fetch.php?media=en:svo-tensor-dataset.tar.gz svo-tensor-dataset.tar.gz\ntar -xzvf svo-tensor-dataset.tar.gz\nDIR=SVO-tensor-dataset\nfor f in ${DIR}/svo_data*.dat;\ndo\n fn=${DIR}/ft_$(basename $f)\n awk '{print \"0_\"$1,\"1_\"$3,\"__label__\"$2;}' < ${f} > ${fn};\ndone\ncat ${DIR}/ft_*train*.dat ${DIR}/ft_*valid*.dat > ${DIR}/ft_svo_data-valid+train.dat\n" }, { "path": "scripts/kbcompletion/eval.cpp", "content": "/**\n * Copyright (c) 2017-present, Facebook, Inc.\n * All rights reserved.\n *\n * This source code is licensed under the MIT license found in the\n * LICENSE file in the root directory of this source tree.\n */\n\n#include \n#include \n#include \n#include \n#include \n\nstd::string EOS = \"\";\n\nbool readWord(std::istream& in, std::string& word)\n{\n char c;\n std::streambuf& sb = *in.rdbuf();\n word.clear();\n while ((c = sb.sbumpc()) != EOF) {\n if (c == ' ' || c == '\\n' || c == '\\r' || c == '\\t' || c == '\\v' ||\n c == '\\f' || c == '\\0') {\n if (word.empty()) {\n if (c == '\\n') {\n word += EOS;\n return true;\n }\n continue;\n } else {\n if (c == '\\n')\n sb.sungetc();\n return true;\n }\n }\n word.push_back(c);\n }\n in.get();\n return !word.empty();\n}\n\nint main(int argc, char** argv) {\n int k = 10;\n if (argc < 4) {\n std::cerr<<\"eval []\"< > KB;\n\n while (kbf.peek() != EOF) {\n std::string label, key, word;\n while (readWord(kbf, word)) {\n if (word == EOS) {break;}\n if (word.find(\"__label__\") == 0) {label = word;}\n else {key += \"|\" + word;}\n }\n KB[key][label] = true;\n }\n kbf.close();\n\n double precision = 0.0;\n int32_t nexamples = 0;\n while (predf.peek() != EOF || gtf.peek() != EOF) {\n if (predf.peek() == EOF || gtf.peek() == EOF) {\n std::cerr<<\"pred / gt files have diff sizes\"< /dev/null | awk '{if(NR==3) print \"raw hit@10=\"$2}'\n\necho \"computing filtered hit@10...\"\n$ft predict ${model}.bin $DIR/ft_freebase_mtr100_mte100-test.txt 20000 > $pred\n./eval $pred ${DIR}/ft_freebase_mtr100_mte100-test.txt $DIR/ft_freebase_mtr100_mte100-full.txt 10 | awk '{if(NR==2) print \"filtered hit@10=\"$2}'\n\necho \"---- train+val ----\"\n\n$ft supervised -input $DIR/ft_freebase_mtr100_mte100-valid+train.txt \\\n -dim ${dim} -epoch ${dim} -output ${model} -lr .2 -thread 20 -loss ns -neg ${neg} -minCount 0\n\necho \"computing raw hits@10...\"\n$ft test ${model}.bin $DIR/ft_freebase_mtr100_mte100-test.txt 10 2> /dev/null | awk '{if(NR==3) print \"raw hit@10=\"$2}'\n\necho \"computing filtered hit@10...\"\n$ft predict ${model}.bin $DIR/ft_freebase_mtr100_mte100-test.txt 20000 > $pred\n./eval $pred ${DIR}/ft_freebase_mtr100_mte100-test.txt $DIR/ft_freebase_mtr100_mte100-full.txt 10 | awk '{if(NR==2) print \"filtered hit@10=\"$2}'\n" }, { "path": "scripts/kbcompletion/fb15k237.sh", "content": "#!/usr/bin/env bash\n#\n# copyright (c) 2017-present, facebook, inc.\n# all rights reserved.\n#\n# this source code is licensed under the MIT license found in the\n# license file in the root directory of this source tree.\n#\n# script for FB15k237\nDIR=data/Release/\nFASTTEXTDIR=../../\n\n# compile\n\npushd $FASTTEXTDIR\nmake opt\npopd\nft=${FASTTEXTDIR}/fasttext\n\ng++ -std=c++0x eval.cpp -o eval\n\n## Train model and test it on validation:\n\npred=data/fb237pred\nmodel=data/fb15k237\ndim=50\nepoch=10\nneg=500\n\necho \"---- train ----\"\n$ft supervised -input $DIR/ft_train.txt \\\n -dim $dim -epoch $epoch -output ${model} -lr .2 -thread 20 -loss ns -neg $neg -minCount 0\n\necho \"computing filtered hit@10...\"\n$ft predict ${model}.bin $DIR/ft_test.txt 20000 > $pred\n./eval $pred ${DIR}/ft_test.txt $DIR/ft_full.txt 10 | awk '{if(NR==2) print \"filtered hit@10=\"$2}'\n\necho \"---- train+val ----\"\n\n$ft supervised -input $DIR/ft_valid+train.txt \\\n -dim ${dim} -epoch ${dim} -output ${model} -lr .2 -thread 20 -loss ns -neg ${neg} -minCount 0\n\necho \"computing filtered hit@10...\"\n$ft predict ${model}.bin $DIR/ft_test.txt 20000 > $pred\n./eval $pred ${DIR}/ft_test.txt $DIR/ft_full.txt 10 | awk '{if(NR==2) print \"filtered hit@10=\"$2}'\n" }, { "path": "scripts/kbcompletion/svo.sh", "content": "#!/usr/bin/env bash\n#\n# copyright (c) 2017-present, facebook, inc.\n# all rights reserved.\n#\n# this source code is licensed under the MIT license found in the\n# license file in the root directory of this source tree.\n#\n# script for SVO\nDIR=data/SVO-tensor-dataset\nFASTTEXTDIR=../../\n\n# compile\npushd $FASTTEXTDIR\nmake opt\npopd\nft=${FASTTEXTDIR}/fasttext\n\n## Train model and test it on validation:\n\ndim=200\nepoch=3\nmodel=svo\n\necho \"---- train ----\"\ntime $ft supervised -input ${DIR}/ft_svo_data_train_1000000.dat \\\n -dim $dim -epoch $epoch -output ${model} -lr .2 -thread 20\n\necho \"computing raw hit@5%...\"\n$ft test ${model}.bin ${DIR}/ft_svo_data_test_250000.dat 227 2> /dev/null | awk '{if(NR==3) print \"raw hit@5%=\"$2}'\n\n\necho \"---- train + valid ----\"\ntime $ft supervised -input ${DIR}/ft_svo_data-valid+train.dat \\\n -dim $dim -epoch $epoch -output ${model} -lr .2 -thread 20\n\necho \"computing raw hit@5%...\"\n$ft test ${model}.bin ${DIR}/ft_svo_data_test_250000.dat 227 2> /dev/null | awk '{if(NR==3) print \"raw hit@5%=\"$2}'\n" }, { "path": "scripts/kbcompletion/wn18.sh", "content": "#!/usr/bin/env bash\n#\n# copyright (c) 2017-present, facebook, inc.\n# all rights reserved.\n#\n# this source code is licensed under the MIT license found in the\n# license file in the root directory of this source tree.\n#\n# script for WN11\nDIR=data/wordnet-mlj12/\nFASTTEXTDIR=../../\n\n# compile\n\npushd $FASTTEXTDIR\nmake opt\npopd\nft=${FASTTEXTDIR}/fasttext\n\ng++ -std=c++0x eval.cpp -o eval\n\n# Train model and test it:\ndim=100\nepoch=100\nneg=500\nmodel=data/wn\npred=data/wnpred\n\necho \"---- train ----\"\n$ft supervised -input ${DIR}/ft_wordnet-mlj12-train.txt \\\n -dim $dim -epoch $epoch -output ${model} -lr .2 -thread 20 -loss ns -neg $neg\n\necho \"computing raw hits@10...\"\n$ft test ${model}.bin ${DIR}/ft_wordnet-mlj12-test.txt 10 2> /dev/null | awk '{if(NR==3) print \"raw hit@10 = \"$2}'\n\necho \"computing filtered hit@10...\"\n$ft predict ${model}.bin ${DIR}/ft_wordnet-mlj12-test.txt 20000 > $pred\n./eval $pred ${DIR}/ft_wordnet-mlj12-test.txt $DIR/ft_wordnet-mlj12-full.txt 10 | awk '{if(NR==2) print \"filtered hit@10 = \"$2}'\n\necho \"---- train+val ----\"\n$ft supervised -input ${DIR}/ft_wordnet-mlj12-valid+train.txt \\\n -dim $dim -epoch $epoch -output ${model} -lr .2 -thread 20 -loss ns -neg $neg\n\necho \"computing raw hits@10...\"\n$ft test ${model}.bin ${DIR}/ft_wordnet-mlj12-test.txt 10 2> /dev/null | awk '{if(NR==3) print \"raw hit@10 = \"$2}'\n\necho \"computing filtered hit@10...\"\n$ft predict ${model}.bin ${DIR}/ft_wordnet-mlj12-test.txt 20000 > $pred\n./eval $pred ${DIR}/ft_wordnet-mlj12-test.txt $DIR/ft_wordnet-mlj12-full.txt 10 | awk '{if(NR==2) print \"filtered hit@10 = \"$2}'\n" }, { "path": "scripts/quantization/quantization-results.sh", "content": "#!/usr/bin/env bash\n#\n# Copyright (c) 2016-present, Facebook, Inc.\n# All rights reserved.\n#\n# This source code is licensed under the MIT license found in the\n# LICENSE file in the root directory of this source tree.\n#\n\n# This script applies quantization to the models from Table 1 in:\n# Bag of Tricks for Efficient Text Classification, arXiv 1607.01759, 2016\n\nset -e\n\nDATASET=(\n ag_news\n sogou_news\n dbpedia\n yelp_review_polarity\n yelp_review_full\n yahoo_answers\n amazon_review_full\n amazon_review_polarity\n)\n\n# These learning rates were chosen by validation on a subset of the training set.\nLR=( 0.25 0.5 0.5 0.1 0.1 0.1 0.05 0.05 )\n\nRESULTDIR=result\nDATADIR=data\n\necho 'Warning! Make sure you run the classification-results.sh script before this one'\necho 'Otherwise you can expect the commands in this script to fail'\n\nfor i in {0..7}\ndo\n echo \"Working on dataset ${DATASET[i]}\"\n ../../fasttext quantize -input \"${DATADIR}/${DATASET[i]}.train\" \\\n -output \"${RESULTDIR}/${DATASET[i]}\" -lr \"${LR[i]}\" \\\n -thread 4 -qnorm -retrain -epoch 5 -cutoff 100000 > /dev/null\n ../../fasttext test \"${RESULTDIR}/${DATASET[i]}.ftz\" \\\n \"${DATADIR}/${DATASET[i]}.test\"\ndone\n" }, { "path": "setup.cfg", "content": "[metadata]\ndescription-file = README.md\n" }, { "path": "setup.py", "content": "#!/usr/bin/env python\n\n# Copyright (c) 2017-present, Facebook, Inc.\n# All rights reserved.\n#\n# This source code is licensed under the MIT license found in the\n# LICENSE file in the root directory of this source tree.\n#\n\nfrom __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\nfrom __future__ import unicode_literals\n\nfrom setuptools import setup, Extension\nfrom setuptools.command.build_ext import build_ext\nimport sys\nimport setuptools\nimport os\nimport subprocess\nimport platform\nimport io\n\n__version__ = \"0.9.2\"\nFASTTEXT_SRC = \"src\"\n\n# Based on https://github.com/pybind/python_example\n\n\nclass get_pybind_include:\n \"\"\"Helper class to determine the pybind11 include path\n\n The purpose of this class is to postpone importing pybind11\n until it is actually installed, so that the ``get_include()``\n method can be invoked.\"\"\"\n\n def __init__(self, user=False):\n try:\n pass\n except ImportError:\n if subprocess.call([sys.executable, \"-m\", \"pip\", \"install\", \"pybind11\"]):\n raise RuntimeError(\"pybind11 install failed.\")\n\n self.user = user\n\n def __str__(self):\n import pybind11\n\n return pybind11.get_include(self.user)\n\n\ntry:\n coverage_index = sys.argv.index(\"--coverage\")\nexcept ValueError:\n coverage = False\nelse:\n del sys.argv[coverage_index]\n coverage = True\n\nfasttext_src_files = map(str, os.listdir(FASTTEXT_SRC))\nfasttext_src_cc = list(filter(lambda x: x.endswith(\".cc\"), fasttext_src_files))\n\nfasttext_src_cc = list(\n map(lambda x: str(os.path.join(FASTTEXT_SRC, x)), fasttext_src_cc)\n)\n\next_modules = [\n Extension(\n str(\"fasttext_pybind\"),\n [\n str(\"python/fasttext_module/fasttext/pybind/fasttext_pybind.cc\"),\n ]\n + fasttext_src_cc,\n include_dirs=[\n # Path to pybind11 headers\n get_pybind_include(),\n get_pybind_include(user=True),\n # Path to fasttext source code\n FASTTEXT_SRC,\n ],\n language=\"c++\",\n extra_compile_args=[\n \"-O0 -fno-inline -fprofile-arcs -pthread -march=native\"\n if coverage\n else \"-O3 -funroll-loops -pthread -march=native\"\n ],\n ),\n]\n\n\n# As of Python 3.6, CCompiler has a `has_flag` method.\n# cf http://bugs.python.org/issue26689\ndef has_flag(compiler, flags):\n \"\"\"Return a boolean indicating whether a flag name is supported on\n the specified compiler.\n \"\"\"\n import tempfile\n\n with tempfile.NamedTemporaryFile(\"w\", suffix=\".cpp\") as f:\n f.write(\"int main (int argc, char **argv) { return 0; }\")\n try:\n compiler.compile([f.name], extra_postargs=flags)\n except setuptools.distutils.errors.CompileError:\n return False\n return True\n\n\ndef cpp_flag(compiler):\n \"\"\"Return the -std=c++17 compiler flag.\"\"\"\n standards = [\"-std=c++17\"]\n for standard in standards:\n if has_flag(compiler, [standard]):\n return standard\n raise RuntimeError(\"Unsupported compiler -- at least C++17 support \" \"is needed!\")\n\n\nclass BuildExt(build_ext):\n \"\"\"A custom build extension for adding compiler-specific options.\"\"\"\n\n c_opts = {\n \"msvc\": [\"/EHsc\"],\n \"unix\": [],\n }\n\n def build_extensions(self):\n if sys.platform == \"darwin\":\n mac_osx_version = float(\".\".join(platform.mac_ver()[0].split(\".\")[:2]))\n os.environ[\"MACOSX_DEPLOYMENT_TARGET\"] = str(mac_osx_version)\n all_flags = [\"-stdlib=libc++\", \"-mmacosx-version-min=10.7\"]\n if has_flag(self.compiler, [all_flags[0]]):\n self.c_opts[\"unix\"] += [all_flags[0]]\n elif has_flag(self.compiler, all_flags):\n self.c_opts[\"unix\"] += all_flags\n else:\n raise RuntimeError(\n \"libc++ is needed! Failed to compile with {} and {}.\".format(\n \" \".join(all_flags), all_flags[0]\n )\n )\n ct = self.compiler.compiler_type\n opts = self.c_opts.get(ct, [])\n extra_link_args = []\n\n if coverage:\n coverage_option = \"--coverage\"\n opts.append(coverage_option)\n extra_link_args.append(coverage_option)\n\n if ct == \"unix\":\n opts.append('-DVERSION_INFO=\"%s\"' % self.distribution.get_version())\n opts.append(cpp_flag(self.compiler))\n if has_flag(self.compiler, [\"-fvisibility=hidden\"]):\n opts.append(\"-fvisibility=hidden\")\n elif ct == \"msvc\":\n opts.append('/DVERSION_INFO=\\\\\"%s\\\\\"' % self.distribution.get_version())\n for ext in self.extensions:\n ext.extra_compile_args = opts\n ext.extra_link_args = extra_link_args\n build_ext.build_extensions(self)\n\n\ndef _get_readme():\n \"\"\"\n Use pandoc to generate rst from md.\n pandoc --from=markdown --to=rst --output=python/README.rst python/README.md\n \"\"\"\n with io.open(\"python/README.rst\", encoding=\"utf-8\") as fid:\n return fid.read()\n\n\nsetup(\n name=\"fasttext\",\n version=__version__,\n author=\"Onur Celebi\",\n author_email=\"celebio@fb.com\",\n description=\"fasttext Python bindings\",\n long_description=_get_readme(),\n ext_modules=ext_modules,\n url=\"https://github.com/facebookresearch/fastText\",\n license=\"MIT\",\n classifiers=[\n \"Development Status :: 3 - Alpha\",\n \"Intended Audience :: Developers\",\n \"Intended Audience :: Science/Research\",\n \"License :: OSI Approved :: MIT License\",\n \"Programming Language :: Python :: 2.7\",\n \"Programming Language :: Python :: 3.4\",\n \"Programming Language :: Python :: 3.5\",\n \"Programming Language :: Python :: 3.6\",\n \"Topic :: Software Development\",\n \"Topic :: Scientific/Engineering\",\n \"Operating System :: Microsoft :: Windows\",\n \"Operating System :: POSIX\",\n \"Operating System :: Unix\",\n \"Operating System :: MacOS\",\n ],\n install_requires=[\"pybind11>=2.2\", \"setuptools >= 0.7.0\", \"numpy\"],\n cmdclass={\"build_ext\": BuildExt},\n packages=[\n str(\"fasttext\"),\n str(\"fasttext.util\"),\n str(\"fasttext.tests\"),\n ],\n package_dir={str(\"\"): str(\"python/fasttext_module\")},\n zip_safe=False,\n)\n" }, { "path": "src/aligned.h", "content": "#pragma once\n#include \n#include \n#ifdef _MSC_VER\n// Ensure _HAS_EXCEPTIONS is defined\n#include \n#include \n#endif\n\n#if !((defined(_MSC_VER) && !defined(__clang__)) ? (_HAS_EXCEPTIONS) : (__EXCEPTIONS))\n#include \n#endif\n\n// Aligned simple vector.\n\nnamespace intgemm {\n\ntemplate class AlignedVector {\n public:\n AlignedVector() : mem_(nullptr), size_(0) {}\n\n explicit AlignedVector(std::size_t size, std::size_t alignment = 64 /* CPU cares about this */)\n : size_(size) {\n#ifdef _MSC_VER\n mem_ = static_cast(_aligned_malloc(size * sizeof(T), alignment));\n if (!mem_) {\n# if (defined(_MSC_VER) && !defined(__clang__)) ? (_HAS_EXCEPTIONS) : (__EXCEPTIONS)\n throw std::bad_alloc();\n# else\n std::abort();\n# endif\n }\n#else\n if (posix_memalign(reinterpret_cast(&mem_), alignment, size * sizeof(T))) {\n# if (defined(_MSC_VER) && !defined(__clang__)) ? (_HAS_EXCEPTIONS) : (__EXCEPTIONS)\n throw std::bad_alloc();\n# else\n std::abort();\n# endif\n }\n#endif\n }\n\n template AlignedVector(InputIt first, InputIt last) \n : AlignedVector(last - first) {\n std::copy(first, last, begin());\n }\n\n AlignedVector(AlignedVector &&from) noexcept : mem_(from.mem_), size_(from.size_) {\n from.mem_ = nullptr;\n from.size_ = 0;\n }\n\n AlignedVector &operator=(AlignedVector &&from) {\n if (this == &from) return *this;\n release();\n mem_ = from.mem_;\n size_ = from.size_;\n from.mem_ = nullptr;\n from.size_ = 0;\n return *this;\n }\n\n AlignedVector(const AlignedVector&) = delete;\n AlignedVector& operator=(const AlignedVector&) = delete;\n\n ~AlignedVector() { release(); }\n\n std::size_t size() const { return size_; }\n\n T &operator[](std::size_t offset) { return mem_[offset]; }\n const T &operator[](std::size_t offset) const { return mem_[offset]; }\n\n T *begin() { return mem_; }\n const T *begin() const { return mem_; }\n T *end() { return mem_ + size_; }\n const T *end() const { return mem_ + size_; }\n\n T *data() { return mem_; }\n const T *data() const { return mem_; }\n\n template \n ReturnType *as() { return reinterpret_cast(mem_); }\n\n private:\n T *mem_;\n std::size_t size_;\n\n void release() {\n#ifdef _MSC_VER\n _aligned_free(mem_);\n#else\n std::free(mem_);\n#endif\n }\n};\n\n} // namespace intgemm\n" }, { "path": "src/args.cc", "content": "/**\n * Copyright (c) 2016-present, Facebook, Inc.\n * All rights reserved.\n *\n * This source code is licensed under the MIT license found in the\n * LICENSE file in the root directory of this source tree.\n */\n\n#include \"args.h\"\n\n#include \n#include \n\n#include \n#include \n#include \n#include \n\nnamespace fasttext {\n\nArgs::Args() {\n lr = 0.05;\n dim = 100;\n ws = 5;\n epoch = 5;\n minCount = 5;\n minCountLabel = 0;\n neg = 5;\n wordNgrams = 1;\n loss = loss_name::ns;\n model = model_name::sg;\n bucket = 2000000;\n minn = 3;\n maxn = 6;\n thread = 12;\n lrUpdateRate = 100;\n t = 1e-4;\n label = \"__label__\";\n verbose = 2;\n pretrainedVectors = \"\";\n saveOutput = false;\n seed = 0;\n\n qout = false;\n retrain = false;\n qnorm = false;\n cutoff = 0;\n dsub = 2;\n\n autotuneValidationFile = \"\";\n autotuneMetric = \"f1\";\n autotunePredictions = 1;\n autotuneDuration = 60 * 5; // 5 minutes\n autotuneModelSize = \"\";\n}\n\nstd::string Args::lossToString(loss_name ln) const {\n switch (ln) {\n case loss_name::hs:\n return \"hs\";\n case loss_name::ns:\n return \"ns\";\n case loss_name::softmax:\n return \"softmax\";\n case loss_name::ova:\n return \"one-vs-all\";\n }\n return \"Unknown loss!\"; // should never happen\n}\n\nstd::string Args::boolToString(bool b) const {\n if (b) {\n return \"true\";\n } else {\n return \"false\";\n }\n}\n\nstd::string Args::modelToString(model_name mn) const {\n switch (mn) {\n case model_name::cbow:\n return \"cbow\";\n case model_name::sg:\n return \"sg\";\n case model_name::sup:\n return \"sup\";\n }\n return \"Unknown model name!\"; // should never happen\n}\n\nstd::string Args::metricToString(metric_name mn) const {\n switch (mn) {\n case metric_name::f1score:\n return \"f1score\";\n case metric_name::f1scoreLabel:\n return \"f1scoreLabel\";\n case metric_name::precisionAtRecall:\n return \"precisionAtRecall\";\n case metric_name::precisionAtRecallLabel:\n return \"precisionAtRecallLabel\";\n case metric_name::recallAtPrecision:\n return \"recallAtPrecision\";\n case metric_name::recallAtPrecisionLabel:\n return \"recallAtPrecisionLabel\";\n }\n return \"Unknown metric name!\"; // should never happen\n}\n\nvoid Args::parseArgs(const std::vector& args) {\n const std::string& command(args[1]);\n if (command == \"supervised\") {\n model = model_name::sup;\n loss = loss_name::softmax;\n minCount = 1;\n minn = 0;\n maxn = 0;\n lr = 0.1;\n } else if (command == \"cbow\") {\n model = model_name::cbow;\n }\n for (int ai = 2; ai < args.size(); ai += 2) {\n if (args[ai][0] != '-') {\n std::cerr << \"Provided argument without a dash! Usage:\" << std::endl;\n printHelp();\n exit(EXIT_FAILURE);\n }\n try {\n setManual(args[ai].substr(1));\n\n if (args[ai] == \"-h\") {\n std::cerr << \"Here is the help! Usage:\" << std::endl;\n printHelp();\n exit(EXIT_FAILURE);\n } else if (args[ai] == \"-input\") {\n input = std::string(args.at(ai + 1));\n } else if (args[ai] == \"-output\") {\n output = std::string(args.at(ai + 1));\n } else if (args[ai] == \"-lr\") {\n lr = std::stof(args.at(ai + 1));\n } else if (args[ai] == \"-lrUpdateRate\") {\n lrUpdateRate = std::stoi(args.at(ai + 1));\n } else if (args[ai] == \"-dim\") {\n dim = std::stoi(args.at(ai + 1));\n } else if (args[ai] == \"-ws\") {\n ws = std::stoi(args.at(ai + 1));\n } else if (args[ai] == \"-epoch\") {\n epoch = std::stoi(args.at(ai + 1));\n } else if (args[ai] == \"-minCount\") {\n minCount = std::stoi(args.at(ai + 1));\n } else if (args[ai] == \"-minCountLabel\") {\n minCountLabel = std::stoi(args.at(ai + 1));\n } else if (args[ai] == \"-neg\") {\n neg = std::stoi(args.at(ai + 1));\n } else if (args[ai] == \"-wordNgrams\") {\n wordNgrams = std::stoi(args.at(ai + 1));\n } else if (args[ai] == \"-loss\") {\n if (args.at(ai + 1) == \"hs\") {\n loss = loss_name::hs;\n } else if (args.at(ai + 1) == \"ns\") {\n loss = loss_name::ns;\n } else if (args.at(ai + 1) == \"softmax\") {\n loss = loss_name::softmax;\n } else if (\n args.at(ai + 1) == \"one-vs-all\" || args.at(ai + 1) == \"ova\") {\n loss = loss_name::ova;\n } else {\n std::cerr << \"Unknown loss: \" << args.at(ai + 1) << std::endl;\n printHelp();\n exit(EXIT_FAILURE);\n }\n } else if (args[ai] == \"-bucket\") {\n bucket = std::stoi(args.at(ai + 1));\n } else if (args[ai] == \"-minn\") {\n minn = std::stoi(args.at(ai + 1));\n } else if (args[ai] == \"-maxn\") {\n maxn = std::stoi(args.at(ai + 1));\n } else if (args[ai] == \"-thread\") {\n thread = std::stoi(args.at(ai + 1));\n } else if (args[ai] == \"-t\") {\n t = std::stof(args.at(ai + 1));\n } else if (args[ai] == \"-label\") {\n label = std::string(args.at(ai + 1));\n } else if (args[ai] == \"-verbose\") {\n verbose = std::stoi(args.at(ai + 1));\n } else if (args[ai] == \"-pretrainedVectors\") {\n pretrainedVectors = std::string(args.at(ai + 1));\n } else if (args[ai] == \"-saveOutput\") {\n saveOutput = true;\n ai--;\n } else if (args[ai] == \"-seed\") {\n seed = std::stoi(args.at(ai + 1));\n } else if (args[ai] == \"-qnorm\") {\n qnorm = true;\n ai--;\n } else if (args[ai] == \"-retrain\") {\n retrain = true;\n ai--;\n } else if (args[ai] == \"-qout\") {\n qout = true;\n ai--;\n } else if (args[ai] == \"-cutoff\") {\n cutoff = std::stoi(args.at(ai + 1));\n } else if (args[ai] == \"-dsub\") {\n dsub = std::stoi(args.at(ai + 1));\n } else if (args[ai] == \"-autotune-validation\") {\n autotuneValidationFile = std::string(args.at(ai + 1));\n } else if (args[ai] == \"-autotune-metric\") {\n autotuneMetric = std::string(args.at(ai + 1));\n getAutotuneMetric(); // throws exception if not able to parse\n getAutotuneMetricLabel(); // throws exception if not able to parse\n } else if (args[ai] == \"-autotune-predictions\") {\n autotunePredictions = std::stoi(args.at(ai + 1));\n } else if (args[ai] == \"-autotune-duration\") {\n autotuneDuration = std::stoi(args.at(ai + 1));\n } else if (args[ai] == \"-autotune-modelsize\") {\n autotuneModelSize = std::string(args.at(ai + 1));\n } else {\n std::cerr << \"Unknown argument: \" << args[ai] << std::endl;\n printHelp();\n exit(EXIT_FAILURE);\n }\n } catch (std::out_of_range) {\n std::cerr << args[ai] << \" is missing an argument\" << std::endl;\n printHelp();\n exit(EXIT_FAILURE);\n }\n }\n if (input.empty() || output.empty()) {\n std::cerr << \"Empty input or output path.\" << std::endl;\n printHelp();\n exit(EXIT_FAILURE);\n }\n if (wordNgrams <= 1 && maxn == 0 && !hasAutotune()) {\n bucket = 0;\n }\n}\n\nvoid Args::printHelp() {\n printBasicHelp();\n printDictionaryHelp();\n printTrainingHelp();\n printAutotuneHelp();\n printQuantizationHelp();\n}\n\nvoid Args::printBasicHelp() {\n std::cerr << \"\\nThe following arguments are mandatory:\\n\"\n << \" -input training file path\\n\"\n << \" -output output file path\\n\"\n << \"\\nThe following arguments are optional:\\n\"\n << \" -verbose verbosity level [\" << verbose << \"]\\n\";\n}\n\nvoid Args::printDictionaryHelp() {\n std::cerr << \"\\nThe following arguments for the dictionary are optional:\\n\"\n << \" -minCount minimal number of word occurences [\"\n << minCount << \"]\\n\"\n << \" -minCountLabel minimal number of label occurences [\"\n << minCountLabel << \"]\\n\"\n << \" -wordNgrams max length of word ngram [\" << wordNgrams\n << \"]\\n\"\n << \" -bucket number of buckets [\" << bucket << \"]\\n\"\n << \" -minn min length of char ngram [\" << minn\n << \"]\\n\"\n << \" -maxn max length of char ngram [\" << maxn\n << \"]\\n\"\n << \" -t sampling threshold [\" << t << \"]\\n\"\n << \" -label labels prefix [\" << label << \"]\\n\";\n}\n\nvoid Args::printTrainingHelp() {\n std::cerr\n << \"\\nThe following arguments for training are optional:\\n\"\n << \" -lr learning rate [\" << lr << \"]\\n\"\n << \" -lrUpdateRate change the rate of updates for the learning \"\n \"rate [\"\n << lrUpdateRate << \"]\\n\"\n << \" -dim size of word vectors [\" << dim << \"]\\n\"\n << \" -ws size of the context window [\" << ws << \"]\\n\"\n << \" -epoch number of epochs [\" << epoch << \"]\\n\"\n << \" -neg number of negatives sampled [\" << neg << \"]\\n\"\n << \" -loss loss function {ns, hs, softmax, one-vs-all} [\"\n << lossToString(loss) << \"]\\n\"\n << \" -thread number of threads (set to 1 to ensure \"\n \"reproducible results) [\"\n << thread << \"]\\n\"\n << \" -pretrainedVectors pretrained word vectors for supervised \"\n \"learning [\"\n << pretrainedVectors << \"]\\n\"\n << \" -saveOutput whether output params should be saved [\"\n << boolToString(saveOutput) << \"]\\n\"\n << \" -seed random generator seed [\" << seed << \"]\\n\";\n}\n\nvoid Args::printAutotuneHelp() {\n std::cerr << \"\\nThe following arguments are for autotune:\\n\"\n << \" -autotune-validation validation file to be used \"\n \"for evaluation\\n\"\n << \" -autotune-metric metric objective {f1, \"\n \"f1:labelname} [\"\n << autotuneMetric << \"]\\n\"\n << \" -autotune-predictions number of predictions used \"\n \"for evaluation [\"\n << autotunePredictions << \"]\\n\"\n << \" -autotune-duration maximum duration in seconds [\"\n << autotuneDuration << \"]\\n\"\n << \" -autotune-modelsize constraint model file size [\"\n << autotuneModelSize << \"] (empty = do not quantize)\\n\";\n}\n\nvoid Args::printQuantizationHelp() {\n std::cerr\n << \"\\nThe following arguments for quantization are optional:\\n\"\n << \" -cutoff number of words and ngrams to retain [\"\n << cutoff << \"]\\n\"\n << \" -retrain whether embeddings are finetuned if a cutoff \"\n \"is applied [\"\n << boolToString(retrain) << \"]\\n\"\n << \" -qnorm whether the norm is quantized separately [\"\n << boolToString(qnorm) << \"]\\n\"\n << \" -qout whether the classifier is quantized [\"\n << boolToString(qout) << \"]\\n\"\n << \" -dsub size of each sub-vector [\" << dsub << \"]\\n\";\n}\n\nvoid Args::save(std::ostream& out) {\n out.write((char*)&(dim), sizeof(int));\n out.write((char*)&(ws), sizeof(int));\n out.write((char*)&(epoch), sizeof(int));\n out.write((char*)&(minCount), sizeof(int));\n out.write((char*)&(neg), sizeof(int));\n out.write((char*)&(wordNgrams), sizeof(int));\n out.write((char*)&(loss), sizeof(loss_name));\n out.write((char*)&(model), sizeof(model_name));\n out.write((char*)&(bucket), sizeof(int));\n out.write((char*)&(minn), sizeof(int));\n out.write((char*)&(maxn), sizeof(int));\n out.write((char*)&(lrUpdateRate), sizeof(int));\n out.write((char*)&(t), sizeof(double));\n}\n\nvoid Args::load(std::istream& in) {\n in.read((char*)&(dim), sizeof(int));\n in.read((char*)&(ws), sizeof(int));\n in.read((char*)&(epoch), sizeof(int));\n in.read((char*)&(minCount), sizeof(int));\n in.read((char*)&(neg), sizeof(int));\n in.read((char*)&(wordNgrams), sizeof(int));\n in.read((char*)&(loss), sizeof(loss_name));\n in.read((char*)&(model), sizeof(model_name));\n in.read((char*)&(bucket), sizeof(int));\n in.read((char*)&(minn), sizeof(int));\n in.read((char*)&(maxn), sizeof(int));\n in.read((char*)&(lrUpdateRate), sizeof(int));\n in.read((char*)&(t), sizeof(double));\n}\n\nvoid Args::dump(std::ostream& out) const {\n out << \"dim\"\n << \" \" << dim << std::endl;\n out << \"ws\"\n << \" \" << ws << std::endl;\n out << \"epoch\"\n << \" \" << epoch << std::endl;\n out << \"minCount\"\n << \" \" << minCount << std::endl;\n out << \"neg\"\n << \" \" << neg << std::endl;\n out << \"wordNgrams\"\n << \" \" << wordNgrams << std::endl;\n out << \"loss\"\n << \" \" << lossToString(loss) << std::endl;\n out << \"model\"\n << \" \" << modelToString(model) << std::endl;\n out << \"bucket\"\n << \" \" << bucket << std::endl;\n out << \"minn\"\n << \" \" << minn << std::endl;\n out << \"maxn\"\n << \" \" << maxn << std::endl;\n out << \"lrUpdateRate\"\n << \" \" << lrUpdateRate << std::endl;\n out << \"t\"\n << \" \" << t << std::endl;\n}\n\nbool Args::hasAutotune() const {\n return !autotuneValidationFile.empty();\n}\n\nbool Args::isManual(const std::string& argName) const {\n return (manualArgs_.count(argName) != 0);\n}\n\nvoid Args::setManual(const std::string& argName) {\n manualArgs_.emplace(argName);\n}\n\nmetric_name Args::getAutotuneMetric() const {\n if (autotuneMetric.substr(0, 3) == \"f1:\") {\n return metric_name::f1scoreLabel;\n } else if (autotuneMetric == \"f1\") {\n return metric_name::f1score;\n } else if (autotuneMetric.substr(0, 18) == \"precisionAtRecall:\") {\n size_t semicolon = autotuneMetric.find(':', 18);\n if (semicolon != std::string::npos) {\n return metric_name::precisionAtRecallLabel;\n }\n return metric_name::precisionAtRecall;\n } else if (autotuneMetric.substr(0, 18) == \"recallAtPrecision:\") {\n size_t semicolon = autotuneMetric.find(':', 18);\n if (semicolon != std::string::npos) {\n return metric_name::recallAtPrecisionLabel;\n }\n return metric_name::recallAtPrecision;\n }\n throw std::runtime_error(\"Unknown metric : \" + autotuneMetric);\n}\n\nstd::string Args::getAutotuneMetricLabel() const {\n metric_name metric = getAutotuneMetric();\n std::string label;\n if (metric == metric_name::f1scoreLabel) {\n label = autotuneMetric.substr(3);\n } else if (\n metric == metric_name::precisionAtRecallLabel ||\n metric == metric_name::recallAtPrecisionLabel) {\n size_t semicolon = autotuneMetric.find(':', 18);\n label = autotuneMetric.substr(semicolon + 1);\n } else {\n return label;\n }\n\n if (label.empty()) {\n throw std::runtime_error(\"Empty metric label : \" + autotuneMetric);\n }\n return label;\n}\n\ndouble Args::getAutotuneMetricValue() const {\n metric_name metric = getAutotuneMetric();\n double value = 0.0;\n if (metric == metric_name::precisionAtRecallLabel ||\n metric == metric_name::precisionAtRecall ||\n metric == metric_name::recallAtPrecisionLabel ||\n metric == metric_name::recallAtPrecision) {\n size_t firstSemicolon = 18; // semicolon position in \"precisionAtRecall:\"\n size_t secondSemicolon = autotuneMetric.find(':', firstSemicolon);\n const std::string valueStr =\n autotuneMetric.substr(firstSemicolon, secondSemicolon - firstSemicolon);\n value = std::stof(valueStr) / 100.0;\n }\n return value;\n}\n\nint64_t Args::getAutotuneModelSize() const {\n std::string modelSize = autotuneModelSize;\n if (modelSize.empty()) {\n return Args::kUnlimitedModelSize;\n }\n std::unordered_map units = {\n {'k', 1000},\n {'K', 1000},\n {'m', 1000000},\n {'M', 1000000},\n {'g', 1000000000},\n {'G', 1000000000},\n };\n uint64_t multiplier = 1;\n char lastCharacter = modelSize.back();\n if (units.count(lastCharacter)) {\n multiplier = units[lastCharacter];\n modelSize = modelSize.substr(0, modelSize.size() - 1);\n }\n uint64_t size = 0;\n size_t nonNumericCharacter = 0;\n bool parseError = false;\n try {\n size = std::stol(modelSize, &nonNumericCharacter);\n } catch (std::invalid_argument&) {\n parseError = true;\n }\n if (!parseError && nonNumericCharacter != modelSize.size()) {\n parseError = true;\n }\n if (parseError) {\n throw std::invalid_argument(\n \"Unable to parse model size \" + autotuneModelSize);\n }\n\n return size * multiplier;\n}\n\n} // namespace fasttext\n" }, { "path": "src/args.h", "content": "/**\n * Copyright (c) 2016-present, Facebook, Inc.\n * All rights reserved.\n *\n * This source code is licensed under the MIT license found in the\n * LICENSE file in the root directory of this source tree.\n */\n\n#pragma once\n\n#include \n#include \n#include \n#include \n#include \n\nnamespace fasttext {\n\nenum class model_name : int { cbow = 1, sg, sup };\nenum class loss_name : int { hs = 1, ns, softmax, ova };\nenum class metric_name : int {\n f1score = 1,\n f1scoreLabel,\n precisionAtRecall,\n precisionAtRecallLabel,\n recallAtPrecision,\n recallAtPrecisionLabel\n};\n\nclass Args {\n protected:\n std::string boolToString(bool) const;\n std::string modelToString(model_name) const;\n std::string metricToString(metric_name) const;\n std::unordered_set manualArgs_;\n\n public:\n Args();\n std::string input;\n std::string output;\n double lr;\n int lrUpdateRate;\n int dim;\n int ws;\n int epoch;\n int minCount;\n int minCountLabel;\n int neg;\n int wordNgrams;\n loss_name loss;\n model_name model;\n int bucket;\n int minn;\n int maxn;\n int thread;\n double t;\n std::string label;\n int verbose;\n std::string pretrainedVectors;\n bool saveOutput;\n int seed;\n\n bool qout;\n bool retrain;\n bool qnorm;\n size_t cutoff;\n size_t dsub;\n\n std::string autotuneValidationFile;\n std::string autotuneMetric;\n int autotunePredictions;\n int autotuneDuration;\n std::string autotuneModelSize;\n\n void parseArgs(const std::vector& args);\n void printHelp();\n void printBasicHelp();\n void printDictionaryHelp();\n void printTrainingHelp();\n void printAutotuneHelp();\n void printQuantizationHelp();\n void save(std::ostream&);\n void load(std::istream&);\n void dump(std::ostream&) const;\n bool hasAutotune() const;\n bool isManual(const std::string& argName) const;\n void setManual(const std::string& argName);\n std::string lossToString(loss_name) const;\n metric_name getAutotuneMetric() const;\n std::string getAutotuneMetricLabel() const;\n double getAutotuneMetricValue() const;\n int64_t getAutotuneModelSize() const;\n\n static constexpr double kUnlimitedModelSize = -1.0;\n};\n} // namespace fasttext\n" }, { "path": "src/autotune.cc", "content": "/**\n * Copyright (c) 2016-present, Facebook, Inc.\n * All rights reserved.\n *\n * This source code is licensed under the MIT license found in the\n * LICENSE file in the root directory of this source tree.\n */\n\n#include \"autotune.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#define LOG_VAL(name, val) \\\n if (autotuneArgs.verbose > 2) { \\\n std::cout << #name \" = \" << val << std::endl; \\\n }\n#define LOG_VAL_NAN(name, val) \\\n if (autotuneArgs.verbose > 2) { \\\n if (std::isnan(val)) { \\\n std::cout << #name \" = NaN\" << std::endl; \\\n } else { \\\n std::cout << #name \" = \" << val << std::endl; \\\n } \\\n }\n\nnamespace {\n\nstd::function interruptSignalHandler;\n\nvoid signalHandler(int signal) {\n if (signal == SIGINT) {\n interruptSignalHandler();\n }\n}\n\nclass ElapsedTimeMarker {\n std::chrono::steady_clock::time_point start_;\n\n public:\n ElapsedTimeMarker() {\n start_ = std::chrono::steady_clock::now();\n }\n double getElapsed() {\n return fasttext::utils::getDuration(\n start_, std::chrono::steady_clock::now());\n }\n};\n\n} // namespace\n\nnamespace fasttext {\n\nconstexpr double kUnknownBestScore = -1.0;\nconstexpr int kCutoffLimit = 256;\n\ntemplate \nT getArgGauss(\n T val,\n std::minstd_rand& rng,\n double startSigma,\n double endSigma,\n double t,\n bool linear) {\n T returnValue;\n const double stddev = startSigma -\n ((startSigma - endSigma) / 0.5) *\n std::min(0.5, std::max((t - 0.25), 0.0));\n\n std::normal_distribution normal(0.0, stddev);\n\n const double coeff = normal(rng);\n double updateCoeff = 0.0;\n\n if (linear) {\n updateCoeff = coeff;\n returnValue = static_cast(updateCoeff + val);\n } else {\n updateCoeff = std::pow(2.0, coeff);\n returnValue = static_cast(updateCoeff * val);\n }\n\n return returnValue;\n}\n\ntemplate \nT updateArgGauss(\n T val,\n T min,\n T max,\n double startSigma,\n double endSigma,\n double t,\n bool linear,\n std::minstd_rand& rng) {\n T retVal = getArgGauss(val, rng, startSigma, endSigma, t, linear);\n if (retVal > max) {\n retVal = max;\n }\n if (retVal < min) {\n retVal = min;\n }\n return retVal;\n}\n\nAutotuneStrategy::AutotuneStrategy(\n const Args& originalArgs,\n std::minstd_rand::result_type seed)\n : bestArgs_(),\n maxDuration_(originalArgs.autotuneDuration),\n rng_(seed),\n trials_(0),\n bestMinnIndex_(0),\n bestDsubExponent_(1),\n bestNonzeroBucket_(2000000),\n originalBucket_(originalArgs.bucket) {\n minnChoices_ = {0, 2, 3};\n updateBest(originalArgs);\n}\n\nArgs AutotuneStrategy::ask(double elapsed) {\n const double t = std::min(1.0, elapsed / maxDuration_);\n trials_++;\n\n if (trials_ == 1) {\n return bestArgs_;\n }\n\n Args args = bestArgs_;\n\n if (!args.isManual(\"epoch\")) {\n args.epoch = updateArgGauss(args.epoch, 1, 100, 2.8, 2.5, t, false, rng_);\n }\n if (!args.isManual(\"lr\")) {\n args.lr = updateArgGauss(args.lr, 0.01, 5.0, 1.9, 1.0, t, false, rng_);\n };\n if (!args.isManual(\"dim\")) {\n args.dim = updateArgGauss(args.dim, 1, 1000, 1.4, 0.3, t, false, rng_);\n }\n if (!args.isManual(\"wordNgrams\")) {\n args.wordNgrams =\n updateArgGauss(args.wordNgrams, 1, 5, 4.3, 2.4, t, true, rng_);\n }\n if (!args.isManual(\"dsub\")) {\n int dsubExponent =\n updateArgGauss(bestDsubExponent_, 1, 4, 2.0, 1.0, t, true, rng_);\n args.dsub = (1 << dsubExponent);\n }\n if (!args.isManual(\"minn\")) {\n int minnIndex = updateArgGauss(\n bestMinnIndex_,\n 0,\n static_cast(minnChoices_.size() - 1),\n 4.0,\n 1.4,\n t,\n true,\n rng_);\n args.minn = minnChoices_[minnIndex];\n }\n if (!args.isManual(\"maxn\")) {\n if (args.minn == 0) {\n args.maxn = 0;\n } else {\n args.maxn = args.minn + 3;\n }\n }\n if (!args.isManual(\"bucket\")) {\n int nonZeroBucket = updateArgGauss(\n bestNonzeroBucket_, 10000, 10000000, 2.0, 1.5, t, false, rng_);\n args.bucket = nonZeroBucket;\n } else {\n args.bucket = originalBucket_;\n }\n if (args.wordNgrams <= 1 && args.maxn == 0) {\n args.bucket = 0;\n }\n if (!args.isManual(\"loss\")) {\n args.loss = loss_name::softmax;\n }\n\n return args;\n}\n\nint AutotuneStrategy::getIndex(int val, const std::vector& choices) {\n auto found = std::find(choices.begin(), choices.end(), val);\n int ind = 0;\n if (found != choices.end()) {\n ind = std::distance(choices.begin(), found);\n }\n return ind;\n}\n\nvoid AutotuneStrategy::updateBest(const Args& args) {\n bestArgs_ = args;\n bestMinnIndex_ = getIndex(args.minn, minnChoices_);\n bestDsubExponent_ = log2(args.dsub);\n if (args.bucket != 0) {\n bestNonzeroBucket_ = args.bucket;\n }\n}\n\nAutotune::Autotune(const std::shared_ptr& fastText)\n : fastText_(fastText),\n elapsed_(0.),\n bestScore_(0.),\n trials_(0),\n sizeConstraintFailed_(0),\n continueTraining_(false),\n strategy_(),\n timer_() {}\n\nvoid Autotune::printInfo(double maxDuration) {\n double progress = elapsed_ * 100 / maxDuration;\n progress = std::min(progress, 100.0);\n\n std::cerr << \"\\r\";\n std::cerr << std::fixed;\n std::cerr << \"Progress: \";\n std::cerr << std::setprecision(1) << std::setw(5) << progress << \"%\";\n std::cerr << \" Trials: \" << std::setw(4) << trials_;\n std::cerr << \" Best score: \" << std::setw(9) << std::setprecision(6);\n if (bestScore_ == kUnknownBestScore) {\n std::cerr << \"unknown\";\n } else {\n std::cerr << bestScore_;\n }\n std::cerr << \" ETA: \"\n << utils::ClockPrint(std::max(maxDuration - elapsed_, 0.0));\n std::cerr << std::flush;\n}\n\nvoid Autotune::timer(\n const std::chrono::steady_clock::time_point& start,\n double maxDuration) {\n elapsed_ = 0.0;\n while (keepTraining(maxDuration)) {\n std::this_thread::sleep_for(std::chrono::milliseconds(500));\n elapsed_ = utils::getDuration(start, std::chrono::steady_clock::now());\n printInfo(maxDuration);\n }\n abort();\n}\n\nbool Autotune::keepTraining(double maxDuration) const {\n return continueTraining_ && elapsed_ < maxDuration;\n}\n\nvoid Autotune::abort() {\n if (continueTraining_) {\n continueTraining_ = false;\n fastText_->abort();\n }\n}\n\nvoid Autotune::startTimer(const Args& args) {\n std::chrono::steady_clock::time_point start =\n std::chrono::steady_clock::now();\n timer_ = std::thread([=]() { timer(start, args.autotuneDuration); });\n bestScore_ = kUnknownBestScore;\n trials_ = 0;\n continueTraining_ = true;\n\n auto previousSignalHandler = std::signal(SIGINT, signalHandler);\n interruptSignalHandler = [&]() {\n std::signal(SIGINT, previousSignalHandler);\n std::cerr << std::endl << \"Aborting autotune...\" << std::endl;\n abort();\n };\n}\n\ndouble Autotune::getMetricScore(\n Meter& meter,\n const metric_name& metricName,\n const double metricValue,\n const std::string& metricLabel) const {\n double score = 0.0;\n int32_t labelId = -1;\n if (!metricLabel.empty()) {\n labelId = fastText_->getLabelId(metricLabel);\n if (labelId == -1) {\n throw std::runtime_error(\"Unknown autotune metric label\");\n }\n }\n if (metricName == metric_name::f1score) {\n score = meter.f1Score();\n } else if (metricName == metric_name::f1scoreLabel) {\n score = meter.f1Score(labelId);\n } else if (metricName == metric_name::precisionAtRecall) {\n score = meter.precisionAtRecall(metricValue);\n } else if (metricName == metric_name::precisionAtRecallLabel) {\n score = meter.precisionAtRecall(labelId, metricValue);\n } else if (metricName == metric_name::recallAtPrecision) {\n score = meter.recallAtPrecision(metricValue);\n } else if (metricName == metric_name::recallAtPrecisionLabel) {\n score = meter.recallAtPrecision(labelId, metricValue);\n } else {\n throw std::runtime_error(\"Unknown metric\");\n }\n return score;\n}\n\nvoid Autotune::printArgs(const Args& args, const Args& autotuneArgs) {\n LOG_VAL(epoch, args.epoch)\n LOG_VAL(lr, args.lr)\n LOG_VAL(dim, args.dim)\n LOG_VAL(minCount, args.minCount)\n LOG_VAL(wordNgrams, args.wordNgrams)\n LOG_VAL(minn, args.minn)\n LOG_VAL(maxn, args.maxn)\n LOG_VAL(bucket, args.bucket)\n LOG_VAL(dsub, args.dsub)\n LOG_VAL(loss, args.lossToString(args.loss))\n}\n\nint Autotune::getCutoffForFileSize(\n bool qout,\n bool qnorm,\n int dsub,\n int64_t fileSize) const {\n int64_t outModelSize = 0;\n const int64_t outM = fastText_->getOutputMatrix()->size(0);\n const int64_t outN = fastText_->getOutputMatrix()->size(1);\n if (qout) {\n const int64_t outputPqSize = 16 + 4 * (outN * (1 << 8));\n outModelSize =\n 21 + (outM * ((outN + 2 - 1) / 2)) + outputPqSize + (qnorm ? outM : 0);\n } else {\n outModelSize = 16 + 4 * (outM * outN);\n }\n const int64_t dim = fastText_->getInputMatrix()->size(1);\n\n int target = (fileSize - (107) - 4 * (1 << 8) * dim - outModelSize);\n int cutoff = target / ((dim + dsub - 1) / dsub + (qnorm ? 1 : 0) + 10);\n\n return std::max(cutoff, kCutoffLimit);\n}\n\nbool Autotune::quantize(Args& args, const Args& autotuneArgs) {\n if (autotuneArgs.getAutotuneModelSize() == Args::kUnlimitedModelSize) {\n return true;\n }\n auto outputSize = fastText_->getOutputMatrix()->size(0);\n\n args.qnorm = true;\n args.qout = (outputSize >= kCutoffLimit);\n args.retrain = true;\n args.cutoff = getCutoffForFileSize(\n args.qout, args.qnorm, args.dsub, autotuneArgs.getAutotuneModelSize());\n LOG_VAL(cutoff, args.cutoff);\n if (args.cutoff == kCutoffLimit) {\n return false;\n }\n fastText_->quantize(args);\n\n return true;\n}\n\nvoid Autotune::printSkippedArgs(const Args& autotuneArgs) {\n std::unordered_set argsToCheck = {\"epoch\",\n \"lr\",\n \"dim\",\n \"wordNgrams\",\n \"loss\",\n \"bucket\",\n \"minn\",\n \"maxn\",\n \"dsub\"};\n for (const auto& arg : argsToCheck) {\n if (autotuneArgs.isManual(arg)) {\n std::cerr << \"Warning : \" << arg\n << \" is manually set to a specific value. \"\n << \"It will not be automatically optimized.\" << std::endl;\n }\n }\n}\n\nvoid Autotune::train(const Args& autotuneArgs) {\n std::ifstream validationFileStream(autotuneArgs.autotuneValidationFile);\n if (!validationFileStream.is_open()) {\n throw std::invalid_argument(\"Validation file cannot be opened!\");\n }\n printSkippedArgs(autotuneArgs);\n\n bool sizeConstraintWarning = false;\n int verbose = autotuneArgs.verbose;\n Args bestTrainArgs(autotuneArgs);\n Args trainArgs(autotuneArgs);\n trainArgs.verbose = 0;\n strategy_ = std::unique_ptr(\n new AutotuneStrategy(trainArgs, autotuneArgs.seed));\n startTimer(autotuneArgs);\n\n while (keepTraining(autotuneArgs.autotuneDuration)) {\n trials_++;\n\n trainArgs = strategy_->ask(elapsed_);\n LOG_VAL(Trial, trials_)\n printArgs(trainArgs, autotuneArgs);\n ElapsedTimeMarker elapsedTimeMarker;\n double currentScore = std::numeric_limits::quiet_NaN();\n try {\n fastText_->train(trainArgs);\n bool sizeConstraintOK = quantize(trainArgs, autotuneArgs);\n if (sizeConstraintOK) {\n const auto& metricLabel = autotuneArgs.getAutotuneMetricLabel();\n Meter meter(!metricLabel.empty());\n fastText_->test(\n validationFileStream, autotuneArgs.autotunePredictions, 0.0, meter);\n\n currentScore = getMetricScore(\n meter,\n autotuneArgs.getAutotuneMetric(),\n autotuneArgs.getAutotuneMetricValue(),\n metricLabel);\n\n if (bestScore_ == kUnknownBestScore || (currentScore > bestScore_)) {\n bestTrainArgs = trainArgs;\n bestScore_ = currentScore;\n strategy_->updateBest(bestTrainArgs);\n }\n } else {\n sizeConstraintFailed_++;\n if (!sizeConstraintWarning && trials_ > 10 &&\n sizeConstraintFailed_ > (trials_ / 2)) {\n sizeConstraintWarning = true;\n std::cerr << std::endl\n << \"Warning : requested model size is probably too small. \"\n \"You may want to increase `autotune-modelsize`.\"\n << std::endl;\n }\n }\n } catch (DenseMatrix::EncounteredNaNError&) {\n // ignore diverging loss and go on\n } catch (std::bad_alloc&) {\n // ignore parameter samples asking too much memory\n } catch (TimeoutError&) {\n break;\n } catch (FastText::AbortError&) {\n break;\n }\n LOG_VAL_NAN(currentScore, currentScore)\n LOG_VAL(train took, elapsedTimeMarker.getElapsed())\n }\n if (timer_.joinable()) {\n timer_.join();\n }\n\n if (bestScore_ == kUnknownBestScore) {\n std::string errorMessage;\n if (sizeConstraintWarning) {\n errorMessage =\n \"Couldn't fulfil model size constraint: please increase \"\n \"`autotune-modelsize`.\";\n } else {\n errorMessage =\n \"Didn't have enough time to train once: please increase \"\n \"`autotune-duration`.\";\n }\n throw std::runtime_error(errorMessage);\n } else {\n std::cerr << std::endl;\n std::cerr << \"Training again with best arguments\" << std::endl;\n bestTrainArgs.verbose = verbose;\n LOG_VAL(Best selected args, 0)\n printArgs(bestTrainArgs, autotuneArgs);\n fastText_->train(bestTrainArgs);\n quantize(bestTrainArgs, autotuneArgs);\n }\n}\n\n} // namespace fasttext\n" }, { "path": "src/autotune.h", "content": "/**\n * Copyright (c) 2016-present, Facebook, Inc.\n * All rights reserved.\n *\n * This source code is licensed under the MIT license found in the\n * LICENSE file in the root directory of this source tree.\n */\n\n#pragma once\n\n#include \n#include \n#include \n#include \n#include \n\n#include \"args.h\"\n#include \"fasttext.h\"\n\nnamespace fasttext {\n\nclass AutotuneStrategy {\n private:\n Args bestArgs_;\n int maxDuration_;\n std::minstd_rand rng_;\n int trials_;\n int bestMinnIndex_;\n int bestDsubExponent_;\n int bestNonzeroBucket_;\n int originalBucket_;\n std::vector minnChoices_;\n int getIndex(int val, const std::vector& choices);\n\n public:\n explicit AutotuneStrategy(\n const Args& args,\n std::minstd_rand::result_type seed);\n Args ask(double elapsed);\n void updateBest(const Args& args);\n};\n\nclass Autotune {\n protected:\n std::shared_ptr fastText_;\n double elapsed_;\n double bestScore_;\n int32_t trials_;\n int32_t sizeConstraintFailed_;\n std::atomic continueTraining_;\n std::unique_ptr strategy_;\n std::thread timer_;\n\n bool keepTraining(double maxDuration) const;\n void printInfo(double maxDuration);\n void timer(\n const std::chrono::steady_clock::time_point& start,\n double maxDuration);\n void abort();\n void startTimer(const Args& args);\n double getMetricScore(\n Meter& meter,\n const metric_name& metricName,\n const double metricValue,\n const std::string& metricLabel) const;\n void printArgs(const Args& args, const Args& autotuneArgs);\n void printSkippedArgs(const Args& autotuneArgs);\n bool quantize(Args& args, const Args& autotuneArgs);\n int getCutoffForFileSize(bool qout, bool qnorm, int dsub, int64_t fileSize)\n const;\n\n class TimeoutError : public std::runtime_error {\n public:\n TimeoutError() : std::runtime_error(\"Autotune timed out.\") {}\n };\n\n public:\n Autotune() = delete;\n explicit Autotune(const std::shared_ptr& fastText);\n Autotune(const Autotune&) = delete;\n Autotune(Autotune&&) = delete;\n Autotune& operator=(const Autotune&) = delete;\n Autotune& operator=(Autotune&&) = delete;\n ~Autotune() noexcept = default;\n\n void train(const Args& args);\n};\n\n} // namespace fasttext\n" }, { "path": "src/densematrix.cc", "content": "/**\n * Copyright (c) 2016-present, Facebook, Inc.\n * All rights reserved.\n *\n * This source code is licensed under the MIT license found in the\n * LICENSE file in the root directory of this source tree.\n */\n\n#include \"densematrix.h\"\n\n#include \n#include \n#include \n#include \n#include \"utils.h\"\n#include \"vector.h\"\n\n#if defined(__AVX512F__) || defined(__AVX__) || defined(__SSE__)\n#include \n#endif\n\nnamespace fasttext {\n\nDenseMatrix::DenseMatrix() : DenseMatrix(0, 0) {}\n\nDenseMatrix::DenseMatrix(int64_t m, int64_t n) : Matrix(m, n), data_(m * n) {}\n\nDenseMatrix::DenseMatrix(DenseMatrix&& other) noexcept\n : Matrix(other.m_, other.n_), data_(std::move(other.data_)) {}\n\nDenseMatrix::DenseMatrix(int64_t m, int64_t n, real* dataPtr)\n : Matrix(m, n), data_(dataPtr, dataPtr + (m * n)) {}\n\nvoid DenseMatrix::zero() {\n std::fill(data_.begin(), data_.end(), 0.0);\n}\n\nvoid DenseMatrix::uniformThread(real a, int block, int32_t seed) {\n std::minstd_rand rng(block + seed);\n std::uniform_real_distribution<> uniform(-a, a);\n int64_t blockSize = (m_ * n_) / 10;\n for (int64_t i = blockSize * block;\n i < (m_ * n_) && i < blockSize * (block + 1);\n i++) {\n data_[i] = uniform(rng);\n }\n}\n\nvoid DenseMatrix::uniform(real a, unsigned int thread, int32_t seed) {\n if (thread > 1) {\n std::vector threads;\n for (int i = 0; i < thread; i++) {\n threads.push_back(std::thread([=]() { uniformThread(a, i, seed); }));\n }\n for (int32_t i = 0; i < threads.size(); i++) {\n threads[i].join();\n }\n } else {\n // webassembly can't instantiate `std::thread`\n uniformThread(a, 0, seed);\n }\n}\n\nvoid DenseMatrix::multiplyRow(const Vector& nums, int64_t ib, int64_t ie) {\n if (ie == -1) {\n ie = m_;\n }\n assert(ie <= nums.size());\n for (auto i = ib; i < ie; i++) {\n real n = nums[i - ib];\n if (n != 0) {\n for (auto j = 0; j < n_; j++) {\n at(i, j) *= n;\n }\n }\n }\n}\n\nvoid DenseMatrix::divideRow(const Vector& denoms, int64_t ib, int64_t ie) {\n if (ie == -1) {\n ie = m_;\n }\n assert(ie <= denoms.size());\n for (auto i = ib; i < ie; i++) {\n real n = denoms[i - ib];\n if (n != 0) {\n for (auto j = 0; j < n_; j++) {\n at(i, j) /= n;\n }\n }\n }\n}\n\nreal DenseMatrix::l2NormRow(int64_t i) const {\n auto norm = 0.0;\n for (auto j = 0; j < n_; j++) {\n norm += at(i, j) * at(i, j);\n }\n if (std::isnan(norm)) {\n throw EncounteredNaNError();\n }\n return std::sqrt(norm);\n}\n\nvoid DenseMatrix::l2NormRow(Vector& norms) const {\n assert(norms.size() == m_);\n for (auto i = 0; i < m_; i++) {\n norms[i] = l2NormRow(i);\n }\n}\n\nreal DenseMatrix::dotRow(const Vector& vec, int64_t i) const {\n assert(i >= 0);\n assert(i < m_);\n assert(vec.size() == n_);\n real d = 0.0;\n for (int64_t j = 0; j < n_; j++) {\n d += at(i, j) * vec[j];\n }\n if (std::isnan(d)) {\n throw EncounteredNaNError();\n }\n return d;\n}\n\nvoid DenseMatrix::addVectorToRow(const Vector& vec, int64_t i, real a) {\n assert(i >= 0);\n assert(i < m_);\n assert(vec.size() == n_);\n for (int64_t j = 0; j < n_; j++) {\n data_[i * n_ + j] += a * vec[j];\n }\n}\n\nvoid DenseMatrix::addRowToVector(Vector& x, int32_t i) const {\n assert(i >= 0);\n assert(i < this->size(0));\n assert(x.size() == this->size(1));\n for (int64_t j = 0; j < n_; j++) {\n x[j] += at(i, j);\n }\n}\n\nvoid DenseMatrix::addRowToVector(Vector& x, int32_t i, real a) const {\n assert(i >= 0);\n assert(i < this->size(0));\n assert(x.size() == this->size(1));\n for (int64_t j = 0; j < n_; j++) {\n x[j] += a * at(i, j);\n }\n}\n\n/* Abstract over AVX512F, AVX, and SSE intrinsics, using the one available on this machine. */\n#if defined(__AVX512F__)\nusing Register = __m512;\ninline Register Add(Register first, Register second) { return _mm512_add_ps(first, second); }\ninline Register Set1(float to) { return _mm512_set1_ps(to); }\ninline Register Multiply(Register first, Register second) { return _mm512_mul_ps(first, second); }\n#elif defined(__AVX__)\nusing Register = __m256;\ninline Register Add(Register first, Register second) { return _mm256_add_ps(first, second); }\ninline Register Set1(float to) { return _mm256_set1_ps(to); }\ninline Register Multiply(Register first, Register second) { return _mm256_mul_ps(first, second); }\n#elif defined(__SSE__)\nusing Register = __m128;\ninline Register Add(Register first, Register second) { return _mm_add_ps(first, second); }\ninline Register Set1(float to) { return _mm_set1_ps(to); }\ninline Register Multiply(Register first, Register second) { return _mm_mul_ps(first, second); }\n#endif\n\n/* Faster routine for averaging rows of a matrix on x86.\n * The idea here is to keep the accumulators in registers if possible. */\n#if defined(__AVX512F__) || defined(__AVX__) || defined(__SSE__)\ntemplate void averageRowsFast(Vector& x, const std::vector& rows, const DenseMatrix &matrix) {\n // Columns must be a multiple of how many floats fit in a register.\n static_assert(Cols % (sizeof(Register) / 4) == 0);\n constexpr unsigned RegisterCount = Cols / (sizeof(Register) / 4);\n // These should be aligned by aligned.h\n assert(reinterpret_cast(x.data()) % sizeof(Register) == 0);\n assert(reinterpret_cast(matrix.data()) % sizeof(Register) == 0);\n\n // Guard against empty list of rows with default NaN behavior.\n if (rows.empty()) {\n x.zero();\n x.mul(1.0 / rows.size());\n return;\n }\n\n // Copy the first row to accumulation registers.\n Register accum[RegisterCount];\n auto row = rows.cbegin();\n const Register *base = reinterpret_cast(matrix.data() + matrix.cols() * *row);\n for (unsigned i = 0; i < RegisterCount; ++i) {\n accum[i] = base[i];\n }\n // Add the rows after the first.\n for (++row; row != rows.cend(); ++row) {\n base = reinterpret_cast(matrix.data() + matrix.cols() * *row);\n for (unsigned i = 0; i < RegisterCount; ++i) {\n accum[i] = Add(accum[i], base[i]);\n }\n }\n // Multiply by (1.0 / rows.size()) and write to x.\n Register mul = Set1(1.0 / rows.size());\n for (unsigned i = 0; i < RegisterCount; ++i) {\n reinterpret_cast(x.data())[i] = Multiply(accum[i], mul);\n }\n}\n#endif\n\nvoid DenseMatrix::averageRowsToVector(Vector& x, const std::vector& rows) const {\n#if defined(__AVX512F__) || defined(__AVX__) || defined(__SSE__)\n switch (cols()) {\n case 512:\n // Maximum number that can fit all in registers on AVX512F.\n averageRowsFast<512>(x, rows, *this);\n return;\n case 256:\n averageRowsFast<256>(x, rows, *this);\n return;\n case 64:\n averageRowsFast<64>(x, rows, *this);\n return;\n case 32:\n averageRowsFast<32>(x, rows, *this);\n return;\n case 16:\n averageRowsFast<16>(x, rows, *this);\n return;\n }\n#endif\n x.zero();\n for (auto it = rows.cbegin(); it != rows.cend(); ++it) {\n addRowToVector(x, *it);\n }\n x.mul(1.0 / rows.size());\n}\n\nvoid DenseMatrix::save(std::ostream& out) const {\n out.write((char*)&m_, sizeof(int64_t));\n out.write((char*)&n_, sizeof(int64_t));\n out.write((char*)data_.data(), m_ * n_ * sizeof(real));\n}\n\nvoid DenseMatrix::load(std::istream& in) {\n in.read((char*)&m_, sizeof(int64_t));\n in.read((char*)&n_, sizeof(int64_t));\n data_ = intgemm::AlignedVector(m_ * n_);\n in.read((char*)data_.data(), m_ * n_ * sizeof(real));\n}\n\nvoid DenseMatrix::dump(std::ostream& out) const {\n out << m_ << \" \" << n_ << std::endl;\n for (int64_t i = 0; i < m_; i++) {\n for (int64_t j = 0; j < n_; j++) {\n if (j > 0) {\n out << \" \";\n }\n out << at(i, j);\n }\n out << std::endl;\n }\n};\n\n} // namespace fasttext\n" }, { "path": "src/densematrix.h", "content": "/**\n * Copyright (c) 2016-present, Facebook, Inc.\n * All rights reserved.\n *\n * This source code is licensed under the MIT license found in the\n * LICENSE file in the root directory of this source tree.\n */\n\n#pragma once\n\n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \"aligned.h\"\n#include \"matrix.h\"\n#include \"real.h\"\n\nnamespace fasttext {\n\nclass Vector;\n\nclass DenseMatrix : public Matrix {\n protected:\n intgemm::AlignedVector data_;\n void uniformThread(real, int, int32_t);\n\n public:\n DenseMatrix();\n explicit DenseMatrix(int64_t, int64_t);\n explicit DenseMatrix(int64_t m, int64_t n, real* dataPtr);\n DenseMatrix(const DenseMatrix&) = default;\n DenseMatrix(DenseMatrix&&) noexcept;\n DenseMatrix& operator=(const DenseMatrix&) = delete;\n DenseMatrix& operator=(DenseMatrix&&) = delete;\n virtual ~DenseMatrix() noexcept override = default;\n\n inline real* data() {\n return data_.data();\n }\n inline const real* data() const {\n return data_.data();\n }\n\n inline const real& at(int64_t i, int64_t j) const {\n assert(i * n_ + j < data_.size());\n return data_[i * n_ + j];\n };\n inline real& at(int64_t i, int64_t j) {\n return data_[i * n_ + j];\n };\n\n inline int64_t rows() const {\n return m_;\n }\n inline int64_t cols() const {\n return n_;\n }\n void zero();\n void uniform(real, unsigned int, int32_t);\n\n void multiplyRow(const Vector& nums, int64_t ib = 0, int64_t ie = -1);\n void divideRow(const Vector& denoms, int64_t ib = 0, int64_t ie = -1);\n\n real l2NormRow(int64_t i) const;\n void l2NormRow(Vector& norms) const;\n\n real dotRow(const Vector&, int64_t) const override;\n void addVectorToRow(const Vector&, int64_t, real) override;\n void addRowToVector(Vector& x, int32_t i) const override;\n void addRowToVector(Vector& x, int32_t i, real a) const override;\n void averageRowsToVector(Vector& x, const std::vector& rows) const override;\n void save(std::ostream&) const override;\n void load(std::istream&) override;\n void dump(std::ostream&) const override;\n\n class EncounteredNaNError : public std::runtime_error {\n public:\n EncounteredNaNError() : std::runtime_error(\"Encountered NaN.\") {}\n };\n};\n} // namespace fasttext\n" }, { "path": "src/dictionary.cc", "content": "/**\n * Copyright (c) 2016-present, Facebook, Inc.\n * All rights reserved.\n *\n * This source code is licensed under the MIT license found in the\n * LICENSE file in the root directory of this source tree.\n */\n\n#include \"dictionary.h\"\n\n#include \n\n#include \n#include \n#include \n#include \n#include \n#include \n\nnamespace fasttext {\n\nconst std::string Dictionary::EOS = \"\";\nconst std::string Dictionary::BOW = \"<\";\nconst std::string Dictionary::EOW = \">\";\n\nDictionary::Dictionary(std::shared_ptr args)\n : args_(args),\n word2int_(MAX_VOCAB_SIZE, -1),\n size_(0),\n nwords_(0),\n nlabels_(0),\n ntokens_(0),\n pruneidx_size_(-1) {}\n\nDictionary::Dictionary(std::shared_ptr args, std::istream& in)\n : args_(args),\n size_(0),\n nwords_(0),\n nlabels_(0),\n ntokens_(0),\n pruneidx_size_(-1) {\n load(in);\n}\n\nint32_t Dictionary::find(const std::string_view w) const {\n return find(w, hash(w));\n}\n\nint32_t Dictionary::find(const std::string_view w, uint32_t h) const {\n int32_t word2intsize = word2int_.size();\n int32_t id = h % word2intsize;\n while (word2int_[id] != -1 && words_[word2int_[id]].word != w) {\n id = (id + 1) % word2intsize;\n }\n return id;\n}\n\nvoid Dictionary::add(const std::string& w) {\n int32_t h = find(w);\n ntokens_++;\n if (word2int_[h] == -1) {\n entry e;\n e.word = w;\n e.count = 1;\n e.type = getType(w);\n words_.push_back(e);\n word2int_[h] = size_++;\n } else {\n words_[word2int_[h]].count++;\n }\n}\n\nint32_t Dictionary::nwords() const {\n return nwords_;\n}\n\nint32_t Dictionary::nlabels() const {\n return nlabels_;\n}\n\nint64_t Dictionary::ntokens() const {\n return ntokens_;\n}\n\nconst std::vector& Dictionary::getSubwords(int32_t i) const {\n assert(i >= 0);\n assert(i < nwords_);\n return words_[i].subwords;\n}\n\nconst std::vector Dictionary::getSubwords(\n const std::string& word) const {\n int32_t i = getId(word);\n if (i >= 0) {\n return getSubwords(i);\n }\n std::vector ngrams;\n if (word != EOS) {\n computeSubwords(BOW + word + EOW, ngrams);\n }\n return ngrams;\n}\n\nvoid Dictionary::getSubwords(\n const std::string& word,\n std::vector& ngrams,\n std::vector& substrings) const {\n int32_t i = getId(word);\n ngrams.clear();\n substrings.clear();\n if (i >= 0) {\n ngrams.push_back(i);\n substrings.push_back(words_[i].word);\n }\n if (word != EOS) {\n computeSubwords(BOW + word + EOW, ngrams, &substrings);\n }\n}\n\nbool Dictionary::discard(int32_t id, real rand) const {\n assert(id >= 0);\n assert(id < nwords_);\n if (args_->model == model_name::sup) {\n return false;\n }\n return rand > pdiscard_[id];\n}\n\nint32_t Dictionary::getId(const std::string_view w, uint32_t h) const {\n int32_t id = find(w, h);\n return word2int_[id];\n}\n\nint32_t Dictionary::getId(const std::string_view w) const {\n int32_t h = find(w);\n return word2int_[h];\n}\n\nentry_type Dictionary::getType(int32_t id) const {\n assert(id >= 0);\n assert(id < size_);\n return words_[id].type;\n}\n\nentry_type Dictionary::getType(const std::string_view w) const {\n return (w.find(args_->label) == 0) ? entry_type::label : entry_type::word;\n}\n\nstd::string Dictionary::getWord(int32_t id) const {\n assert(id >= 0);\n assert(id < size_);\n return words_[id].word;\n}\n\n// The correct implementation of fnv should be:\n// h = h ^ uint32_t(uint8_t(str[i]));\n// Unfortunately, earlier version of fasttext used\n// h = h ^ uint32_t(str[i]);\n// which is undefined behavior (as char can be signed or unsigned).\n// Since all fasttext models that were already released were trained\n// using signed char, we fixed the hash function to make models\n// compatible whatever compiler is used.\nuint32_t Dictionary::hash(const std::string_view str) const {\n uint32_t h = 2166136261;\n for (size_t i = 0; i < str.size(); i++) {\n h = h ^ uint32_t(int8_t(str[i]));\n h = h * 16777619;\n }\n return h;\n}\n\nvoid Dictionary::computeSubwords(\n const std::string& word,\n std::vector& ngrams,\n std::vector* substrings) const {\n for (size_t i = 0; i < word.size(); i++) {\n std::string ngram;\n if ((word[i] & 0xC0) == 0x80) {\n continue;\n }\n for (size_t j = i, n = 1; j < word.size() && n <= args_->maxn; n++) {\n ngram.push_back(word[j++]);\n while (j < word.size() && (word[j] & 0xC0) == 0x80) {\n ngram.push_back(word[j++]);\n }\n if (n >= args_->minn && !(n == 1 && (i == 0 || j == word.size()))) {\n int32_t h = hash(ngram) % args_->bucket;\n pushHash(ngrams, h);\n if (substrings) {\n substrings->push_back(ngram);\n }\n }\n }\n }\n}\n\nvoid Dictionary::initNgrams() {\n for (size_t i = 0; i < size_; i++) {\n std::string word = BOW + words_[i].word + EOW;\n words_[i].subwords.clear();\n words_[i].subwords.push_back(i);\n if (words_[i].word != EOS) {\n computeSubwords(word, words_[i].subwords);\n }\n }\n}\n\nbool Dictionary::readWord(std::istream& in, std::string& word) const {\n int c;\n std::streambuf& sb = *in.rdbuf();\n word.clear();\n while ((c = sb.sbumpc()) != EOF) {\n if (c == ' ' || c == '\\n' || c == '\\r' || c == '\\t' || c == '\\v' ||\n c == '\\f' || c == '\\0') {\n if (word.empty()) {\n if (c == '\\n') {\n word += EOS;\n return true;\n }\n continue;\n } else {\n if (c == '\\n')\n sb.sungetc();\n return true;\n }\n }\n word.push_back(c);\n }\n // trigger eofbit\n in.get();\n return !word.empty();\n}\n\nvoid Dictionary::readFromFile(std::istream& in) {\n std::string word;\n int64_t minThreshold = 1;\n while (readWord(in, word)) {\n add(word);\n if (ntokens_ % 1000000 == 0 && args_->verbose > 1) {\n std::cerr << \"\\rRead \" << ntokens_ / 1000000 << \"M words\" << std::flush;\n }\n if (size_ > 0.75 * MAX_VOCAB_SIZE) {\n minThreshold++;\n threshold(minThreshold, minThreshold);\n }\n }\n threshold(args_->minCount, args_->minCountLabel);\n initTableDiscard();\n initNgrams();\n if (args_->verbose > 0) {\n std::cerr << \"\\rRead \" << ntokens_ / 1000000 << \"M words\" << std::endl;\n std::cerr << \"Number of words: \" << nwords_ << std::endl;\n std::cerr << \"Number of labels: \" << nlabels_ << std::endl;\n }\n if (size_ == 0) {\n throw std::invalid_argument(\n \"Empty vocabulary. Try a smaller -minCount value.\");\n }\n}\n\nvoid Dictionary::threshold(int64_t t, int64_t tl) {\n sort(words_.begin(), words_.end(), [](const entry& e1, const entry& e2) {\n if (e1.type != e2.type) {\n return e1.type < e2.type;\n }\n return e1.count > e2.count;\n });\n words_.erase(\n remove_if(\n words_.begin(),\n words_.end(),\n [&](const entry& e) {\n return (e.type == entry_type::word && e.count < t) ||\n (e.type == entry_type::label && e.count < tl);\n }),\n words_.end());\n words_.shrink_to_fit();\n size_ = 0;\n nwords_ = 0;\n nlabels_ = 0;\n std::fill(word2int_.begin(), word2int_.end(), -1);\n for (auto it = words_.begin(); it != words_.end(); ++it) {\n int32_t h = find(it->word);\n word2int_[h] = size_++;\n if (it->type == entry_type::word) {\n nwords_++;\n }\n if (it->type == entry_type::label) {\n nlabels_++;\n }\n }\n}\n\nvoid Dictionary::initTableDiscard() {\n pdiscard_.resize(size_);\n for (size_t i = 0; i < size_; i++) {\n real f = real(words_[i].count) / real(ntokens_);\n pdiscard_[i] = std::sqrt(args_->t / f) + args_->t / f;\n }\n}\n\nstd::vector Dictionary::getCounts(entry_type type) const {\n std::vector counts;\n for (auto& w : words_) {\n if (w.type == type) {\n counts.push_back(w.count);\n }\n }\n return counts;\n}\n\nvoid Dictionary::addWordNgrams(\n std::vector& line,\n const std::vector& hashes,\n int32_t n) const {\n for (int32_t i = 0; i < hashes.size(); i++) {\n uint64_t h = hashes[i];\n for (int32_t j = i + 1; j < hashes.size() && j < i + n; j++) {\n h = h * 116049371 + hashes[j];\n pushHash(line, h % args_->bucket);\n }\n }\n}\n\nvoid Dictionary::addSubwords(\n std::vector& line,\n const std::string_view token,\n int32_t wid) const {\n if (wid < 0) { // out of vocab\n if (token != EOS) {\n std::string concat;\n concat.reserve(BOW.size() + token.size() + EOW.size());\n concat += BOW;\n concat.append(token.data(), token.size());\n concat += EOW;\n computeSubwords(concat, line);\n }\n } else {\n if (args_->maxn <= 0) { // in vocab w/o subwords\n line.push_back(wid);\n } else { // in vocab w/ subwords\n const std::vector& ngrams = getSubwords(wid);\n line.insert(line.end(), ngrams.cbegin(), ngrams.cend());\n }\n }\n}\n\nvoid Dictionary::reset(std::istream& in) const {\n if (in.eof()) {\n in.clear();\n in.seekg(std::streampos(0));\n }\n}\n\nint32_t Dictionary::getLine(\n std::istream& in,\n std::vector& words,\n std::minstd_rand& rng) const {\n std::uniform_real_distribution<> uniform(0, 1);\n std::string token;\n int32_t ntokens = 0;\n\n reset(in);\n words.clear();\n while (readWord(in, token)) {\n int32_t h = find(token);\n int32_t wid = word2int_[h];\n if (wid < 0) {\n continue;\n }\n\n ntokens++;\n if (getType(wid) == entry_type::word && !discard(wid, uniform(rng))) {\n words.push_back(wid);\n }\n if (ntokens > MAX_LINE_SIZE || token == EOS) {\n break;\n }\n }\n return ntokens;\n}\n\nint32_t Dictionary::getLine(\n std::istream& in,\n std::vector& words,\n std::vector& labels) const {\n std::vector word_hashes;\n std::string token;\n int32_t ntokens = 0;\n\n reset(in);\n words.clear();\n labels.clear();\n while (readWord(in, token)) {\n uint32_t h = hash(token);\n int32_t wid = getId(token, h);\n entry_type type = wid < 0 ? getType(token) : getType(wid);\n\n ntokens++;\n if (type == entry_type::word) {\n addSubwords(words, token, wid);\n word_hashes.push_back(h);\n } else if (type == entry_type::label && wid >= 0) {\n labels.push_back(wid - nwords_);\n }\n if (token == EOS) {\n break;\n }\n }\n addWordNgrams(words, word_hashes, args_->wordNgrams);\n return ntokens;\n}\n\nnamespace {\nbool readWordNoNewline(std::string_view& in, std::string_view& word) {\n const std::string_view spaces(\" \\n\\r\\t\\v\\f\\0\");\n std::string_view::size_type begin = in.find_first_not_of(spaces);\n if (begin == std::string_view::npos) {\n in.remove_prefix(in.size());\n return false;\n }\n in.remove_prefix(begin);\n word = in.substr(0, in.find_first_of(spaces));\n in.remove_prefix(word.size());\n return true;\n}\n} // namespace\n\nint32_t Dictionary::getStringNoNewline(\n std::string_view in,\n std::vector& words,\n std::vector& labels) const {\n std::vector word_hashes;\n std::string_view token;\n int32_t ntokens = 0;\n\n words.clear();\n labels.clear();\n while (readWordNoNewline(in, token)) {\n uint32_t h = hash(token);\n int32_t wid = getId(token, h);\n entry_type type = wid < 0 ? getType(token) : getType(wid);\n\n ntokens++;\n if (type == entry_type::word) {\n addSubwords(words, token, wid);\n word_hashes.push_back(h);\n } else if (type == entry_type::label && wid >= 0) {\n labels.push_back(wid - nwords_);\n }\n if (token == EOS) {\n break;\n }\n }\n addWordNgrams(words, word_hashes, args_->wordNgrams);\n return ntokens;\n}\n\nvoid Dictionary::pushHash(std::vector& hashes, int32_t id) const {\n if (pruneidx_size_ == 0 || id < 0) {\n return;\n }\n if (pruneidx_size_ > 0) {\n if (pruneidx_.count(id)) {\n id = pruneidx_.at(id);\n } else {\n return;\n }\n }\n hashes.push_back(nwords_ + id);\n}\n\nstd::string Dictionary::getLabel(int32_t lid) const {\n if (lid < 0 || lid >= nlabels_) {\n throw std::invalid_argument(\n \"Label id is out of range [0, \" + std::to_string(nlabels_) + \"]\");\n }\n return words_[lid + nwords_].word;\n}\n\nvoid Dictionary::save(std::ostream& out) const {\n out.write((char*)&size_, sizeof(int32_t));\n out.write((char*)&nwords_, sizeof(int32_t));\n out.write((char*)&nlabels_, sizeof(int32_t));\n out.write((char*)&ntokens_, sizeof(int64_t));\n out.write((char*)&pruneidx_size_, sizeof(int64_t));\n for (int32_t i = 0; i < size_; i++) {\n entry e = words_[i];\n out.write(e.word.data(), e.word.size() * sizeof(char));\n out.put(0);\n out.write((char*)&(e.count), sizeof(int64_t));\n out.write((char*)&(e.type), sizeof(entry_type));\n }\n for (const auto pair : pruneidx_) {\n out.write((char*)&(pair.first), sizeof(int32_t));\n out.write((char*)&(pair.second), sizeof(int32_t));\n }\n}\n\nvoid Dictionary::load(std::istream& in) {\n words_.clear();\n in.read((char*)&size_, sizeof(int32_t));\n in.read((char*)&nwords_, sizeof(int32_t));\n in.read((char*)&nlabels_, sizeof(int32_t));\n in.read((char*)&ntokens_, sizeof(int64_t));\n in.read((char*)&pruneidx_size_, sizeof(int64_t));\n for (int32_t i = 0; i < size_; i++) {\n char c;\n entry e;\n while ((c = in.get()) != 0) {\n e.word.push_back(c);\n }\n in.read((char*)&e.count, sizeof(int64_t));\n in.read((char*)&e.type, sizeof(entry_type));\n words_.push_back(e);\n }\n pruneidx_.clear();\n for (int32_t i = 0; i < pruneidx_size_; i++) {\n int32_t first;\n int32_t second;\n in.read((char*)&first, sizeof(int32_t));\n in.read((char*)&second, sizeof(int32_t));\n pruneidx_[first] = second;\n }\n initTableDiscard();\n initNgrams();\n\n int32_t word2intsize = std::ceil(size_ / 0.7);\n word2int_.assign(word2intsize, -1);\n for (int32_t i = 0; i < size_; i++) {\n word2int_[find(words_[i].word)] = i;\n }\n}\n\nvoid Dictionary::init() {\n initTableDiscard();\n initNgrams();\n}\n\nvoid Dictionary::prune(std::vector& idx) {\n std::vector words, ngrams;\n for (auto it = idx.cbegin(); it != idx.cend(); ++it) {\n if (*it < nwords_) {\n words.push_back(*it);\n } else {\n ngrams.push_back(*it);\n }\n }\n std::sort(words.begin(), words.end());\n idx = words;\n\n if (ngrams.size() != 0) {\n int32_t j = 0;\n for (const auto ngram : ngrams) {\n pruneidx_[ngram - nwords_] = j;\n j++;\n }\n idx.insert(idx.end(), ngrams.begin(), ngrams.end());\n }\n pruneidx_size_ = pruneidx_.size();\n\n std::fill(word2int_.begin(), word2int_.end(), -1);\n\n int32_t j = 0;\n for (int32_t i = 0; i < words_.size(); i++) {\n if (getType(i) == entry_type::label ||\n (j < words.size() && words[j] == i)) {\n words_[j] = words_[i];\n word2int_[find(words_[j].word)] = j;\n j++;\n }\n }\n nwords_ = words.size();\n size_ = nwords_ + nlabels_;\n words_.erase(words_.begin() + size_, words_.end());\n initNgrams();\n}\n\nvoid Dictionary::dump(std::ostream& out) const {\n out << words_.size() << std::endl;\n for (auto it : words_) {\n std::string entryType = \"word\";\n if (it.type == entry_type::label) {\n entryType = \"label\";\n }\n out << it.word << \" \" << it.count << \" \" << entryType << std::endl;\n }\n}\n\n} // namespace fasttext\n" }, { "path": "src/dictionary.h", "content": "/**\n * Copyright (c) 2016-present, Facebook, Inc.\n * All rights reserved.\n *\n * This source code is licensed under the MIT license found in the\n * LICENSE file in the root directory of this source tree.\n */\n\n#pragma once\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \"args.h\"\n#include \"real.h\"\n\nnamespace fasttext {\n\ntypedef int32_t id_type;\nenum class entry_type : int8_t { word = 0, label = 1 };\n\nstruct entry {\n std::string word;\n int64_t count;\n entry_type type;\n std::vector subwords;\n};\n\nclass Dictionary {\n protected:\n static const int32_t MAX_VOCAB_SIZE = 30000000;\n static const int32_t MAX_LINE_SIZE = 1024;\n\n int32_t find(const std::string_view) const;\n int32_t find(const std::string_view, uint32_t h) const;\n void initTableDiscard();\n void initNgrams();\n void reset(std::istream&) const;\n void pushHash(std::vector&, int32_t) const;\n void addSubwords(std::vector&, const std::string_view, int32_t) const;\n\n std::shared_ptr args_;\n std::vector word2int_;\n std::vector words_;\n\n std::vector pdiscard_;\n int32_t size_;\n int32_t nwords_;\n int32_t nlabels_;\n int64_t ntokens_;\n\n int64_t pruneidx_size_;\n std::unordered_map pruneidx_;\n void addWordNgrams(\n std::vector& line,\n const std::vector& hashes,\n int32_t n) const;\n\n public:\n static const std::string EOS;\n static const std::string BOW;\n static const std::string EOW;\n\n explicit Dictionary(std::shared_ptr);\n explicit Dictionary(std::shared_ptr, std::istream&);\n int32_t nwords() const;\n int32_t nlabels() const;\n int64_t ntokens() const;\n int32_t getId(const std::string_view) const;\n int32_t getId(const std::string_view, uint32_t h) const;\n entry_type getType(int32_t) const;\n entry_type getType(const std::string_view) const;\n bool discard(int32_t, real) const;\n std::string getWord(int32_t) const;\n const std::vector& getSubwords(int32_t) const;\n const std::vector getSubwords(const std::string&) const;\n void getSubwords(\n const std::string&,\n std::vector&,\n std::vector&) const;\n void computeSubwords(\n const std::string&,\n std::vector&,\n std::vector* substrings = nullptr) const;\n uint32_t hash(const std::string_view str) const;\n void add(const std::string&);\n bool readWord(std::istream&, std::string&) const;\n void readFromFile(std::istream&);\n std::string getLabel(int32_t) const;\n void save(std::ostream&) const;\n void load(std::istream&);\n std::vector getCounts(entry_type) const;\n int32_t getLine(std::istream&, std::vector&, std::vector&)\n const;\n int32_t getLine(std::istream&, std::vector&, std::minstd_rand&)\n const;\n int32_t getStringNoNewline(std::string_view, std::vector&,\n std::vector&) const;\n void threshold(int64_t, int64_t);\n void prune(std::vector&);\n bool isPruned() {\n return pruneidx_size_ >= 0;\n }\n void dump(std::ostream&) const;\n void init();\n};\n\n} // namespace fasttext\n" }, { "path": "src/fasttext.cc", "content": "/**\n * Copyright (c) 2016-present, Facebook, Inc.\n * All rights reserved.\n *\n * This source code is licensed under the MIT license found in the\n * LICENSE file in the root directory of this source tree.\n */\n\n#include \"fasttext.h\"\n#include \"loss.h\"\n#include \"quantmatrix.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\nnamespace fasttext {\n\nconstexpr int32_t FASTTEXT_VERSION = 12; /* Version 1b */\nconstexpr int32_t FASTTEXT_FILEFORMAT_MAGIC_INT32 = 793712314;\n\nbool comparePairs(\n const std::pair& l,\n const std::pair& r);\n\nstd::shared_ptr FastText::createLoss(std::shared_ptr& output) {\n loss_name lossName = args_->loss;\n switch (lossName) {\n case loss_name::hs:\n return std::make_shared(\n output, getTargetCounts());\n case loss_name::ns:\n return std::make_shared(\n output, args_->neg, getTargetCounts());\n case loss_name::softmax:\n return std::make_shared(output);\n case loss_name::ova:\n return std::make_shared(output);\n default:\n throw std::runtime_error(\"Unknown loss\");\n }\n}\n\nFastText::FastText()\n : quant_(false), wordVectors_(nullptr), trainException_(nullptr) {}\n\nvoid FastText::addInputVector(Vector& vec, int32_t ind) const {\n vec.addRow(*input_, ind);\n}\n\nstd::shared_ptr FastText::getDictionary() const {\n return dict_;\n}\n\nconst Args FastText::getArgs() const {\n return *args_.get();\n}\n\nstd::shared_ptr FastText::getInputMatrix() const {\n if (quant_) {\n throw std::runtime_error(\"Can't export quantized matrix\");\n }\n assert(input_.get());\n return std::dynamic_pointer_cast(input_);\n}\n\nvoid FastText::setMatrices(\n const std::shared_ptr& inputMatrix,\n const std::shared_ptr& outputMatrix) {\n assert(input_->size(1) == output_->size(1));\n\n input_ = std::dynamic_pointer_cast(inputMatrix);\n output_ = std::dynamic_pointer_cast(outputMatrix);\n wordVectors_.reset();\n args_->dim = input_->size(1);\n\n buildModel();\n}\n\nstd::shared_ptr FastText::getOutputMatrix() const {\n if (quant_ && args_->qout) {\n throw std::runtime_error(\"Can't export quantized matrix\");\n }\n assert(output_.get());\n return std::dynamic_pointer_cast(output_);\n}\n\nint32_t FastText::getWordId(const std::string& word) const {\n return dict_->getId(word);\n}\n\nint32_t FastText::getSubwordId(const std::string& subword) const {\n int32_t h = dict_->hash(subword) % args_->bucket;\n return dict_->nwords() + h;\n}\n\nint32_t FastText::getLabelId(const std::string& label) const {\n int32_t labelId = dict_->getId(label);\n if (labelId != -1) {\n labelId -= dict_->nwords();\n }\n return labelId;\n}\n\nvoid FastText::getWordVector(Vector& vec, const std::string& word) const {\n const std::vector& ngrams = dict_->getSubwords(word);\n vec.zero();\n for (int i = 0; i < ngrams.size(); i++) {\n addInputVector(vec, ngrams[i]);\n }\n if (ngrams.size() > 0) {\n vec.mul(1.0 / ngrams.size());\n }\n}\n\nvoid FastText::getSubwordVector(Vector& vec, const std::string& subword) const {\n vec.zero();\n int32_t h = dict_->hash(subword) % args_->bucket;\n h = h + dict_->nwords();\n addInputVector(vec, h);\n}\n\nvoid FastText::saveVectors(const std::string& filename) {\n if (!input_ || !output_) {\n throw std::runtime_error(\"Model never trained\");\n }\n std::ofstream ofs(filename);\n if (!ofs.is_open()) {\n throw std::invalid_argument(\n filename + \" cannot be opened for saving vectors!\");\n }\n ofs << dict_->nwords() << \" \" << args_->dim << std::endl;\n Vector vec(args_->dim);\n for (int32_t i = 0; i < dict_->nwords(); i++) {\n std::string word = dict_->getWord(i);\n getWordVector(vec, word);\n ofs << word << \" \" << vec << std::endl;\n }\n ofs.close();\n}\n\nvoid FastText::saveOutput(const std::string& filename) {\n std::ofstream ofs(filename);\n if (!ofs.is_open()) {\n throw std::invalid_argument(\n filename + \" cannot be opened for saving vectors!\");\n }\n if (quant_) {\n throw std::invalid_argument(\n \"Option -saveOutput is not supported for quantized models.\");\n }\n int32_t n =\n (args_->model == model_name::sup) ? dict_->nlabels() : dict_->nwords();\n ofs << n << \" \" << args_->dim << std::endl;\n Vector vec(args_->dim);\n for (int32_t i = 0; i < n; i++) {\n std::string word = (args_->model == model_name::sup) ? dict_->getLabel(i)\n : dict_->getWord(i);\n vec.zero();\n vec.addRow(*output_, i);\n ofs << word << \" \" << vec << std::endl;\n }\n ofs.close();\n}\n\nbool FastText::checkModel(std::istream& in) {\n int32_t magic;\n in.read((char*)&(magic), sizeof(int32_t));\n if (magic != FASTTEXT_FILEFORMAT_MAGIC_INT32) {\n return false;\n }\n in.read((char*)&(version), sizeof(int32_t));\n if (version > FASTTEXT_VERSION) {\n return false;\n }\n return true;\n}\n\nvoid FastText::signModel(std::ostream& out) {\n const int32_t magic = FASTTEXT_FILEFORMAT_MAGIC_INT32;\n const int32_t version = FASTTEXT_VERSION;\n out.write((char*)&(magic), sizeof(int32_t));\n out.write((char*)&(version), sizeof(int32_t));\n}\n\nvoid FastText::saveModel(const std::string& filename) {\n std::ofstream ofs(filename, std::ofstream::binary);\n if (!ofs.is_open()) {\n throw std::invalid_argument(filename + \" cannot be opened for saving!\");\n }\n if (!input_ || !output_) {\n throw std::runtime_error(\"Model never trained\");\n }\n signModel(ofs);\n args_->save(ofs);\n dict_->save(ofs);\n\n ofs.write((char*)&(quant_), sizeof(bool));\n input_->save(ofs);\n\n ofs.write((char*)&(args_->qout), sizeof(bool));\n output_->save(ofs);\n\n ofs.close();\n}\n\nvoid FastText::loadModel(const std::string& filename) {\n std::ifstream ifs(filename, std::ifstream::binary);\n if (!ifs.is_open()) {\n throw std::invalid_argument(filename + \" cannot be opened for loading!\");\n }\n if (!checkModel(ifs)) {\n throw std::invalid_argument(filename + \" has wrong file format!\");\n }\n loadModel(ifs);\n ifs.close();\n}\n\nstd::vector FastText::getTargetCounts() const {\n if (args_->model == model_name::sup) {\n return dict_->getCounts(entry_type::label);\n } else {\n return dict_->getCounts(entry_type::word);\n }\n}\n\nvoid FastText::buildModel() {\n auto loss = createLoss(output_);\n bool normalizeGradient = (args_->model == model_name::sup);\n model_ = std::make_shared(input_, output_, loss, normalizeGradient);\n}\n\nvoid FastText::loadModel(std::istream& in) {\n args_ = std::make_shared();\n input_ = std::make_shared();\n output_ = std::make_shared();\n args_->load(in);\n if (version == 11 && args_->model == model_name::sup) {\n // backward compatibility: old supervised models do not use char ngrams.\n args_->maxn = 0;\n }\n dict_ = std::make_shared(args_, in);\n\n bool quant_input;\n in.read((char*)&quant_input, sizeof(bool));\n if (quant_input) {\n quant_ = true;\n input_ = std::make_shared();\n }\n input_->load(in);\n\n if (!quant_input && dict_->isPruned()) {\n throw std::invalid_argument(\n \"Invalid model file.\\n\"\n \"Please download the updated model from www.fasttext.cc.\\n\"\n \"See issue #332 on Github for more information.\\n\");\n }\n\n in.read((char*)&args_->qout, sizeof(bool));\n if (quant_ && args_->qout) {\n output_ = std::make_shared();\n }\n output_->load(in);\n\n buildModel();\n}\n\nstd::tuple FastText::progressInfo(real progress) {\n double t = utils::getDuration(start_, std::chrono::steady_clock::now());\n double lr = args_->lr * (1.0 - progress);\n double wst = 0;\n\n int64_t eta = 2592000; // Default to one month in seconds (720 * 3600)\n\n if (progress > 0 && t >= 0) {\n eta = t * (1 - progress) / progress;\n wst = double(tokenCount_) / t / args_->thread;\n }\n\n return std::tuple(wst, lr, eta);\n}\n\nvoid FastText::printInfo(real progress, real loss, std::ostream& log_stream) {\n double wst;\n double lr;\n int64_t eta;\n std::tie(wst, lr, eta) = progressInfo(progress);\n\n log_stream << std::fixed;\n log_stream << \"Progress: \";\n log_stream << std::setprecision(1) << std::setw(5) << (progress * 100) << \"%\";\n log_stream << \" words/sec/thread: \" << std::setw(7) << int64_t(wst);\n log_stream << \" lr: \" << std::setw(9) << std::setprecision(6) << lr;\n log_stream << \" avg.loss: \" << std::setw(9) << std::setprecision(6) << loss;\n log_stream << \" ETA: \" << utils::ClockPrint(eta);\n log_stream << std::flush;\n}\n\nstd::vector FastText::selectEmbeddings(int32_t cutoff) const {\n std::shared_ptr input =\n std::dynamic_pointer_cast(input_);\n Vector norms(input->size(0));\n input->l2NormRow(norms);\n std::vector idx(input->size(0), 0);\n std::iota(idx.begin(), idx.end(), 0);\n auto eosid = dict_->getId(Dictionary::EOS);\n std::sort(idx.begin(), idx.end(), [&norms, eosid](size_t i1, size_t i2) {\n if (i1 == eosid && i2 == eosid) { // satisfy strict weak ordering\n return false;\n }\n return eosid == i1 || (eosid != i2 && norms[i1] > norms[i2]);\n });\n idx.erase(idx.begin() + cutoff, idx.end());\n return idx;\n}\n\nvoid FastText::quantize(const Args& qargs, const TrainCallback& callback) {\n if (args_->model != model_name::sup) {\n throw std::invalid_argument(\n \"For now we only support quantization of supervised models\");\n }\n args_->input = qargs.input;\n args_->qout = qargs.qout;\n args_->output = qargs.output;\n std::shared_ptr input =\n std::dynamic_pointer_cast(input_);\n std::shared_ptr output =\n std::dynamic_pointer_cast(output_);\n bool normalizeGradient = (args_->model == model_name::sup);\n\n if (qargs.cutoff > 0 && qargs.cutoff < input->size(0)) {\n auto idx = selectEmbeddings(qargs.cutoff);\n dict_->prune(idx);\n std::shared_ptr ninput =\n std::make_shared(idx.size(), args_->dim);\n for (auto i = 0; i < idx.size(); i++) {\n for (auto j = 0; j < args_->dim; j++) {\n ninput->at(i, j) = input->at(idx[i], j);\n }\n }\n input = ninput;\n if (qargs.retrain) {\n args_->epoch = qargs.epoch;\n args_->lr = qargs.lr;\n args_->thread = qargs.thread;\n args_->verbose = qargs.verbose;\n auto loss = createLoss(output_);\n model_ = std::make_shared(input, output, loss, normalizeGradient);\n startThreads(callback);\n }\n }\n input_ = std::make_shared(\n std::move(*(input.get())), qargs.dsub, qargs.qnorm);\n\n if (args_->qout) {\n output_ = std::make_shared(\n std::move(*(output.get())), 2, qargs.qnorm);\n }\n quant_ = true;\n auto loss = createLoss(output_);\n model_ = std::make_shared(input_, output_, loss, normalizeGradient);\n}\n\nvoid FastText::supervised(\n Model::State& state,\n real lr,\n const std::vector& line,\n const std::vector& labels) {\n if (labels.size() == 0 || line.size() == 0) {\n return;\n }\n if (args_->loss == loss_name::ova) {\n model_->update(line, labels, Model::kAllLabelsAsTarget, lr, state);\n } else {\n std::uniform_int_distribution<> uniform(0, labels.size() - 1);\n int32_t i = uniform(state.rng);\n model_->update(line, labels, i, lr, state);\n }\n}\n\nvoid FastText::cbow(\n Model::State& state,\n real lr,\n const std::vector& line) {\n std::vector bow;\n std::uniform_int_distribution<> uniform(1, args_->ws);\n for (int32_t w = 0; w < line.size(); w++) {\n int32_t boundary = uniform(state.rng);\n bow.clear();\n for (int32_t c = -boundary; c <= boundary; c++) {\n if (c != 0 && w + c >= 0 && w + c < line.size()) {\n const std::vector& ngrams = dict_->getSubwords(line[w + c]);\n bow.insert(bow.end(), ngrams.cbegin(), ngrams.cend());\n }\n }\n model_->update(bow, line, w, lr, state);\n }\n}\n\nvoid FastText::skipgram(\n Model::State& state,\n real lr,\n const std::vector& line) {\n std::uniform_int_distribution<> uniform(1, args_->ws);\n for (int32_t w = 0; w < line.size(); w++) {\n int32_t boundary = uniform(state.rng);\n const std::vector& ngrams = dict_->getSubwords(line[w]);\n for (int32_t c = -boundary; c <= boundary; c++) {\n if (c != 0 && w + c >= 0 && w + c < line.size()) {\n model_->update(ngrams, line, w + c, lr, state);\n }\n }\n }\n}\n\nstd::tuple\nFastText::test(std::istream& in, int32_t k, real threshold) {\n Meter meter(false);\n test(in, k, threshold, meter);\n\n return std::tuple(\n meter.nexamples(), meter.precision(), meter.recall());\n}\n\nvoid FastText::test(std::istream& in, int32_t k, real threshold, Meter& meter)\n const {\n std::vector line;\n std::vector labels;\n Predictions predictions;\n Model::State state(args_->dim, dict_->nlabels(), 0);\n in.clear();\n in.seekg(0, std::ios_base::beg);\n\n while (in.peek() != EOF) {\n line.clear();\n labels.clear();\n dict_->getLine(in, line, labels);\n\n if (!labels.empty() && !line.empty()) {\n predictions.clear();\n predict(k, line, predictions, threshold);\n meter.log(labels, predictions);\n }\n }\n}\n\nvoid FastText::predict(\n int32_t k,\n const std::vector& words,\n Predictions& predictions,\n real threshold) const {\n if (words.empty()) {\n return;\n }\n Model::State state(args_->dim, dict_->nlabels(), 0);\n if (args_->model != model_name::sup) {\n throw std::invalid_argument(\"Model needs to be supervised for prediction!\");\n }\n model_->predict(words, k, threshold, predictions, state);\n}\n\nbool FastText::predictLine(\n std::istream& in,\n std::vector>& predictions,\n int32_t k,\n real threshold) const {\n predictions.clear();\n if (in.peek() == EOF) {\n return false;\n }\n\n std::vector words, labels;\n dict_->getLine(in, words, labels);\n Predictions linePredictions;\n predict(k, words, linePredictions, threshold);\n for (const auto& p : linePredictions) {\n predictions.push_back(\n std::make_pair(std::exp(p.first), dict_->getLabel(p.second)));\n }\n\n return true;\n}\n\nvoid FastText::getSentenceVector(std::istream& in, fasttext::Vector& svec) {\n svec.zero();\n if (args_->model == model_name::sup) {\n std::vector line, labels;\n dict_->getLine(in, line, labels);\n for (int32_t i = 0; i < line.size(); i++) {\n addInputVector(svec, line[i]);\n }\n if (!line.empty()) {\n svec.mul(1.0 / line.size());\n }\n } else {\n Vector vec(args_->dim);\n std::string sentence;\n std::getline(in, sentence);\n std::istringstream iss(sentence);\n std::string word;\n int32_t count = 0;\n while (iss >> word) {\n getWordVector(vec, word);\n real norm = vec.norm();\n if (norm > 0) {\n vec.mul(1.0 / norm);\n svec.addVector(vec);\n count++;\n }\n }\n if (count > 0) {\n svec.mul(1.0 / count);\n }\n }\n}\n\nstd::vector> FastText::getNgramVectors(\n const std::string& word) const {\n std::vector> result;\n std::vector ngrams;\n std::vector substrings;\n dict_->getSubwords(word, ngrams, substrings);\n assert(ngrams.size() <= substrings.size());\n for (int32_t i = 0; i < ngrams.size(); i++) {\n Vector vec(args_->dim);\n if (ngrams[i] >= 0) {\n vec.addRow(*input_, ngrams[i]);\n }\n result.emplace_back(substrings[i], std::move(vec));\n }\n return result;\n}\n\nvoid FastText::precomputeWordVectors(DenseMatrix& wordVectors) {\n Vector vec(args_->dim);\n wordVectors.zero();\n for (int32_t i = 0; i < dict_->nwords(); i++) {\n std::string word = dict_->getWord(i);\n getWordVector(vec, word);\n real norm = vec.norm();\n if (norm > 0) {\n wordVectors.addVectorToRow(vec, i, 1.0 / norm);\n }\n }\n}\n\nvoid FastText::lazyComputeWordVectors() {\n if (!wordVectors_) {\n wordVectors_ = std::unique_ptr(\n new DenseMatrix(dict_->nwords(), args_->dim));\n precomputeWordVectors(*wordVectors_);\n }\n}\n\nstd::vector> FastText::getNN(\n const std::string& word,\n int32_t k) {\n Vector query(args_->dim);\n\n getWordVector(query, word);\n\n lazyComputeWordVectors();\n assert(wordVectors_);\n return getNN(*wordVectors_, query, k, {word});\n}\n\nstd::vector> FastText::getNN(\n const DenseMatrix& wordVectors,\n const Vector& query,\n int32_t k,\n const std::set& banSet) {\n std::vector> heap;\n\n real queryNorm = query.norm();\n if (std::abs(queryNorm) < 1e-8) {\n queryNorm = 1;\n }\n\n for (int32_t i = 0; i < dict_->nwords(); i++) {\n std::string word = dict_->getWord(i);\n if (banSet.find(word) == banSet.end()) {\n real dp = wordVectors.dotRow(query, i);\n real similarity = dp / queryNorm;\n\n if (heap.size() == k && similarity < heap.front().first) {\n continue;\n }\n heap.push_back(std::make_pair(similarity, word));\n std::push_heap(heap.begin(), heap.end(), comparePairs);\n if (heap.size() > k) {\n std::pop_heap(heap.begin(), heap.end(), comparePairs);\n heap.pop_back();\n }\n }\n }\n std::sort_heap(heap.begin(), heap.end(), comparePairs);\n\n return heap;\n}\n\nstd::vector> FastText::getAnalogies(\n int32_t k,\n const std::string& wordA,\n const std::string& wordB,\n const std::string& wordC) {\n Vector query(args_->dim);\n query.zero();\n\n Vector buffer(args_->dim);\n getWordVector(buffer, wordA);\n query.addVector(buffer, 1.0 / (buffer.norm() + 1e-8));\n getWordVector(buffer, wordB);\n query.addVector(buffer, -1.0 / (buffer.norm() + 1e-8));\n getWordVector(buffer, wordC);\n query.addVector(buffer, 1.0 / (buffer.norm() + 1e-8));\n\n lazyComputeWordVectors();\n assert(wordVectors_);\n return getNN(*wordVectors_, query, k, {wordA, wordB, wordC});\n}\n\nbool FastText::keepTraining(const int64_t ntokens) const {\n return tokenCount_ < args_->epoch * ntokens && !trainException_;\n}\n\nvoid FastText::trainThread(int32_t threadId, const TrainCallback& callback) {\n std::ifstream ifs(args_->input);\n utils::seek(ifs, threadId * utils::size(ifs) / args_->thread);\n\n Model::State state(args_->dim, output_->size(0), threadId + args_->seed);\n\n const int64_t ntokens = dict_->ntokens();\n int64_t localTokenCount = 0;\n std::vector line, labels;\n uint64_t callbackCounter = 0;\n try {\n while (keepTraining(ntokens)) {\n real progress = real(tokenCount_) / (args_->epoch * ntokens);\n if (callback && ((callbackCounter++ % 64) == 0)) {\n double wst;\n double lr;\n int64_t eta;\n std::tie(wst, lr, eta) =\n progressInfo(progress);\n callback(progress, loss_, wst, lr, eta);\n }\n real lr = args_->lr * (1.0 - progress);\n if (args_->model == model_name::sup) {\n localTokenCount += dict_->getLine(ifs, line, labels);\n supervised(state, lr, line, labels);\n } else if (args_->model == model_name::cbow) {\n localTokenCount += dict_->getLine(ifs, line, state.rng);\n cbow(state, lr, line);\n } else if (args_->model == model_name::sg) {\n localTokenCount += dict_->getLine(ifs, line, state.rng);\n skipgram(state, lr, line);\n }\n if (localTokenCount > args_->lrUpdateRate) {\n tokenCount_ += localTokenCount;\n localTokenCount = 0;\n if (threadId == 0 && args_->verbose > 1) {\n loss_ = state.getLoss();\n }\n }\n }\n } catch (DenseMatrix::EncounteredNaNError&) {\n trainException_ = std::current_exception();\n }\n if (threadId == 0)\n loss_ = state.getLoss();\n ifs.close();\n}\n\nstd::shared_ptr FastText::getInputMatrixFromFile(\n const std::string& filename) const {\n std::ifstream in(filename);\n std::vector words;\n std::shared_ptr mat; // temp. matrix for pretrained vectors\n int64_t n, dim;\n if (!in.is_open()) {\n throw std::invalid_argument(filename + \" cannot be opened for loading!\");\n }\n in >> n >> dim;\n if (dim != args_->dim) {\n throw std::invalid_argument(\n \"Dimension of pretrained vectors (\" + std::to_string(dim) +\n \") does not match dimension (\" + std::to_string(args_->dim) + \")!\");\n }\n mat = std::make_shared(n, dim);\n for (size_t i = 0; i < n; i++) {\n std::string word;\n in >> word;\n words.push_back(word);\n dict_->add(word);\n for (size_t j = 0; j < dim; j++) {\n in >> mat->at(i, j);\n }\n }\n in.close();\n\n dict_->threshold(1, 0);\n dict_->init();\n std::shared_ptr input = std::make_shared(\n dict_->nwords() + args_->bucket, args_->dim);\n input->uniform(1.0 / args_->dim, args_->thread, args_->seed);\n\n for (size_t i = 0; i < n; i++) {\n int32_t idx = dict_->getId(words[i]);\n if (idx < 0 || idx >= dict_->nwords()) {\n continue;\n }\n for (size_t j = 0; j < dim; j++) {\n input->at(idx, j) = mat->at(i, j);\n }\n }\n return input;\n}\n\nstd::shared_ptr FastText::createRandomMatrix() const {\n std::shared_ptr input = std::make_shared(\n dict_->nwords() + args_->bucket, args_->dim);\n input->uniform(1.0 / args_->dim, args_->thread, args_->seed);\n\n return input;\n}\n\nstd::shared_ptr FastText::createTrainOutputMatrix() const {\n int64_t m =\n (args_->model == model_name::sup) ? dict_->nlabels() : dict_->nwords();\n std::shared_ptr output =\n std::make_shared(m, args_->dim);\n output->zero();\n\n return output;\n}\n\nvoid FastText::train(const Args& args, const TrainCallback& callback) {\n args_ = std::make_shared(args);\n dict_ = std::make_shared(args_);\n if (args_->input == \"-\") {\n // manage expectations\n throw std::invalid_argument(\"Cannot use stdin for training!\");\n }\n std::ifstream ifs(args_->input);\n if (!ifs.is_open()) {\n throw std::invalid_argument(\n args_->input + \" cannot be opened for training!\");\n }\n dict_->readFromFile(ifs);\n ifs.close();\n\n if (!args_->pretrainedVectors.empty()) {\n input_ = getInputMatrixFromFile(args_->pretrainedVectors);\n } else {\n input_ = createRandomMatrix();\n }\n output_ = createTrainOutputMatrix();\n quant_ = false;\n auto loss = createLoss(output_);\n bool normalizeGradient = (args_->model == model_name::sup);\n model_ = std::make_shared(input_, output_, loss, normalizeGradient);\n startThreads(callback);\n}\n\nvoid FastText::abort() {\n try {\n throw AbortError();\n } catch (AbortError&) {\n trainException_ = std::current_exception();\n }\n}\n\nvoid FastText::startThreads(const TrainCallback& callback) {\n start_ = std::chrono::steady_clock::now();\n tokenCount_ = 0;\n loss_ = -1;\n trainException_ = nullptr;\n std::vector threads;\n if (args_->thread > 1) {\n for (int32_t i = 0; i < args_->thread; i++) {\n threads.push_back(std::thread([=]() { trainThread(i, callback); }));\n }\n } else {\n // webassembly can't instantiate `std::thread`\n trainThread(0, callback);\n }\n const int64_t ntokens = dict_->ntokens();\n // Same condition as trainThread\n while (keepTraining(ntokens)) {\n std::this_thread::sleep_for(std::chrono::milliseconds(100));\n if (loss_ >= 0 && args_->verbose > 1) {\n real progress = real(tokenCount_) / (args_->epoch * ntokens);\n std::cerr << \"\\r\";\n printInfo(progress, loss_, std::cerr);\n }\n }\n for (int32_t i = 0; i < threads.size(); i++) {\n threads[i].join();\n }\n if (trainException_) {\n std::exception_ptr exception = trainException_;\n trainException_ = nullptr;\n std::rethrow_exception(exception);\n }\n if (args_->verbose > 0) {\n std::cerr << \"\\r\";\n printInfo(1.0, loss_, std::cerr);\n std::cerr << std::endl;\n }\n}\n\nint FastText::getDimension() const {\n return args_->dim;\n}\n\nbool FastText::isQuant() const {\n return quant_;\n}\n\nbool comparePairs(\n const std::pair& l,\n const std::pair& r) {\n return l.first > r.first;\n}\n\n} // namespace fasttext\n" }, { "path": "src/fasttext.h", "content": "/**\n * Copyright (c) 2016-present, Facebook, Inc.\n * All rights reserved.\n *\n * This source code is licensed under the MIT license found in the\n * LICENSE file in the root directory of this source tree.\n */\n\n#pragma once\n\n#include \n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#include \"args.h\"\n#include \"densematrix.h\"\n#include \"dictionary.h\"\n#include \"matrix.h\"\n#include \"meter.h\"\n#include \"model.h\"\n#include \"real.h\"\n#include \"utils.h\"\n#include \"vector.h\"\n\nnamespace fasttext {\n\nclass FastText {\n public:\n using TrainCallback =\n std::function;\n\n protected:\n std::shared_ptr args_;\n std::shared_ptr dict_;\n std::shared_ptr input_;\n std::shared_ptr output_;\n std::shared_ptr model_;\n std::atomic tokenCount_{};\n std::atomic loss_{};\n std::chrono::steady_clock::time_point start_;\n bool quant_;\n int32_t version;\n std::unique_ptr wordVectors_;\n std::exception_ptr trainException_;\n\n void signModel(std::ostream&);\n bool checkModel(std::istream&);\n void startThreads(const TrainCallback& callback = {});\n void addInputVector(Vector&, int32_t) const;\n void trainThread(int32_t, const TrainCallback& callback);\n std::vector> getNN(\n const DenseMatrix& wordVectors,\n const Vector& queryVec,\n int32_t k,\n const std::set& banSet);\n void lazyComputeWordVectors();\n void printInfo(real, real, std::ostream&);\n std::shared_ptr getInputMatrixFromFile(const std::string&) const;\n std::shared_ptr createRandomMatrix() const;\n std::shared_ptr createTrainOutputMatrix() const;\n std::vector getTargetCounts() const;\n std::shared_ptr createLoss(std::shared_ptr& output);\n void supervised(\n Model::State& state,\n real lr,\n const std::vector& line,\n const std::vector& labels);\n void cbow(Model::State& state, real lr, const std::vector& line);\n void skipgram(Model::State& state, real lr, const std::vector& line);\n std::vector selectEmbeddings(int32_t cutoff) const;\n void precomputeWordVectors(DenseMatrix& wordVectors);\n bool keepTraining(const int64_t ntokens) const;\n void buildModel();\n std::tuple progressInfo(real progress);\n\n public:\n FastText();\n\n int32_t getWordId(const std::string& word) const;\n\n int32_t getSubwordId(const std::string& subword) const;\n\n int32_t getLabelId(const std::string& label) const;\n\n void getWordVector(Vector& vec, const std::string& word) const;\n\n void getSubwordVector(Vector& vec, const std::string& subword) const;\n\n inline void getInputVector(Vector& vec, int32_t ind) {\n vec.zero();\n addInputVector(vec, ind);\n }\n\n const Args getArgs() const;\n\n std::shared_ptr getDictionary() const;\n\n std::shared_ptr getInputMatrix() const;\n\n void setMatrices(\n const std::shared_ptr& inputMatrix,\n const std::shared_ptr& outputMatrix);\n\n std::shared_ptr getOutputMatrix() const;\n\n void saveVectors(const std::string& filename);\n\n void saveModel(const std::string& filename);\n\n void saveOutput(const std::string& filename);\n\n void loadModel(std::istream& in);\n\n void loadModel(const std::string& filename);\n\n void getSentenceVector(std::istream& in, Vector& vec);\n\n void quantize(const Args& qargs, const TrainCallback& callback = {});\n\n std::tuple\n test(std::istream& in, int32_t k, real threshold = 0.0);\n\n void test(std::istream& in, int32_t k, real threshold, Meter& meter) const;\n\n void predict(\n int32_t k,\n const std::vector& words,\n Predictions& predictions,\n real threshold = 0.0) const;\n\n bool predictLine(\n std::istream& in,\n std::vector>& predictions,\n int32_t k,\n real threshold) const;\n\n std::vector> getNgramVectors(\n const std::string& word) const;\n\n std::vector> getNN(\n const std::string& word,\n int32_t k);\n\n std::vector> getAnalogies(\n int32_t k,\n const std::string& wordA,\n const std::string& wordB,\n const std::string& wordC);\n\n void train(const Args& args, const TrainCallback& callback = {});\n\n void abort();\n\n int getDimension() const;\n\n bool isQuant() const;\n\n class AbortError : public std::runtime_error {\n public:\n AbortError() : std::runtime_error(\"Aborted.\") {}\n };\n};\n} // namespace fasttext\n" }, { "path": "src/loss.cc", "content": "/**\n * Copyright (c) 2016-present, Facebook, Inc.\n * All rights reserved.\n *\n * This source code is licensed under the MIT license found in the\n * LICENSE file in the root directory of this source tree.\n */\n\n#include \"loss.h\"\n#include \"utils.h\"\n\n#include \n\nnamespace fasttext {\n\nconstexpr int64_t SIGMOID_TABLE_SIZE = 512;\nconstexpr int64_t MAX_SIGMOID = 8;\nconstexpr int64_t LOG_TABLE_SIZE = 512;\n\nbool comparePairs(\n const std::pair& l,\n const std::pair& r) {\n return l.first > r.first;\n}\n\nreal std_log(real x) {\n return std::log(x + 1e-5);\n}\n\nLoss::Loss(std::shared_ptr& wo) : wo_(wo) {\n t_sigmoid_.reserve(SIGMOID_TABLE_SIZE + 1);\n for (int i = 0; i < SIGMOID_TABLE_SIZE + 1; i++) {\n real x = real(i * 2 * MAX_SIGMOID) / SIGMOID_TABLE_SIZE - MAX_SIGMOID;\n t_sigmoid_.push_back(1.0 / (1.0 + std::exp(-x)));\n }\n\n t_log_.reserve(LOG_TABLE_SIZE + 1);\n for (int i = 0; i < LOG_TABLE_SIZE + 1; i++) {\n real x = (real(i) + 1e-5) / LOG_TABLE_SIZE;\n t_log_.push_back(std::log(x));\n }\n}\n\nreal Loss::log(real x) const {\n if (x > 1.0) {\n return 0.0;\n }\n int64_t i = int64_t(x * LOG_TABLE_SIZE);\n return t_log_[i];\n}\n\nreal Loss::sigmoid(real x) const {\n if (x < -MAX_SIGMOID) {\n return 0.0;\n } else if (x > MAX_SIGMOID) {\n return 1.0;\n } else {\n int64_t i =\n int64_t((x + MAX_SIGMOID) * SIGMOID_TABLE_SIZE / MAX_SIGMOID / 2);\n return t_sigmoid_[i];\n }\n}\n\nvoid Loss::predict(\n int32_t k,\n real threshold,\n Predictions& heap,\n Model::State& state) const {\n computeOutput(state);\n findKBest(k, threshold, heap, state.output);\n std::sort_heap(heap.begin(), heap.end(), comparePairs);\n}\n\nvoid Loss::findKBest(\n int32_t k,\n real threshold,\n Predictions& heap,\n const Vector& output) const {\n for (int32_t i = 0; i < output.size(); i++) {\n if (output[i] < threshold) {\n continue;\n }\n if (heap.size() == k && std_log(output[i]) < heap.front().first) {\n continue;\n }\n heap.push_back(std::make_pair(std_log(output[i]), i));\n std::push_heap(heap.begin(), heap.end(), comparePairs);\n if (heap.size() > k) {\n std::pop_heap(heap.begin(), heap.end(), comparePairs);\n heap.pop_back();\n }\n }\n}\n\nBinaryLogisticLoss::BinaryLogisticLoss(std::shared_ptr& wo)\n : Loss(wo) {}\n\nreal BinaryLogisticLoss::binaryLogistic(\n int32_t target,\n Model::State& state,\n bool labelIsPositive,\n real lr,\n bool backprop) const {\n real score = sigmoid(wo_->dotRow(state.hidden, target));\n if (backprop) {\n real alpha = lr * (real(labelIsPositive) - score);\n state.grad.addRow(*wo_, target, alpha);\n wo_->addVectorToRow(state.hidden, target, alpha);\n }\n if (labelIsPositive) {\n return -log(score);\n } else {\n return -log(1.0 - score);\n }\n}\n\nvoid BinaryLogisticLoss::computeOutput(Model::State& state) const {\n Vector& output = state.output;\n output.mul(*wo_, state.hidden);\n int32_t osz = output.size();\n for (int32_t i = 0; i < osz; i++) {\n output[i] = sigmoid(output[i]);\n }\n}\n\nOneVsAllLoss::OneVsAllLoss(std::shared_ptr& wo)\n : BinaryLogisticLoss(wo) {}\n\nreal OneVsAllLoss::forward(\n const std::vector& targets,\n int32_t /* we take all targets here */,\n Model::State& state,\n real lr,\n bool backprop) {\n real loss = 0.0;\n int32_t osz = state.output.size();\n for (int32_t i = 0; i < osz; i++) {\n bool isMatch = utils::contains(targets, i);\n loss += binaryLogistic(i, state, isMatch, lr, backprop);\n }\n\n return loss;\n}\n\nNegativeSamplingLoss::NegativeSamplingLoss(\n std::shared_ptr& wo,\n int neg,\n const std::vector& targetCounts)\n : BinaryLogisticLoss(wo), neg_(neg), negatives_(), uniform_() {\n real z = 0.0;\n for (size_t i = 0; i < targetCounts.size(); i++) {\n z += pow(targetCounts[i], 0.5);\n }\n for (size_t i = 0; i < targetCounts.size(); i++) {\n real c = pow(targetCounts[i], 0.5);\n for (size_t j = 0; j < c * NegativeSamplingLoss::NEGATIVE_TABLE_SIZE / z;\n j++) {\n negatives_.push_back(i);\n }\n }\n uniform_ = std::uniform_int_distribution(0, negatives_.size() - 1);\n}\n\nreal NegativeSamplingLoss::forward(\n const std::vector& targets,\n int32_t targetIndex,\n Model::State& state,\n real lr,\n bool backprop) {\n assert(targetIndex >= 0);\n assert(targetIndex < targets.size());\n int32_t target = targets[targetIndex];\n real loss = binaryLogistic(target, state, true, lr, backprop);\n\n for (int32_t n = 0; n < neg_; n++) {\n auto negativeTarget = getNegative(target, state.rng);\n loss += binaryLogistic(negativeTarget, state, false, lr, backprop);\n }\n return loss;\n}\n\nint32_t NegativeSamplingLoss::getNegative(\n int32_t target,\n std::minstd_rand& rng) {\n int32_t negative;\n do {\n negative = negatives_[uniform_(rng)];\n } while (target == negative);\n return negative;\n}\n\nHierarchicalSoftmaxLoss::HierarchicalSoftmaxLoss(\n std::shared_ptr& wo,\n const std::vector& targetCounts)\n : BinaryLogisticLoss(wo),\n paths_(),\n codes_(),\n tree_(),\n osz_(targetCounts.size()) {\n buildTree(targetCounts);\n}\n\nvoid HierarchicalSoftmaxLoss::buildTree(const std::vector& counts) {\n tree_.resize(2 * osz_ - 1);\n for (int32_t i = 0; i < 2 * osz_ - 1; i++) {\n tree_[i].parent = -1;\n tree_[i].left = -1;\n tree_[i].right = -1;\n tree_[i].count = 1e15;\n tree_[i].binary = false;\n }\n for (int32_t i = 0; i < osz_; i++) {\n tree_[i].count = counts[i];\n }\n int32_t leaf = osz_ - 1;\n int32_t node = osz_;\n for (int32_t i = osz_; i < 2 * osz_ - 1; i++) {\n int32_t mini[2] = {0};\n for (int32_t j = 0; j < 2; j++) {\n if (leaf >= 0 && tree_[leaf].count < tree_[node].count) {\n mini[j] = leaf--;\n } else {\n mini[j] = node++;\n }\n }\n tree_[i].left = mini[0];\n tree_[i].right = mini[1];\n tree_[i].count = tree_[mini[0]].count + tree_[mini[1]].count;\n tree_[mini[0]].parent = i;\n tree_[mini[1]].parent = i;\n tree_[mini[1]].binary = true;\n }\n for (int32_t i = 0; i < osz_; i++) {\n std::vector path;\n std::vector code;\n int32_t j = i;\n while (tree_[j].parent != -1) {\n path.push_back(tree_[j].parent - osz_);\n code.push_back(tree_[j].binary);\n j = tree_[j].parent;\n }\n paths_.push_back(path);\n codes_.push_back(code);\n }\n}\n\nreal HierarchicalSoftmaxLoss::forward(\n const std::vector& targets,\n int32_t targetIndex,\n Model::State& state,\n real lr,\n bool backprop) {\n real loss = 0.0;\n int32_t target = targets[targetIndex];\n const std::vector& binaryCode = codes_[target];\n const std::vector& pathToRoot = paths_[target];\n for (int32_t i = 0; i < pathToRoot.size(); i++) {\n loss += binaryLogistic(pathToRoot[i], state, binaryCode[i], lr, backprop);\n }\n return loss;\n}\n\nvoid HierarchicalSoftmaxLoss::predict(\n int32_t k,\n real threshold,\n Predictions& heap,\n Model::State& state) const {\n dfs(k, threshold, 2 * osz_ - 2, 0.0, heap, state.hidden);\n std::sort_heap(heap.begin(), heap.end(), comparePairs);\n}\n\nvoid HierarchicalSoftmaxLoss::dfs(\n int32_t k,\n real threshold,\n int32_t node,\n real score,\n Predictions& heap,\n const Vector& hidden) const {\n if (score < std_log(threshold)) {\n return;\n }\n if (heap.size() == k && score < heap.front().first) {\n return;\n }\n\n if (tree_[node].left == -1 && tree_[node].right == -1) {\n heap.push_back(std::make_pair(score, node));\n std::push_heap(heap.begin(), heap.end(), comparePairs);\n if (heap.size() > k) {\n std::pop_heap(heap.begin(), heap.end(), comparePairs);\n heap.pop_back();\n }\n return;\n }\n\n real f = wo_->dotRow(hidden, node - osz_);\n f = 1. / (1 + std::exp(-f));\n\n dfs(k, threshold, tree_[node].left, score + std_log(1.0 - f), heap, hidden);\n dfs(k, threshold, tree_[node].right, score + std_log(f), heap, hidden);\n}\n\nSoftmaxLoss::SoftmaxLoss(std::shared_ptr& wo) : Loss(wo) {}\n\nvoid SoftmaxLoss::computeOutput(Model::State& state) const {\n Vector& output = state.output;\n output.mul(*wo_, state.hidden);\n real max = output[0], z = 0.0;\n int32_t osz = output.size();\n for (int32_t i = 0; i < osz; i++) {\n max = std::max(output[i], max);\n }\n for (int32_t i = 0; i < osz; i++) {\n output[i] = exp(output[i] - max);\n z += output[i];\n }\n for (int32_t i = 0; i < osz; i++) {\n output[i] /= z;\n }\n}\n\nreal SoftmaxLoss::forward(\n const std::vector& targets,\n int32_t targetIndex,\n Model::State& state,\n real lr,\n bool backprop) {\n computeOutput(state);\n\n assert(targetIndex >= 0);\n assert(targetIndex < targets.size());\n int32_t target = targets[targetIndex];\n\n if (backprop) {\n int32_t osz = wo_->size(0);\n for (int32_t i = 0; i < osz; i++) {\n real label = (i == target) ? 1.0 : 0.0;\n real alpha = lr * (label - state.output[i]);\n state.grad.addRow(*wo_, i, alpha);\n wo_->addVectorToRow(state.hidden, i, alpha);\n }\n }\n return -log(state.output[target]);\n};\n\n} // namespace fasttext\n" }, { "path": "src/loss.h", "content": "/**\n * Copyright (c) 2016-present, Facebook, Inc.\n * All rights reserved.\n *\n * This source code is licensed under the MIT license found in the\n * LICENSE file in the root directory of this source tree.\n */\n\n#pragma once\n\n#include \n#include \n#include \n\n#include \"matrix.h\"\n#include \"model.h\"\n#include \"real.h\"\n#include \"utils.h\"\n#include \"vector.h\"\n\nnamespace fasttext {\n\nclass Loss {\n private:\n void findKBest(\n int32_t k,\n real threshold,\n Predictions& heap,\n const Vector& output) const;\n\n protected:\n std::vector t_sigmoid_;\n std::vector t_log_;\n std::shared_ptr& wo_;\n\n real log(real x) const;\n real sigmoid(real x) const;\n\n public:\n explicit Loss(std::shared_ptr& wo);\n virtual ~Loss() = default;\n\n virtual real forward(\n const std::vector& targets,\n int32_t targetIndex,\n Model::State& state,\n real lr,\n bool backprop) = 0;\n virtual void computeOutput(Model::State& state) const = 0;\n\n virtual void predict(\n int32_t /*k*/,\n real /*threshold*/,\n Predictions& /*heap*/,\n Model::State& /*state*/) const;\n};\n\nclass BinaryLogisticLoss : public Loss {\n protected:\n real binaryLogistic(\n int32_t target,\n Model::State& state,\n bool labelIsPositive,\n real lr,\n bool backprop) const;\n\n public:\n explicit BinaryLogisticLoss(std::shared_ptr& wo);\n virtual ~BinaryLogisticLoss() noexcept override = default;\n void computeOutput(Model::State& state) const override;\n};\n\nclass OneVsAllLoss : public BinaryLogisticLoss {\n public:\n explicit OneVsAllLoss(std::shared_ptr& wo);\n ~OneVsAllLoss() noexcept override = default;\n real forward(\n const std::vector& targets,\n int32_t targetIndex,\n Model::State& state,\n real lr,\n bool backprop) override;\n};\n\nclass NegativeSamplingLoss : public BinaryLogisticLoss {\n protected:\n static const int32_t NEGATIVE_TABLE_SIZE = 10000000;\n\n int neg_;\n std::vector negatives_;\n std::uniform_int_distribution uniform_;\n int32_t getNegative(int32_t target, std::minstd_rand& rng);\n\n public:\n explicit NegativeSamplingLoss(\n std::shared_ptr& wo,\n int neg,\n const std::vector& targetCounts);\n ~NegativeSamplingLoss() noexcept override = default;\n\n real forward(\n const std::vector& targets,\n int32_t targetIndex,\n Model::State& state,\n real lr,\n bool backprop) override;\n};\n\nclass HierarchicalSoftmaxLoss : public BinaryLogisticLoss {\n protected:\n struct Node {\n int32_t parent;\n int32_t left;\n int32_t right;\n int64_t count;\n bool binary;\n };\n\n std::vector> paths_;\n std::vector> codes_;\n std::vector tree_;\n int32_t osz_;\n void buildTree(const std::vector& counts);\n void dfs(\n int32_t k,\n real threshold,\n int32_t node,\n real score,\n Predictions& heap,\n const Vector& hidden) const;\n\n public:\n explicit HierarchicalSoftmaxLoss(\n std::shared_ptr& wo,\n const std::vector& counts);\n ~HierarchicalSoftmaxLoss() noexcept override = default;\n real forward(\n const std::vector& targets,\n int32_t targetIndex,\n Model::State& state,\n real lr,\n bool backprop) override;\n void predict(\n int32_t k,\n real threshold,\n Predictions& heap,\n Model::State& state) const override;\n};\n\nclass SoftmaxLoss : public Loss {\n public:\n explicit SoftmaxLoss(std::shared_ptr& wo);\n ~SoftmaxLoss() noexcept override = default;\n real forward(\n const std::vector& targets,\n int32_t targetIndex,\n Model::State& state,\n real lr,\n bool backprop) override;\n void computeOutput(Model::State& state) const override;\n};\n\n} // namespace fasttext\n" }, { "path": "src/main.cc", "content": "/**\n * Copyright (c) 2016-present, Facebook, Inc.\n * All rights reserved.\n *\n * This source code is licensed under the MIT license found in the\n * LICENSE file in the root directory of this source tree.\n */\n\n#include \n#include \n#include \n#include \n#include \"args.h\"\n#include \"autotune.h\"\n#include \"fasttext.h\"\n\nusing namespace fasttext;\n\nvoid printUsage() {\n std::cerr\n << \"usage: fasttext \\n\\n\"\n << \"The commands supported by fasttext are:\\n\\n\"\n << \" supervised train a supervised classifier\\n\"\n << \" quantize quantize a model to reduce the memory \"\n \"usage\\n\"\n << \" test evaluate a supervised classifier\\n\"\n << \" test-label print labels with precision and recall \"\n \"scores\\n\"\n << \" predict predict most likely labels\\n\"\n << \" predict-prob predict most likely labels with \"\n \"probabilities\\n\"\n << \" skipgram train a skipgram model\\n\"\n << \" cbow train a cbow model\\n\"\n << \" print-word-vectors print word vectors given a trained model\\n\"\n << \" print-sentence-vectors print sentence vectors given a trained \"\n \"model\\n\"\n << \" print-ngrams print ngrams given a trained model and \"\n \"word\\n\"\n << \" nn query for nearest neighbors\\n\"\n << \" analogies query for analogies\\n\"\n << \" dump dump arguments,dictionary,input/output \"\n \"vectors\\n\"\n << std::endl;\n}\n\nvoid printQuantizeUsage() {\n std::cerr << \"usage: fasttext quantize \" << std::endl;\n}\n\nvoid printTestUsage() {\n std::cerr\n << \"usage: fasttext test [] []\\n\\n\"\n << \" model filename\\n\"\n << \" test data filename (if -, read from stdin)\\n\"\n << \" (optional; 1 by default) predict top k labels\\n\"\n << \" (optional; 0.0 by default) probability threshold\\n\"\n << std::endl;\n}\n\nvoid printPredictUsage() {\n std::cerr\n << \"usage: fasttext predict[-prob] [] []\\n\\n\"\n << \" model filename\\n\"\n << \" test data filename (if -, read from stdin)\\n\"\n << \" (optional; 1 by default) predict top k labels\\n\"\n << \" (optional; 0.0 by default) probability threshold\\n\"\n << std::endl;\n}\n\nvoid printTestLabelUsage() {\n std::cerr\n << \"usage: fasttext test-label [] []\\n\\n\"\n << \" model filename\\n\"\n << \" test data filename\\n\"\n << \" (optional; 1 by default) predict top k labels\\n\"\n << \" (optional; 0.0 by default) probability threshold\\n\"\n << std::endl;\n}\n\nvoid printPrintWordVectorsUsage() {\n std::cerr << \"usage: fasttext print-word-vectors \\n\\n\"\n << \" model filename\\n\"\n << std::endl;\n}\n\nvoid printPrintSentenceVectorsUsage() {\n std::cerr << \"usage: fasttext print-sentence-vectors \\n\\n\"\n << \" model filename\\n\"\n << std::endl;\n}\n\nvoid printPrintNgramsUsage() {\n std::cerr << \"usage: fasttext print-ngrams \\n\\n\"\n << \" model filename\\n\"\n << \" word to print\\n\"\n << std::endl;\n}\n\nvoid quantize(const std::vector& args) {\n Args a = Args();\n if (args.size() < 3) {\n printQuantizeUsage();\n a.printHelp();\n exit(EXIT_FAILURE);\n }\n a.parseArgs(args);\n FastText fasttext;\n // parseArgs checks if a->output is given.\n fasttext.loadModel(a.output + \".bin\");\n fasttext.quantize(a);\n fasttext.saveModel(a.output + \".ftz\");\n exit(0);\n}\n\nvoid printNNUsage() {\n std::cout << \"usage: fasttext nn \\n\\n\"\n << \" model filename\\n\"\n << \" (optional; 10 by default) predict top k labels\\n\"\n << std::endl;\n}\n\nvoid printAnalogiesUsage() {\n std::cout << \"usage: fasttext analogies \\n\\n\"\n << \" model filename\\n\"\n << \" (optional; 10 by default) predict top k labels\\n\"\n << std::endl;\n}\n\nvoid printDumpUsage() {\n std::cout << \"usage: fasttext dump