[
  {
    "path": ".gitattributes",
    "content": "*.txt linguist-detectable=false\ndata/* lingusit-detectable=false"
  },
  {
    "path": ".gitignore",
    "content": "# Byte-compiled / optimized / DLL files\n__pycache__/\n*.py[cod]\n*$py.class\n\n# C extensions\n*.so\n\n# Distribution / packaging\n.Python\nbuild/\ndevelop-eggs/\ndist/\ndownloads/\neggs/\n.eggs/\nlib/\nlib64/\nparts/\nsdist/\nvar/\nwheels/\nshare/python-wheels/\n*.egg-info/\n.installed.cfg\n*.egg\nMANIFEST\n\n# PyInstaller\n#  Usually these files are written by a python script from a template\n#  before PyInstaller builds the exe, so as to inject date/other infos into it.\n*.manifest\n*.spec\n\n# Installer logs\npip-log.txt\npip-delete-this-directory.txt\n\n# Unit test / coverage reports\nhtmlcov/\n.tox/\n.nox/\n.coverage\n.coverage.*\n.cache\nnosetests.xml\ncoverage.xml\n*.cover\n*.py,cover\n.hypothesis/\n.pytest_cache/\ncover/\n\n# Translations\n*.mo\n*.pot\n\n# Django stuff:\n*.log\nlocal_settings.py\ndb.sqlite3\ndb.sqlite3-journal\n\n# Flask stuff:\ninstance/\n.webassets-cache\n\n# Scrapy stuff:\n.scrapy\n\n# Sphinx documentation\ndocs/_build/\n\n# PyBuilder\n.pybuilder/\ntarget/\n\n# Jupyter Notebook\n.ipynb_checkpoints\n\n# IPython\nprofile_default/\nipython_config.py\n\n# pyenv\n#   For a library or package, you might want to ignore these files since the code is\n#   intended to run in multiple environments; otherwise, check them in:\n# .python-version\n\n# pipenv\n#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.\n#   However, in case of collaboration, if having platform-specific dependencies or dependencies\n#   having no cross-platform support, pipenv may install dependencies that don't work, or not\n#   install all needed dependencies.\n#Pipfile.lock\n\n# poetry\n#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.\n#   This is especially recommended for binary packages to ensure reproducibility, and is more\n#   commonly ignored for libraries.\n#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control\n#poetry.lock\n\n# pdm\n#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.\n#pdm.lock\n#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it\n#   in version control.\n#   https://pdm.fming.dev/#use-with-ide\n.pdm.toml\n\n# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm\n__pypackages__/\n\n# Celery stuff\ncelerybeat-schedule\ncelerybeat.pid\n\n# SageMath parsed files\n*.sage.py\n\n# Environments\n.env\n.venv\nenv/\nvenv/\nENV/\nenv.bak/\nvenv.bak/\n\n# Spyder project settings\n.spyderproject\n.spyproject\n\n# Rope project settings\n.ropeproject\n\n# mkdocs documentation\n/site\n\n# mypy\n.mypy_cache/\n.dmypy.json\ndmypy.json\n\n# Pyre type checker\n.pyre/\n\n# pytype static type analyzer\n.pytype/\n\n# Cython debug symbols\ncython_debug/\n\n# PyCharm\n#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can\n#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore\n#  and can be added to the global gitignore or merged into this file.  For a more nuclear\n#  option (not recommended) you can uncomment the following to ignore the entire idea folder.\n#.idea/\n\nopenai.config\nanthropic.config\nbard.config\n\n*.wp_source\n*.wp_target\n\ngptzero.ipynb\nsymbolic_data\nsymbolic_data_four\n\ninput.txt\nroberta/models\nresults.csv\nquestion.csv\nanswer.csv\nplayground.ipynb\nt_data\n\ndata/wp/raw\ndata/reuter/raw\ndata/essay/raw\n\nroberta/models_old\nroberta/models_bak\nroberta/models*\nsymbolic_data*\nt_data*\n\neve.csv\ngpt_zero.ipynb\ndetect_gpt.ipynb\nzero_shot.ipynb\nghostbuster_stats.ipynb\n*.ipynb\n\ntemp\ndata\nrebuttal.py"
  },
  {
    "path": "LICENSE",
    "content": "Creative Commons Legal Code\n\nAttribution 3.0 Unported\n\n    CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE\n    LEGAL SERVICES. DISTRIBUTION OF THIS LICENSE DOES NOT CREATE AN\n    ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS\n    INFORMATION ON AN \"AS-IS\" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES\n    REGARDING THE INFORMATION PROVIDED, AND DISCLAIMS LIABILITY FOR\n    DAMAGES RESULTING FROM ITS USE.\n\nLicense\n\nTHE WORK (AS DEFINED BELOW) IS PROVIDED UNDER THE TERMS OF THIS CREATIVE\nCOMMONS PUBLIC LICENSE (\"CCPL\" OR \"LICENSE\"). THE WORK IS PROTECTED BY\nCOPYRIGHT AND/OR OTHER APPLICABLE LAW. ANY USE OF THE WORK OTHER THAN AS\nAUTHORIZED UNDER THIS LICENSE OR COPYRIGHT LAW IS PROHIBITED.\n\nBY EXERCISING ANY RIGHTS TO THE WORK PROVIDED HERE, YOU ACCEPT AND AGREE\nTO BE BOUND BY THE TERMS OF THIS LICENSE. TO THE EXTENT THIS LICENSE MAY\nBE CONSIDERED TO BE A CONTRACT, THE LICENSOR GRANTS YOU THE RIGHTS\nCONTAINED HERE IN CONSIDERATION OF YOUR ACCEPTANCE OF SUCH TERMS AND\nCONDITIONS.\n\n1. Definitions\n\n a. \"Adaptation\" means a work based upon the Work, or upon the Work and\n    other pre-existing works, such as a translation, adaptation,\n    derivative work, arrangement of music or other alterations of a\n    literary or artistic work, or phonogram or performance and includes\n    cinematographic adaptations or any other form in which the Work may be\n    recast, transformed, or adapted including in any form recognizably\n    derived from the original, except that a work that constitutes a\n    Collection will not be considered an Adaptation for the purpose of\n    this License. For the avoidance of doubt, where the Work is a musical\n    work, performance or phonogram, the synchronization of the Work in\n    timed-relation with a moving image (\"synching\") will be considered an\n    Adaptation for the purpose of this License.\n b. \"Collection\" means a collection of literary or artistic works, such as\n    encyclopedias and anthologies, or performances, phonograms or\n    broadcasts, or other works or subject matter other than works listed\n    in Section 1(f) below, which, by reason of the selection and\n    arrangement of their contents, constitute intellectual creations, in\n    which the Work is included in its entirety in unmodified form along\n    with one or more other contributions, each constituting separate and\n    independent works in themselves, which together are assembled into a\n    collective whole. A work that constitutes a Collection will not be\n    considered an Adaptation (as defined above) for the purposes of this\n    License.\n c. \"Distribute\" means to make available to the public the original and\n    copies of the Work or Adaptation, as appropriate, through sale or\n    other transfer of ownership.\n d. \"Licensor\" means the individual, individuals, entity or entities that\n    offer(s) the Work under the terms of this License.\n e. \"Original Author\" means, in the case of a literary or artistic work,\n    the individual, individuals, entity or entities who created the Work\n    or if no individual or entity can be identified, the publisher; and in\n    addition (i) in the case of a performance the actors, singers,\n    musicians, dancers, and other persons who act, sing, deliver, declaim,\n    play in, interpret or otherwise perform literary or artistic works or\n    expressions of folklore; (ii) in the case of a phonogram the producer\n    being the person or legal entity who first fixes the sounds of a\n    performance or other sounds; and, (iii) in the case of broadcasts, the\n    organization that transmits the broadcast.\n f. \"Work\" means the literary and/or artistic work offered under the terms\n    of this License including without limitation any production in the\n    literary, scientific and artistic domain, whatever may be the mode or\n    form of its expression including digital form, such as a book,\n    pamphlet and other writing; a lecture, address, sermon or other work\n    of the same nature; a dramatic or dramatico-musical work; a\n    choreographic work or entertainment in dumb show; a musical\n    composition with or without words; a cinematographic work to which are\n    assimilated works expressed by a process analogous to cinematography;\n    a work of drawing, painting, architecture, sculpture, engraving or\n    lithography; a photographic work to which are assimilated works\n    expressed by a process analogous to photography; a work of applied\n    art; an illustration, map, plan, sketch or three-dimensional work\n    relative to geography, topography, architecture or science; a\n    performance; a broadcast; a phonogram; a compilation of data to the\n    extent it is protected as a copyrightable work; or a work performed by\n    a variety or circus performer to the extent it is not otherwise\n    considered a literary or artistic work.\n g. \"You\" means an individual or entity exercising rights under this\n    License who has not previously violated the terms of this License with\n    respect to the Work, or who has received express permission from the\n    Licensor to exercise rights under this License despite a previous\n    violation.\n h. \"Publicly Perform\" means to perform public recitations of the Work and\n    to communicate to the public those public recitations, by any means or\n    process, including by wire or wireless means or public digital\n    performances; to make available to the public Works in such a way that\n    members of the public may access these Works from a place and at a\n    place individually chosen by them; to perform the Work to the public\n    by any means or process and the communication to the public of the\n    performances of the Work, including by public digital performance; to\n    broadcast and rebroadcast the Work by any means including signs,\n    sounds or images.\n i. \"Reproduce\" means to make copies of the Work by any means including\n    without limitation by sound or visual recordings and the right of\n    fixation and reproducing fixations of the Work, including storage of a\n    protected performance or phonogram in digital form or other electronic\n    medium.\n\n2. Fair Dealing Rights. Nothing in this License is intended to reduce,\nlimit, or restrict any uses free from copyright or rights arising from\nlimitations or exceptions that are provided for in connection with the\ncopyright protection under copyright law or other applicable laws.\n\n3. License Grant. Subject to the terms and conditions of this License,\nLicensor hereby grants You a worldwide, royalty-free, non-exclusive,\nperpetual (for the duration of the applicable copyright) license to\nexercise the rights in the Work as stated below:\n\n a. to Reproduce the Work, to incorporate the Work into one or more\n    Collections, and to Reproduce the Work as incorporated in the\n    Collections;\n b. to create and Reproduce Adaptations provided that any such Adaptation,\n    including any translation in any medium, takes reasonable steps to\n    clearly label, demarcate or otherwise identify that changes were made\n    to the original Work. For example, a translation could be marked \"The\n    original work was translated from English to Spanish,\" or a\n    modification could indicate \"The original work has been modified.\";\n c. to Distribute and Publicly Perform the Work including as incorporated\n    in Collections; and,\n d. to Distribute and Publicly Perform Adaptations.\n e. For the avoidance of doubt:\n\n     i. Non-waivable Compulsory License Schemes. In those jurisdictions in\n        which the right to collect royalties through any statutory or\n        compulsory licensing scheme cannot be waived, the Licensor\n        reserves the exclusive right to collect such royalties for any\n        exercise by You of the rights granted under this License;\n    ii. Waivable Compulsory License Schemes. In those jurisdictions in\n        which the right to collect royalties through any statutory or\n        compulsory licensing scheme can be waived, the Licensor waives the\n        exclusive right to collect such royalties for any exercise by You\n        of the rights granted under this License; and,\n   iii. Voluntary License Schemes. The Licensor waives the right to\n        collect royalties, whether individually or, in the event that the\n        Licensor is a member of a collecting society that administers\n        voluntary licensing schemes, via that society, from any exercise\n        by You of the rights granted under this License.\n\nThe above rights may be exercised in all media and formats whether now\nknown or hereafter devised. The above rights include the right to make\nsuch modifications as are technically necessary to exercise the rights in\nother media and formats. Subject to Section 8(f), all rights not expressly\ngranted by Licensor are hereby reserved.\n\n4. Restrictions. The license granted in Section 3 above is expressly made\nsubject to and limited by the following restrictions:\n\n a. You may Distribute or Publicly Perform the Work only under the terms\n    of this License. You must include a copy of, or the Uniform Resource\n    Identifier (URI) for, this License with every copy of the Work You\n    Distribute or Publicly Perform. You may not offer or impose any terms\n    on the Work that restrict the terms of this License or the ability of\n    the recipient of the Work to exercise the rights granted to that\n    recipient under the terms of the License. You may not sublicense the\n    Work. You must keep intact all notices that refer to this License and\n    to the disclaimer of warranties with every copy of the Work You\n    Distribute or Publicly Perform. When You Distribute or Publicly\n    Perform the Work, You may not impose any effective technological\n    measures on the Work that restrict the ability of a recipient of the\n    Work from You to exercise the rights granted to that recipient under\n    the terms of the License. This Section 4(a) applies to the Work as\n    incorporated in a Collection, but this does not require the Collection\n    apart from the Work itself to be made subject to the terms of this\n    License. If You create a Collection, upon notice from any Licensor You\n    must, to the extent practicable, remove from the Collection any credit\n    as required by Section 4(b), as requested. If You create an\n    Adaptation, upon notice from any Licensor You must, to the extent\n    practicable, remove from the Adaptation any credit as required by\n    Section 4(b), as requested.\n b. If You Distribute, or Publicly Perform the Work or any Adaptations or\n    Collections, You must, unless a request has been made pursuant to\n    Section 4(a), keep intact all copyright notices for the Work and\n    provide, reasonable to the medium or means You are utilizing: (i) the\n    name of the Original Author (or pseudonym, if applicable) if supplied,\n    and/or if the Original Author and/or Licensor designate another party\n    or parties (e.g., a sponsor institute, publishing entity, journal) for\n    attribution (\"Attribution Parties\") in Licensor's copyright notice,\n    terms of service or by other reasonable means, the name of such party\n    or parties; (ii) the title of the Work if supplied; (iii) to the\n    extent reasonably practicable, the URI, if any, that Licensor\n    specifies to be associated with the Work, unless such URI does not\n    refer to the copyright notice or licensing information for the Work;\n    and (iv) , consistent with Section 3(b), in the case of an Adaptation,\n    a credit identifying the use of the Work in the Adaptation (e.g.,\n    \"French translation of the Work by Original Author,\" or \"Screenplay\n    based on original Work by Original Author\"). The credit required by\n    this Section 4 (b) may be implemented in any reasonable manner;\n    provided, however, that in the case of a Adaptation or Collection, at\n    a minimum such credit will appear, if a credit for all contributing\n    authors of the Adaptation or Collection appears, then as part of these\n    credits and in a manner at least as prominent as the credits for the\n    other contributing authors. For the avoidance of doubt, You may only\n    use the credit required by this Section for the purpose of attribution\n    in the manner set out above and, by exercising Your rights under this\n    License, You may not implicitly or explicitly assert or imply any\n    connection with, sponsorship or endorsement by the Original Author,\n    Licensor and/or Attribution Parties, as appropriate, of You or Your\n    use of the Work, without the separate, express prior written\n    permission of the Original Author, Licensor and/or Attribution\n    Parties.\n c. Except as otherwise agreed in writing by the Licensor or as may be\n    otherwise permitted by applicable law, if You Reproduce, Distribute or\n    Publicly Perform the Work either by itself or as part of any\n    Adaptations or Collections, You must not distort, mutilate, modify or\n    take other derogatory action in relation to the Work which would be\n    prejudicial to the Original Author's honor or reputation. Licensor\n    agrees that in those jurisdictions (e.g. Japan), in which any exercise\n    of the right granted in Section 3(b) of this License (the right to\n    make Adaptations) would be deemed to be a distortion, mutilation,\n    modification or other derogatory action prejudicial to the Original\n    Author's honor and reputation, the Licensor will waive or not assert,\n    as appropriate, this Section, to the fullest extent permitted by the\n    applicable national law, to enable You to reasonably exercise Your\n    right under Section 3(b) of this License (right to make Adaptations)\n    but not otherwise.\n\n5. Representations, Warranties and Disclaimer\n\nUNLESS OTHERWISE MUTUALLY AGREED TO BY THE PARTIES IN WRITING, LICENSOR\nOFFERS THE WORK AS-IS AND MAKES NO REPRESENTATIONS OR WARRANTIES OF ANY\nKIND CONCERNING THE WORK, EXPRESS, IMPLIED, STATUTORY OR OTHERWISE,\nINCLUDING, WITHOUT LIMITATION, WARRANTIES OF TITLE, MERCHANTIBILITY,\nFITNESS FOR A PARTICULAR PURPOSE, NONINFRINGEMENT, OR THE ABSENCE OF\nLATENT OR OTHER DEFECTS, ACCURACY, OR THE PRESENCE OF ABSENCE OF ERRORS,\nWHETHER OR NOT DISCOVERABLE. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION\nOF IMPLIED WARRANTIES, SO SUCH EXCLUSION MAY NOT APPLY TO YOU.\n\n6. Limitation on Liability. EXCEPT TO THE EXTENT REQUIRED BY APPLICABLE\nLAW, IN NO EVENT WILL LICENSOR BE LIABLE TO YOU ON ANY LEGAL THEORY FOR\nANY SPECIAL, INCIDENTAL, CONSEQUENTIAL, PUNITIVE OR EXEMPLARY DAMAGES\nARISING OUT OF THIS LICENSE OR THE USE OF THE WORK, EVEN IF LICENSOR HAS\nBEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.\n\n7. Termination\n\n a. This License and the rights granted hereunder will terminate\n    automatically upon any breach by You of the terms of this License.\n    Individuals or entities who have received Adaptations or Collections\n    from You under this License, however, will not have their licenses\n    terminated provided such individuals or entities remain in full\n    compliance with those licenses. Sections 1, 2, 5, 6, 7, and 8 will\n    survive any termination of this License.\n b. Subject to the above terms and conditions, the license granted here is\n    perpetual (for the duration of the applicable copyright in the Work).\n    Notwithstanding the above, Licensor reserves the right to release the\n    Work under different license terms or to stop distributing the Work at\n    any time; provided, however that any such election will not serve to\n    withdraw this License (or any other license that has been, or is\n    required to be, granted under the terms of this License), and this\n    License will continue in full force and effect unless terminated as\n    stated above.\n\n8. Miscellaneous\n\n a. Each time You Distribute or Publicly Perform the Work or a Collection,\n    the Licensor offers to the recipient a license to the Work on the same\n    terms and conditions as the license granted to You under this License.\n b. Each time You Distribute or Publicly Perform an Adaptation, Licensor\n    offers to the recipient a license to the original Work on the same\n    terms and conditions as the license granted to You under this License.\n c. If any provision of this License is invalid or unenforceable under\n    applicable law, it shall not affect the validity or enforceability of\n    the remainder of the terms of this License, and without further action\n    by the parties to this agreement, such provision shall be reformed to\n    the minimum extent necessary to make such provision valid and\n    enforceable.\n d. No term or provision of this License shall be deemed waived and no\n    breach consented to unless such waiver or consent shall be in writing\n    and signed by the party to be charged with such waiver or consent.\n e. This License constitutes the entire agreement between the parties with\n    respect to the Work licensed here. There are no understandings,\n    agreements or representations with respect to the Work not specified\n    here. Licensor shall not be bound by any additional provisions that\n    may appear in any communication from You. This License may not be\n    modified without the mutual written agreement of the Licensor and You.\n f. The rights granted under, and the subject matter referenced, in this\n    License were drafted utilizing the terminology of the Berne Convention\n    for the Protection of Literary and Artistic Works (as amended on\n    September 28, 1979), the Rome Convention of 1961, the WIPO Copyright\n    Treaty of 1996, the WIPO Performances and Phonograms Treaty of 1996\n    and the Universal Copyright Convention (as revised on July 24, 1971).\n    These rights and subject matter take effect in the relevant\n    jurisdiction in which the License terms are sought to be enforced\n    according to the corresponding provisions of the implementation of\n    those treaty provisions in the applicable national law. If the\n    standard suite of rights granted under applicable copyright law\n    includes additional rights not granted under this License, such\n    additional rights are deemed to be included in the License; this\n    License is not intended to restrict the license of any rights under\n    applicable law.\n\n\nCreative Commons Notice\n\n    Creative Commons is not a party to this License, and makes no warranty\n    whatsoever in connection with the Work. Creative Commons will not be\n    liable to You or any party on any legal theory for any damages\n    whatsoever, including without limitation any general, special,\n    incidental or consequential damages arising in connection to this\n    license. Notwithstanding the foregoing two (2) sentences, if Creative\n    Commons has expressly identified itself as the Licensor hereunder, it\n    shall have all rights and obligations of Licensor.\n\n    Except for the limited purpose of indicating to the public that the\n    Work is licensed under the CCPL, Creative Commons does not authorize\n    the use by either party of the trademark \"Creative Commons\" or any\n    related trademark or logo of Creative Commons without the prior\n    written consent of Creative Commons. Any permitted use will be in\n    compliance with Creative Commons' then-current trademark usage\n    guidelines, as may be published on its website or otherwise made\n    available upon request from time to time. For the avoidance of doubt,\n    this trademark restriction does not form part of this License.\n\n    Creative Commons may be contacted at https://creativecommons.org/."
  },
  {
    "path": "README.md",
    "content": "# Ghostbuster: Detecting Text <i>Ghostwritten</i> by Large Language Models <a href=\"https://arxiv.org/abs/2305.15047\">[paper]</a> <a href=\"https://ghostbuster.app\">[demo]</a> <a href=\"https://github.com/vivek3141/ghostbuster-data\">[data]</a>\n\nWe introduce Ghostbuster, a state-of-the-art system for detecting AI-generated text. Our method works by passing documents through a series of weaker language models, running a structured search over possible combinations of their features, and then training a classifier on the selected features to predict whether documents are AI-generated. \n\n<p align=\"center\">\n<img src=\"docs/figure.jpg\" width=75%>\n</p>\n\nCrucially, Ghostbuster does not require access to token probabilities from the target model, making it useful for detecting text generated by black-box models or unknown model versions.\n\nWe compare Ghostbuster to a variety of existing detectors, including DetectGPT and GPTZero, as well as a new RoBERTa baseline. Ghostbuster achieves 99.0 F1 when evaluated across domains, which is 5.9 F1 higher than the best preexisting model. It also outperforms all previous approaches in generalization across writing domains (+7.5 F1), prompting strategies (+2.1 F1), and language models (+4.4 F1).\n\n## Datasets\n\nIn conjunction with our model, we release three new datasets of human- and AI-generated text as detection benchmarks in the domains of student essays, creative writing, and news articles. These can be found under the `data` folder. Each dataset also contains a `logprobs` folder, which consists of files of the form `X-ada.txt` and `X-davinci.txt`, corresponding to token-wise logprobs when running the document `X.txt` under ada and davinci.\n\nWe note that any usages of our data prior to Nov 14 2023 would likely be using our old dataset. In order to access this, please revert to the commit hash `604d85c`.\n\n# Installation\n\nEach of our files are pickled with `python3.10`, so we highly reccomend creating a new conda environment as follows:\n\n```\nconda create -n ghostbuster python=3.10\nconda activate ghostbuster\n```\n\nThen, clone the reponsitory:\n\n```\ngit clone git@github.com:vivek3141/ghostbuster.git\ncd ghostbuster\n```\n\nLastly, install the dependencies and the package:\n\n```\npip install -r requirements.txt\npip install -e .\n```\n\nYou may also need to open a `python` shell to install the following nltk `brown` model:\n```python\nimport nltk\nnltk.download('brown')\n```\n\n# Usage\n\nIn order to run a standalone text through Ghostbuster, we provide a `classify.py` file with the following usage:\n\n```\npython3 classify.py --file INPUT_FILE_HERE --openai_key OPENAI_KEY\n```\n\nTo run the experiment files, create a file called `openai.config` in the main directory with the following template:\n```javascript\n{\n    \"organization\": ORGANIZATION,\n    \"api_key\": API_KEY\n}\n```\n\nThen, you must generate the cached symbolic data file. This consists of a feature vector for every single feature found through our \"symbolic search\" method. Running these commands will create binary files in the root directory:\n```\npython train.py --generate_symbolic_data_four\npython train.py --generate_symbolic_data_eval\n```\nThese commands should take a couple hours to run! Then, you can run any of the experiments listed in the `run.py` file.\n\n## Disclaimer\n\nGhostbuster’s training data, which consists of news, student essay, and creative writing data, is not representative of all writing styles or topics and contains predominantly British and American English text. If you wish to apply Ghostbuster to real-world cases of potential off-limits usage of text generation, such as identifying ChatGPT-written student essays, be wary that incorrect predictions by Ghostbuster are particularly likely in the following cases:\n\n<ul>\n<li> For shorter text\n<li> In domains that are further from those on which Ghostbuster was trained (e.g., text messages)\n<li> For text in varieties of English besides American and British English, or in non-English languages\n<li> For text written by non-native speakers of English\n<li> For AI-generated text that has been edited or paraphrased by a human\n</ul>\n\nNo AI-generated text detector is 100% accurate; we strongly discourage incorporation of Ghostbuster into any systems that automatically penalize students or other writers for alleged usage of text generation without human intervention. Privacy: Please be aware that all inputs to Ghostbuster are sent to the OpenAI API, and we also save the inputs for internal testing purposes. Though we will not distribute the data publicly, we cannot guarantee the privacy of any inputs to Ghostbuster."
  },
  {
    "path": "classify.py",
    "content": "import numpy as np\nimport dill as pickle\nimport tiktoken\nimport openai\nimport argparse\n\nfrom sklearn.linear_model import LogisticRegression\nfrom utils.featurize import normalize, t_featurize_logprobs, score_ngram\nfrom utils.symbolic import train_trigram, get_words, vec_functions, scalar_functions\n\nparser = argparse.ArgumentParser()\nparser.add_argument(\"--file\", type=str, default=\"input.txt\")\nparser.add_argument(\"--openai_key\", type=str, default=\"\")\nargs = parser.parse_args()\n\nif args.openai_key != \"\":\n    openai.api_key = args.openai_key\n\nfile = args.file\nMAX_TOKENS = 2047\nbest_features = open(\"model/features.txt\").read().strip().split(\"\\n\")\n\n# Load davinci tokenizer\nenc = tiktoken.encoding_for_model(\"davinci\")\n\n# Load model\nmodel = pickle.load(open(\"model/model\", \"rb\"))\nmu = pickle.load(open(\"model/mu\", \"rb\"))\nsigma = pickle.load(open(\"model/sigma\", \"rb\"))\n\n# Load data and featurize\nwith open(file) as f:\n    doc = f.read().strip()\n    # Strip data to first MAX_TOKENS tokens\n    tokens = enc.encode(doc)[:MAX_TOKENS]\n    doc = enc.decode(tokens).strip()\n\n    print(f\"Input: {doc}\")\n\n# Train trigram\nprint(\"Loading Trigram...\")\n\ntrigram_model = train_trigram()\n\ntrigram = np.array(score_ngram(doc, trigram_model, enc.encode, n=3, strip_first=False))\nunigram = np.array(score_ngram(doc, trigram_model.base, enc.encode, n=1, strip_first=False))\n\nresponse = openai.Completion.create(\n    model=\"ada\",\n    prompt=\"<|endoftext|>\" + doc,\n    max_tokens=0,\n    echo=True,\n    logprobs=1,\n)\nada = np.array(list(map(lambda x: np.exp(x), response[\"choices\"][0][\"logprobs\"][\"token_logprobs\"][1:])))\n\nresponse = openai.Completion.create(\n    model=\"davinci\",\n    prompt=\"<|endoftext|>\" + doc,\n    max_tokens=0,\n    echo=True,\n    logprobs=1,\n)\ndavinci = np.array(list(map(lambda x: np.exp(x), response[\"choices\"][0][\"logprobs\"][\"token_logprobs\"][1:])))\n\nsubwords = response[\"choices\"][0][\"logprobs\"][\"tokens\"][1:]\ngpt2_map = {\"\\n\": \"Ċ\", \"\\t\": \"ĉ\", \" \": \"Ġ\"}\nfor i in range(len(subwords)):\n    for k, v in gpt2_map.items():\n        subwords[i] = subwords[i].replace(k, v)\n\nt_features = t_featurize_logprobs(davinci, ada, subwords)\n\nvector_map = {\n    \"davinci-logprobs\": davinci,\n    \"ada-logprobs\": ada,\n    \"trigram-logprobs\": trigram,\n    \"unigram-logprobs\": unigram\n}\n\nexp_features = []\nfor exp in best_features:\n\n    exp_tokens = get_words(exp)\n    curr = vector_map[exp_tokens[0]]\n\n    for i in range(1, len(exp_tokens)):\n        if exp_tokens[i] in vec_functions:\n            next_vec = vector_map[exp_tokens[i+1]]\n            curr = vec_functions[exp_tokens[i]](curr, next_vec)\n        elif exp_tokens[i] in scalar_functions:\n            exp_features.append(scalar_functions[exp_tokens[i]](curr))\n            break\n\ndata = (np.array(t_features + exp_features) - mu) / sigma\npreds = model.predict_proba(data.reshape(-1, 1).T)[:, 1]\n\nprint(f\"Prediction: {preds}\")\n"
  },
  {
    "path": "generate.py",
    "content": "import argparse\nimport openai\nimport re\nimport tqdm\nimport os\nimport math\nimport nltk\nimport numpy as np\nimport string\nimport torch\n\nfrom nltk.corpus import wordnet\nfrom datasets import load_dataset\nfrom nltk.tokenize.treebank import TreebankWordDetokenizer\nfrom tenacity import (\n    retry,\n    stop_after_attempt,\n    wait_random_exponential,\n)\nfrom transformers import PegasusForConditionalGeneration, PegasusTokenizer\nfrom transformers import AutoTokenizer, AutoModelForCausalLM\n\nfrom utils.generate import generate_documents\nfrom utils.write_logprobs import write_logprobs, write_llama_logprobs\nfrom utils.symbolic import convert_file_to_logprob_file\nfrom utils.load import Dataset, get_generate_dataset\n\n\nnltk.download(\"wordnet\")\nnltk.download(\"omw-1.4\")\n\n\nllama_tokenizer = AutoTokenizer.from_pretrained(\"meta-llama/Llama-2-7b-hf\")\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(\"Using device:\", device)\n\ndatasets = [\n    Dataset(\"normal\", \"data/wp/human\"),\n    Dataset(\"normal\", \"data/wp/gpt\"),\n    Dataset(\"author\", \"data/reuter/human\"),\n    Dataset(\"author\", \"data/reuter/gpt\"),\n    Dataset(\"normal\", \"data/essay/human\"),\n    Dataset(\"normal\", \"data/essay/gpt\"),\n]\ngenerate_dataset_fn = get_generate_dataset(*datasets)\n\nprompt_types = [\"gpt\", \"gpt_prompt1\", \"gpt_prompt2\", \"gpt_writing\", \"gpt_semantic\"]\nhtml_replacements = [\n    (\"&amp;\", \"&\"),\n    (\"&lt;\", \"<\"),\n    (\"&gt;\", \">\"),\n    (\"&quot;\", '\"'),\n    (\"&apos;\", \"'\"),\n]\n\nperturb_char_names = [\n    \"char_basic\",\n    \"char_space\",\n    \"char_cap\",\n    \"word_adj\",\n    \"word_syn\",\n]\nperturb_char_sizes = [0, 1, 2, 3, 4, 5, 10, 20, 50, 100, 200]\n\nperturb_sent_names = [\"sent_adj\", \"sent_paraph\", \"para_adj\", \"para_paraph\"]\nperturb_sent_sizes = list(range(11))\n\n\ndef closest_synonym(word):\n    synonyms = wordnet.synsets(word)\n    if not synonyms:\n        return None  # Return None if there are no synonyms\n    closest_synset = synonyms[0]  # Assume the first synset is the closest\n    for synset in synonyms[1:]:\n        # Update closest_synset if we find a synset with more lemmas (synonyms)\n        if len(synset.lemmas()) > len(closest_synset.lemmas()):\n            closest_synset = synset\n    # Return the name of the lemma from the closest synset\n    # that is not the same as the input word\n    for lemma in closest_synset.lemmas():\n        if lemma.name() != word:\n            return lemma.name()\n    return None\n\n\ndef html_replace(text):\n    for replacement in html_replacements:\n        text = text.replace(replacement[0], replacement[1])\n    return text\n\n\n@retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(6))\ndef openai_backoff(**kwargs):\n    return openai.ChatCompletion.create(**kwargs)\n\n\ndef round_to_100(n):\n    return int(round(n / 100.0)) * 100\n\n\ndef get_wp_prompts(words, prompt):\n    return [\n        f'Write a story in {words} words to the prompt \"{prompt}.\"',\n        f'You are an author, who is writing a story in response to the prompt \"{prompt}.\" What would you write in {words} words?',\n        f'Hi! I\\'m trying to write a {words}-word story on the following prompt: \"{prompt}.\" Could you please draft something for me?',\n        f'Please help me write a short story in response to the prompt \"{prompt}.\"',\n        f'Write a {words}-word story in the style of a beginner writer in response to the prompt \"{prompt}.\"',\n        f'Write a story with very short sentences in {words} words to the prompt \"{prompt}.\"',\n    ]\n\n\ndef get_reuter_prompts(words, headline):\n    return [\n        f'Write a news article in {words} words based on the headline \"{headline}.\"',\n        f'You are a news reporter, who is writing an article with the headline \"{headline}.\" What would you write in {words} words?',\n        f'Hi! I\\'m trying to write a {words}-word news article based on the following headline: \"{headline}.\" Could you please draft something for me?',\n        f'Please help me write a New York Times article for the headline \"{headline}.\"',\n        f'Write a {words}-word news article in the style of a New York Times article based on the headline \"{headline}.\"',\n        f'Write a news article with very short sentences in {words} words based on the headline \"{headline}.\"',\n    ]\n\n\ndef get_essay_prompts(words, prompts):\n    return [\n        f'Write an essay in {words} words to the prompt \"{prompt}.\"',\n        f'You are a student, who is writing an essay in response to the prompt \"{prompt}.\" What would you write in {words} words?',\n        f'Hi! I\\'m trying to write a {words}-word essay based on the following prompt: \"{prompt}.\" Could you please draft something for me?',\n        f'Please help me write an essay in response to the prompt \"{prompt}.\"',\n        f\"Write a {words}-word essay in the style of a high-school student  in response to the following prompt: {prompt}.\",\n        f'Write an essay with very short sentences in {words} words to the prompt \"{prompt}.\"',\n    ]\n\n\ndef generate_logprobs(generate_dataset_fn, llama_7b_model=None, llama_13b_model=None):\n    files = generate_dataset_fn(lambda f: f)\n\n    for file in tqdm.tqdm(files):\n        if \"logprobs\" in file:\n            continue\n\n        base_path = os.path.dirname(file) + \"/logprobs\"\n        if not os.path.exists(base_path):\n            os.mkdir(base_path)\n\n        with open(file, \"r\") as f:\n            doc = f.read().strip()\n\n        davinci_file = convert_file_to_logprob_file(file, \"davinci\")\n        if not os.path.exists(davinci_file):\n            write_logprobs(doc, davinci_file, \"davinci\")\n\n        ada_file = convert_file_to_logprob_file(file, \"ada\")\n        if not os.path.exists(ada_file):\n            write_logprobs(doc, ada_file, \"ada\")\n\n        llama_7b_file = convert_file_to_logprob_file(file, \"llama-7b\")\n        if llama_7b_model and not os.path.exists(llama_7b_file):\n            write_llama_logprobs(doc, llama_7b_file, llama_7b_model)\n\n        llama_13b_file = convert_file_to_logprob_file(file, \"llama-13b\")\n        if llama_13b_model and not os.path.exists(llama_13b_file):\n            write_llama_logprobs(doc, llama_13b_file, llama_13b_model)\n\n\nif __name__ == \"__main__\":\n    parser = argparse.ArgumentParser()\n\n    parser.add_argument(\"--seed\", type=int, default=0)\n\n    parser.add_argument(\"--wp_prompts\", action=\"store_true\")\n    parser.add_argument(\"--wp_human\", action=\"store_true\")\n    parser.add_argument(\"--wp_gpt\", action=\"store_true\")\n\n    parser.add_argument(\"--reuter_human\", action=\"store_true\")\n    parser.add_argument(\"--reuter_gpt\", action=\"store_true\")\n\n    parser.add_argument(\"--essay_prompts\", action=\"store_true\")\n    parser.add_argument(\"--essay_human\", action=\"store_true\")\n    parser.add_argument(\"--essay_gpt\", action=\"store_true\")\n\n    parser.add_argument(\"--logprobs\", action=\"store_true\")\n    parser.add_argument(\"--logprob_other\", action=\"store_true\")\n    parser.add_argument(\"--logprob_llama\", action=\"store_true\")\n\n    parser.add_argument(\"--gen_perturb_char\", action=\"store_true\")\n    parser.add_argument(\"--logprob_perturb_char\", action=\"store_true\")\n\n    parser.add_argument(\"--gen_perturb_sent\", action=\"store_true\")\n    parser.add_argument(\"--logprob_perturb_sent\", action=\"store_true\")\n\n    args = parser.parse_args()\n\n    if args.wp_prompts:\n\n        def format_prompt(p):\n            p = re.sub(r\"\\[.*\\]\", \"\", p)\n            p = re.sub(r\"\\\\n\", \" \", p)\n            p = re.sub(r\"\\\\t\", \" \", p)\n            p = re.sub(r\"\\s+\", \" \", p)\n            return p.strip()\n\n        with open(\"data/wp/raw/train.wp_source\", \"r\") as f:\n            num_lines_read = 0\n\n            print(\"Generating and writing WP prompts...\")\n\n            pbar = tqdm.tqdm(total=1000)\n            for prompt in f:\n                if num_lines_read >= 1000:\n                    break\n\n                input_prompt = format_prompt(prompt)\n\n                response = openai_backoff(\n                    model=\"gpt-3.5-turbo\",\n                    messages=[\n                        {\n                            \"role\": \"user\",\n                            \"content\": f\"Remove all the formatting in this prompt:\\n\\n{input_prompt}\",\n                        }\n                    ],\n                )\n                reply = response[\"choices\"][0][\"message\"][\"content\"].strip()\n\n                with open(f\"data/wp/prompts/{num_lines_read + 1}.txt\", \"w\") as f:\n                    f.write(reply)\n\n                num_lines_read += 1\n                pbar.update(1)\n\n            pbar.close()\n\n    if args.wp_human:\n        print(\"Formatting Human WP documents...\")\n\n        with open(\"data/wp/raw/train.wp_target\", \"r\") as f:\n            num_lines_read = 0\n\n            pbar = tqdm.tqdm(total=1000)\n            for doc in f:\n                if num_lines_read >= 1000:\n                    break\n\n                doc = doc.strip()\n                tokens = doc.split(\" \")\n\n                replace = [\n                    [\"<newline>\", \"\\n\"],\n                ]\n                for r in replace:\n                    tokens = [t.replace(r[0], r[1]) for t in tokens]\n\n                detokenizer = TreebankWordDetokenizer()\n                formatted_doc = detokenizer.detokenize(tokens)\n\n                formatted_doc = \"\\n\".join(\n                    [i.strip() for i in formatted_doc.split(\"\\n\")]\n                )\n                formatted_doc = formatted_doc.replace(\"\\n\\n\", \"\\n\")\n                formatted_doc = formatted_doc.replace(\"\\n\\n\", \"\\n\")\n\n                formatted_doc = formatted_doc.replace(\" .\", \".\")\n                formatted_doc = formatted_doc.replace(\" ’ \", \"'\")\n\n                formatted_doc = formatted_doc.replace(\" ”\", '\"')\n                formatted_doc = formatted_doc.replace(\"“ \", '\"')\n\n                formatted_doc = html_replace(formatted_doc)\n\n                with open(f\"data/wp/human/{num_lines_read + 1}.txt\", \"w\") as f:\n                    f.write(formatted_doc)\n\n                num_lines_read += 1\n                pbar.update(1)\n\n            pbar.close()\n\n    if args.wp_gpt:\n        print(\"Generating GPT WP documents...\")\n\n        for idx in tqdm.tqdm(range(1, 1001)):\n            with open(f\"data/wp/prompts/{idx}.txt\", \"r\") as f:\n                prompt = f.read().strip()\n\n            with open(f\"data/wp/human/{idx}.txt\", \"r\") as f:\n                words = round_to_100(len(f.read().split(\" \")))\n\n            prompts = get_wp_prompts(words, prompt)\n\n            for type, prompt in zip(prompt_types, prompts):\n                if os.path.exists(f\"data/wp/{type}/{idx}.txt\"):\n                    continue\n\n                response = openai_backoff(\n                    model=\"gpt-3.5-turbo\",\n                    messages=[\n                        {\n                            \"role\": \"user\",\n                            \"content\": prompt,\n                        }\n                    ],\n                )\n\n                reply = response[\"choices\"][0][\"message\"][\"content\"].strip()\n                reply = reply.replace(\"\\n\\n\", \"\\n\")\n\n                with open(f\"data/wp/{type}/{idx}.txt\", \"w\") as f:\n                    f.write(reply)\n\n    if args.reuter_human:\n        reuter_replace = [\"--\", \"202-898-8312\", \"((\", \"($1=\", \"(A$\", \"Reuters Chicago\"]\n\n        authors = os.listdir(\"data/reuter/raw/C50train\")\n        print(\"Formatting Human Reuters documents...\")\n\n        for author in tqdm.tqdm(authors):\n            if not os.path.exists(f\"data/reuter/human/{author}\"):\n                os.makedirs(f\"data/reuter/human/{author}\")\n\n            files = [\n                f\"data/reuter/raw/C50train/{author}/{i}\"\n                for i in os.listdir(f\"data/reuter/raw/C50train/{author}\")\n            ] + [\n                f\"data/reuter/raw/C50test/{author}/{i}\"\n                for i in os.listdir(f\"data/reuter/raw/C50test/{author}\")\n            ]\n\n            for n, file in enumerate(files[:20]):\n                with open(file, \"r\") as f:\n                    doc = f.read().strip()\n                    doc = doc.replace(\"\\n\\n\", \"\\n\")\n\n                    lines = doc.split(\"\\n\")\n                    if any([i in lines[-1] for i in reuter_replace]):\n                        lines = lines[:-1]\n                    doc = \"\\n\".join(lines)\n                    doc = html_replace(doc)\n\n                    with open(f\"data/reuter/human/{author}/{n+1}.txt\", \"w\") as f:\n                        f.write(doc.strip())\n\n    if args.reuter_gpt:\n        print(\"Generating GPT Reuters documents...\")\n\n        authors = os.listdir(\"data/reuter/human\")\n        for author in tqdm.tqdm(authors):\n            for idx in range(1, 21):\n                with open(f\"data/reuter/human/{author}/{idx}.txt\", \"r\") as f:\n                    words = round_to_100(len(f.read().split(\" \")))\n\n                with open(f\"data/reuter/gpt/{author}/headlines/{idx}.txt\", \"r\") as f:\n                    headline = f.read().strip()\n\n                prompts = get_reuter_prompts(words, headline)\n\n                for type, prompt in zip(prompt_types, prompts):\n                    if not os.path.exists(f\"data/reuter/{type}/{author}\"):\n                        os.makedirs(f\"data/reuter/{type}/{author}\")\n\n                    if os.path.exists(f\"data/reuter/{type}/{author}/{idx}.txt\"):\n                        continue\n\n                    response = openai_backoff(\n                        model=\"gpt-3.5-turbo\",\n                        messages=[\n                            {\n                                \"role\": \"user\",\n                                \"content\": prompt,\n                            }\n                        ],\n                    )\n\n                    reply = response[\"choices\"][0][\"message\"][\"content\"].strip()\n                    reply = reply.replace(\"\\n\\n\", \"\\n\")\n\n                    lines = reply.split(\"\\n\")\n                    if any([i in lines[0].lower() for i in [\"sure\", \"certainly\"]]):\n                        reply = \"\\n\".join(lines[1:])\n\n                    lines = reply.split(\"\\n\")\n                    if any([i in lines[0].lower() for i in [\"title\"]]):\n                        reply = \"\\n\".join(lines[1:])\n\n                    with open(f\"data/reuter/{type}/{author}/{idx}.txt\", \"w\") as f:\n                        f.write(reply)\n\n    if args.essay_human or args.essay_gpt:\n        essay_dataset = load_dataset(\"qwedsacf/ivypanda-essays\")\n\n    if args.essay_human:\n        print(\"Formatting Human Essay documents...\")\n\n        num_documents, idx = 0, 0\n        pbar = tqdm.tqdm(total=1000)\n\n        while num_documents < 1000:\n            essay = essay_dataset[\"train\"][idx]\n            essay = essay[\"TEXT\"].strip()\n            essay = essay[essay.index(\"\\n\") + 1 :]\n\n            idx += 1\n\n            if \"table of contents\" in essay.lower():\n                continue\n\n            essay = essay.replace(\"\\n\\n\", \"\\n\")\n            lines = essay.split(\"\\n\")\n\n            doc = []\n            for line in lines:\n                if any(\n                    [\n                        i in line.lower()\n                        for i in [\n                            \"references\",\n                            \"reference\",\n                            \"work cited\",\n                            \"works cited\",\n                            \"bibliography\",\n                        ]\n                    ]\n                ):\n                    break\n                doc.append(line)\n            doc = \"\\n\".join(doc)\n\n            with open(f\"data/essay/human/{num_documents + 1}.txt\", \"w\") as f:\n                f.write(doc.strip())\n\n            num_documents += 1\n            pbar.update(1)\n\n    if args.essay_prompts:\n        print(\"Generating Essay prompts...\")\n\n        for idx in tqdm.tqdm(range(1, 1001)):\n            with open(f\"data/essay/human/{idx}.txt\", \"r\") as f:\n                doc = f.read().strip()\n\n            response = openai_backoff(\n                model=\"gpt-3.5-turbo\",\n                messages=[\n                    {\n                        \"role\": \"user\",\n                        \"content\": f\"Given the following essay, write a prompt for it:\\n\\n{' '.join(doc.split(' ')[:500])}\",\n                    }\n                ],\n            )\n            reply = response[\"choices\"][0][\"message\"][\"content\"].strip()\n            reply = reply.replace(\"Prompt: \", \"\").strip()\n\n            with open(f\"data/essay/prompts/{idx}.txt\", \"w\") as f:\n                f.write(reply)\n\n    if args.essay_gpt:\n        print(\"Generating GPT Essay documents...\")\n\n        for type in prompt_types:\n            if not os.path.exists(f\"data/essay/{type}\"):\n                os.makedirs(f\"data/essay/{type}\")\n\n        for idx in tqdm.tqdm(range(1, 1001)):\n            with open(f\"data/essay/prompts/{idx}.txt\", \"r\") as f:\n                prompt = f.read().strip()\n\n            with open(f\"data/essay/human/{idx}.txt\", \"r\") as f:\n                words = round_to_100(len(f.read().split(\" \")))\n\n            prompts = get_essay_prompts(words, prompt)\n\n            for type, prompt in zip(prompt_types, prompts):\n                if os.path.exists(f\"data/essay/{type}/{idx}.txt\"):\n                    continue\n\n                response = openai_backoff(\n                    model=\"gpt-3.5-turbo\",\n                    messages=[\n                        {\n                            \"role\": \"user\",\n                            \"content\": prompt,\n                        }\n                    ],\n                )\n\n                reply = response[\"choices\"][0][\"message\"][\"content\"].strip()\n                reply = reply.replace(\"\\n\\n\", \"\\n\")\n\n                lines = reply.split(\"\\n\")\n                if any([i in lines[0].lower() for i in [\"sure\", \"certainly\"]]):\n                    reply = \"\\n\".join(lines[1:])\n\n                lines = reply.split(\"\\n\")\n                if any([i in lines[0].lower() for i in [\"title\"]]):\n                    reply = \"\\n\".join(lines[1:])\n\n                with open(f\"data/essay/{type}/{idx}.txt\", \"w\") as f:\n                    f.write(reply)\n\n    if args.logprobs:\n        datasets = [\n            Dataset(\"normal\", \"data/wp/human\"),\n            Dataset(\"normal\", \"data/wp/gpt\"),\n            Dataset(\"author\", \"data/reuter/human\"),\n            Dataset(\"author\", \"data/reuter/gpt\"),\n            Dataset(\"normal\", \"data/essay/human\"),\n            Dataset(\"normal\", \"data/essay/gpt\"),\n        ]\n        generate_logprobs(get_generate_dataset(*datasets))\n\n    if args.logprob_other:\n        other_datasets = [\n            Dataset(\"normal\", \"data/other/ets\"),\n            Dataset(\"normal\", \"data/other/lang8\"),\n            Dataset(\"normal\", \"data/other/pelic\"),\n            Dataset(\"normal\", \"data/other/gptzero/gpt\"),\n            Dataset(\"normal\", \"data/other/gptzero/human\"),\n            Dataset(\"normal\", \"data/other/toefl91\"),\n            Dataset(\"normal\", \"data/other/undetectable\"),\n        ]\n\n        generate_logprobs(get_generate_dataset(*other_datasets))\n\n    if args.logprob_llama:\n        print(\"Loading LLAMA...\")\n        # llama_7b = AutoModelForCausalLM.from_pretrained(\"meta-llama/Llama-2-7b-hf\").to(\n        #     device\n        # )\n        llama_13b = AutoModelForCausalLM.from_pretrained(\"TheBloke/Llama-2-13B-AWQ\").to(\n            device\n        )\n        print(\"LLAMA Loaded\")\n\n        datasets = [\n            Dataset(\"normal\", \"data/wp/human\"),\n            Dataset(\"normal\", \"data/wp/gpt\"),\n            Dataset(\"author\", \"data/reuter/human\"),\n            Dataset(\"author\", \"data/reuter/gpt\"),\n            Dataset(\"normal\", \"data/essay/human\"),\n            Dataset(\"normal\", \"data/essay/gpt\"),\n        ]\n        generate_logprobs(\n            get_generate_dataset(*datasets),\n            # llama_7b_model=llama_7b,\n            llama_13b_model=llama_13b,\n        )\n\n    if args.gen_perturb_char:\n\n        def perturb_char_basic(doc, n=1):\n            if len(doc) < 2:\n                return doc\n\n            for _ in range(n):\n                peturb_type = np.random.choice([\"swap\", \"delete\", \"insert\"])\n                if peturb_type == \"swap\":\n                    idx = np.random.randint(len(doc) - 1)\n                    doc = doc[:idx] + doc[idx + 1] + doc[idx] + doc[idx + 2 :]\n                elif peturb_type == \"delete\" and len(doc) > 1:\n                    idx = np.random.randint(len(doc))\n                    doc = doc[:idx] + doc[idx + 1 :]\n                elif peturb_type == \"insert\":\n                    idx = np.random.randint(len(doc))\n                    doc = (\n                        doc[:idx]\n                        + np.random.choice(list(string.ascii_letters))\n                        + doc[idx:]\n                    )\n            return doc\n\n        def perturb_char_space(doc, n=1):\n            if len(doc) < 2:\n                return doc\n\n            for _ in range(n):\n                perturb_type = np.random.choice([\"insert\", \"delete\"])\n                if perturb_type == \"insert\":\n                    idx = np.random.randint(len(doc))\n                    doc = doc[:idx] + \" \" + doc[idx:]\n                elif perturb_type == \"delete\":\n                    space_indices = [\n                        idx for idx, c in enumerate(doc) if c == \" \" or c == \"\\n\"\n                    ]\n                    if len(space_indices) > 0:\n                        idx = np.random.choice(space_indices)\n                        doc = doc[:idx] + doc[idx + 1 :]\n            return doc\n\n        def perturb_char_cap(doc, n=1):\n            if len(doc) < 2:\n                return doc\n\n            for _ in range(n):\n                idx = np.random.randint(len(doc))\n                if doc[idx].isalpha():\n                    if doc[idx].isupper():\n                        doc = doc[:idx] + doc[idx].lower() + doc[idx + 1 :]\n                    else:\n                        doc = doc[:idx] + doc[idx].upper() + doc[idx + 1 :]\n            return doc\n\n        def perturb_word_adj(doc, n=1):\n            words = doc.split(\" \")\n            if len(words) < 2:\n                return doc\n\n            for _ in range(n):\n                idx = np.random.randint(len(words) - 1)\n                words[idx], words[idx + 1] = words[idx + 1], words[idx]\n            doc = \" \".join(words)\n\n            return doc\n\n        def perturb_word_syn(doc, n=1):\n            words = doc.split(\" \")\n            if len(words) < 2:\n                return doc\n\n            for _ in range(n):\n                idx = np.random.randint(len(words))\n                word = words[idx]\n                synonym = closest_synonym(word)\n                if synonym:\n                    words[idx] = synonym\n            doc = \" \".join(words)\n\n            return doc\n\n        perturb_char_word_fns = {\n            \"char_basic\": perturb_char_basic,\n            \"char_space\": perturb_char_space,\n            \"char_cap\": perturb_char_cap,\n            \"word_adj\": perturb_word_adj,\n            \"word_syn\": perturb_word_syn,\n        }\n\n        if not os.path.exists(\"data/perturb\"):\n            os.makedirs(\"data/perturb\")\n\n        np.random.seed(args.seed)\n        # Construct the test/train split. Seed of 0 ensures seriality across\n        # all files performing the same split.\n        indices = np.arange(6000)\n        np.random.shuffle(indices)\n\n        train, test = (\n            indices[: math.floor(0.8 * len(indices))],\n            indices[math.floor(0.8 * len(indices)) :],\n        )\n\n        # [4320 2006 5689 ... 4256 5807 4875] [5378 5980 5395 ... 1653 2607 2732]\n        print(\"Train/Test Split:\", train, test)\n        files = generate_dataset_fn(lambda f: f, verbose=False)\n\n        indices = np.arange(len(test))\n        np.random.shuffle(indices)\n        indices = indices[:200]\n\n        labels = []\n        for file in files[test][indices]:\n            if \"human\" in file and \"gpt\" not in file:\n                labels.append(0)\n            elif \"gpt\" in file and \"human\" not in file:\n                labels.append(1)\n            else:\n                raise ValueError(\"Invalid file name\")\n\n        with open(\"data/perturb/labels.txt\", \"w\") as f:\n            f.write(\"\\n\".join([str(i) for i in labels]))\n\n        # Generate the perturbed documents\n        num_perturb = [0, 1, 2, 3, 4, 5, 10, 20, 50, 100, 200]\n        for n in tqdm.tqdm(num_perturb):\n            for perturb_type, func in perturb_char_word_fns.items():\n                if not os.path.exists(f\"data/perturb/{perturb_type}/{n}\"):\n                    os.makedirs(f\"data/perturb/{perturb_type}/{n}\")\n\n                for idx, file in enumerate(files[test][indices]):\n                    with open(file, \"r\") as f:\n                        doc = f.read().strip()\n\n                    perturb_doc = func(doc, n=n)\n                    with open(f\"data/perturb/{perturb_type}/{n}/{idx}.txt\", \"w\") as f:\n                        f.write(perturb_doc)\n\n    if args.logprob_perturb_char:\n        perturb_datasets = [\n            Dataset(\"normal\", f\"data/perturb/{perturb_type}/{n}\")\n            for perturb_type in perturb_char_names\n            for n in perturb_char_sizes\n        ]\n\n        generate_logprobs(get_generate_dataset(*perturb_datasets))\n\n    if args.gen_perturb_sent:\n        if torch.cuda.is_available():\n            device = \"cuda\"\n            print(\"Using GPU\")\n        else:\n            device = \"cpu\"\n            print(\"Using CPU\")\n\n        tokenizer = PegasusTokenizer.from_pretrained(\"tuner007/pegasus_paraphrase\")\n        model = PegasusForConditionalGeneration.from_pretrained(\n            \"tuner007/pegasus_paraphrase\"\n        ).to(device)\n\n        def paraphrase(text):\n            batch = tokenizer(\n                [text], truncation=True, padding=\"longest\", return_tensors=\"pt\"\n            ).to(device)\n            translated = model.generate(**batch)\n            tgt_text = tokenizer.batch_decode(translated, skip_special_tokens=True)\n            return tgt_text[0]\n\n        def perturb_sent_adj(doc, n=1):\n            \"\"\"\n            Randomly swap n pairs of adjacent sentences in the document\n            \"\"\"\n            doc = nltk.sent_tokenize(doc)\n            if len(doc) < 2:\n                return (\" \".join(doc)).strip()\n\n            for _ in range(n):\n                idx = np.random.randint(len(doc) - 1)\n                doc[idx], doc[idx + 1] = doc[idx + 1], doc[idx]\n\n            return (\" \".join(doc)).strip()\n\n        def perturb_sent_paraph(doc, n=1):\n            \"\"\"\n            Randomly paraphrase n sentences in the document\n            \"\"\"\n            doc = nltk.sent_tokenize(doc)\n            if len(doc) < 1:\n                return (\" \".join(doc)).strip()\n\n            for _ in range(n):\n                idx = np.random.randint(len(doc))\n                doc[idx] = paraphrase(doc[idx])\n\n            return (\" \".join(doc)).strip()\n\n        def perturb_para_adj(doc, n=1):\n            \"\"\"\n            Randomly swap n pairs of adjacent paragraphs in the document\n            \"\"\"\n            doc = doc.split(\"\\n\")\n            if len(doc) < 2:\n                return \"\\n\".join(doc)\n\n            for _ in range(n):\n                idx = np.random.randint(len(doc) - 1)\n                doc[idx], doc[idx + 1] = doc[idx + 1], doc[idx]\n            return \"\\n\".join(doc)\n\n        def perturb_para_paraph(doc, n=1):\n            \"\"\"\n            Randomly paraphrase n paragraphs in the document\n            \"\"\"\n            doc = doc.split(\"\\n\")\n            if len(doc) < 1:\n                return \"\\n\".join(doc)\n\n            for _ in range(n):\n                idx = np.random.randint(len(doc))\n                doc[idx] = paraphrase(doc[idx])\n\n            return \"\\n\".join(doc)\n\n        perturb_sent_fns = {\n            \"sent_adj\": perturb_sent_adj,\n            \"sent_paraph\": perturb_sent_paraph,\n            \"para_adj\": perturb_para_adj,\n            \"para_paraph\": perturb_para_paraph,\n        }\n\n        if not os.path.exists(\"data/perturb\"):\n            os.makedirs(\"data/perturb\")\n\n        np.random.seed(args.seed)\n        # Construct the test/train split. Seed of 0 ensures seriality across\n        # all files performing the same split.\n        indices = np.arange(6000)\n        np.random.shuffle(indices)\n\n        train, test = (\n            indices[: math.floor(0.8 * len(indices))],\n            indices[math.floor(0.8 * len(indices)) :],\n        )\n\n        # [4320 2006 5689 ... 4256 5807 4875] [5378 5980 5395 ... 1653 2607 2732]\n        print(\"Train/Test Split:\", train, test)\n        files = generate_dataset_fn(lambda f: f, verbose=False)\n\n        indices = np.arange(len(test))\n        np.random.shuffle(indices)\n        indices = indices[:200]\n\n        labels = []\n        for file in files[test][indices]:\n            if \"human\" in file and \"gpt\" not in file:\n                labels.append(0)\n            elif \"gpt\" in file and \"human\" not in file:\n                labels.append(1)\n            else:\n                raise ValueError(\"Invalid file name\")\n\n        with open(\"data/perturb/labels.txt\", \"w\") as f:\n            f.write(\"\\n\".join([str(i) for i in labels]))\n\n        # Generate the perturbed documents\n        num_perturb = list(range(11))\n        for n in tqdm.tqdm(num_perturb):\n            for perturb_type, func in perturb_sent_fns.items():\n                if not os.path.exists(f\"data/perturb/{perturb_type}/{n}\"):\n                    os.makedirs(f\"data/perturb/{perturb_type}/{n}\")\n\n                for idx, file in enumerate(files[test][indices]):\n                    with open(file, \"r\") as f:\n                        doc = f.read().strip()\n\n                    perturb_doc = func(doc, n=n)\n                    with open(f\"data/perturb/{perturb_type}/{n}/{idx}.txt\", \"w\") as f:\n                        f.write(perturb_doc)\n\n    if args.logprob_perturb_sent:\n        perturb_datasets = [\n            Dataset(\"normal\", f\"data/perturb/{perturb_type}/{n}\")\n            for perturb_type in perturb_sent_names\n            for n in perturb_sent_sizes\n        ]\n\n        generate_logprobs(get_generate_dataset(*perturb_datasets))\n"
  },
  {
    "path": "llama.py",
    "content": "import os\nimport argparse\n\nfrom utils.featurize import convert_file_to_logprob_file, get_logprobs\nfrom utils.load import Dataset, get_generate_dataset\nfrom utils.n_gram import TrigramBackoff\nfrom utils.featurize import select_features, normalize\nfrom utils.symbolic import vec_functions, scalar_functions\n\nfrom transformers import AutoTokenizer\nfrom collections import defaultdict\n\nimport math\nimport tqdm\nimport numpy as np\nimport dill as pickle\n\nfrom nltk.util import ngrams\nfrom nltk.corpus import brown\nfrom nltk.tokenize import word_tokenize\n\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.metrics import f1_score\n\ndatasets = [\n    Dataset(\"normal\", \"data/wp/human\"),\n    Dataset(\"normal\", \"data/wp/gpt\"),\n    Dataset(\"author\", \"data/reuter/human\"),\n    Dataset(\"author\", \"data/reuter/gpt\"),\n    Dataset(\"normal\", \"data/essay/human\"),\n    Dataset(\"normal\", \"data/essay/gpt\"),\n]\n\nbest_features = [\n    \"trigram-logprobs v-add unigram-logprobs v-> llama-logprobs s-var\",\n    \"trigram-logprobs v-div unigram-logprobs v-div trigram-logprobs s-avg-top-25\",\n    \"unigram-logprobs v-mul llama-logprobs s-avg\",\n    \"trigram-logprobs v-mul unigram-logprobs v-div trigram-logprobs s-avg\",\n    \"trigram-logprobs v-< unigram-logprobs v-mul llama-logprobs s-avg-top-25\",\n    \"trigram-logprobs v-mul unigram-logprobs v-sub llama-logprobs s-min\",\n    \"trigram-logprobs v-mul unigram-logprobs s-avg\",\n    \"trigram-logprobs v-< unigram-logprobs v-sub llama-logprobs s-avg\",\n    \"trigram-logprobs v-> unigram-logprobs v-add llama-logprobs s-avg\",\n    \"trigram-logprobs v-div llama-logprobs v-div trigram-logprobs s-min\",\n]\n\nmodels = [\"gpt\"]\ndomains = [\"wp\", \"reuter\", \"essay\"]\neval_domains = [\"claude\", \"gpt_prompt1\", \"gpt_prompt2\", \"gpt_writing\", \"gpt_semantic\"]\n\n\nvectors = [\"llama-logprobs\", \"unigram-logprobs\", \"trigram-logprobs\"]\n\nparser = argparse.ArgumentParser()\nparser.add_argument(\"--feature_select\", action=\"store_true\")\nparser.add_argument(\"--classify\", action=\"store_true\")\n\nargs = parser.parse_args()\ntokenizer = AutoTokenizer.from_pretrained(\"meta-llama/Llama-2-7b-hf\")\n\nsentences = brown.sents()\n\ntokenized_corpus = []\nfor sentence in tqdm.tqdm(sentences):\n    tokens = tokenizer(\" \".join(sentence))[\"input_ids\"]\n    tokenized_corpus += tokens\n\ntrigram = TrigramBackoff(tokenized_corpus)\n\n\nvec_combinations = defaultdict(list)\nfor vec1 in range(len(vectors)):\n    for vec2 in range(vec1):\n        for func in vec_functions:\n            if func != \"v-div\":\n                vec_combinations[vectors[vec1]].append(f\"{func} {vectors[vec2]}\")\n\nfor vec1 in vectors:\n    for vec2 in vectors:\n        if vec1 != vec2:\n            vec_combinations[vec1].append(f\"v-div {vec2}\")\n\n\ndef get_words(exp):\n    \"\"\"\n    Splits up expression into words, to be individually processed\n    \"\"\"\n    return exp.split(\" \")\n\n\ndef backtrack_functions(\n    max_depth=2,\n):\n    \"\"\"\n    Backtrack all possible features.\n    \"\"\"\n\n    def helper(prev, depth):\n        if depth >= max_depth:\n            return []\n\n        all_funcs = []\n        prev_word = get_words(prev)[-1]\n\n        for func in scalar_functions:\n            all_funcs.append(f\"{prev} {func}\")\n\n        for comb in vec_combinations[prev_word]:\n            all_funcs += helper(f\"{prev} {comb}\", depth + 1)\n\n        return all_funcs\n\n    ret = []\n    for vec in vectors:\n        ret += helper(vec, 0)\n    return ret\n\n\ndef score_ngram(doc, model, tokenizer, n=3):\n    \"\"\"\n    Returns vector of ngram probabilities given document, model and tokenizer\n    \"\"\"\n    scores = []\n    tokens = (\n        tokenizer(doc.strip())[1:] if n == 1 else (n - 2) * [2] + tokenizer(doc.strip())\n    )\n\n    for i in ngrams(tokens, n):\n        scores.append(model.n_gram_probability(i))\n\n    return np.array(scores)\n\n\ndef get_all_logprobs(\n    generate_dataset,\n    preprocess=lambda x: x.strip(),\n    verbose=True,\n    trigram=None,\n    tokenizer=None,\n    num_tokens=2047,\n):\n    llama_logprobs = {}\n    trigram_logprobs, unigram_logprobs = {}, {}\n\n    if verbose:\n        print(\"Loading logprobs into memory\")\n\n    file_names = generate_dataset(lambda file: file, verbose=False)\n    to_iter = tqdm.tqdm(file_names) if verbose else file_names\n\n    for file in to_iter:\n        if \"logprobs\" in file:\n            continue\n\n        with open(file, \"r\") as f:\n            doc = preprocess(f.read())\n        llama_logprobs[file] = get_logprobs(\n            convert_file_to_logprob_file(file, \"llama-7b\")\n        )[:num_tokens]\n        trigram_logprobs[file] = score_ngram(doc, trigram, tokenizer, n=3)[:num_tokens]\n        unigram_logprobs[file] = score_ngram(doc, trigram.base, tokenizer, n=1)[\n            :num_tokens\n        ]\n\n    return llama_logprobs, trigram_logprobs, unigram_logprobs\n\n\nall_funcs = backtrack_functions(max_depth=3)\nnp.random.seed(0)\n\n# Construct the test/train split. Seed of 0 ensures seriality across\n# all files performing the same split.\nindices = np.arange(6000)\nnp.random.shuffle(indices)\n\ntrain, test = (\n    indices[: math.floor(0.8 * len(indices))],\n    indices[math.floor(0.8 * len(indices)) :],\n)\n\n# [4320 2006 5689 ... 4256 5807 4875] [5378 5980 5395 ... 1653 2607 2732]\nprint(\"Train/Test Split:\", train, test)\n\ngenerate_dataset_fn = get_generate_dataset(*datasets)\nlabels = generate_dataset_fn(\n    lambda file: 1 if any([m in file for m in [\"gpt\", \"claude\"]]) else 0\n)\n\n\n# Construct all indices\ndef get_indices(filter_fn):\n    where = np.where(generate_dataset_fn(filter_fn))[0]\n\n    curr_train = [i for i in train if i in where]\n    curr_test = [i for i in test if i in where]\n\n    return curr_train, curr_test\n\n\nindices_dict = {}\n\nfor model in models + [\"human\"]:\n    train_indices, test_indices = get_indices(\n        lambda file: 1 if model in file else 0,\n    )\n\n    indices_dict[f\"{model}_train\"] = train_indices\n    indices_dict[f\"{model}_test\"] = test_indices\n\nfor model in models + [\"human\"]:\n    for domain in domains:\n        train_key = f\"{model}_{domain}_train\"\n        test_key = f\"{model}_{domain}_test\"\n\n        train_indices, test_indices = get_indices(\n            lambda file: 1 if domain in file and model in file else 0,\n        )\n\n        indices_dict[train_key] = train_indices\n        indices_dict[test_key] = test_indices\n\nif args.feature_select:\n    (\n        llama_logprobs,\n        trigram_logprobs,\n        unigram_logprobs,\n    ) = get_all_logprobs(\n        generate_dataset_fn,\n        verbose=True,\n        tokenizer=lambda x: tokenizer(x)[\"input_ids\"],\n        trigram=trigram,\n    )\n\n    vector_map = {\n        \"llama-logprobs\": lambda file: llama_logprobs[file],\n        \"trigram-logprobs\": lambda file: trigram_logprobs[file],\n        \"unigram-logprobs\": lambda file: unigram_logprobs[file],\n    }\n\n    def calc_features(file, exp):\n        exp_tokens = get_words(exp)\n        curr = vector_map[exp_tokens[0]](file)\n\n        for i in range(1, len(exp_tokens)):\n            if exp_tokens[i] in vec_functions:\n                next_vec = vector_map[exp_tokens[i + 1]](file)\n                curr = vec_functions[exp_tokens[i]](curr, next_vec)\n            elif exp_tokens[i] in scalar_functions:\n                return scalar_functions[exp_tokens[i]](curr)\n\n    print(\"Preparing exp_to_data\")\n    exp_to_data = {}\n    for exp in tqdm.tqdm(all_funcs):\n        exp_to_data[exp] = generate_dataset_fn(\n            lambda file: calc_features(file, exp)\n        ).reshape(-1, 1)\n\n    select_features(exp_to_data, labels, verbose=True, to_normalize=True, indices=train)\n\nif args.classify:\n    (\n        llama_logprobs,\n        trigram_logprobs,\n        unigram_logprobs,\n    ) = get_all_logprobs(\n        generate_dataset_fn,\n        verbose=True,\n        tokenizer=lambda x: tokenizer(x)[\"input_ids\"],\n        trigram=trigram,\n    )\n\n    vector_map = {\n        \"llama-logprobs\": lambda file: llama_logprobs[file],\n        \"trigram-logprobs\": lambda file: trigram_logprobs[file],\n        \"unigram-logprobs\": lambda file: unigram_logprobs[file],\n    }\n\n    def get_exp_featurize(best_features, vector_map):\n        def calc_features(file, exp):\n            exp_tokens = get_words(exp)\n            curr = vector_map[exp_tokens[0]](file)\n\n            for i in range(1, len(exp_tokens)):\n                if exp_tokens[i] in vec_functions:\n                    next_vec = vector_map[exp_tokens[i + 1]](file)\n                    curr = vec_functions[exp_tokens[i]](curr, next_vec)\n                elif exp_tokens[i] in scalar_functions:\n                    return scalar_functions[exp_tokens[i]](curr)\n\n        def exp_featurize(file):\n            return np.array([calc_features(file, exp) for exp in best_features])\n\n        return exp_featurize\n\n    data = generate_dataset_fn(get_exp_featurize(best_features, vector_map))\n    data = normalize(data)\n\n    def train_llama(data, train, test):\n        model = LogisticRegression()\n        model.fit(data[train], labels[train])\n        return f1_score(labels[test], model.predict(data[test]))\n\n    print(\n        f\"In-Domain: {train_llama(data, indices_dict['gpt_train'] + indices_dict['human_train'], indices_dict['gpt_test'] + indices_dict['human_test'])}\"\n    )\n\n    for test_domain in domains:\n        train_indices = []\n        for train_domain in domains:\n            if train_domain == test_domain:\n                continue\n\n            train_indices += (\n                indices_dict[f\"gpt_{train_domain}_train\"]\n                + indices_dict[f\"human_{train_domain}_train\"]\n            )\n\n        print(\n            f\"Out-Domain ({test_domain}): {train_llama(data, train_indices, indices_dict[f'gpt_{test_domain}_test'] + indices_dict[f'human_{test_domain}_test'])}\"\n        )\n"
  },
  {
    "path": "model/features.txt",
    "content": "unigram-logprobs v-> davinci-logprobs s-var\nunigram-logprobs v-sub davinci-logprobs v-div unigram-logprobs s-avg-top-25\ntrigram-logprobs v-> ada-logprobs v-sub davinci-logprobs s-var\nunigram-logprobs v-> trigram-logprobs v-< davinci-logprobs s-avg\nunigram-logprobs v-< trigram-logprobs v-mul ada-logprobs s-avg\nunigram-logprobs v-< trigram-logprobs v-add ada-logprobs s-max\ntrigram-logprobs v-< ada-logprobs v-> davinci-logprobs s-var\ntrigram-logprobs v-> ada-logprobs v-> davinci-logprobs s-avg-top-25\nada-logprobs v-sub davinci-logprobs s-l2\ndavinci-logprobs s-var\ntrigram-logprobs v-< davinci-logprobs v-div unigram-logprobs s-l2\ntrigram-logprobs v-> ada-logprobs v-mul davinci-logprobs s-avg\n"
  },
  {
    "path": "requirements.txt",
    "content": "tqdm\nscikit-learn\nnumpy\ntenacity\nopenai==0.28.1\ntorch\ntiktoken\nflask\ntabulate\ndill\nnltk\ndatasets==2.16.1\ntransformers==4.36.2\nmatplotlib"
  },
  {
    "path": "results/best_features_custom.txt",
    "content": "trigram-logprobs v-< davinci-logprobs v-div unigram-logprobs s-l2\ntrigram-logprobs v-add ada-logprobs v-< davinci-logprobs s-avg-top-25\ntrigram-logprobs v-sub ada-logprobs v-> davinci-logprobs s-var\ntrigram-logprobs v-div ada-logprobs s-avg\ntrigram-logprobs v-div ada-logprobs v-div davinci-logprobs s-avg\ntrigram-logprobs v-div unigram-logprobs v-> ada-logprobs s-l2\nunigram-logprobs v-mul ada-logprobs v-sub davinci-logprobs s-len\nunigram-logprobs v-< ada-logprobs v-> davinci-logprobs s-avg-top-25\nunigram-logprobs v-< ada-logprobs v-div davinci-logprobs s-min\nunigram-logprobs v-add trigram-logprobs v-add davinci-logprobs s-min"
  },
  {
    "path": "results/best_features_essay.txt",
    "content": "unigram-logprobs v-div ada-logprobs v-> davinci-logprobs s-avg\ndavinci-logprobs s-avg\nada-logprobs v-> davinci-logprobs v-div trigram-logprobs s-avg\ntrigram-logprobs v-> davinci-logprobs s-var\n"
  },
  {
    "path": "results/best_features_four.txt",
    "content": "unigram-logprobs v-add trigram-logprobs v-> ada-logprobs v-sub davinci-logprobs s-var\nada-logprobs v-sub davinci-logprobs v-div ada-logprobs v-< davinci-logprobs s-var\nunigram-logprobs v-> trigram-logprobs v-sub ada-logprobs v-add davinci-logprobs s-avg-top-25\nunigram-logprobs v-> trigram-logprobs v-< ada-logprobs v-add davinci-logprobs s-avg\ntrigram-logprobs v-mul davinci-logprobs v-div unigram-logprobs v-< trigram-logprobs s-var\ntrigram-logprobs v-> davinci-logprobs v-div unigram-logprobs v-div davinci-logprobs s-var\nunigram-logprobs v-< trigram-logprobs v-div unigram-logprobs v-add ada-logprobs s-var\nada-logprobs v-div trigram-logprobs v-sub ada-logprobs v-div davinci-logprobs s-max\n"
  },
  {
    "path": "results/best_features_no_gpt.txt",
    "content": "unigram-logprobs s-var\ntrigram-logprobs v-div unigram-logprobs v-sub trigram-logprobs s-min\ntrigram-logprobs v-div unigram-logprobs v-div trigram-logprobs s-min\nunigram-logprobs v-> trigram-logprobs s-avg\nunigram-logprobs v-< trigram-logprobs v-div unigram-logprobs s-avg\nunigram-logprobs v-mul trigram-logprobs s-avg-top-25\nunigram-logprobs v-mul trigram-logprobs s-l2\ntrigram-logprobs s-l2\ntrigram-logprobs v-div unigram-logprobs v-sub trigram-logprobs s-l2\n"
  },
  {
    "path": "results/best_features_old.txt",
    "content": "unigram-logprobs v-div ada-logprobs v-> davinci-logprobs s-avg\nunigram-logprobs v-> ada-logprobs v-sub davinci-logprobs s-var\nunigram-logprobs s-avg\nada-logprobs s-min\ndavinci-logprobs v-div trigram-logprobs v-mul ada-logprobs s-avg\nunigram-logprobs v-> davinci-logprobs s-min\nunigram-logprobs v-> ada-logprobs s-var\nada-logprobs v-sub davinci-logprobs s-avg\nunigram-logprobs v-mul ada-logprobs s-avg\ntrigram-logprobs v-< ada-logprobs s-avg\nunigram-logprobs v-< trigram-logprobs s-var\nunigram-logprobs v-> ada-logprobs v-> davinci-logprobs s-var\nunigram-logprobs v-add trigram-logprobs v-> davinci-logprobs s-avg\nunigram-logprobs v-add ada-logprobs v-sub davinci-logprobs s-avg\ndavinci-logprobs v-div trigram-logprobs v-add davinci-logprobs s-min\n"
  },
  {
    "path": "results/best_features_one.txt",
    "content": "ada-logprobs s-avg\ndavinci-logprobs s-var\ndavinci-logprobs s-min\ntrigram-logprobs s-var\ndavinci-logprobs s-len\ndavinci-logprobs s-max\n"
  },
  {
    "path": "results/best_features_only_ada.txt",
    "content": "unigram-logprobs v-add trigram-logprobs v-< ada-logprobs s-var\nunigram-logprobs v-sub ada-logprobs s-avg-top-25\nada-logprobs s-var\nunigram-logprobs v-mul ada-logprobs s-avg\nunigram-logprobs v-> trigram-logprobs v-div ada-logprobs s-avg\nunigram-logprobs v-< trigram-logprobs v-mul ada-logprobs s-avg\nunigram-logprobs s-avg\ntrigram-logprobs v-div unigram-logprobs v-div ada-logprobs s-max\n"
  },
  {
    "path": "results/best_features_reuter.txt",
    "content": "unigram-logprobs v-> ada-logprobs v-sub davinci-logprobs s-var\ntrigram-logprobs v-div ada-logprobs v-> davinci-logprobs s-avg\nada-logprobs v-sub davinci-logprobs s-var\nunigram-logprobs v-> trigram-logprobs v-add ada-logprobs s-avg\nada-logprobs v-> davinci-logprobs v-div trigram-logprobs s-avg-top-25\n"
  },
  {
    "path": "results/best_features_three.txt",
    "content": "unigram-logprobs v-add trigram-logprobs v-< davinci-logprobs s-avg\nunigram-logprobs v-> ada-logprobs v-sub davinci-logprobs s-var\nunigram-logprobs v-sub ada-logprobs v-add davinci-logprobs s-avg-top-25\nada-logprobs v-> davinci-logprobs v-div ada-logprobs s-avg\ntrigram-logprobs v-div ada-logprobs v-< davinci-logprobs s-avg\nunigram-logprobs v-mul davinci-logprobs s-max\nunigram-logprobs v-sub ada-logprobs v-mul davinci-logprobs s-avg\nunigram-logprobs v-mul trigram-logprobs v-sub ada-logprobs s-var\ntrigram-logprobs v-sub davinci-logprobs v-div unigram-logprobs s-max"
  },
  {
    "path": "results/best_features_two.txt",
    "content": "unigram-logprobs v-< davinci-logprobs s-var\ntrigram-logprobs v-> ada-logprobs s-avg\nada-logprobs v-sub davinci-logprobs s-min\nunigram-logprobs v-sub ada-logprobs s-l2\nunigram-logprobs v-sub davinci-logprobs s-avg-top-25\nunigram-logprobs v-mul davinci-logprobs s-l2\nada-logprobs v-< davinci-logprobs s-avg\ntrigram-logprobs v-mul ada-logprobs s-l2\n"
  },
  {
    "path": "results/best_features_wp.txt",
    "content": "unigram-logprobs v-add trigram-logprobs v-< davinci-logprobs s-avg\nunigram-logprobs v-> trigram-logprobs v-> davinci-logprobs s-avg\nunigram-logprobs v-sub ada-logprobs v-> davinci-logprobs s-avg-top-25\nunigram-logprobs v-mul davinci-logprobs s-var\nada-logprobs v-div unigram-logprobs v-add ada-logprobs s-max\nunigram-logprobs v-sub davinci-logprobs v-div unigram-logprobs s-avg-top-25\ndavinci-logprobs v-div ada-logprobs v-sub davinci-logprobs s-var\n"
  },
  {
    "path": "results/ghostbuster.csv",
    "content": "Model Type,Experiment,Accuracy,F1,AUC\r\nGhostbuster (Depth One),In-Domain (wp),0.948052,0.946524,0.979872\r\nGhostbuster (Depth One),In-Domain (reuter),0.995215,0.995215,0.999496\r\nGhostbuster (Depth One),In-Domain (essay),0.97733,0.97852,0.985001\r\nGhostbuster (Depth One),In-Domain,0.936667,0.936772,0.979032\r\nGhostbuster (Depth One),Out-Domain (wp),0.836364,0.813056,0.926055\r\nGhostbuster (Depth One),Out-Domain (reuter),0.935407,0.937063,0.983769\r\nGhostbuster (Depth One),Out-Domain (essay),0.848866,0.87395,0.962745\r\nGhostbuster (Depth One),Out-Domain (claude),0.841725,0.896527,0.957148\r\nGhostbuster (Depth One),Out-Domain (gpt_prompt1),0.908762,0.942857,0.975045\r\nGhostbuster (Depth One),Out-Domain (gpt_prompt2),0.910431,0.943961,0.974575\r\nGhostbuster (Depth One),Out-Domain (gpt_writing),0.899305,0.936558,0.972838\r\nGhostbuster (Depth One),Out-Domain (gpt_semantic),0.958275,0.974654,0.985868\r\nGhostbuster (Depth Two),In-Domain (wp),0.984416,0.983784,0.998109\r\nGhostbuster (Depth Two),In-Domain (reuter),0.995215,0.995215,0.999725\r\nGhostbuster (Depth Two),In-Domain (essay),0.997481,0.997625,0.999924\r\nGhostbuster (Depth Two),In-Domain,0.983333,0.983471,0.997694\r\nGhostbuster (Depth Two),Out-Domain (wp),0.955844,0.952381,0.995461\r\nGhostbuster (Depth Two),Out-Domain (reuter),0.95933,0.958838,0.991644\r\nGhostbuster (Depth Two),Out-Domain (essay),0.921914,0.931264,0.9904\r\nGhostbuster (Depth Two),Out-Domain (claude),0.85758,0.907044,0.987045\r\nGhostbuster (Depth Two),Out-Domain (gpt_prompt1),0.991099,0.994656,0.999174\r\nGhostbuster (Depth Two),Out-Domain (gpt_prompt2),0.991099,0.994656,0.999431\r\nGhostbuster (Depth Two),Out-Domain (gpt_writing),0.99388,0.996332,0.999353\r\nGhostbuster (Depth Two),Out-Domain (gpt_semantic),0.994159,0.996499,0.9994\r\nGhostbuster (Depth Three),In-Domain (wp),0.984416,0.983784,0.999406\r\nGhostbuster (Depth Three),In-Domain (reuter),0.995215,0.995192,0.999657\r\nGhostbuster (Depth Three),In-Domain (essay),0.994962,0.995261,0.99972\r\nGhostbuster (Depth Three),In-Domain,0.99,0.990066,0.998942\r\nGhostbuster (Depth Three),Out-Domain (wp),0.953247,0.949153,0.996407\r\nGhostbuster (Depth Three),Out-Domain (reuter),0.978469,0.978622,0.996474\r\nGhostbuster (Depth Three),Out-Domain (essay),0.974811,0.976744,0.999491\r\nGhostbuster (Depth Three),Out-Domain (claude),0.878999,0.921973,0.986922\r\nGhostbuster (Depth Three),Out-Domain (gpt_prompt1),0.989152,0.993468,0.998737\r\nGhostbuster (Depth Three),Out-Domain (gpt_prompt2),0.992768,0.995655,0.999135\r\nGhostbuster (Depth Three),Out-Domain (gpt_writing),0.994715,0.996829,0.998888\r\nGhostbuster (Depth Three),Out-Domain (gpt_semantic),0.995828,0.997498,0.999452\r\nGhostbuster (Depth Four),In-Domain (wp),0.979221,0.978261,0.997001\r\nGhostbuster (Depth Four),In-Domain (reuter),0.995215,0.995192,0.999886\r\nGhostbuster (Depth Four),In-Domain (essay),0.994962,0.995261,0.998803\r\nGhostbuster (Depth Four),In-Domain,0.9825,0.982688,0.996741\r\nGhostbuster (Depth Four),Out-Domain (wp),0.961039,0.958217,0.994272\r\nGhostbuster (Depth Four),Out-Domain (reuter),0.964115,0.9642,0.993681\r\nGhostbuster (Depth Four),Out-Domain (essay),0.97733,0.979021,0.997377\r\nGhostbuster (N-Gram Only),In-Domain (wp),0.94026,0.936986,0.985951\r\nGhostbuster (N-Gram Only),In-Domain (reuter),0.91866,0.917874,0.974085\r\nGhostbuster (N-Gram Only),In-Domain (essay),0.962217,0.964871,0.996562\r\nGhostbuster (N-Gram Only),In-Domain,0.88,0.88216,0.94893\r\nGhostbuster (N-Gram Only),Out-Domain (wp),0.818182,0.785276,0.919328\r\nGhostbuster (N-Gram Only),Out-Domain (reuter),0.73445,0.700809,0.806735\r\nGhostbuster (N-Gram Only),Out-Domain (essay),0.65995,0.754991,0.921416\r\nGhostbuster (N-Gram Only),Out-Domain (claude),0.836161,0.894198,0.932834\r\nGhostbuster (N-Gram Only),Out-Domain (gpt_prompt1),0.90904,0.943901,0.963673\r\nGhostbuster (N-Gram Only),Out-Domain (gpt_prompt2),0.901252,0.938804,0.958407\r\nGhostbuster (N-Gram Only),Out-Domain (gpt_writing),0.901252,0.938804,0.959402\r\nGhostbuster (N-Gram Only),Out-Domain (gpt_semantic),0.90904,0.943901,0.961151\r\nGhostbuster (N-Gram and Ada),In-Domain (wp),0.994805,0.994624,0.999568\r\nGhostbuster (N-Gram and Ada),In-Domain (reuter),0.992823,0.992771,0.999908\r\nGhostbuster (N-Gram and Ada),In-Domain (essay),0.997481,0.997625,0.99972\r\nGhostbuster (N-Gram and Ada),In-Domain,0.9875,0.987654,0.998267\r\nGhostbuster (N-Gram and Ada),Out-Domain (wp),0.914286,0.902655,0.996218\r\nGhostbuster (N-Gram and Ada),Out-Domain (reuter),0.973684,0.973747,0.995948\r\nGhostbuster (N-Gram and Ada),Out-Domain (essay),0.906801,0.919037,0.998243\r\nGhostbuster (N-Gram and Ada),Out-Domain (claude),0.717385,0.796962,0.965743\r\nGhostbuster (N-Gram and Ada),Out-Domain (gpt_prompt1),0.984145,0.990441,0.997964\r\nGhostbuster (N-Gram and Ada),Out-Domain (gpt_prompt2),0.983866,0.990272,0.998183\r\nGhostbuster (N-Gram and Ada),Out-Domain (gpt_writing),0.989986,0.993984,0.998328\r\nGhostbuster (N-Gram and Ada),Out-Domain (gpt_semantic),0.989708,0.993816,0.998736\r\n\"Ghostbuster (Depth Three, No Handcrafted)\",In-Domain (wp),0.98961,0.989189,0.998595\r\n\"Ghostbuster (Depth Three, No Handcrafted)\",In-Domain (reuter),0.990431,0.990431,0.999565\r\n\"Ghostbuster (Depth Three, No Handcrafted)\",In-Domain (essay),0.994962,0.995261,0.999796\r\n\"Ghostbuster (Depth Three, No Handcrafted)\",In-Domain,0.989167,0.989247,0.998692\r\n\"Ghostbuster (Depth Three, No Handcrafted)\",Out-Domain (wp),0.94026,0.934097,0.995218\r\n\"Ghostbuster (Depth Three, No Handcrafted)\",Out-Domain (reuter),0.978469,0.97852,0.996108\r\n\"Ghostbuster (Depth Three, No Handcrafted)\",Out-Domain (essay),0.972292,0.974478,0.999618\r\n\"Ghostbuster (Depth Three, No Handcrafted)\",Out-Domain (claude),0.888734,0.928724,0.987969\r\n\"Ghostbuster (Depth Three, No Handcrafted)\",Out-Domain (gpt_prompt1),0.987204,0.99229,0.998831\r\n\"Ghostbuster (Depth Three, No Handcrafted)\",Out-Domain (gpt_prompt2),0.991655,0.994985,0.999216\r\n\"Ghostbuster (Depth Three, No Handcrafted)\",Out-Domain (gpt_writing),0.99388,0.996327,0.999087\r\n\"Ghostbuster (Depth Three, No Handcrafted)\",Out-Domain (gpt_semantic),0.994437,0.996662,0.99959\r\nGhostbuster (No Symbolic),In-Domain (wp),0.797403,0.793651,0.884665\r\nGhostbuster (No Symbolic),In-Domain (reuter),0.880383,0.878641,0.935784\r\nGhostbuster (No Symbolic),In-Domain (essay),0.937028,0.940618,0.988592\r\nGhostbuster (No Symbolic),In-Domain,0.803333,0.805281,0.896296\r\nGhostbuster (No Symbolic),Out-Domain (wp),0.742857,0.772414,0.869725\r\nGhostbuster (No Symbolic),Out-Domain (reuter),0.767943,0.758105,0.851377\r\nGhostbuster (No Symbolic),Out-Domain (essay),0.755668,0.771765,0.863305\r\nGhostbuster (No Symbolic),Out-Domain (claude),0.807789,0.875428,0.893528\r\nGhostbuster (No Symbolic),Out-Domain (gpt_prompt1),0.779138,0.854151,0.868459\r\nGhostbuster (No Symbolic),Out-Domain (gpt_prompt2),0.731015,0.816543,0.843016\r\nGhostbuster (No Symbolic),Out-Domain (gpt_writing),0.689569,0.782116,0.821094\r\nGhostbuster (No Symbolic),Out-Domain (gpt_semantic),0.85007,0.905422,0.914689\r\n"
  },
  {
    "path": "results/other.csv",
    "content": "Model Type,Experiment,Accuracy,F1,AUC\r\nGhostbuster,Out-Domain (lang8),0.955,0.0,-1\r\nRoBERTa,Out-Domain (lang8),0.986,0.0,-1\r\nGhostbuster,Out-Domain (gptzero),0.9,0.9,0.9684\r\nGhostbuster,Out-Domain (ets),0.999,0.0,-1\r\nGhostbuster,In-Domain (ets),1.0,0.0,-1\r\nRoBERTa,Out-Domain (ets),0.981,0.0,-1\r\n"
  },
  {
    "path": "results/perturb.csv",
    "content": "Model Type,Experiment,Accuracy,F1,AUC\nGhostbuster,\"Out-Domain (letter, 0)\",0.985,0.983957,0.998095\nGhostbuster,\"Out-Domain (letter, 10)\",0.905,0.888889,0.989474\nGhostbuster,\"Out-Domain (letter, 25)\",0.655,0.429752,0.970125\nGhostbuster,\"Out-Domain (letter, 50)\",0.535,0.041237,0.957594\nGhostbuster,\"Out-Domain (letter, 100)\",0.525,0.0,0.937343\nGhostbuster,\"Out-Domain (letter, 200)\",0.525,0.0,0.899348\nGhostbuster,\"Out-Domain (word, 0)\",0.985,0.983957,0.998095\nGhostbuster,\"Out-Domain (word, 10)\",0.82,0.766234,0.967519\nGhostbuster,\"Out-Domain (word, 25)\",0.555,0.118812,0.94807\nGhostbuster,\"Out-Domain (word, 50)\",0.525,0.0,0.918095\nGhostbuster,\"Out-Domain (word, 100)\",0.525,0.0,0.922907\nGhostbuster,\"Out-Domain (word, 200)\",0.525,0.0,0.915088\nGhostbuster,\"Out-Domain (sentences, 0)\",0.98,0.978723,0.997794\nGhostbuster,\"Out-Domain (sentences, 10)\",0.985,0.983957,0.994386\nGhostbuster,\"Out-Domain (sentences, 25)\",0.98,0.978723,0.993083\nGhostbuster,\"Out-Domain (sentences, 50)\",0.975,0.972973,0.995789\nGhostbuster,\"Out-Domain (sentences, 100)\",0.965,0.961749,0.994586\nGhostbuster,\"Out-Domain (sentences, 200)\",0.975,0.973262,0.994787\nGhostbuster,\"Out-Domain (paragraphs, 0)\",0.98,0.978723,0.998095\nGhostbuster,\"Out-Domain (paragraphs, 10)\",0.985,0.983957,0.997895\nGhostbuster,\"Out-Domain (paragraphs, 25)\",0.98,0.978495,0.997594\nGhostbuster,\"Out-Domain (paragraphs, 50)\",0.975,0.973262,0.997794\nGhostbuster,\"Out-Domain (paragraphs, 100)\",0.975,0.972973,0.998897\nGhostbuster,\"Out-Domain (paragraphs, 200)\",0.975,0.973262,0.998897\n"
  },
  {
    "path": "results/roberta.csv",
    "content": "Model Type,Experiment,Accuracy,F1,AUC\r\nRoBERTa,In-Domain (wp),0.81039,0.820639,0.853947\r\nRoBERTa,In-Domain (reuter),0.933014,0.929648,0.992697\r\nRoBERTa,In-Domain (essay),0.962217,0.964706,0.993863\r\nRoBERTa,In-Domain,0.903333,0.905691,0.957186\r\nRoBERTa,Out-Domain (wp),0.974026,0.972376,0.998055\r\nRoBERTa,Out-Domain (reuter),0.801435,0.832998,0.988233\r\nRoBERTa,Out-Domain (essay),0.531486,0.693069,0.935803\r\nRoBERTa,Out-Domain (claude),0.815021,0.87845,0.908494\r\nRoBERTa,Out-Domain (gpt_prompt1),0.951043,0.97047,0.973395\r\nRoBERTa,Out-Domain (gpt_prompt2),0.956885,0.974085,0.975836\r\nRoBERTa,Out-Domain (gpt_writing),0.974965,0.985114,0.979904\r\nRoBERTa,Out-Domain (gpt_semantic),0.943533,0.965785,0.970273\r\n"
  },
  {
    "path": "roberta/roberta_results.csv",
    "content": "Model Type,Experiment,Accuracy,F1,AUC\r\nRoBERTa,In-Domain,0.985833,0.986145,0.99995\r\nRoBERTa,In-Domain (wp),0.963636,0.963731,0.999838\r\nRoBERTa,Out-Domain (wp),0.974026,0.973545,0.999271\r\nRoBERTa,In-Domain (reuter),1.0,1.0,1.0\r\nRoBERTa,Out-Domain (reuter),0.543062,0.15859,0.865456\r\nRoBERTa,In-Domain (essay),0.992443,0.992908,1.0\r\nRoBERTa,Out-Domain (essay),0.561713,0.707071,0.879081\r\n"
  },
  {
    "path": "roberta/run_roberta.py",
    "content": "import argparse\nimport math\nimport numpy as np\nimport torch\nimport tqdm\nimport csv\n\nfrom transformers import (\n    RobertaTokenizer,\n    RobertaForSequenceClassification,\n)\nfrom utils.load import get_generate_dataset, Dataset\n\nfrom sklearn.metrics import (\n    accuracy_score,\n    f1_score,\n    roc_auc_score,\n)\nfrom torch.utils.data import Dataset as TorchDataset, DataLoader\nfrom torch.nn import functional as F\n\nif torch.cuda.is_available():\n    print(\"Using CUDA...\")\n    device = torch.device(\"cuda\")\nelse:\n    print(\"Using CPU...\")\n    device = torch.device(\"cpu\")\n\nmodels = [\"gpt\", \"claude\"]\ndomains = [\"wp\", \"reuter\", \"essay\"]\n\nroberta_tokenizer = RobertaTokenizer.from_pretrained(\"roberta-base\")\n\nwp_dataset = [\n    Dataset(\"normal\", \"../data/wp/human\"),\n    Dataset(\"normal\", \"../data/wp/gpt\"),\n]\n\nreuter_dataset = [\n    Dataset(\"author\", \"../data/reuter/human\"),\n    Dataset(\"author\", \"../data/reuter/gpt\"),\n]\n\nessay_dataset = [\n    Dataset(\"normal\", \"../data/essay/human\"),\n    Dataset(\"normal\", \"../data/essay/gpt\"),\n]\n\n\nclass RobertaDataset(TorchDataset):\n    def __init__(self, texts, labels):\n        self.texts = texts\n        self.labels = labels\n\n    def __len__(self):\n        return len(self.labels)\n\n    def __getitem__(self, idx):\n        encoding = roberta_tokenizer(\n            self.texts[idx],\n            return_tensors=\"pt\",\n            truncation=True,\n            padding=\"max_length\",\n            max_length=512,\n        )\n\n        return {\n            \"input_ids\": encoding[\"input_ids\"].squeeze().to(device),\n            \"attention_mask\": encoding[\"attention_mask\"].squeeze().to(device),\n            \"labels\": self.labels[idx],\n        }\n\n\ndef get_scores(labels, probabilities, calibrated=False, precision=6):\n    if calibrated:\n        threshold = sorted(probabilities)[len(labels) - sum(labels) - 1]\n    else:\n        threshold = 0.5\n\n    assert len(labels) == len(probabilities)\n\n    if sum(labels) == 0:\n        return (\n            round(accuracy_score(labels, probabilities > threshold), precision),\n            round(f1_score(labels, probabilities > threshold), precision),\n            -1,\n        )\n\n    return (\n        round(accuracy_score(labels, probabilities > threshold), precision),\n        round(f1_score(labels, probabilities > threshold), precision),\n        round(roc_auc_score(labels, probabilities), precision),\n    )\n\n\ndef train_roberta_model(train_text, train_labels, output_dir, max_epochs=1):\n    roberta_model = RobertaForSequenceClassification.from_pretrained(\n        \"roberta-base\", num_labels=2\n    ).to(device)\n\n    optimizer = torch.optim.SGD(roberta_model.parameters(), lr=0.001)\n    loss_fn = torch.nn.CrossEntropyLoss()\n\n    print(\"Fine-tuning RoBERTa...\")\n\n    indices = np.arange(len(train_text))\n    np.random.shuffle(indices)\n\n    train, valid = (\n        indices[: math.floor(0.8 * len(indices))],\n        indices[math.floor(0.8 * len(indices)) :],\n    )\n\n    train_labels = np.array(train_labels)\n    train_dataset = RobertaDataset([train_text[i] for i in train], train_labels[train])\n    val_dataset = RobertaDataset([train_text[i] for i in valid], train_labels[valid])\n\n    train_loader = DataLoader(train_dataset, batch_size=8, shuffle=True)\n    val_loader = DataLoader(val_dataset, batch_size=8, shuffle=True)\n\n    prev_val_loss = float(\"inf\")\n\n    for epoch in range(max_epochs):\n        roberta_model.train()\n\n        for batch in tqdm.tqdm(train_loader):\n            optimizer.zero_grad()\n            outputs = roberta_model(**batch)\n            loss = loss_fn(outputs.logits.to(device), batch[\"labels\"].to(device))\n            loss.backward()\n            optimizer.step()\n\n            del outputs, loss, batch\n\n        roberta_model.eval()\n\n        val_loss = 0\n        for batch in tqdm.tqdm(val_loader):\n            outputs = roberta_model(**batch)\n            loss = loss_fn(outputs.logits.to(device), batch[\"labels\"].to(device))\n            val_loss += loss.item()\n\n            del outputs, loss, batch\n\n        val_loss /= len(val_loader)\n\n        print(f\"Epoch {epoch + 1} Validation Loss: {val_loss}\")\n\n        if val_loss > prev_val_loss:\n            break\n\n        prev_val_loss = val_loss\n        roberta_model.save_pretrained(output_dir)\n\n\ndef run_roberta_model(model_name, texts, labels):\n    roberta_model = RobertaForSequenceClassification.from_pretrained(\n        f\"models/roberta_{model_name}\", num_labels=2\n    ).to(device)\n\n    roberta_model.eval()\n\n    probs = []\n    for text in tqdm.tqdm(texts):\n        encoding = roberta_tokenizer(\n            text,\n            return_tensors=\"pt\",\n            truncation=True,\n            padding=\"max_length\",\n            max_length=512,\n        )\n        encoding = {k: v.to(device) for k, v in encoding.items()}\n\n        with torch.no_grad():\n            outputs = roberta_model(**encoding)\n\n        probs.append(float(F.softmax(outputs.logits, dim=1)[0][1].item()))\n\n        del encoding, outputs\n\n    return get_scores(np.array(labels), np.array(probs), calibrated=False)\n\n\nif __name__ == \"__main__\":\n    parser = argparse.ArgumentParser()\n    parser.add_argument(\"--train\", action=\"store_true\")\n    parser.add_argument(\"--run\", action=\"store_true\")\n    parser.add_argument(\"--seed\", type=int, default=0)\n    parser.add_argument(\"--output_file\", default=\"roberta_results.csv\")\n    args = parser.parse_args()\n\n    np.random.seed(args.seed)\n    # Construct the test/train split. Seed of 0 ensures seriality across\n    # all files performing the same split.\n    indices = np.arange(6000)\n    np.random.shuffle(indices)\n\n    train, test = (\n        indices[: math.floor(0.8 * len(indices))],\n        indices[math.floor(0.8 * len(indices)) :],\n    )\n\n    # [4320 2006 5689 ... 4256 5807 4875] [5378 5980 5395 ... 1653 2607 2732]\n    print(\"Train/Test Split:\", train, test)\n\n    generate_dataset = get_generate_dataset(\n        *wp_dataset, *reuter_dataset, *essay_dataset\n    )\n    labels = generate_dataset(lambda f: \"gpt\" in f)\n    assert len(labels) // 2 == sum(labels)\n    if args.train:\n\n        def train_roberta_gen(\n            gen_fn, out_dir, indices=None, max_epochs=2, filter_fn=lambda x: True\n        ):\n            train_text, train_labels = [], []\n            if indices is not None:\n                files = gen_fn(lambda f: f)[indices]\n            else:\n                files = gen_fn(lambda f: f)\n\n            for file in files:\n                if not filter_fn(file):\n                    continue\n\n                with open(file) as f:\n                    text = f.read()\n\n                train_text.append(text)\n                train_labels.append(int(\"gpt\" in file))\n\n            train_roberta_model(\n                train_text, train_labels, out_dir, max_epochs=max_epochs\n            )\n\n        gen_fn_all = get_generate_dataset(*wp_dataset, *reuter_dataset, *essay_dataset)\n        gen_fn_wp = get_generate_dataset(*reuter_dataset, *essay_dataset)\n        gen_fn_reuter = get_generate_dataset(*wp_dataset, *essay_dataset)\n        gen_fn_essay = get_generate_dataset(*wp_dataset, *reuter_dataset)\n\n        train_roberta_gen(gen_fn_all, \"models/roberta_gpt\", indices=train)\n        train_roberta_gen(gen_fn_wp, \"models/roberta_wp\")\n        train_roberta_gen(gen_fn_reuter, \"models/roberta_reuter\")\n        train_roberta_gen(gen_fn_essay, \"models/roberta_essay\")\n\n    if args.run:\n        results_table = [\n            [\"Model Type\", \"Experiment\", \"Accuracy\", \"F1\", \"AUC\"],\n        ]\n\n        def get_data(gen_fn, indices=None, filter_fn=lambda f: True):\n            if indices is not None:\n                files = gen_fn(lambda f: f)[indices]\n            else:\n                files = gen_fn(lambda f: f)\n\n            texts, labels = [], []\n            for file in files:\n                if not filter_fn(file):\n                    continue\n\n                with open(file) as f:\n                    text = f.read()\n\n                texts.append(text)\n                labels.append(int(\"gpt\" in file))\n\n            return texts, labels\n\n        gen_fn_all = get_generate_dataset(*wp_dataset, *reuter_dataset, *essay_dataset)\n        gen_fn_wp = get_generate_dataset(*wp_dataset)\n        gen_fn_reuter = get_generate_dataset(*reuter_dataset)\n        gen_fn_essay = get_generate_dataset(*essay_dataset)\n\n        results_table.append(\n            [\n                \"RoBERTa\",\n                \"In-Domain\",\n                *run_roberta_model(\"gpt\", *get_data(gen_fn_all, test)),\n            ]\n        )\n\n        for domain in [\"wp\", \"reuter\", \"essay\"]:\n            results_table.append(\n                [\n                    \"RoBERTa\",\n                    f\"In-Domain ({domain})\",\n                    *run_roberta_model(\n                        \"gpt\" if domain != \"reuter\" else \"only_reuter\",\n                        *get_data(gen_fn_all, test, lambda x: domain in x),\n                    ),\n                ]\n            )\n\n            results_table.append(\n                [\n                    \"RoBERTa\",\n                    f\"Out-Domain ({domain})\",\n                    *run_roberta_model(\n                        domain, *get_data(gen_fn_all, test, lambda x: domain in x)\n                    ),\n                ]\n            )\n\n        if len(results_table) > 1:\n            with open(args.output_file, \"w\") as f:\n                writer = csv.writer(f)\n                writer.writerows(results_table)\n\n            print(f\"Saved results to {args.output_file}\")\n"
  },
  {
    "path": "roberta/train.py",
    "content": "import argparse\nimport math\nimport numpy as np\nimport dill as pickle\nimport torch\nimport tqdm\nimport itertools\n\nfrom transformers import (\n    RobertaTokenizer,\n    RobertaForSequenceClassification,\n)\n\nfrom utils.load import get_generate_dataset, Dataset\n\nfrom torch.utils.data import Dataset as TorchDataset, DataLoader\n\nif torch.cuda.is_available():\n    print(\"Using CUDA...\")\n    device = torch.device(\"cuda\")\nelse:\n    print(\"Using CPU...\")\n    device = torch.device(\"cpu\")\n\nmodels = [\"gpt\", \"claude\"]\ndomains = [\"wp\", \"reuter\", \"essay\"]\n\nroberta_tokenizer = RobertaTokenizer.from_pretrained(\"roberta-base\")\n\nwp_dataset = [\n    Dataset(\"normal\", \"../data/wp/human\"),\n    Dataset(\"normal\", \"../data/wp/gpt\"),\n]\n\nreuter_dataset = [\n    Dataset(\"author\", \"../data/reuter/human\"),\n    Dataset(\"author\", \"../data/reuter/gpt\"),\n]\n\nessay_dataset = [\n    Dataset(\"normal\", \"../data/essay/human\"),\n    Dataset(\"normal\", \"../data/essay/gpt\"),\n]\n\n\nclass RobertaDataset(TorchDataset):\n    def __init__(self, texts, labels):\n        self.texts = texts\n        self.labels = labels\n\n    def __len__(self):\n        return len(self.labels)\n\n    def __getitem__(self, idx):\n        encoding = roberta_tokenizer(\n            self.texts[idx],\n            return_tensors=\"pt\",\n            truncation=True,\n            padding=\"max_length\",\n            max_length=512,\n        )\n\n        return {\n            \"input_ids\": encoding[\"input_ids\"].squeeze().to(device),\n            \"attention_mask\": encoding[\"attention_mask\"].squeeze().to(device),\n            \"labels\": self.labels[idx],\n        }\n\n\ndef train_roberta_model(train_text, train_labels, output_dir):\n    roberta_model = RobertaForSequenceClassification.from_pretrained(\n        \"roberta-large\", num_labels=2\n    ).to(device)\n\n    optimizer = torch.optim.AdamW(roberta_model.parameters(), lr=1e-5)\n    loss_fn = torch.nn.CrossEntropyLoss()\n\n    print(\"Fine-tuning RoBERTa...\")\n\n    indices = np.arange(len(train_text))\n    np.random.shuffle(indices)\n\n    train, valid = (\n        indices[: math.floor(0.8 * len(indices))],\n        indices[math.floor(0.8 * len(indices)) :],\n    )\n\n    train_labels = np.array(train_labels)\n    train_dataset = RobertaDataset([train_text[i] for i in train], train_labels[train])\n    val_dataset = RobertaDataset([train_text[i] for i in valid], train_labels[valid])\n\n    train_loader = DataLoader(train_dataset, batch_size=8, shuffle=True)\n    val_loader = DataLoader(val_dataset, batch_size=8, shuffle=True)\n\n    prev_val_loss = float(\"inf\")\n\n    for epoch in range(1):\n        roberta_model.train()\n\n        for batch in tqdm.tqdm(train_loader):\n            optimizer.zero_grad()\n            outputs = roberta_model(**batch)\n            loss = loss_fn(outputs.logits.to(device), batch[\"labels\"].to(device))\n            loss.backward()\n            optimizer.step()\n\n            del outputs, loss, batch\n\n        roberta_model.eval()\n\n        val_loss = 0\n        for batch in tqdm.tqdm(val_loader):\n            outputs = roberta_model(**batch)\n            loss = loss_fn(outputs.logits.to(device), batch[\"labels\"].to(device))\n            val_loss += loss.item()\n\n            del outputs, loss, batch\n\n        val_loss /= len(val_loader)\n\n        print(f\"Epoch {epoch + 1} Validation Loss: {val_loss}\")\n\n        if val_loss > prev_val_loss:\n            break\n\n        prev_val_loss = val_loss\n        roberta_model.save_pretrained(output_dir)\n\n\nif __name__ == \"__main__\":\n    parser = argparse.ArgumentParser()\n    parser.add_argument(\"--seed\", type=int, default=0)\n    parser.add_argument(\"--train_all\", action=\"store_true\")\n    args = parser.parse_args()\n\n    np.random.seed(args.seed)\n\n    datasets = [\n        *wp_dataset,\n        *reuter_dataset,\n        *essay_dataset,\n    ]\n    generate_dataset_fn = get_generate_dataset(*datasets)\n\n    files = generate_dataset_fn(lambda x: x)\n    labels = generate_dataset_fn(\n        lambda file: 1 if any([m in file for m in [\"gpt\", \"claude\"]]) else 0\n    )\n    indices = np.arange(len(labels))\n\n    # [4320 2006 5689 ... 4256 5807 4875] [5378 5980 5395 ... 1653 2607 2732]\n    np.random.shuffle(indices)\n    train, test = (\n        indices[: math.floor(0.8 * len(indices))],\n        indices[math.floor(0.8 * len(indices)) :],\n    )\n    print(\"Train/Test Split:\", train, test)\n\n    # Construct all indices\n    def get_indices(filter_fn):\n        where = np.where(generate_dataset_fn(filter_fn))[0]\n\n        curr_train = [i for i in train if i in where]\n        curr_test = [i for i in test if i in where]\n\n        return curr_train, curr_test\n\n    def get_texts(indices):\n        texts = []\n        for file in files[indices]:\n            with open(file) as f:\n                texts.append(f.read())\n        return texts\n\n    indices_dict = {}\n\n    for model in models + [\"human\"]:\n        train_indices, test_indices = get_indices(\n            lambda file: 1 if model in file else 0\n        )\n\n        indices_dict[f\"{model}_train\"] = train_indices\n        indices_dict[f\"{model}_test\"] = test_indices\n\n    for model in models + [\"human\"]:\n        for domain in domains:\n            train_key = f\"{model}_{domain}_train\"\n            test_key = f\"{model}_{domain}_test\"\n\n            train_indices, test_indices = get_indices(\n                lambda file: 1 if domain in file and model in file else 0\n            )\n\n            indices_dict[train_key] = train_indices\n            indices_dict[test_key] = test_indices\n\n    if args.train_all:\n        train_indices = []\n        for domain in domains:\n            train_indices += (\n                indices_dict[f\"gpt_{domain}_train\"]\n                + indices_dict[f\"human_{domain}_train\"]\n            )\n\n        print(\"Training on GPT Data\")\n        print(\"# of Training Examples:\", len(train_indices))\n        train_roberta_model(\n            get_texts(train_indices),\n            labels[train_indices],\n            f\"models/roberta_gpt\",\n        )\n\n        for test_domain in domains:\n            train_indices = []\n            for train_domain in domains:\n                if train_domain == test_domain:\n                    continue\n\n                train_indices += (\n                    indices_dict[f\"gpt_{train_domain}_train\"]\n                    + indices_dict[f\"human_{train_domain}_train\"]\n                )\n\n            print(f\"Training on OOD {test_domain} Data\")\n            print(\"# of Training Examples:\", len(train_indices))\n            train_roberta_model(\n                get_texts(train_indices),\n                labels[train_indices],\n                f\"models/roberta_{test_domain}\",\n            )\n"
  },
  {
    "path": "run.py",
    "content": "# Built-In Imports\nimport csv\nimport itertools\nimport math\nimport os\nfrom collections import defaultdict\n\n# External Imports\nimport argparse\nimport dill as pickle\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport tiktoken\nimport tqdm\n\n# Torch imports\nimport torch\nimport torch.nn.functional as F\n\nfrom torch.utils.data import Dataset as TorchDataset\nfrom transformers import RobertaForSequenceClassification, RobertaTokenizer\n\n# Sklearn imports\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.metrics import accuracy_score, f1_score, roc_auc_score\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.calibration import CalibratedClassifierCV\n\n# Local Imports\nfrom utils.featurize import normalize, t_featurize, select_features\nfrom utils.symbolic import get_all_logprobs, get_exp_featurize, backtrack_functions\nfrom utils.load import Dataset, get_generate_dataset\n\nfrom generate import perturb_char_names, perturb_char_sizes\nfrom generate import perturb_sent_names, perturb_sent_sizes\n\nmodels = [\"gpt\"]\ndomains = [\"wp\", \"reuter\", \"essay\"]\neval_domains = [\"claude\", \"gpt_prompt1\", \"gpt_prompt2\", \"gpt_writing\", \"gpt_semantic\"]\n\nif torch.cuda.is_available():\n    print(\"Using CUDA...\")\n    device = torch.device(\"cuda\")\nelse:\n    print(\"Using CPU...\")\n    device = torch.device(\"cpu\")\n\nbest_features_map = {}\n\nfor file in os.listdir(\"results\"):\n    if file.startswith(\"best_features\"):\n        with open(f\"results/{file}\") as f:\n            best_features_map[file[:-4]] = f.read().strip().split(\"\\n\")\n\nprint(\"Loading trigram model...\")\ntrigram_model = pickle.load(\n    open(\"model/trigram_model.pkl\", \"rb\"), pickle.HIGHEST_PROTOCOL\n)\ntokenizer = tiktoken.encoding_for_model(\"davinci\").encode\n\nprint(\"Loading features...\")\nexp_to_data = pickle.load(open(\"symbolic_data_gpt\", \"rb\"))\nt_data = pickle.load(open(\"t_data\", \"rb\"))\n\nprint(\"Loading eval data...\")\n# exp_to_data_eval = pickle.load(open(\"symbolic_data_eval\", \"rb\"))\n# t_data_eval = pickle.load(open(\"t_data_eval\", \"rb\"))\n\nroberta_tokenizer = RobertaTokenizer.from_pretrained(\"roberta-base\")\n\ndatasets = [\n    Dataset(\"normal\", \"data/wp/human\"),\n    Dataset(\"normal\", \"data/wp/gpt\"),\n    Dataset(\"author\", \"data/reuter/human\"),\n    Dataset(\"author\", \"data/reuter/gpt\"),\n    Dataset(\"normal\", \"data/essay/human\"),\n    Dataset(\"normal\", \"data/essay/gpt\"),\n]\n\neval_dataset = [\n    Dataset(\"normal\", \"data/wp/claude\"),\n    Dataset(\"author\", \"data/reuter/claude\"),\n    Dataset(\"normal\", \"data/essay/claude\"),\n    Dataset(\"normal\", \"data/wp/gpt_prompt1\"),\n    Dataset(\"author\", \"data/reuter/gpt_prompt1\"),\n    Dataset(\"normal\", \"data/essay/gpt_prompt1\"),\n    Dataset(\"normal\", \"data/wp/gpt_prompt2\"),\n    Dataset(\"author\", \"data/reuter/gpt_prompt2\"),\n    Dataset(\"normal\", \"data/essay/gpt_prompt2\"),\n    Dataset(\"normal\", \"data/wp/gpt_writing\"),\n    Dataset(\"author\", \"data/reuter/gpt_writing\"),\n    Dataset(\"normal\", \"data/essay/gpt_writing\"),\n    Dataset(\"normal\", \"data/wp/gpt_semantic\"),\n    Dataset(\"author\", \"data/reuter/gpt_semantic\"),\n    Dataset(\"normal\", \"data/essay/gpt_semantic\"),\n]\n\nother_dataset = []\n\n\nclass RobertaDataset(TorchDataset):\n    def __init__(self, texts, labels):\n        self.texts = texts\n        self.labels = labels\n\n    def __len__(self):\n        return len(self.labels)\n\n    def __getitem__(self, idx):\n        encoding = roberta_tokenizer(\n            self.texts[idx],\n            return_tensors=\"pt\",\n            truncation=True,\n            padding=\"max_length\",\n            max_length=512,\n        )\n        return {\n            \"input_ids\": encoding[\"input_ids\"].squeeze().to(device),\n            \"attention_mask\": encoding[\"attention_mask\"].squeeze().to(device),\n            \"labels\": self.labels[idx],\n        }\n\n\ndef get_scores(labels, probabilities, calibrated=False, precision=6):\n    if calibrated:\n        threshold = sorted(probabilities)[len(labels) - sum(labels) - 1]\n    else:\n        threshold = 0.5\n\n    assert len(labels) == len(probabilities)\n\n    if sum(labels) == 0 or sum(labels) == len(labels):\n        return (\n            round(accuracy_score(labels, probabilities > threshold), precision),\n            round(f1_score(labels, probabilities > threshold), precision),\n            -1,\n        )\n\n    return (\n        round(accuracy_score(labels, probabilities > threshold), precision),\n        round(f1_score(labels, probabilities > threshold), precision),\n        round(roc_auc_score(labels, probabilities), precision),\n    )\n\n\nif __name__ == \"__main__\":\n    parser = argparse.ArgumentParser()\n\n    parser.add_argument(\"--claude\", action=\"store_true\")\n\n    parser.add_argument(\"--roberta\", action=\"store_true\")\n    parser.add_argument(\"--perplexity_only\", action=\"store_true\")\n\n    parser.add_argument(\"--ghostbuster\", action=\"store_true\")\n\n    parser.add_argument(\"--ghostbuster_depth_one\", action=\"store_true\")\n    parser.add_argument(\"--ghostbuster_depth_two\", action=\"store_true\")\n    parser.add_argument(\"--ghostbuster_depth_three\", action=\"store_true\")\n    parser.add_argument(\"--ghostbuster_depth_four\", action=\"store_true\")\n\n    parser.add_argument(\"--ghostbuster_random\", action=\"store_true\")\n    parser.add_argument(\"--ghostbuster_no_gpt\", action=\"store_true\")\n    parser.add_argument(\"--ghostbuster_no_handcrafted\", action=\"store_true\")\n    parser.add_argument(\"--ghostbuster_no_symbolic\", action=\"store_true\")\n    parser.add_argument(\"--ghostbuster_only_ada\", action=\"store_true\")\n    parser.add_argument(\"--ghostbuster_custom\", action=\"store_true\")\n    parser.add_argument(\"--ghostbuster_other_eval\", action=\"store_true\")\n\n    parser.add_argument(\"--ghostbuster_vary_training_data\", action=\"store_true\")\n    parser.add_argument(\"--ghostbuster_vary_document_size\", action=\"store_true\")\n\n    parser.add_argument(\"--hyperparameter_search\", action=\"store_true\")\n\n    parser.add_argument(\"--perturb\", action=\"store_true\")\n    parser.add_argument(\"--calibration\", action=\"store_true\")\n\n    parser.add_argument(\"--toefl\", action=\"store_true\")\n\n    parser.add_argument(\"--seed\", type=int, default=0)\n    parser.add_argument(\"--output_file\", type=str, default=\"results.csv\")\n    args = parser.parse_args()\n\n    np.random.seed(args.seed)\n    # Construct the test/train split. Seed of 0 ensures seriality across\n    # all files performing the same split.\n    indices = np.arange(6000)\n    np.random.shuffle(indices)\n\n    train, test = (\n        indices[: math.floor(0.8 * len(indices))],\n        indices[math.floor(0.8 * len(indices)) :],\n    )\n\n    # [4320 2006 5689 ... 4256 5807 4875] [5378 5980 5395 ... 1653 2607 2732]\n    print(\"Train/Test Split:\", train, test)\n\n    # Results table, outputted to args.output_file.\n    # Example Row: [\"Ghostbuster (No GPT)\", \"WP\", \"gpt_wp\", 0.5, 0.5, 0.5]\n    results_table = [\n        [\"Model Type\", \"Experiment\", \"Accuracy\", \"F1\", \"AUC\"],\n    ]\n\n    # Construct the generate_dataset_fn. This function takes in a featurize function,\n    # which is a mapping from a file location (str) to a desired feature vector.\n\n    generate_dataset_fn_gpt = get_generate_dataset(*datasets)\n    generate_dataset_fn_eval = get_generate_dataset(*eval_dataset)\n\n    # t_data_eval = generate_dataset_fn_eval(t_featurize, verbose=True)\n    # pickle.dump(t_data_eval, open(\"t_data_eval\", \"wb\"), pickle.HIGHEST_PROTOCOL)\n\n    generate_dataset_fn = get_generate_dataset(*datasets, *eval_dataset)\n\n    # t_data = generate_dataset_fn(t_featurize, verbose=True)\n    # pickle.dump(t_data, open(\"t_data\", \"wb\"), pickle.HIGHEST_PROTOCOL)\n\n    def get_featurized_data(best_features, gpt_only=False):\n        gpt_data = np.concatenate(\n            [t_data] + [exp_to_data[i] for i in best_features], axis=1\n        )\n        if gpt_only:\n            return gpt_data\n\n        eval_data = np.concatenate(\n            [t_data_eval] + [exp_to_data_eval[i] for i in best_features], axis=1\n        )\n        return np.concatenate([gpt_data, eval_data], axis=0)\n\n    # Construct all indices\n    def get_indices(filter_fn):\n        where = np.where(generate_dataset_fn_gpt(filter_fn))[0]\n\n        curr_train = [i for i in train if i in where]\n        curr_test = [i for i in test if i in where]\n\n        return curr_train, curr_test\n\n    indices_dict = {}\n\n    for model in models + [\"human\"]:\n        train_indices, test_indices = get_indices(\n            lambda file: 1 if model in file else 0,\n        )\n\n        indices_dict[f\"{model}_train\"] = train_indices\n        indices_dict[f\"{model}_test\"] = test_indices\n\n    for model in models + [\"human\"]:\n        for domain in domains:\n            train_key = f\"{model}_{domain}_train\"\n            test_key = f\"{model}_{domain}_test\"\n\n            train_indices, test_indices = get_indices(\n                lambda file: 1 if domain in file and model in file else 0,\n            )\n\n            indices_dict[train_key] = train_indices\n            indices_dict[test_key] = test_indices\n\n    for key in eval_domains:\n        where = np.where(generate_dataset_fn(lambda file: 1 if key in file else 0))[0]\n        assert len(where) == 3000\n\n        indices_dict[f\"{key}_test\"] = list(where)\n\n    files = generate_dataset_fn(lambda x: x)\n    labels = generate_dataset_fn(\n        lambda file: 1 if any([m in file for m in [\"gpt\", \"claude\"]]) else 0\n    )\n\n    def get_roberta_predictions(data, train, test, domain):\n        print(f\"Loading model roberta/models/roberta_{domain}...\")\n\n        roberta_model = RobertaForSequenceClassification.from_pretrained(\n            f\"roberta/models/roberta_{domain}\", num_labels=2\n        )\n        roberta_model.to(device)\n\n        test_labels = labels[test]\n        test_predictions = []\n\n        roberta_model.eval()\n        with torch.no_grad():\n            for file in tqdm.tqdm(files[test]):\n                with open(file) as f:\n                    text = f.read()\n                inputs = roberta_tokenizer(\n                    text,\n                    return_tensors=\"pt\",\n                    truncation=True,\n                    padding=\"max_length\",\n                    max_length=512,\n                )\n                inputs = {k: v.to(device) for k, v in inputs.items()}\n                outputs = roberta_model(**inputs)\n                test_predictions.append(\n                    float(F.softmax(outputs.logits, dim=1)[0][1].item())\n                )\n\n        return get_scores(np.array(test_labels), np.array(test_predictions))\n\n    def train_ghostbuster(data, train, test, domain):\n        model = LogisticRegression()\n        model.fit(data[train], labels[train])\n        probs = model.predict_proba(data[test])[:, 1]\n\n        return get_scores(labels[test], probs)\n\n    def train_perplexity(data, train, test, domain):\n        features = data[train][:, -1].reshape(-1, 1)\n        threshold = sorted(features)[len(features) - sum(labels[train]) - 1]\n        probs = (data[test][:, -1] > threshold).astype(float)\n        return get_scores(labels[test], probs)\n\n    def run_experiment(best_features, model_name, train_fn, gpt_only=True):\n        gpt_data = get_featurized_data(best_features, gpt_only=True)\n        _, mu, sigma = normalize(gpt_data, ret_mu_sigma=True)\n\n        data = normalize(\n            get_featurized_data(best_features, gpt_only=gpt_only), mu=mu, sigma=sigma\n        )\n\n        print(f\"Running {model_name} Predictions...\")\n\n        train_indices, test_indices = [], []\n        for domain in domains:\n            train_indices += (\n                indices_dict[f\"gpt_{domain}_train\"]\n                + indices_dict[f\"human_{domain}_train\"]\n            )\n            test_indices += (\n                indices_dict[f\"gpt_{domain}_test\"]\n                + indices_dict[f\"human_{domain}_test\"]\n            )\n\n            results_table.append(\n                [\n                    model_name,\n                    f\"In-Domain ({domain})\",\n                    *train_fn(\n                        data,\n                        indices_dict[f\"gpt_{domain}_train\"]\n                        + indices_dict[f\"human_{domain}_train\"],\n                        indices_dict[f\"gpt_{domain}_test\"]\n                        + indices_dict[f\"human_{domain}_test\"],\n                        \"gpt\",\n                    ),\n                ]\n            )\n\n        results_table.append(\n            [\n                model_name,\n                \"In-Domain\",\n                *train_fn(data, train_indices, test_indices, \"gpt\"),\n            ]\n        )\n\n        for test_domain in domains:\n            train_indices = []\n            for train_domain in domains:\n                if train_domain == test_domain:\n                    continue\n\n                train_indices += (\n                    indices_dict[f\"gpt_{train_domain}_train\"]\n                    + indices_dict[f\"human_{train_domain}_train\"]\n                )\n\n            results_table.append(\n                [\n                    model_name,\n                    f\"Out-Domain ({test_domain})\",\n                    *train_fn(\n                        data,\n                        train_indices,\n                        indices_dict[f\"gpt_{test_domain}_test\"]\n                        + indices_dict[f\"human_{test_domain}_test\"],\n                        test_domain,\n                    ),\n                ]\n            )\n\n        if gpt_only:\n            return\n\n        train_indices, test_indices = [], []\n        for domain in domains:\n            train_indices += (\n                indices_dict[f\"gpt_{domain}_train\"]\n                + indices_dict[f\"human_{domain}_train\"]\n            )\n            test_indices += indices_dict[f\"human_{domain}_test\"]\n\n        for domain in eval_domains:\n            curr_test_indices = list(indices_dict[f\"{domain}_test\"]) + test_indices\n\n            results_table.append(\n                [\n                    model_name,\n                    f\"Out-Domain ({domain})\",\n                    *train_fn(data, train_indices, curr_test_indices, \"gpt\"),\n                ]\n            )\n\n    if args.perplexity_only:\n        run_experiment(\n            [\"davinci-logprobs s-avg\"],\n            \"Perplexity-Only\",\n            train_perplexity,\n        )\n\n    if args.roberta:\n        run_experiment([], \"RoBERTa\", get_roberta_predictions, gpt_only=True)\n\n    if args.ghostbuster_depth_one or args.ghostbuster:\n        run_experiment(\n            best_features_map[\"best_features_one\"],\n            \"Ghostbuster (Depth One)\",\n            train_ghostbuster,\n        )\n\n    if args.ghostbuster_depth_two or args.ghostbuster:\n        run_experiment(\n            best_features_map[\"best_features_two\"],\n            \"Ghostbuster (Depth Two)\",\n            train_ghostbuster,\n        )\n\n    if args.ghostbuster_depth_three or args.ghostbuster:\n        run_experiment(\n            best_features_map[\"best_features_three\"],\n            \"Ghostbuster (Depth Three)\",\n            train_ghostbuster,\n        )\n\n    if args.ghostbuster_depth_four or args.ghostbuster:\n        run_experiment(\n            best_features_map[\"best_features_four\"],\n            \"Ghostbuster (Depth Four)\",\n            train_ghostbuster,\n            gpt_only=True,\n        )\n\n    if args.ghostbuster_no_gpt or args.ghostbuster:\n        run_experiment(\n            best_features_map[\"best_features_no_gpt\"],\n            \"Ghostbuster (N-Gram Only)\",\n            train_ghostbuster,\n        )\n\n    if args.ghostbuster_only_ada or args.ghostbuster:\n        run_experiment(\n            best_features_map[\"best_features_only_ada\"],\n            \"Ghostbuster (N-Gram and Ada)\",\n            train_ghostbuster,\n        )\n\n    if args.ghostbuster_random or args.ghostbuster:\n        all_features = backtrack_functions(max_depth=3)\n        random_features = np.random.choice(all_features, 10, replace=False)\n\n        run_experiment(\n            random_features,\n            \"Ghostbuster (Random)\",\n            train_ghostbuster,\n        )\n\n    if args.ghostbuster_custom:\n        run_experiment(\n            best_features_map[\"best_features_custom\"],\n            \"Ghostbuster (Custom)\",\n            train_ghostbuster,\n        )\n\n    if args.ghostbuster_no_handcrafted or args.ghostbuster:\n\n        def train_ghostbuster_no_handcrafted(data, train, test, domain):\n            data = data[:, 7:]\n            return train_ghostbuster(data, train, test, domain)\n\n        run_experiment(\n            best_features_map[\"best_features_three\"],\n            \"Ghostbuster (Depth Three, No Handcrafted)\",\n            train_ghostbuster_no_handcrafted,\n        )\n\n    if args.ghostbuster_no_symbolic or args.ghostbuster:\n\n        def train_ghostbuster_no_symbolic(data, train, test, domain):\n            data = data[:, :7]\n            return train_ghostbuster(data, train, test, domain)\n\n        run_experiment(\n            best_features_map[\"best_features_three\"],\n            \"Ghostbuster (No Symbolic)\",\n            train_ghostbuster_no_symbolic,\n        )\n\n    if args.ghostbuster_other_eval:\n        data, mu, sigma = normalize(\n            get_featurized_data(\n                best_features_map[\"best_features_three\"], gpt_only=True\n            ),\n            ret_mu_sigma=True,\n        )\n\n        model = LogisticRegression()\n        model.fit(\n            data[indices_dict[\"gpt_train\"] + indices_dict[\"human_train\"]],\n            labels[indices_dict[\"gpt_train\"] + indices_dict[\"human_train\"]],\n        )\n\n        # Get roberta results on ets\n        roberta_model = RobertaForSequenceClassification.from_pretrained(\n            f\"roberta/models/roberta_gpt\", num_labels=2\n        )\n        roberta_model.to(device)\n\n        print(\n            get_scores(\n                labels[indices_dict[\"gpt_test\"] + indices_dict[\"human_test\"]],\n                model.predict_proba(\n                    data[indices_dict[\"gpt_test\"] + indices_dict[\"human_test\"]]\n                )[:, 1],\n            )\n        )\n\n        other_datasets = [\n            Dataset(\"normal\", \"data/other/lang8\"),\n            Dataset(\"normal\", \"data/other/pelic\"),\n            Dataset(\"normal\", \"data/other/gptzero/gpt\"),\n            Dataset(\"normal\", \"data/other/gptzero/human\"),\n        ]\n\n        def get_data(generate_dataset_fn, best_features):\n            davinci, ada, trigram, unigram = get_all_logprobs(\n                generate_dataset_fn,\n                trigram=trigram_model,\n                tokenizer=tokenizer,\n            )\n            vector_map = {\n                \"davinci-logprobs\": lambda file: davinci[file],\n                \"ada-logprobs\": lambda file: ada[file],\n                \"trigram-logprobs\": lambda file: trigram[file],\n                \"unigram-logprobs\": lambda file: unigram[file],\n            }\n            exp_featurize = get_exp_featurize(best_features, vector_map)\n            exp_data = generate_dataset_fn(exp_featurize)\n            return exp_data\n\n        def evaluate_on_dataset(\n            model,\n            best_features,\n            curr_labels,\n            generate_dataset_fn,\n            dataset_name,\n            train=False,\n            to_normalize=True,\n        ):\n            data, mu, sigma = normalize(\n                get_featurized_data(best_features, gpt_only=True),\n                ret_mu_sigma=True,\n            )\n\n            t_data = generate_dataset_fn(t_featurize, verbose=True)\n            exp_data = get_data(generate_dataset_fn, best_features)\n\n            if to_normalize:\n                curr_data = normalize(\n                    np.concatenate([t_data, exp_data], axis=1), mu=mu, sigma=sigma\n                )\n            else:\n                curr_data = np.concatenate([t_data, exp_data], axis=1)\n\n            if train:\n                indices = np.arange(len(curr_data))\n                np.random.shuffle(indices)\n\n                train_indices = indices[: math.floor(0.8 * len(indices))]\n                test_indices = indices[math.floor(0.8 * len(indices)) :]\n\n                curr_train_data = np.concatenate(\n                    [\n                        curr_data[train_indices],\n                        data[indices_dict[\"gpt_train\"] + indices_dict[\"human_train\"]],\n                    ],\n                    axis=0,\n                )\n                curr_train_labels = np.concatenate(\n                    [\n                        curr_labels[train_indices],\n                        labels[indices_dict[\"gpt_train\"] + indices_dict[\"human_train\"]],\n                    ],\n                    axis=0,\n                )\n\n                model = LogisticRegression()\n                model.fit(curr_train_data, curr_train_labels)\n\n                probs = model.predict_proba(curr_data[test_indices])[:, 1]\n                results_table.append(\n                    [\n                        \"Ghostbuster\",\n                        f\"In-Domain ({dataset_name})\",\n                        *get_scores(curr_labels[test_indices], probs),\n                    ]\n                )\n            else:\n                probs = model.predict_proba(curr_data)[:, 1]\n                results_table.append(\n                    [\n                        \"Ghostbuster\",\n                        f\"Out-Domain ({dataset_name})\",\n                        *get_scores(curr_labels, probs),\n                    ]\n                )\n\n        for dataset in [\"lang8\"]:\n            gen_fn = get_generate_dataset(Dataset(\"normal\", f\"data/other/{dataset}\"))\n            curr_labels = gen_fn(lambda _: 0)\n\n            evaluate_on_dataset(\n                model,\n                best_features_map[\"best_features_three\"],\n                curr_labels,\n                gen_fn,\n                dataset,\n            )\n\n            exp_data = get_data(gen_fn, [\"davinci-logprobs s-avg\"])\n\n            model_p = LogisticRegression()\n            model_p.fit(\n                exp_to_data[\"davinci-logprobs s-avg\"][\n                    indices_dict[\"gpt_train\"] + indices_dict[\"human_train\"]\n                ],\n                labels[indices_dict[\"gpt_train\"] + indices_dict[\"human_train\"]],\n            )\n\n            probs = model_p.predict_proba(exp_data)[:, 1]\n\n            results_table.append(\n                [\n                    \"Perplexity Only\",\n                    f\"Out-Domain (lang8)\",\n                    *get_scores(curr_labels, probs),\n                ]\n            )\n\n            # Evaluate roberta\n            roberta_test = RobertaDataset(\n                gen_fn(lambda file: open(file).read()),\n                gen_fn(lambda _: 0),\n            )\n\n            roberta_test_loader = torch.utils.data.DataLoader(\n                roberta_test, batch_size=1, shuffle=False\n            )\n\n            roberta_model.eval()\n\n            roberta_probs = []\n            with torch.no_grad():\n                for batch in tqdm.tqdm(roberta_test_loader):\n                    inputs = {k: v.to(device) for k, v in batch.items()}\n                    outputs = roberta_model(**inputs)\n                    roberta_probs.append(\n                        float(F.softmax(outputs.logits, dim=1)[0][1].item())\n                    )\n\n            results_table.append(\n                [\n                    \"RoBERTa\",\n                    f\"Out-Domain ({dataset})\",\n                    *get_scores(gen_fn(lambda _: 0), np.array(roberta_probs)),\n                ]\n            )\n\n        gpt_zero = get_generate_dataset(\n            Dataset(\"normal\", f\"data/other/gptzero/human\"),\n            Dataset(\"normal\", f\"data/other/gptzero/gpt\"),\n        )\n        curr_labels = np.array([0] * 50 + [1] * 50)\n\n        evaluate_on_dataset(\n            model,\n            best_features_map[\"best_features_three\"],\n            curr_labels,\n            gpt_zero,\n            \"gptzero\",\n        )\n\n        evaluate_on_dataset(\n            model,\n            best_features_map[\"best_features_three\"],\n            [1] * 100,\n            get_generate_dataset(Dataset(\"normal\", \"data/other/undetectable\")),\n            \"undetectable\",\n        )\n\n        gen_ets = get_generate_dataset(Dataset(\"normal\", f\"data/other/ets\"))\n        curr_labels = gen_ets(lambda _: 0)\n\n        evaluate_on_dataset(\n            model,\n            best_features_map[\"best_features_three\"],\n            curr_labels,\n            gen_ets,\n            \"ets\",\n        )\n\n        evaluate_on_dataset(\n            model,\n            best_features_map[\"best_features_three\"],\n            curr_labels,\n            gen_ets,\n            \"ets\",\n            train=True,\n        )\n\n        exp_data = get_data(gen_ets, [\"davinci-logprobs s-avg\"])\n\n        model = LogisticRegression()\n        model.fit(\n            exp_to_data[\"davinci-logprobs s-avg\"][\n                indices_dict[\"gpt_train\"] + indices_dict[\"human_train\"]\n            ],\n            labels[indices_dict[\"gpt_train\"] + indices_dict[\"human_train\"]],\n        )\n\n        probs = model.predict_proba(exp_data)[:, 1]\n\n        results_table.append(\n            [\n                \"Perplexity Only\",\n                f\"Out-Domain (ets)\",\n                *get_scores(curr_labels, probs),\n            ]\n        )\n\n        roberta_test = RobertaDataset(\n            gen_ets(lambda file: open(file).read()),\n            gen_ets(lambda _: 0),\n        )\n        roberta_test_loader = torch.utils.data.DataLoader(\n            roberta_test, batch_size=1, shuffle=False\n        )\n\n        roberta_model.eval()\n        roberta_probs = []\n        with torch.no_grad():\n            for batch in tqdm.tqdm(roberta_test_loader):\n                inputs = {k: v.to(device) for k, v in batch.items()}\n                outputs = roberta_model(**inputs)\n                roberta_probs.append(\n                    float(F.softmax(outputs.logits, dim=1)[0][1].item())\n                )\n\n        results_table.append(\n            [\n                \"RoBERTa\",\n                f\"Out-Domain (ets)\",\n                *get_scores(gen_ets(lambda _: 0), np.array(roberta_probs)),\n            ]\n        )\n\n    if args.toefl:\n\n        def get_data(generate_dataset_fn, best_features):\n            davinci, ada, trigram, unigram = get_all_logprobs(\n                generate_dataset_fn,\n                trigram=trigram_model,\n                tokenizer=tokenizer,\n            )\n            vector_map = {\n                \"davinci-logprobs\": lambda file: davinci[file],\n                \"ada-logprobs\": lambda file: ada[file],\n                \"trigram-logprobs\": lambda file: trigram[file],\n                \"unigram-logprobs\": lambda file: unigram[file],\n            }\n            exp_featurize = get_exp_featurize(best_features, vector_map)\n            exp_data = generate_dataset_fn(exp_featurize)\n\n            t_data = generate_dataset_fn(t_featurize, verbose=True)\n\n            return np.concatenate([t_data, exp_data], axis=1)\n\n        # Evaluate on data contained in data/other/toefl91\n        data = get_featurized_data(best_features_map[\"best_features_three\"])\n        data, mu, sigma = normalize(data, ret_mu_sigma=True)\n\n        model = LogisticRegression()\n        model.fit(\n            data[indices_dict[\"gpt_train\"] + indices_dict[\"human_train\"]],\n            labels[indices_dict[\"gpt_train\"] + indices_dict[\"human_train\"]],\n        )\n        print(\n            f\"Model F1: {f1_score(labels[indices_dict['gpt_test'] + indices_dict['human_test']], model.predict(data[indices_dict['gpt_test'] + indices_dict['human_test']]))}\"\n        )\n\n        toefl = get_generate_dataset(Dataset(\"normal\", \"data/other/toefl91\"))\n        toefl_labels = toefl(lambda _: 0)\n        toefl_data = get_data(toefl, best_features_map[\"best_features_three\"])\n        toefl_data = normalize(toefl_data, mu=mu, sigma=sigma)\n\n        results_table.append(\n            [\n                \"Ghostbuster\",\n                f\"Out-Domain (toefl)\",\n                accuracy_score(toefl_labels, model.predict(toefl_data)),\n            ]\n        )\n\n        # Do with perplexity only\n        perplxity_data = get_featurized_data([\"davinci-logprobs s-avg\"], gpt_only=True)\n\n        perplexity_model = LogisticRegression()\n        perplexity_model.fit(\n            perplxity_data[indices_dict[\"gpt_train\"] + indices_dict[\"human_train\"]],\n            labels[indices_dict[\"gpt_train\"] + indices_dict[\"human_train\"]],\n        )\n\n        toefl_data = get_data(toefl, [\"davinci-logprobs s-avg\"])\n\n        results_table.append(\n            [\n                \"Perplexity Only\",\n                f\"Out-Domain (toefl)\",\n                accuracy_score(toefl_labels, perplexity_model.predict(toefl_data)),\n            ]\n        )\n\n        # Do RoBERTa\n        roberta_model = RobertaForSequenceClassification.from_pretrained(\n            f\"roberta/models/roberta_gpt\", num_labels=2\n        )\n        roberta_model.to(device)\n\n        roberta_predictions = []\n        with torch.no_grad():\n            for file in tqdm.tqdm(toefl(lambda file: file)):\n                with open(file) as f:\n                    text = f.read()\n                inputs = roberta_tokenizer(\n                    text,\n                    return_tensors=\"pt\",\n                    truncation=True,\n                    padding=\"max_length\",\n                    max_length=512,\n                )\n                inputs = {k: v.to(device) for k, v in inputs.items()}\n                outputs = roberta_model(**inputs)\n                roberta_predictions.append(\n                    float(F.softmax(outputs.logits, dim=1)[0][1].item())\n                )\n\n        results_table.append(\n            [\n                \"RoBERTa\",\n                f\"Out-Domain (toefl)\",\n                accuracy_score(toefl_labels, np.array(roberta_predictions) > 0.5),\n            ]\n        )\n\n        results_table.append([\"GPT Zero\", f\"Out-Domain (toefl)\", 0.923077])\n\n        results_table.append([\"DetectGPT\", f\"Out-Domain (toefl)\", 0.6373626373626373])\n\n    if args.ghostbuster_vary_training_data:\n        exp_to_data_three = pickle.load(open(\"symbolic_data_gpt\", \"rb\"))\n\n        train_indices = indices_dict[\"gpt_train\"] + indices_dict[\"human_train\"]\n        test_indices = indices_dict[\"gpt_test\"] + indices_dict[\"human_test\"]\n        np.random.shuffle(train_indices)\n\n        claude_test_indices = (\n            list(indices_dict[\"claude_test\"]) + indices_dict[\"human_test\"]\n        )\n\n        scores = []\n        train_sizes = [int(125 * (2**i)) for i in range(6)] + [len(train_indices)]\n        print(train_sizes)\n\n        for size in tqdm.tqdm(train_sizes):\n            print(f\"Now running size: {size}\")\n\n            curr_train_indices = train_indices[:size]\n            curr_best_features = select_features(\n                exp_to_data_three,\n                labels,\n                verbose=True,\n                to_normalize=True,\n                indices=curr_train_indices,\n            )\n            data = normalize(get_featurized_data(curr_best_features))\n\n            curr_score_vec = []\n            print(data[curr_train_indices].shape)\n\n            model = LogisticRegression()\n            model.fit(data[curr_train_indices], labels[curr_train_indices])\n\n            curr_score_vec.append(\n                f1_score(labels[test_indices], model.predict(data[test_indices]))\n            )\n\n            curr_score_vec.append(\n                f1_score(\n                    labels[claude_test_indices],\n                    model.predict(data[claude_test_indices]),\n                )\n            )\n\n            for test_domain in domains:\n                domain_train_indices = []\n\n                for train_domain in domains:\n                    if train_domain == test_domain:\n                        continue\n\n                    domain_train_indices += (\n                        indices_dict[f\"gpt_{train_domain}_train\"]\n                        + indices_dict[f\"human_{train_domain}_train\"]\n                    )\n\n                domain_train_indices = np.intersect1d(\n                    domain_train_indices, curr_train_indices\n                )\n\n                domain_test_indices = (\n                    indices_dict[f\"gpt_{test_domain}_test\"]\n                    + indices_dict[f\"human_{test_domain}_test\"]\n                )\n\n                model = LogisticRegression()\n                model.fit(data[domain_train_indices], labels[domain_train_indices])\n\n                curr_score_vec.append(\n                    f1_score(\n                        labels[domain_test_indices],\n                        model.predict(data[domain_test_indices]),\n                    )\n                )\n            scores.append(curr_score_vec)\n\n        scores = np.array(scores)\n        np.save(\"results/training_size.npy\", scores)\n\n        plt.plot(train_sizes, scores[:, 0], label=\"In-Domain\")\n        plt.plot(train_sizes, scores[:, 1], label=\"Out-Domain (Claude)\")\n        plt.plot(train_sizes, scores[:, 2], label=\"Out-Domain (WP)\")\n        plt.plot(train_sizes, scores[:, 3], label=\"Out-Domain (Reuter)\")\n        plt.plot(train_sizes, scores[:, 4], label=\"Out-Domain (Essay)\")\n\n        plt.xlabel(\"Training Size (# of Documents)\")\n        plt.ylabel(\"F1 Score\")\n\n        plt.legend()\n        plt.savefig(\"results/training_size.png\")\n\n    if args.ghostbuster_vary_document_size:\n        token_sizes = [10, 25, 50, 100, 250, 500, 1000]\n        scores = []\n\n        train_indices = indices_dict[\"gpt_train\"] + indices_dict[\"human_train\"]\n        test_indices = indices_dict[\"gpt_test\"] + indices_dict[\"human_test\"]\n        claude_test_indices = (\n            list(indices_dict[\"claude_test\"]) + indices_dict[\"human_test\"]\n        )\n\n        data = get_featurized_data(best_features_map[\"best_features_three\"])\n        data, mu, sigma = normalize(data, ret_mu_sigma=True)\n\n        for num_tokens in tqdm.tqdm(token_sizes):\n            print(f\"Now running size: {num_tokens}\")\n\n            curr_t_data = generate_dataset_fn(\n                lambda file: t_featurize(file, num_tokens=num_tokens), verbose=True\n            )\n            davinci, ada, trigram, unigram = get_all_logprobs(\n                generate_dataset_fn,\n                trigram=trigram_model,\n                tokenizer=tokenizer,\n                num_tokens=num_tokens,\n            )\n\n            vector_map = {\n                \"davinci-logprobs\": lambda file: davinci[file],\n                \"ada-logprobs\": lambda file: ada[file],\n                \"trigram-logprobs\": lambda file: trigram[file],\n                \"unigram-logprobs\": lambda file: unigram[file],\n            }\n            exp_featurize = get_exp_featurize(\n                best_features_map[\"best_features_three\"], vector_map\n            )\n            curr_exp_data = generate_dataset_fn(exp_featurize)\n            curr_data = np.concatenate([curr_t_data, curr_exp_data], axis=1)\n            curr_data = normalize(curr_data, mu=mu, sigma=sigma)\n\n            curr_score_vec = []\n            print(data.shape)\n\n            model = LogisticRegression()\n            model.fit(data[train_indices], labels[train_indices])\n\n            curr_score_vec.append(\n                f1_score(labels[test_indices], model.predict(curr_data[test_indices]))\n            )\n\n            curr_score_vec.append(\n                f1_score(\n                    labels[claude_test_indices],\n                    model.predict(curr_data[claude_test_indices]),\n                )\n            )\n\n            for test_domain in domains:\n                domain_train_indices = []\n\n                for train_domain in domains:\n                    if train_domain == test_domain:\n                        continue\n\n                    domain_train_indices += (\n                        indices_dict[f\"gpt_{train_domain}_train\"]\n                        + indices_dict[f\"human_{train_domain}_train\"]\n                    )\n\n                domain_train_indices = np.intersect1d(\n                    domain_train_indices, train_indices\n                )\n\n                domain_test_indices = (\n                    indices_dict[f\"gpt_{test_domain}_test\"]\n                    + indices_dict[f\"human_{test_domain}_test\"]\n                )\n\n                model = LogisticRegression()\n                model.fit(data[domain_train_indices], labels[domain_train_indices])\n\n                curr_score_vec.append(\n                    f1_score(\n                        labels[domain_test_indices],\n                        model.predict(curr_data[domain_test_indices]),\n                    )\n                )\n\n            scores.append(curr_score_vec)\n\n            print(curr_score_vec)\n\n        scores = np.array(scores)\n        np.save(\"results/document_size.npy\", scores)\n\n        plt.plot(token_sizes, scores[:, 0], label=\"In-Domain\")\n        plt.plot(token_sizes, scores[:, 1], label=\"Out-Domain (Claude)\")\n        plt.plot(token_sizes, scores[:, 2], label=\"Out-Domain (WP)\")\n        plt.plot(token_sizes, scores[:, 3], label=\"Out-Domain (Reuter)\")\n        plt.plot(token_sizes, scores[:, 4], label=\"Out-Domain (Essay)\")\n\n        plt.xlabel(\"Document Size (# of Tokens)\")\n        plt.ylabel(\"F1 Score\")\n\n        plt.legend()\n\n        plt.savefig(\"results/document_size.png\")\n\n    if args.hyperparameter_search:\n        data = normalize(get_featurized_data(best_features_map[\"best_features_three\"]))\n\n        param_grid = {\n            \"C\": [\n                0.01,\n                0.1,\n                0.125,\n                0.25,\n                0.375,\n                0.5,\n                0.675,\n                0.75,\n                0.875,\n                1,\n                2,\n                4,\n                8,\n                10,\n            ],\n            \"penalty\": [\"l1\", \"l2\", \"elasticnet\", None],\n        }\n\n        model = LogisticRegression()\n        grid_search = GridSearchCV(\n            model, param_grid=param_grid, scoring=\"roc_auc\", cv=5, verbose=1\n        )\n\n        grid_search.fit(data[train], labels[train])\n        print(grid_search.best_params_)\n\n        model = LogisticRegression(\n            C=grid_search.best_params_[\"C\"],\n            penalty=grid_search.best_params_[\"penalty\"],\n        )\n        model.fit(data[train], labels[train])\n\n        probs = model.predict_proba(data[test])[:, 1]\n        print(get_scores(labels[test], probs))\n\n    if args.perturb:\n        data = get_featurized_data(best_features_map[\"best_features_three\"])\n        data, mu, sigma = normalize(data, ret_mu_sigma=True)\n\n        model = LogisticRegression()\n        model.fit(\n            data[indices_dict[\"gpt_train\"] + indices_dict[\"human_train\"]],\n            labels[indices_dict[\"gpt_train\"] + indices_dict[\"human_train\"]],\n        )\n\n        with open(\"data/perturb/labels.txt\") as f:\n            perturb_labels = np.array([int(i) for i in f.read().split(\"\\n\")])\n\n        def get_data(generate_dataset_fn, best_features):\n            t_data = generate_dataset_fn(t_featurize, verbose=False)\n\n            davinci, ada, trigram, unigram = get_all_logprobs(\n                generate_dataset_fn,\n                trigram=trigram_model,\n                tokenizer=tokenizer,\n                verbose=False,\n            )\n            vector_map = {\n                \"davinci-logprobs\": lambda file: davinci[file],\n                \"ada-logprobs\": lambda file: ada[file],\n                \"trigram-logprobs\": lambda file: trigram[file],\n                \"unigram-logprobs\": lambda file: unigram[file],\n            }\n            exp_featurize = get_exp_featurize(best_features, vector_map)\n            exp_data = generate_dataset_fn(exp_featurize, verbose=False)\n\n            return np.concatenate([t_data, exp_data], axis=1)\n\n        def get_perturb_data(perturb_names, perturb_sizes, save_file):\n            data = defaultdict(list)\n\n            for perturb_type in tqdm.tqdm(perturb_names):\n                for n in perturb_sizes:\n                    gen_fn = get_generate_dataset(\n                        Dataset(\"normal\", f\"data/perturb/{perturb_type}/{n}\")\n                    )\n                    curr_labels = gen_fn(\n                        lambda file: perturb_labels[\n                            int(os.path.basename(file).split(\".\")[0])\n                        ]\n                    )\n\n                    curr_data = get_data(\n                        gen_fn, best_features_map[\"best_features_three\"]\n                    )\n                    curr_data = (curr_data - mu) / sigma\n                    probs = model.predict_proba(curr_data)[:, 1]\n\n                    results_table.append(\n                        [\n                            \"Ghostbuster\",\n                            f\"Out-Domain ({perturb_type}, {n})\",\n                            *get_scores(curr_labels, probs),\n                        ]\n                    )\n\n                    _, f1, _ = get_scores(curr_labels, probs)\n                    data[perturb_type].append(f1)\n\n            np.save(save_file, data)\n            return data\n\n        perturb_char_data = get_perturb_data(\n            perturb_char_names, perturb_char_sizes, \"results/perturb_char.npy\"\n        )\n\n        for perturb_type in perturb_char_names:\n            plt.plot(\n                perturb_char_sizes,\n                perturb_char_data[perturb_type],\n                label=perturb_type,\n            )\n\n        plt.xlabel(\"Number of Perturbations\")\n        plt.ylabel(\"F1 Score\")\n        plt.legend()\n        plt.savefig(\"results/perturb_char.png\")\n\n        plt.clf()\n\n        perturb_sent_data = get_perturb_data(\n            perturb_sent_names, perturb_sent_sizes, \"results/perturb_sent.npy\"\n        )\n\n        for perturb_type in perturb_sent_names:\n            plt.plot(\n                perturb_sent_sizes,\n                perturb_sent_data[perturb_type],\n                label=perturb_type,\n            )\n\n        plt.xlabel(\"Number of Perturbations\")\n        plt.ylabel(\"F1 Score\")\n        plt.legend()\n        plt.savefig(\"results/perturb_sent.png\")\n\n    if args.calibration:\n\n        def calculate_ece(y_true, y_probs, n_bins=10):\n            \"\"\"Compute ECE\"\"\"\n            bin_lowers = np.linspace(0.0, 1.0 - 1.0 / n_bins, n_bins)\n            bin_uppers = np.linspace(1.0 / n_bins, 1.0, n_bins)\n\n            ece = 0.0\n            for bin_lower, bin_upper in zip(bin_lowers, bin_uppers):\n                in_bin = np.logical_and(bin_lower <= y_probs, y_probs < bin_upper)\n                prop_in_bin = np.mean(in_bin)\n                if prop_in_bin > 0:\n                    y_true_bin = y_true[in_bin]\n                    avg_confidence_in_bin = np.mean(y_probs[in_bin])\n                    avg_accuracy_in_bin = np.mean(y_true_bin)\n                    ece += (\n                        np.abs(avg_accuracy_in_bin - avg_confidence_in_bin)\n                        * prop_in_bin\n                    )\n\n            return ece\n\n        def train_ghostbuster_ece(data, train, test, domain):\n            model = LogisticRegression()\n            model.fit(data[train], labels[train])\n            probs = model.predict_proba(data[test])[:, 1]\n\n            return [calculate_ece(labels[test], probs)]\n\n        def train_ghostbuster_calibrated_ece(data, train, test, domain):\n            clf = LogisticRegression()\n            model = CalibratedClassifierCV(clf, method=\"isotonic\", cv=5)\n            model.fit(data[train], labels[train])\n            probs = model.predict_proba(data[test])[:, 1]\n\n            return [calculate_ece(labels[test], probs)]\n\n        run_experiment(\n            best_features_map[\"best_features_three\"],\n            \"Ghostbuster (Uncalibrated)\",\n            train_ghostbuster_ece,\n        )\n\n        run_experiment(\n            best_features_map[\"best_features_three\"],\n            \"Ghostbuster (Calibrated)\",\n            train_ghostbuster_calibrated_ece,\n        )\n\n    if len(results_table) > 1:\n        # Write data to output csv file\n        with open(args.output_file, \"w\") as f:\n            writer = csv.writer(f)\n            writer.writerows(results_table)\n\n        print(f\"Saved results to {args.output_file}\")\n"
  },
  {
    "path": "setup.py",
    "content": "import os\nfrom setuptools import setup, find_packages\n\nsetup(name=\"ghostbuster\", version=\"1.0\", packages=find_packages())\n"
  },
  {
    "path": "train.py",
    "content": "import argparse\nimport math\nimport numpy as np\nimport tiktoken\nimport dill as pickle\n\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.metrics import f1_score, accuracy_score, roc_auc_score\nfrom sklearn.calibration import CalibratedClassifierCV\n\nfrom tabulate import tabulate\n\nfrom utils.featurize import normalize, t_featurize, select_features\nfrom utils.symbolic import get_all_logprobs, train_trigram, get_exp_featurize\nfrom utils.symbolic import generate_symbolic_data\nfrom utils.load import get_generate_dataset, Dataset\n\n\nwith open(\"results/best_features_four.txt\") as f:\n    best_features = f.read().strip().split(\"\\n\")\n\nprint(\"Loading trigram model...\")\ntrigram_model = pickle.load(\n    open(\"model/trigram_model.pkl\", \"rb\"), pickle.HIGHEST_PROTOCOL\n)\ntokenizer = tiktoken.encoding_for_model(\"davinci\").encode\n\nwp_dataset = [\n    Dataset(\"normal\", \"data/wp/human\"),\n    Dataset(\"normal\", \"data/wp/gpt\"),\n]\n\nreuter_dataset = [\n    Dataset(\"author\", \"data/reuter/human\"),\n    Dataset(\"author\", \"data/reuter/gpt\"),\n]\n\nessay_dataset = [\n    Dataset(\"normal\", \"data/essay/human\"),\n    Dataset(\"normal\", \"data/essay/gpt\"),\n]\n\neval_dataset = [\n    Dataset(\"normal\", \"data/wp/claude\"),\n    Dataset(\"author\", \"data/reuter/claude\"),\n    Dataset(\"normal\", \"data/essay/claude\"),\n    Dataset(\"normal\", \"data/wp/gpt_prompt1\"),\n    Dataset(\"author\", \"data/reuter/gpt_prompt1\"),\n    Dataset(\"normal\", \"data/essay/gpt_prompt1\"),\n    Dataset(\"normal\", \"data/wp/gpt_prompt2\"),\n    Dataset(\"author\", \"data/reuter/gpt_prompt2\"),\n    Dataset(\"normal\", \"data/essay/gpt_prompt2\"),\n    Dataset(\"normal\", \"data/wp/gpt_writing\"),\n    Dataset(\"author\", \"data/reuter/gpt_writing\"),\n    Dataset(\"normal\", \"data/essay/gpt_writing\"),\n    Dataset(\"normal\", \"data/wp/gpt_semantic\"),\n    Dataset(\"author\", \"data/reuter/gpt_semantic\"),\n    Dataset(\"normal\", \"data/essay/gpt_semantic\"),\n]\n\n\ndef get_featurized_data(generate_dataset_fn, best_features):\n    t_data = generate_dataset_fn(t_featurize)\n\n    davinci, ada, trigram, unigram = get_all_logprobs(\n        generate_dataset_fn, trigram=trigram_model, tokenizer=tokenizer\n    )\n\n    vector_map = {\n        \"davinci-logprobs\": lambda file: davinci[file],\n        \"ada-logprobs\": lambda file: ada[file],\n        \"trigram-logprobs\": lambda file: trigram[file],\n        \"unigram-logprobs\": lambda file: unigram[file],\n    }\n    exp_featurize = get_exp_featurize(best_features, vector_map)\n    exp_data = generate_dataset_fn(exp_featurize)\n\n    return np.concatenate([t_data, exp_data], axis=1)\n\n\nif __name__ == \"__main__\":\n    parser = argparse.ArgumentParser()\n    parser.add_argument(\"--generate_symbolic_data\", action=\"store_true\")\n    parser.add_argument(\"--generate_symbolic_data_four\", action=\"store_true\")\n    parser.add_argument(\"--generate_symbolic_data_eval\", action=\"store_true\")\n\n    parser.add_argument(\"--perform_feature_selection\", action=\"store_true\")\n    parser.add_argument(\"--perform_feature_selection_one\", action=\"store_true\")\n    parser.add_argument(\"--perform_feature_selection_two\", action=\"store_true\")\n    parser.add_argument(\"--perform_feature_selection_four\", action=\"store_true\")\n\n    parser.add_argument(\"--perform_feature_selection_only_ada\", action=\"store_true\")\n    parser.add_argument(\"--perform_feature_selection_no_gpt\", action=\"store_true\")\n    parser.add_argument(\"--perform_feature_selection_domain\", action=\"store_true\")\n\n    parser.add_argument(\"--only_include_gpt\", action=\"store_true\")\n    parser.add_argument(\"--train_on_all_data\", action=\"store_true\")\n    parser.add_argument(\"--seed\", type=int, default=0)\n    args = parser.parse_args()\n\n    np.random.seed(args.seed)\n\n    result_table = [[\"F1\", \"Accuracy\", \"AUC\"]]\n\n    datasets = [\n        *wp_dataset,\n        *reuter_dataset,\n        *essay_dataset,\n    ]\n    generate_dataset_fn = get_generate_dataset(*datasets)\n\n    if args.generate_symbolic_data:\n        generate_symbolic_data(\n            generate_dataset_fn,\n            max_depth=3,\n            output_file=\"symbolic_data_gpt\",\n            verbose=True,\n        )\n\n        t_data = generate_dataset_fn(t_featurize)\n        pickle.dump(t_data, open(\"t_data\", \"wb\"))\n\n    if args.generate_symbolic_data_eval:\n        generate_dataset_fn_eval = get_generate_dataset(*eval_dataset)\n        generate_symbolic_data(\n            generate_dataset_fn_eval,\n            max_depth=3,\n            output_file=\"symbolic_data_eval\",\n            verbose=True,\n        )\n\n        t_data_eval = generate_dataset_fn_eval(t_featurize)\n        pickle.dump(t_data_eval, open(\"t_data_eval\", \"wb\"))\n\n    if args.generate_symbolic_data_four:\n        generate_symbolic_data(\n            generate_dataset_fn,\n            max_depth=4,\n            output_file=\"symbolic_data_gpt_four\",\n            verbose=True,\n        )\n\n        t_data = generate_dataset_fn(t_featurize)\n        pickle.dump(t_data, open(\"t_data\", \"wb\"))\n\n    labels = generate_dataset_fn(\n        lambda file: 1 if any([m in file for m in [\"gpt\", \"claude\"]]) else 0\n    )\n\n    indices = np.arange(len(labels))\n    if args.only_include_gpt:\n        where_gpt = np.where(\n            generate_dataset_fn(lambda file: 0 if \"claude\" in file else 1)\n        )[0]\n        indices = indices[where_gpt]\n\n    np.random.shuffle(indices)\n    train, test = (\n        indices[: math.floor(0.8 * len(indices))],\n        indices[math.floor(0.8 * len(indices)) :],\n    )\n    print(\"Train/Test Split\", train, test)\n    print(\"Train Size:\", len(train), \"Valid Size:\", len(test))\n    print(f\"Positive Labels: {sum(labels[indices])}, Total Labels: {len(indices)}\")\n\n    if args.perform_feature_selection:\n        exp_to_data = pickle.load(open(\"symbolic_data_gpt\", \"rb\"))\n        best_features = select_features(\n            exp_to_data, labels, verbose=True, to_normalize=True, indices=train\n        )\n\n        with open(\"results/best_features_three.txt\", \"w\") as f:\n            for feat in best_features:\n                f.write(feat + \"\\n\")\n\n    if args.perform_feature_selection_two:\n        old_exp_to_data = pickle.load(open(\"symbolic_data_gpt\", \"rb\"))\n\n        exp_to_data = {}\n        for key in old_exp_to_data:\n            if len(key.split(\" \")) <= 4:\n                exp_to_data[key] = old_exp_to_data[key]\n\n        best_features = select_features(\n            exp_to_data, labels, verbose=True, to_normalize=True, indices=train\n        )\n\n        with open(\"results/best_features_two.txt\", \"w\") as f:\n            for feat in best_features:\n                f.write(feat + \"\\n\")\n\n    if args.perform_feature_selection_one:\n        old_exp_to_data = pickle.load(open(\"symbolic_data_gpt\", \"rb\"))\n\n        exp_to_data = {}\n        for key in old_exp_to_data:\n            if len(key.split(\" \")) <= 2:\n                exp_to_data[key] = old_exp_to_data[key]\n\n        best_features = select_features(\n            exp_to_data, labels, verbose=True, to_normalize=True, indices=train\n        )\n\n        with open(\"results/best_features_one.txt\", \"w\") as f:\n            for feat in best_features:\n                f.write(feat + \"\\n\")\n\n    if args.perform_feature_selection_four:\n        exp_to_data = pickle.load(open(\"symbolic_data_gpt_four\", \"rb\"))\n\n        best_features = select_features(\n            exp_to_data, labels, verbose=True, to_normalize=True, indices=train\n        )\n\n        with open(\"results/best_features_four.txt\", \"w\") as f:\n            for feat in best_features:\n                f.write(feat + \"\\n\")\n\n    if args.perform_feature_selection_no_gpt:\n        old_exp_to_data = pickle.load(open(\"symbolic_data_gpt\", \"rb\"))\n\n        exp_to_data = {}\n        for key in old_exp_to_data:\n            if \"ada\" not in key and \"davinci\" not in key:\n                exp_to_data[key] = old_exp_to_data[key]\n\n        best_features = select_features(\n            exp_to_data, labels, verbose=True, to_normalize=True, indices=train\n        )\n\n        with open(\"results/best_features_no_gpt.txt\", \"w\") as f:\n            for feat in best_features:\n                f.write(feat + \"\\n\")\n\n    if args.perform_feature_selection_only_ada:\n        old_exp_to_data = pickle.load(open(\"symbolic_data_gpt\", \"rb\"))\n\n        exp_to_data = {}\n        for key in old_exp_to_data:\n            if \"davinci\" not in key:\n                exp_to_data[key] = old_exp_to_data[key]\n\n        best_features = select_features(\n            exp_to_data, labels, verbose=True, to_normalize=True, indices=train\n        )\n\n        with open(\"results/best_features_only_ada.txt\", \"w\") as f:\n            for feat in best_features:\n                f.write(feat + \"\\n\")\n\n    if args.perform_feature_selection_domain:\n        exp_to_data = pickle.load(open(\"symbolic_data_gpt\", \"rb\"))\n\n        wp_indices = np.where(generate_dataset_fn(lambda file: \"wp\" in file))[0]\n        reuter_indices = np.where(generate_dataset_fn(lambda file: \"reuter\" in file))[0]\n        essay_indices = np.where(generate_dataset_fn(lambda file: \"essay\" in file))[0]\n\n        wp_features = select_features(\n            exp_to_data, labels, verbose=True, to_normalize=True, indices=wp_indices\n        )\n        with open(\"results/best_features_wp.txt\", \"w\") as f:\n            for feat in wp_features:\n                f.write(feat + \"\\n\")\n\n        reuter_features = select_features(\n            exp_to_data, labels, verbose=True, to_normalize=True, indices=reuter_indices\n        )\n        with open(\"results/best_features_reuter.txt\", \"w\") as f:\n            for feat in reuter_features:\n                f.write(feat + \"\\n\")\n\n        essay_features = select_features(\n            exp_to_data, labels, verbose=True, to_normalize=True, indices=essay_indices\n        )\n        with open(\"results/best_features_essay.txt\", \"w\") as f:\n            for feat in essay_features:\n                f.write(feat + \"\\n\")\n\n    data, mu, sigma = normalize(\n        get_featurized_data(generate_dataset_fn, best_features), ret_mu_sigma=True\n    )\n    print(f\"Best Features: {best_features}\")\n    print(f\"Data Shape: {data.shape}\")\n\n    base = LogisticRegression()\n    model = CalibratedClassifierCV(base, cv=5)\n\n    if args.train_on_all_data:\n        model.fit(data, labels)\n\n        with open(\"model/features.txt\", \"w\") as f:\n            for feat in best_features:\n                f.write(feat + \"\\n\")\n        pickle.dump(model, open(\"model/model\", \"wb\"))\n        pickle.dump(mu, open(\"model/mu\", \"wb\"))\n        pickle.dump(sigma, open(\"model/sigma\", \"wb\"))\n\n        print(\"Saved model to model/\")\n    else:\n        model.fit(data[train], labels[train])\n\n    predictions = model.predict(data[test])\n    probs = model.predict_proba(data[test])[:, 1]\n\n    result_table.append(\n        [\n            round(f1_score(labels[test], predictions), 3),\n            round(accuracy_score(labels[test], predictions), 3),\n            round(roc_auc_score(labels[test], probs), 3),\n        ]\n    )\n\n    print(tabulate(result_table, headers=\"firstrow\", tablefmt=\"grid\"))\n"
  },
  {
    "path": "utils/__init__.py",
    "content": "import openai\nfrom .write_logprobs import *\nfrom .featurize import *\nfrom .n_gram import *\n\nopenai_path = \"\"\nif os.path.exists(\"../../openai.config\"):\n    openai_path = \"../../openai.config\"\nelif os.path.exists(\"../openai.config\"):\n    openai_path = \"../openai.config\"\nelif os.path.exists(\"openai.config\"):\n    openai_path = \"openai.config\"\n\nif openai_path:\n    openai_config = json.loads(open(openai_path).read())\n    openai.api_key = openai_config[\"api_key\"]\n    openai.organization = openai_config[\"organization\"]\n"
  },
  {
    "path": "utils/featurize.py",
    "content": "import numpy as np\nimport os\nimport tqdm\nfrom nltk import ngrams\nfrom utils.score import k_fold_score\n\n\ndef get_logprobs(file):\n    \"\"\"\n    Returns a vector containing all the logprobs from a given logprobs file\n    \"\"\"\n    logprobs = []\n\n    with open(file) as f:\n        for line in f.read().strip().split(\"\\n\"):\n            line = line.split(\" \")\n            logprobs.append(np.exp(-float(line[1])))\n\n    return np.array(logprobs)\n\n\ndef get_tokens(file):\n    \"\"\"\n    Returns a list of all tokens from a given logprobs file\n    \"\"\"\n    with open(file) as f:\n        tokens = list(map(lambda x: x.split(\" \")[0], f.read().strip().split(\"\\n\")))\n    return tokens\n\n\ndef get_token_len(tokens):\n    \"\"\"\n    Returns a vector of word lengths, in tokens\n    \"\"\"\n    tokens_len = []\n    curr = 0\n\n    for token in tokens:\n        if token[0] == \"Ġ\":\n            tokens_len.append(curr)\n            curr = 1\n        else:\n            curr += 1\n\n    return np.array(tokens_len)\n\n\ndef get_diff(file1, file2):\n    \"\"\"\n    Returns difference in logprobs bewteen file1 and file2\n    \"\"\"\n    return get_logprobs(file1) - get_logprobs(file2)\n\n\ndef convolve(X, window=100):\n    \"\"\"\n    Returns a vector of running average with window size\n    \"\"\"\n    ret = []\n    for i in range(len(X) - window):\n        ret.append(np.mean(X[i : i + window]))\n    return np.array(ret)\n\n\ndef score_ngram(doc, model, tokenizer, n=3, strip_first=False):\n    \"\"\"\n    Returns vector of ngram probabilities given document, model and tokenizer\n    \"\"\"\n    scores = []\n    if strip_first:\n        doc = \" \".join(doc.split()[:1000])\n    for i in ngrams((n - 1) * [50256] + tokenizer(doc.strip()), n):\n        scores.append(model.n_gram_probability(i))\n\n    return np.array(scores)\n\n\ndef normalize(data, mu=None, sigma=None, ret_mu_sigma=False):\n    \"\"\"\n    Normalizes data, where data is a matrix where the first dimension is the number of examples\n    \"\"\"\n    if mu is None:\n        mu = np.mean(data, axis=0)\n    if sigma is None:\n        raw_std = np.std(data, axis=0)\n        sigma = np.ones_like(raw_std)\n        sigma[raw_std != 0] = raw_std[raw_std != 0]\n\n    if ret_mu_sigma:\n        return (data - mu) / sigma, mu, sigma\n    else:\n        return (data - mu) / sigma\n\n\ndef convert_file_to_logprob_file(file_name, model):\n    \"\"\"\n    Removes the extension of file_name, then goes to the logprobs folder of the current directory,\n    and appends a -{model}.txt to it.\n    Example: convert_file_to_logprob_file(\"data/test.txt\", \"davinci\") = \"data/logprobs/test-davinci.txt\"\n    \"\"\"\n    directory = os.path.dirname(file_name)\n    base_name = os.path.basename(file_name)\n\n    file_name_without_ext = os.path.splitext(base_name)[0]\n    logprob_directory = os.path.join(directory, \"logprobs\")\n\n    logprob_file_name = f\"{file_name_without_ext}-{model}.txt\"\n    logprob_file_path = os.path.join(logprob_directory, logprob_file_name)\n\n    return logprob_file_path\n\n\ndef t_featurize_logprobs(davinci_logprobs, ada_logprobs, tokens):\n    X = []\n\n    outliers = []\n    for logprob in davinci_logprobs:\n        if logprob > 3:\n            outliers.append(logprob)\n\n    X.append(len(outliers))\n    outliers += [0] * (50 - len(outliers))\n    X.append(np.mean(outliers[:25]))\n    X.append(np.mean(outliers[25:50]))\n\n    diffs = sorted(davinci_logprobs - ada_logprobs, reverse=True)\n    diffs += [0] * (50 - min(50, len(diffs)))\n    X.append(np.mean(diffs[:25]))\n    X.append(np.mean(diffs[25:]))\n\n    token_len = sorted(get_token_len(tokens), reverse=True)\n    token_len += [0] * (50 - min(50, len(token_len)))\n    X.append(np.mean(token_len[:25]))\n    X.append(np.mean(token_len[25:]))\n\n    return X\n\n\ndef t_featurize(file, num_tokens=2048):\n    \"\"\"\n    Manually handcrafted features for classification.\n    \"\"\"\n    davinci_file = convert_file_to_logprob_file(file, \"davinci\")\n    ada_file = convert_file_to_logprob_file(file, \"ada\")\n\n    davinci_logprobs = get_logprobs(davinci_file)[:num_tokens]\n    ada_logprobs = get_logprobs(ada_file)[:num_tokens]\n    tokens = get_tokens(davinci_file)[:num_tokens]\n\n    return t_featurize_logprobs(davinci_logprobs, ada_logprobs, tokens)\n\n\ndef select_features(exp_to_data, labels, verbose=True, to_normalize=True, indices=None):\n    if to_normalize:\n        normalized_exp_to_data = {}\n        for key in exp_to_data:\n            normalized_exp_to_data[key] = normalize(exp_to_data[key])\n    else:\n        normalized_exp_to_data = exp_to_data\n\n    def get_data(*exp):\n        return np.concatenate([normalized_exp_to_data[e] for e in exp], axis=1)\n\n    val_exp = list(exp_to_data.keys())\n    curr = 0\n    best_features = []\n    i = 0\n\n    while val_exp:\n        best_score, best_exp = -1, \"\"\n\n        for exp in tqdm.tqdm(val_exp) if verbose else val_exp:\n            score = k_fold_score(\n                get_data(*best_features, exp), labels, k=5, indices=indices\n            )\n\n            if score > best_score:\n                best_score = score\n                best_exp = exp\n\n        if verbose:\n            print(\n                f\"Iteration {i}, Current Score: {curr}, \\\n                Best Feature: {best_exp}, New Score: {best_score}\"\n            )\n\n        if best_score <= curr:\n            break\n        else:\n            best_features.append(best_exp)\n            val_exp.remove(best_exp)\n            curr = best_score\n\n        i += 1\n\n    return best_features\n"
  },
  {
    "path": "utils/generate.py",
    "content": "import math\nimport os\nimport tqdm\nimport openai\n\nfrom utils import write_logprobs\nfrom tenacity import (\n    retry,\n    stop_after_attempt,\n    wait_random_exponential,\n)\n\n\ndef round_up(x, base=50):\n    return int(math.ceil(x / 50)) * 50\n\n\n@retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(6))\ndef openai_backoff(**kwargs):\n    return openai.ChatCompletion.create(**kwargs)\n\n\ndef generate_documents(output_dir, prompts, verbose=True, force_regenerate=False):\n    if not os.path.exists(f\"{output_dir}/logprobs\"):\n        os.mkdir(f\"{output_dir}/logprobs\")\n\n    if verbose:\n        print(\"Generating Articles...\")\n\n    for idx, prompt in (enumerate(tqdm.tqdm(prompts)) if verbose else enumerate(prompts)):\n        if os.path.exists(f\"{output_dir}/{idx}.txt\") and not force_regenerate:\n            continue\n\n        response = openai_backoff(\n            model=\"gpt-3.5-turbo\",\n            messages=[\n                {\n                    \"role\": \"user\",\n                            \"content\": prompt,\n                }\n            ],\n        )\n        reply = response[\"choices\"][0][\"message\"][\"content\"].strip()\n\n        with open(f\"{output_dir}/{idx}.txt\", \"w\") as f:\n            f.write(f\"{reply}\")\n\n    if verbose:\n        print(\"Writing logprobs...\")\n\n    for idx, prompt in (enumerate(tqdm.tqdm(prompts)) if verbose else enumerate(prompts)):\n\n        with open(f\"{output_dir}/{idx}.txt\") as f:\n            doc = f.read().strip()\n\n        if not os.path.exists(f\"{output_dir}/logprobs/{idx}-davinci.txt\") and not force_regenerate:\n            write_logprobs(\n                doc, f\"{output_dir}/logprobs/{idx}-davinci.txt\", \"davinci\"\n            )\n        if not os.path.exists(f\"{output_dir}/logprobs/{idx}-ada.txt\") and not force_regenerate:\n            write_logprobs(\n                doc, f\"{output_dir}/logprobs/{idx}-curie.txt\", \"ada\"\n            )\n"
  },
  {
    "path": "utils/load.py",
    "content": "from dataclasses import dataclass\n\nimport tqdm\nimport numpy as np\nimport os\n\n\nDIR_IGNORE = {\"logprobs\", \"prompts\", \"headlines\"}\n\n\n@dataclass\nclass Dataset:\n    type: str\n    path: str\n\n\ndef get_generate_dataset_normal(path: str, verbose=False):\n    files = []\n    to_iter = tqdm.tqdm(os.listdir(path)) if verbose else os.listdir(path)\n\n    for file in to_iter:\n        if file in DIR_IGNORE:\n            continue\n        files.append(f\"{path}/{file}\")\n\n    return files\n\n\ndef get_generate_dataset_author(path: str, author: str, verbose=False):\n    files = []\n\n    if author is None:\n        authors = sorted(os.listdir(path))\n    else:\n        authors = [author]\n\n    to_iter = tqdm.tqdm(authors) if verbose else authors\n\n    for author in to_iter:\n        for file in sorted(os.listdir(f\"{path}/{author}\")):\n            if file in DIR_IGNORE:\n                continue\n            files.append(f\"{path}/{author}/{file}\")\n\n    return files\n\n\ndef get_generate_dataset(*datasets: Dataset):\n    def generate_dataset(featurize, split=None, verbose=False, author=None):\n        files = []\n        for dataset in datasets:\n            if dataset.type == \"normal\":\n                files += get_generate_dataset_normal(dataset.path)\n            elif dataset.type == \"author\":\n                files += get_generate_dataset_author(dataset.path, author=author)\n\n        if split is not None:\n            files = np.array(files)[split]\n\n        data = []\n        files = tqdm.tqdm(files) if verbose else files\n\n        for file in files:\n            if \"logprobs\" in file:\n                continue\n            data.append(featurize(file))\n        return np.array(data)\n\n    return generate_dataset\n"
  },
  {
    "path": "utils/n_gram.py",
    "content": "import tqdm\nfrom collections import defaultdict, Counter\n\n\nclass NGramModel:\n    \"\"\"\n    An n-gram model, where alpha is the laplace smoothing parameter.\n    \"\"\"\n\n    def __init__(self, train_text, n=2, alpha=3e-3, vocab_size=None):\n        self.n = n\n        if vocab_size is None:\n            # Assume GPT tokenizer\n            self.vocab_size = 50257\n\n        self.smoothing = alpha\n        self.smoothing_f = alpha * self.vocab_size\n\n        self.c = defaultdict(lambda: [0, Counter()])\n        for i in tqdm.tqdm(range(len(train_text)-n)):\n            n_gram = tuple(train_text[i:i+n])\n            self.c[n_gram[:-1]][1][n_gram[-1]] += 1\n            self.c[n_gram[:-1]][0] += 1\n        self.n_size = len(self.c)\n\n    def n_gram_probability(self, n_gram):\n        assert len(n_gram) == self.n\n        it = self.c[tuple(n_gram[:-1])]\n        prob = (it[1][n_gram[-1]] + self.smoothing)/(it[0] + self.smoothing_f)\n        return prob\n\n\nclass DiscountBackoffModel(NGramModel):\n    \"\"\"\n    An n-gram model with discounting and backoff. Delta is the discounting parameter.\n    \"\"\"\n\n    def __init__(self, train_text, lower_order_model, n=2, delta=0.9):\n        super().__init__(train_text, n=n)\n        self.lower_order_model = lower_order_model\n        self.discount = delta\n\n    def n_gram_probability(self, n_gram):\n        assert len(n_gram) == self.n\n        it = self.c[tuple(n_gram[:-1])]\n\n        if it[0] == 0:\n            return self.lower_order_model.n_gram_probability(n_gram[1:])\n\n        prob = self.discount * \\\n            (len(it[1])/it[0]) * \\\n            self.lower_order_model.n_gram_probability(n_gram[1:])\n        if it[1][n_gram[-1]] != 0:\n            prob += max(it[1][n_gram[-1]] - self.discount, 0) / it[0]\n\n        return prob\n\n\nclass KneserNeyBaseModel(NGramModel):\n    \"\"\"\n    A Kneser-Ney base model, where n=1.\n    \"\"\"\n\n    def __init__(self, train_text, vocab_size=None):\n        super().__init__(train_text, n=1, vocab_size=vocab_size)\n\n        base_cnt = defaultdict(set)\n        for i in range(1, len(train_text)):\n            base_cnt[train_text[i]].add(train_text[i-1])\n\n        cnt = 0\n        for word in base_cnt:\n            cnt += len(base_cnt[word])\n\n        self.prob = defaultdict(float)\n        for word in base_cnt:\n            self.prob[word] = len(base_cnt[word]) / cnt\n\n    def n_gram_probability(self, n_gram):\n        assert len(n_gram) == 1\n        ret_prob = self.prob[n_gram[0]]\n\n        if ret_prob == 0:\n            return 1 / self.vocab_size\n        else:\n            return ret_prob\n\n\nclass TrigramBackoff:\n    \"\"\"\n    A trigram model with discounting and backoff. Uses a Kneser-Ney base model.\n    \"\"\"\n\n    def __init__(self, train_text, delta=0.9):\n        self.base = KneserNeyBaseModel(train_text)\n        self.bigram = DiscountBackoffModel(\n            train_text, self.base, n=2, delta=delta)\n        self.trigram = DiscountBackoffModel(\n            train_text, self.bigram, n=3, delta=delta)\n\n    def n_gram_probability(self, n_gram):\n        assert len(n_gram) == 3\n        return self.trigram.n_gram_probability(n_gram)\n"
  },
  {
    "path": "utils/score.py",
    "content": "import numpy as np\n\nfrom sklearn.linear_model import LogisticRegression\nfrom torch.utils.data import random_split\n\n\ndef k_fold_score(X, labels, indices=None, k=8, precision=10):\n    if indices is None:\n        indices = np.arange(X.shape[0])\n\n    splits = [len(indices) // k] * k\n    splits[-1] += len(indices) % k\n    k_split = random_split(indices, splits)\n\n    score_sum = 0\n    for i in range(k):\n        train = np.concatenate([np.array(k_split[j]) for j in range(k) if i != j])\n        model = LogisticRegression(C=10, penalty=\"l2\", max_iter=1000)\n        model.fit(X[train], labels[train])\n\n        score_sum += model.score(X[k_split[i]], labels[k_split[i]])\n\n    return round(score_sum / k, precision)\n"
  },
  {
    "path": "utils/symbolic.py",
    "content": "from nltk.util import ngrams\nfrom nltk.corpus import brown\nfrom nltk.tokenize import word_tokenize\n\nimport tqdm\nimport numpy as np\nimport tiktoken\nimport dill as pickle\n\nfrom utils.featurize import *\nfrom utils.n_gram import *\n\nfrom collections import Counter, defaultdict\nfrom sklearn.linear_model import LogisticRegression\n\n\nvec_functions = {\n    \"v-add\": lambda a, b: a + b,\n    \"v-sub\": lambda a, b: a - b,\n    \"v-mul\": lambda a, b: a * b,\n    \"v-div\": lambda a, b: np.divide(\n        a, b, out=np.zeros_like(a), where=(b != 0), casting=\"unsafe\"\n    ),\n    \"v->\": lambda a, b: a > b,\n    \"v-<\": lambda a, b: a < b,\n}\n\nscalar_functions = {\n    \"s-max\": max,\n    \"s-min\": min,\n    \"s-avg\": lambda x: sum(x) / len(x),\n    \"s-avg-top-25\": lambda x: sum(sorted(x, reverse=True)[:25])\n    / len(sorted(x, reverse=True)[:25]),\n    \"s-len\": len,\n    \"s-var\": np.var,\n    \"s-l2\": np.linalg.norm,\n}\n\nvectors = [\"davinci-logprobs\", \"ada-logprobs\", \"trigram-logprobs\", \"unigram-logprobs\"]\n\n# Get vec_combinations\nvec_combinations = defaultdict(list)\nfor vec1 in range(len(vectors)):\n    for vec2 in range(vec1):\n        for func in vec_functions:\n            if func != \"v-div\":\n                vec_combinations[vectors[vec1]].append(f\"{func} {vectors[vec2]}\")\n\nfor vec1 in vectors:\n    for vec2 in vectors:\n        if vec1 != vec2:\n            vec_combinations[vec1].append(f\"v-div {vec2}\")\n\n\ndef get_words(exp):\n    \"\"\"\n    Splits up expression into words, to be individually processed\n    \"\"\"\n    return exp.split(\" \")\n\n\ndef backtrack_functions(\n    vectors=(\n        \"davinci-logprobs\",\n        \"ada-logprobs\",\n        \"trigram-logprobs\",\n        \"unigram-logprobs\",\n    ),\n    max_depth=2,\n):\n    \"\"\"\n    Backtrack all possible features.\n    \"\"\"\n\n    def helper(prev, depth):\n        if depth >= max_depth:\n            return []\n\n        all_funcs = []\n        prev_word = get_words(prev)[-1]\n\n        for func in scalar_functions:\n            all_funcs.append(f\"{prev} {func}\")\n\n        for comb in vec_combinations[prev_word]:\n            all_funcs += helper(f\"{prev} {comb}\", depth + 1)\n\n        return all_funcs\n\n    ret = []\n    for vec in vectors:\n        ret += helper(vec, 0)\n    return ret\n\n\ndef train_trigram(verbose=True, return_tokenizer=False):\n    \"\"\"\n    Trains and returns a trigram model on the brown corpus\n    \"\"\"\n\n    enc = tiktoken.encoding_for_model(\"davinci\")\n    tokenizer = enc.encode\n    vocab_size = enc.n_vocab\n\n    # We use the brown corpus to train the n-gram model\n    sentences = brown.sents()\n\n    if verbose:\n        print(\"Tokenizing corpus...\")\n    tokenized_corpus = []\n    for sentence in tqdm.tqdm(sentences):\n        tokens = tokenizer(\" \".join(sentence))\n        tokenized_corpus += tokens\n\n    if verbose:\n        print(\"\\nTraining n-gram model...\")\n\n    if return_tokenizer:\n        return TrigramBackoff(tokenized_corpus), tokenizer\n    else:\n        return TrigramBackoff(tokenized_corpus)\n\n\ndef get_all_logprobs(\n    generate_dataset,\n    preprocess=lambda x: x.strip(),\n    verbose=True,\n    trigram=None,\n    tokenizer=None,\n    num_tokens=2047,\n):\n    if trigram is None:\n        trigram, tokenizer = train_trigram(verbose=verbose, return_tokenizer=True)\n\n    davinci_logprobs, ada_logprobs = {}, {}\n    trigram_logprobs, unigram_logprobs = {}, {}\n\n    if verbose:\n        print(\"Loading logprobs into memory\")\n\n    file_names = generate_dataset(lambda file: file, verbose=False)\n    to_iter = tqdm.tqdm(file_names) if verbose else file_names\n\n    for file in to_iter:\n        if \"logprobs\" in file:\n            continue\n\n        with open(file, \"r\") as f:\n            doc = preprocess(f.read())\n        davinci_logprobs[file] = get_logprobs(\n            convert_file_to_logprob_file(file, \"davinci\")\n        )[:num_tokens]\n        ada_logprobs[file] = get_logprobs(convert_file_to_logprob_file(file, \"ada\"))[\n            :num_tokens\n        ]\n        trigram_logprobs[file] = score_ngram(doc, trigram, tokenizer, n=3)[:num_tokens]\n        unigram_logprobs[file] = score_ngram(doc, trigram.base, tokenizer, n=1)[\n            :num_tokens\n        ]\n\n    return davinci_logprobs, ada_logprobs, trigram_logprobs, unigram_logprobs\n\n\ndef generate_symbolic_data(\n    generate_dataset,\n    preprocess=lambda x: x,\n    max_depth=2,\n    output_file=\"symbolic_data\",\n    verbose=True,\n    vector_map=None,\n):\n    \"\"\"\n    Brute forces and generates symbolic data from a dataset of text files.\n    \"\"\"\n    if vector_map is None:\n        (\n            davinci_logprobs,\n            ada_logprobs,\n            trigram_logprobs,\n            unigram_logprobs,\n        ) = get_all_logprobs(generate_dataset, preprocess=preprocess, verbose=verbose)\n\n        vector_map = {\n            \"davinci-logprobs\": lambda file: davinci_logprobs[file],\n            \"ada-logprobs\": lambda file: ada_logprobs[file],\n            \"trigram-logprobs\": lambda file: trigram_logprobs[file],\n            \"unigram-logprobs\": lambda file: unigram_logprobs[file],\n        }\n\n        all_funcs = backtrack_functions(max_depth=max_depth)\n\n    if verbose:\n        print(f\"\\nTotal # of Features: {len(all_funcs)}.\")\n        print(\"Sampling 5 features:\")\n        for i in range(5):\n            print(all_funcs[np.random.randint(0, len(all_funcs))])\n        print(\"\\nGenerating datasets...\")\n\n    def calc_features(file, exp):\n        exp_tokens = get_words(exp)\n        curr = vector_map[exp_tokens[0]](file)\n\n        for i in range(1, len(exp_tokens)):\n            if exp_tokens[i] in vec_functions:\n                next_vec = vector_map[exp_tokens[i + 1]](file)\n                curr = vec_functions[exp_tokens[i]](curr, next_vec)\n            elif exp_tokens[i] in scalar_functions:\n                return scalar_functions[exp_tokens[i]](curr)\n\n    exp_to_data = {}\n    for exp in tqdm.tqdm(all_funcs):\n        exp_to_data[exp] = generate_dataset(\n            lambda file: calc_features(file, exp)\n        ).reshape(-1, 1)\n\n    pickle.dump(exp_to_data, open(output_file, \"wb\"))\n\n\ndef get_exp_featurize(best_features, vector_map):\n    def calc_features(file, exp):\n        exp_tokens = get_words(exp)\n        curr = vector_map[exp_tokens[0]](file)\n\n        for i in range(1, len(exp_tokens)):\n            if exp_tokens[i] in vec_functions:\n                next_vec = vector_map[exp_tokens[i + 1]](file)\n                curr = vec_functions[exp_tokens[i]](curr, next_vec)\n            elif exp_tokens[i] in scalar_functions:\n                return scalar_functions[exp_tokens[i]](curr)\n\n    def exp_featurize(file):\n        return np.array([calc_features(file, exp) for exp in best_features])\n\n    return exp_featurize\n"
  },
  {
    "path": "utils/write_logprobs.py",
    "content": "import openai\nimport json\nimport numpy as np\nimport tiktoken\nimport torch\nimport torch.nn.functional as F\nfrom transformers import AutoTokenizer\n\n\ntokenizer = tiktoken.encoding_for_model(\"davinci\")\n\nllama_tokenizer = AutoTokenizer.from_pretrained(\"meta-llama/Llama-2-7b-hf\")\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\nvocab_map = {}\nvocab = llama_tokenizer.vocab\nfor token in vocab:\n    idx = vocab[token]\n    vocab_map[idx] = token\n\n\ndef write_logprobs(text, file, model):\n    \"\"\"\n    Run text under model and write logprobs to file, separated by newline.\n    \"\"\"\n    tokens = tokenizer.encode(text)\n    doc = tokenizer.decode(tokens[:2047])\n\n    response = openai.Completion.create(\n        model=model,\n        prompt=\"<|endoftext|>\" + doc,\n        max_tokens=0,\n        echo=True,\n        logprobs=1,\n    )\n\n    subwords = response[\"choices\"][0][\"logprobs\"][\"tokens\"][1:]\n    subprobs = response[\"choices\"][0][\"logprobs\"][\"token_logprobs\"][1:]\n\n    gpt2_map = {\"\\n\": \"Ċ\", \"\\t\": \"ĉ\", \" \": \"Ġ\"}\n\n    for i in range(len(subwords)):\n        for k, v in gpt2_map.items():\n            subwords[i] = subwords[i].replace(k, v)\n\n    to_write = \"\"\n    for _, (w, p) in enumerate(zip(subwords, subprobs)):\n        to_write += f\"{w} {-p}\\n\"\n\n    with open(file, \"w\") as f:\n        f.write(to_write)\n\n\ndef write_llama_logprobs(text, file, model):\n    with torch.no_grad():\n        encodings = llama_tokenizer(text, return_tensors=\"pt\").to(device)\n        logits = F.softmax(model(encodings[\"input_ids\"]).logits, dim=2)\n\n        tokens = encodings[\"input_ids\"]\n        indices = torch.tensor([[[i] for i in tokens[0]]])[:, 1:, :].to(device)\n\n        subwords = [vocab_map[int(idx)] for idx in encodings[\"input_ids\"][0][1:]]\n        subprobs = (\n            torch.gather(logits[:, :-1, :], dim=2, index=indices)\n            .flatten()\n            .cpu()\n            .detach()\n            .numpy()\n        )\n\n    to_write = \"\"\n    for _, (w, p) in enumerate(zip(subwords, subprobs)):\n        to_write += f\"{w} {-np.log(p)}\\n\"\n\n    with open(file, \"w\") as f:\n        f.write(to_write)\n"
  }
]