[ { "path": ".dockerignore", "content": "# Ignore Python bytecode\n__pycache__/\n*.pyc\n*.pyo\n*.pyd\n\n# Ignore logs and databases\n*.log\n\n# Ignore environments\nvenv/\n.env\n.env.*\n\n# Ignore OS + IDE files\n.DS_Store\n.idea/\n.vscode/\n*.egg-info/\nnode_modules/\n\n# Ignore git + tests\n.git\ntests/\n" }, { "path": ".gitignore", "content": "# Byte-compiled / optimized / DLL files\n__pycache__/\n*.py[cod]\n*$py.class\n\n# C extensions\n*.so\n\n# Distribution / packaging\n.Python\nbuild/\ndevelop-eggs/\ndist/\ndownloads/\neggs/\n.eggs/\nlib/\nlib64/\nparts/\nsdist/\nvar/\nwheels/\nshare/python-wheels/\n*.egg-info/\n.installed.cfg\n*.egg\nMANIFEST\n\n# PyInstaller\n# Usually these files are written by a python script from a template\n# before PyInstaller builds the exe, so as to inject date/other infos into it.\n*.manifest\n*.spec\n\n# Installer logs\npip-log.txt\npip-delete-this-directory.txt\n\n# Unit test / coverage reports\nhtmlcov/\n.tox/\n.nox/\n.coverage\n.coverage.*\n.cache\nnosetests.xml\ncoverage.xml\n*.cover\n*.py,cover\n.hypothesis/\n.pytest_cache/\ncover/\n\n# Translations\n*.mo\n*.pot\n\n# Django stuff:\n*.log\nlocal_settings.py\ndb.sqlite3\ndb.sqlite3-journal\n\n# Flask stuff:\ninstance/\n.webassets-cache\n\n# Scrapy stuff:\n.scrapy\n\n# Sphinx documentation\ndocs/_build/\n\n# PyBuilder\n.pybuilder/\ntarget/\n\n# Jupyter Notebook\n.ipynb_checkpoints\n\n# IPython\nprofile_default/\nipython_config.py\n\n# pyenv\n# For a library or package, you might want to ignore these files since the code is\n# intended to run in multiple environments; otherwise, check them in:\n# .python-version\n\n# pipenv\n# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.\n# However, in case of collaboration, if having platform-specific dependencies or dependencies\n# having no cross-platform support, pipenv may install dependencies that don't work, or not\n# install all needed dependencies.\n#Pipfile.lock\n\n# poetry\n# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.\n# This is especially recommended for binary packages to ensure reproducibility, and is more\n# commonly ignored for libraries.\n# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control\n#poetry.lock\n\n# pdm\n# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.\n#pdm.lock\n# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it\n# in version control.\n# https://pdm.fming.dev/#use-with-ide\n.pdm.toml\n\n# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm\n__pypackages__/\n\n# Celery stuff\ncelerybeat-schedule\ncelerybeat.pid\n\n# SageMath parsed files\n*.sage.py\n\n# Environments\n.env\n.venv\nenv/\nvenv/\nENV/\nenv.bak/\nvenv.bak/\n\n# Spyder project settings\n.spyderproject\n.spyproject\n\n# Rope project settings\n.ropeproject\n\n# mkdocs documentation\n/site\n\n# mypy\n.mypy_cache/\n.dmypy.json\ndmypy.json\n\n# Pyre type checker\n.pyre/\n\n# pytype static type analyzer\n.pytype/\n\n# Cython debug symbols\ncython_debug/\n\n# PyCharm\n# JetBrains specific template is maintained in a separate JetBrains.gitignore that can\n# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore\n# and can be added to the global gitignore or merged into this file. For a more nuclear\n# option (not recommended) you can uncomment the following to ignore the entire idea folder.\n#.idea/\n\n# Mac files\n.AppleDouble/\n.DS_Store\n\n# Vim files\n*.swp\n\n# Ignore user-generated files\nbase_resume.txt\njob_listings.db\njob_listings.db-shm\njob_listings.db-wal\n*.csv\n\n# Ignore vscode configuration\n.vscode/\n\ntest.py" }, { "path": "LICENSE", "content": " Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright [yyyy] [name of copyright owner]\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n" }, { "path": "README.md", "content": "
\n\n ██████╗ ██████╗ ███╗ ███╗███╗ ███╗ █████╗ ███╗ ██╗██████╗ \n ██╔════╝██╔═══██╗████╗ ████║████╗ ████║██╔══██╗████╗ ██║██╔══██╗\n ██║ ██║ ██║██╔████╔██║██╔████╔██║███████║██╔██╗ ██║██║ ██║\n ██║ ██║ ██║██║╚██╔╝██║██║╚██╔╝██║██╔══██║██║╚██╗██║██║ ██║\n ╚██████╗╚██████╔╝██║ ╚═╝ ██║██║ ╚═╝ ██║██║ ██║██║ ╚████║██████╔╝\n ╚═════╝ ╚═════╝ ╚═╝ ╚═╝╚═╝ ╚═╝╚═╝ ╚═╝╚═╝ ╚═══╝╚═════╝ \n \n ██╗ ██████╗ ██████╗ ███████╗ \n ██║██╔═══██╗██╔══██╗██╔════╝ \n ██║██║ ██║██████╔╝███████╗ \n ██ ██║██║ ██║██╔══██╗╚════██║ \n ╚█████╔╝╚██████╔╝██████╔╝███████║ \n ╚════╝ ╚═════╝ ╚═════╝ ╚══════╝ \n\n

📺 Use AI to find the best jobs for your resume and preferences

\n

🧘🏻 A distraction-free, local-first, command line interface to scrape online jobs, and filter them to your needs

\n\n \n \n\n\n\n
\n\n\nUsing AI, Command Jobs makes sure to find only the absolute best matches for your experience, skills and job preferences\n\nStop wasting your time with online tools that are not built for you, the job finder\n\nCommand Jobs is the only job searching tool that runs from where you work, the terminal. And yes, it also doesn't make you read through hundreds of job listings just to find a couple of good matches\n\nThis is just starting out! Follow along as we improve it\n\nTo get started, check out [Quick Start](#quick-start), [Configuration](#configuration) and [Usage](#usage)\n\n\n🙏🏼🤗❤️\n\nNote: If you want to add another source of job listings, [go to this issue](https://github.com/nicobrenner/commandjobs/issues/23) and add it as a suggested source\n\n\n\n\n## Updates\n\n* Optimized docker building and running\n\n* Added new scraper for Workday, currently scraping NVIDIA, CROWDSTRIKE, RED HAT and SALESFORCE.\n * The scraper currently scrapes for all countries on posts no older than a **week** back!\n \n* Building in public:\n * ❤️ If you want to contribute to this project and want to take a crack at writing tests for it, it would be amazing! 🤗 Here's a ticket to write a new test, and a walk-through of the current test code: [Request to create: Test displaying the resume text](https://github.com/nicobrenner/commandjobs/issues/48) 🙏🏼\n\n * Video walkthrough, from `git clone` all the way to finding the best matches\n\n * [![Command Jobs Walkthrough](https://cdn.loom.com/sessions/thumbnails/8034361163004b3e95ada50c91da0143-with-play.gif)](https://www.loom.com/share/8034361163004b3e95ada50c91da0143)\n \n * Here's a little bit of the internals of the application. Very high level overview of the features as well as the database. If you want to see more, or would like a deeper explanation, please create an Issue, thank you\n\n * [![Command Jobs Internals](https://cdn.loom.com/sessions/thumbnails/cf1ad06f82a344f18e3e5a569857d60b-with-play.gif)](https://www.loom.com/share/cf1ad06f82a344f18e3e5a569857d60b)\n\n * Just wrote the first test! 😅 And it's in no small part thanks to Agentic's [Glide](https://glide.agenticlabs.com/task/IqHd0RV), which they recently launched ([see announcement here](https://news.ycombinator.com/item?id=39682183)). I was about to switch from ncurses to [python-prompt-toolkit](https://github.com/prompt-toolkit/python-prompt-toolkit), and failing that from python to Go, so I could build Command Jobs using [Bubble Tea](https://github.com/charmbracelet/bubbletea) 🤩😍🤤\n\n * [![First test with Glide](https://cdn.loom.com/sessions/thumbnails/afd0733ac8dd477cbeea63c8ea6cb363-with-play.gif)](https://www.loom.com/share/afd0733ac8dd477cbeea63c8ea6cb363)\n\n * Check out the amazing [ancv](https://github.com/alexpovel/ancv), a tool for building a really cool ascii version of your resume on the terminal! 🤗 (love the joke with the Venn diagram). Will need to integrate it as a library with Command Jobs\n\n * Tried out [ShellGPT](https://github.com/mattvr/ShellGPT) and made a small PR to highlight its chat interface in the `README`. It's a pretty cool tool to use GPT from the terminal. Next I want to try coding a bit with [aider](https://github.com/paul-gauthier/aider)\n\n * [![ShellGPT](https://cdn.loom.com/sessions/thumbnails/7f415a53cb404cb0a059a9a065addce8-with-play.gif)](https://www.loom.com/share/7f415a53cb404cb0a059a9a065addce8)\n\n * Decided to try to build this project as openly as possible, in that spirit, I just recorded a coding session in which I go through the process of trying to resolve a bug ([issue #12](https://github.com/nicobrenner/commandjobs/issues/12)), and finding 3 other bugs instead!\n\n If you are just getting started with coding, it's also a pretty good overview of a basic software project management. In the video I show the whole workflow of not only writing code, but also managing an environment, dealing with errors, documenting the process in Github, managing git and branches, commiting, pushing and merging code, updating documentation (like now), and sharing/promoting\n\n * [![Trying to solve #12](https://cdn.loom.com/sessions/thumbnails/82196bfcbf0a41d58885c5b3ddc69492-with-play.gif)](https://www.loom.com/share/82196bfcbf0a41d58885c5b3ddc69492)\n\n* Thank you to the Hacker News community for the encouragement, enthusiasm and support. Check out this thread: [Show HN: Tech jobs on the command line](https://news.ycombinator.com/item?id=39621373)\n\n\n## Features\n\n- View and navigate AI-matched job listings directly from the terminal\n ![\"AI job matches\"](docs/commandjobs-ai-matches.png)\n\n- Scrape job listings from \"Ask HN: Who's hiring?\" posts on Hacker News\n\n ![\"Ask HN: Who's hiring?\" March 2024](docs/hn-ask-hn-whos-hiring-march-5-wide-optimized.gif)\n\n- Process listings with GPT to find the best matches for you\n\n * The app asks GPT for each job listing, if it's a good fit for your resume\n * The prompt includes the resume, the job listing, a section for json formating the results, a role description, a job preferences section, and some additional questions\n * You get a filtered list of the best matches for your resume and preferences\n\n\n\n## In the works\n\n- Track job applications directly in the terminal\n- Scrape job listings from additional sources\n- Add cronjob that runs periodically to scrape\n- Alerts about new matches found\n\n- Anything you'd like to see? Please add a ticket\n\n\n## Usage\n\n![\"Command Jobs main menu\"](docs/commandjobs-main-menu.png)\n\nAfter going through the Configuration and successfully running Command Jobs\n\nYou will get a menu with the options below. To navigate the menu, just use the arrow keys and select options with Enter. You can quit at any time by pressing `q`\n\nWhen first running the app, open the Edit Resume section and paste the text of your resume, no need to include your name or contact info (you can see an example resume on `config/base_resume.sample`. Alternatively, you can paste your resume text directly into a `base_resume.txt` file on the base folder of the code\n\nThen, get some job listings into the app by running Scrape \"Ask HN: Who's hiring?\". You can see the first few listings in the Navigate jobs in the local db section (if you want to see more, you can also open `job_listings.db` directly with sqlite3 and check out the contents)\n\nFor the next step, make sure you've reviewed your `.env` file and have adapted the prompts to your preferences for job matching\n\nOnce you have your Resume ready, jobs in the local db and the prompts configured, run Find best matches for resume with AI. That will run through the listings to find a match of your resume and job preferences (for now, it is limited at 5 checks per run, you can modify that through changing the `LIMIT` in the query within `fetch_job_listings()` in `src/database_manager.py`)\n\nWhen the GPT analysis is done, you get access to the AI found X listings match your resume option, where you can navigate the best matches found\n\n\nThe menu includes:\n\n- **Edit Resume**: Add or replace the text of your resume for AI matching\n- **Scrape \"Ask HN: Who's hiring?\"**: Scrape job listings from Hacker News\n- **Navigate jobs in the local db**: Browse listings stored locally\n- **Find best matches for resume with AI**: Match listings to your resume using AI\n- **AI found X listings match your resume**: Review personalized job matches\n\nTo exit the application, press `q`\n\n\n## Quick Start\n\nVideo walkthrough, from `git clone` all the way to finding the best matches\n* [![Command Jobs Walkthrough](https://cdn.loom.com/sessions/thumbnails/8034361163004b3e95ada50c91da0143-with-play.gif)](https://www.loom.com/share/8034361163004b3e95ada50c91da0143)\n\nBelow is the step by step\n\n* Clone the repository:\n\n - `git clone https://github.com/nicobrenner/commandjobs.git`\n - `cd commandjobs`\n\n\n* Run via Docker\n\n 1. Build the Docker image:\n\n - `docker-compose -f docker/docker-compose.yml build`\n\n\n 2. Run the Docker container (make sure you've setup your OpenAI API key in your `.env` file - see [Configuration](#configuration) section below):\n\n - `docker-compose -f docker/docker-compose.yml run --rm app`\n\n\n* (if you don't want to use Docker) Run with Python in a Virtual Environment\n\n 1. Set up a Python virtual environment and activate it:\n\n - `python3 -m venv venv`\n - `source venv/bin/activate`\n\n 2. Install the dependencies:\n\n - `pip install -r config/requirements.txt`\n\n 3. Run the application (make sure you've setup your OpenAI API key in your `.env` file - see [Configuration](#configuration) section below):\n\n - `python src/menu.py`\n\n\n\n## Configuration\n\n1. Create a `.env` file in the root directory of the project by copying the `config/sample.env` file, and adding your OpenAI API key:\n\n `cp config/sample.env .env`\n edit the .env file\n to add your OpenAI API key\n ```\n OPENAI_API_KEY=your_openai_api_key_here\n OPENAI_GPT_MODEL=gpt-4.1-turbo\n\n BASE_RESUME_PATH=base_resume.txt\n HN_START_URL=https://news.ycombinator.com/item?id=45438503&p=1\n\n ...\n ```\n Note: the above HN_START_URL is for October 2025\n\n\n ### Obtaining an OpenAI API Key\n\n If you don't have an OpenAI API key, [follow these instructions](https://openai.com/blog/openai-api) to obtain one.\n\n2. Modify the prompt so that it matches your preferences. The prompt has 5 sections:\n\n * `COMMANDJOBS_ROLE`: list the roles that you are looking for\n ```\n COMMANDJOBS_ROLE=backend engineer, or fullstack engineer, or senior engineer, or senior tech lead, or engineering manager, or senior enginering manager, or founding engineer, or founding fullstack engineer, or something similar\n ```\n \n * `COMMANDJOBS_IDEAL_JOB_QUESTIONS`: explain what is a good fit for you\n ```\n COMMANDJOBS_IDEAL_JOB_QUESTIONS=and the company uses either Ruby, Rails, Ruby on Rails, or Python, the position doesn't require any knowledge or experience in any of the following: {job_requirement_exclusions}, the position is remote, it's for the US and the description matches the resume? (Yes or No), justify the Yes or No about the role being a good fit for the experience of the resume in one sentence.\n ```\n \n * `COMMANDJOBS_EXCLUSIONS`: list things to avoid (this takes some trial and error to get right, iterating with the matches you get each time)\n ```\n COMMANDJOBS_EXCLUSIONS=VMS (video management systems), computer vision systems, Java, C++, C#, Grails, ML, Machine Learning, PyTorch, training models\n ```\n * `COMMANDJOBS_PROMPT`: the prompt includes all the other elements as well as the questions that we want answers about from GPT\n ```\n COMMANDJOBS_PROMPT=Given the below job listing html, and resume text. Listing:\\n{job_html}\\n\\nResume:\\n{resume}\\n\\nPlease provide the following information about the listing: brief 2 sentence summary of the listing, company name, [list of available positions, with individual corresponding links if available], tech stack description, do they use rails? (Yes or No), do they use python? (Yes or No), are the positions remote (not hybrid, not onsite)? (Yes or No), are they hiring in the US? (Yes or No), how to apply to the job? (provide 1 sentence max description, include link or email address if necessary), Does the role prioritize candidates with a background in a specific industry sector (e.g., tech, finance, healthcare)?, does the job seem like a good fit for the resume (Only say Yes if the role is for {roles} {ideal_job_questions}\\n\\nProvide output in JSON format, use this example for reference, always with the same keys, but replace the values with the answers for the previous requests for information: \\n{output_format}\n ```\n * `COMMANDJOBS_OUTPUT_FORMAT`: this specifies the output format for the prompt, including an example to follow - it's important that the structure and fields of the format matches the questions from the prompt\n ```\n COMMANDJOBS_OUTPUT_FORMAT=\"{\\n \\\"small_summary\\\": \\\"Wine and Open Source developers for C-language systems programming\\\",\\n \\\"company_name\\\": \\\"CodeWeavers\\\",\\n \\\"available_positions\\\": [\\n {\\n \\\"position\\\": \\\"Wine and General Open Source Developers\\\",\\n \\\"link\\\": \\\"https://www.codeweavers.com/about/jobs\\\"\\n }\\n ],\\n \\\"tech_stack_description\\\": \\\"C-language systems programming\\\",\\n \\\"use_rails\\\": \\\"No\\\",\\n \\\"use_python\\\": \\\"No\\\",\\n \\\"remote_positions\\\": \\\"Yes\\\",\\n \\\"hiring_in_us\\\": \\\"Yes\\\",\\n \\\"how_to_apply\\\": \\\"Apply through our website, here is the link: https://www.codeweavers.com/about/jobs\\\",\\n \\\"back_ground_with_priority\\\": null,\\n \\\"fit_for_resume\\\": \\\"No\\\",\\n \\\"fit_justification\\\": \\\"The position is for Wine and Open Source developers, neither of which the resume has experience with. The job is remote in the US\\\"\\n }\"\n ```\n\n3. Modify the query with filters for matching jobs.\n\n In the file `src/display_matching_table.py`, the method `__init__` has a variable (`self.good_match_filters`) with the following SQL conditions:\n\n ```sql\n json_valid(gi.answer) = 1\n AND json_extract(gi.answer, '$.fit_for_resume') = 'Yes'\n AND json_extract(gi.answer, '$.remote_positions') = 'Yes'\n AND json_extract(gi.answer, '$.hiring_in_us') <> 'No'\n ```\n\n These 3 conditions represent the default criteria for filtering AI-found matches. Below is the breakdown of the 3 default requirements for a good match:\n\n 1. The AI determined the listing a good match for the resume and preferences\n ```sql\n AND json_extract(gi.answer, '$.fit_for_resume') = 'Yes'\n ```\n\n 2. The role is, or can be, remote\n ```sql\n AND json_extract(gi.answer, '$.remote_positions') = 'Yes'\n ```\n \n 3. The role is hiring in the US (the value can be either Yes or NULL or '', so the condition checks that the field `'$.hiring_in_us'` is not `'No'`)\n ```sql\n AND json_extract(gi.answer, '$.hiring_in_us') <> 'No'\n ```\n\n Note: the database is a sqlite3 database, so you can also just open it `sqlite3 job_listings.db` and then try out a query like the one below, and then experiment to see what you find. Regardless of filtering, all the answers and prompts should be stored in the `gpt_interactions` table (checkout the latest update video about the internals):\n\n ```sql\n SELECT COUNT(gi.job_id)\n FROM gpt_interactions gi\n JOIN job_listings jl ON gi.job_id = jl.id\n WHERE json_valid(gi.answer) = 1\n AND json_extract(gi.answer, '$.fit_for_resume') = 'Yes'\n AND json_extract(gi.answer, '$.remote_positions') = 'Yes'\n AND json_extract(gi.answer, '$.hiring_in_us') <> 'No'\n ```\n\n You should adjust that to your preferences and you can mix and match with the questions/answers you want to get from your prompt\n\n4. Increase the limit of listings to check per batch\n\n The option `COMMANDJOBS_LISTINGS_PER_BATCH` (which should be in your `.env` file, see `sample.env`) determines how many listings are processed each time the menu option \"Find best matches with AI\" is executed. If you are using the default of 10, it means that every time you run the option \"Find best matches\", Command Jobs will make 10 requests to `gpt`. Once you trust the app, I recommend setting the limit to 500, so that the app can process all scraped listings in one go\n\n## Contributing\n\nPriority\n\n* ❤️ If you want to contribute to this project and want to take a crack at writing tests for it, it would be amazing! 🤗 Here's a ticket to write a new test, and a walk-through of the current test code: [Request to create: Test displaying the resume text](https://github.com/nicobrenner/commandjobs/issues/48) 🙏🏼\n\nWe welcome contributions, especially in improving scrapers and enhancing user experience. If you'd like to help, please file an issue or pull request on [our GitHub repository](https://github.com/nicobrenner/commandjobs/issues)\n\nHere's an overview of some of the internals of the app\n\n* [![Command Jobs Internals](https://cdn.loom.com/sessions/thumbnails/cf1ad06f82a344f18e3e5a569857d60b-with-play.gif)](https://www.loom.com/share/cf1ad06f82a344f18e3e5a569857d60b)\n\n\n## Issues\n\nEncounter any issues? Please file them on the [project's GitHub repo](https://github.com/nicobrenner/commandjobs/issues). We appreciate your feedback and contributions to making Command Jobs better!\n\n## License\n\nThis project is open-source and available under the [Apache 2.0 License](LICENSE).\n\n## Related projects\n\n* [ancv](https://github.com/alexpovel/ancv), get a fancy version of your resume in your terminal, very cool\n" }, { "path": "config/base_resume.sample", "content": "Skills\n10+ years: Ruby on Rails | Backend | Frontend | Full-stack | AWS | Postgres | Redis | CI/CD | CircleCI | Javascript | RSpec\n5+ years: Docker | Python | SMS | Twilio | VOIP | SIP\n\nExperience\nCTO/Co-founder\tAutopilotReviews\tSan Francisco / Los Angeles\t10/2014 - Present\n·\tCultivated a robust engineering culture, leading to the successful recruitment and management of a high-performing team\n·\tPioneered the integration of Twilio + A2P10DLC for delivering millions of text messages\n·\tSpearheaded the development of a highly scalable survey SaaS product (Ruby on Rails/PostgreSQL/Redis + JavaScript) \n·\tLed and managed team to setup AWS infrastructure, CI/CD and Agile processes for development\n·\tDrove the development of multiple backend integrations using Python and Selenium\n\nFounding Engineer\tPadlet (YC W13)\tSan Francisco\t09/2013 - 09/2014\n·\tInstrumental in scaling the infrastructure of a Ruby on Rails/PostgreSQL + Angular + Node/Redis stack to support over 1 million registered users and 5,000 concurrent connections\n·\tPlayed a key role in enhancing team capabilities through strategic recruitment and fostering a collaborative environment\n·\tOptimized Postgres performance for high-speed data processing and management, directly contributing to the platform's scalability and efficiency\n·\tBuilt and deployed in-house sensitive media detector, which was fundamental to Padlet’s capacity to grow\n\nCTO/Co-founder\tClickFono\tSantiago, Chile\t03/2008 - 08/2013\n·\tLed the architectural design and server infrastructure setup, incorporating SIP/voice integrations with telecom providers to create the most advanced online SaaS phone platform in Latin America at the time\n·\tBuilt a REST API for voice applications, using Ruby on Rails/PostgreSQL\n·\tTypical clients were top brands in Insurance, Banking, Finance, Retail, Telecommunications\n\nFounding Engineer\tNeedish (acquired by Groupon)\tSantiago, Chile\t03/2007 - 01/2008\n·\tFirst hire, wrote first few versions of the application using CakePHP, setup Postgres database and server infrastructure as well as testing\n\n\nEducation\nUC Berkeley 2004-2005\n1 year EAP program in CS / IEOR\n\nPUC, Chile 2000-2006\nDouble major CS and IEOR engineering degree\n\nSupervised Machine Learning, by Andrew Ng - Coursera 2017\n" }, { "path": "config/requirements.txt", "content": "beautifulsoup4==4.9.3\nrequests==2.25.1\nopenai\npython-dotenv\nwindows-curses; sys_platform == 'win32'\nselenium==4.25.0\nwebdriver-manager==4.0.2" }, { "path": "config/sample.env", "content": "OPENAI_API_KEY=your_openai_api_key_here\nOPENAI_GPT_MODEL=gpt-4.1-nano\nBASE_RESUME_PATH=base_resume.txt\nHN_START_URL=https://news.ycombinator.com/item?id=45438503&p=1\n\nCOMMANDJOBS_LISTINGS_PER_BATCH=10\n\nCOMMANDJOBS_ROLE=backend engineer, or fullstack engineer, or senior engineer, or senior tech lead, or engineering manager, or senior enginering manager, or founding engineer, or founding fullstack engineer, or something similar\n\nCOMMANDJOBS_IDEAL_JOB_QUESTIONS=and the company uses either Ruby, Rails, Ruby on Rails, or Python, the position doesn't require any knowledge or experience in any of the following: {job_requirement_exclusions}, the position is remote, it's for the US and the description matches the resume? (Yes or No), justify the Yes or No about the role being a good fit for the experience of the resume in one sentence.\n\nCOMMANDJOBS_EXCLUSIONS=VMS (video management systems), computer vision systems, Java, C++, C#, Grails, ML, Machine Learning, PyTorch, training models\n\nCOMMANDJOBS_PROMPT=Given the below job listing html, and resume text. Listing:\\n{job_html}\\n\\nResume:\\n{resume}\\n\\nPlease provide the following information about the listing: brief 2 sentence summary of the listing, company name, [list of available positions, with individual corresponding links if available], tech stack description, do they use rails? (Yes or No), do they use python? (Yes or No), are the positions remote (not hybrid, not onsite)? (Yes or No), are they hiring in the US? (Yes or No), how to apply to the job? (provide 1 sentence max description, include link or email address if necessary), Does the role prioritize candidates with a background in a specific industry sector (e.g., tech, finance, healthcare)?, does the job seem like a good fit for the resume (Only say Yes if the role is for {roles} {ideal_job_questions}\\n\\nProvide output in JSON format, use this example for reference, always with the same keys, but replace the values with the answers for the previous requests for information: \\n{output_format}\n\nCOMMANDJOBS_OUTPUT_FORMAT=\"{\\n \\\"small_summary\\\": \\\"Wine and Open Source developers for C-language systems programming\\\",\\n \\\"company_name\\\": \\\"CodeWeavers\\\",\\n \\\"available_positions\\\": [\\n {\\n \\\"position\\\": \\\"Wine and General Open Source Developers\\\",\\n \\\"link\\\": \\\"https://www.codeweavers.com/about/jobs\\\"\\n }\\n ],\\n \\\"tech_stack_description\\\": \\\"C-language systems programming\\\",\\n \\\"use_rails\\\": \\\"No\\\",\\n \\\"use_python\\\": \\\"No\\\",\\n \\\"remote_positions\\\": \\\"Yes\\\",\\n \\\"hiring_in_us\\\": \\\"Yes\\\",\\n \\\"how_to_apply\\\": \\\"Apply through our website, here is the link: https://www.codeweavers.com/about/jobs\\\",\\n \\\"back_ground_with_priority\\\": null,\\n \\\"fit_for_resume\\\": \\\"No\\\",\\n \\\"fit_justification\\\": \\\"The position is for Wine and Open Source developers, neither of which the resume has experience with. The job is remote in the US\\\"\\n }\"\n" }, { "path": "docker/Dockerfile", "content": "# docker/Dockerfile\n\n# Use your prebuilt base image\nFROM commandjobs-base\n\nWORKDIR /commandjobs\n\n# Copy only your actual source code\nCOPY . /commandjobs\n\n# Default command\nCMD [\"python3\", \"src/menu.py\"]\n" }, { "path": "docker/Dockerfile.base", "content": "# docker/Dockerfile.base\n\nFROM python:3.12\n\n# Install system dependencies just once!\nRUN apt-get update && \\\n apt-get install -y wget unzip chromium chromium-driver && \\\n apt-get clean && \\\n rm -rf /var/lib/apt/lists/*\n\n# Set a working directory\nWORKDIR /commandjobs\n\n# Install project Python dependencies (this will still cache very well)\nCOPY config/requirements.txt /commandjobs/config/requirements.txt\nRUN pip3 install --no-cache-dir -r config/requirements.txt\n" }, { "path": "docker/docker-compose.yml", "content": "services:\n base:\n build:\n context: ..\n dockerfile: docker/Dockerfile.base\n image: commandjobs-base:latest\n\n app:\n # Set container & image name\n container_name: commandjobs\n image: commandjobs:1.0\n\n build:\n context: ..\n dockerfile: docker/Dockerfile\n depends_on:\n - base\n\n # Set environment variables\n environment:\n - MENU_APP=src/menu.py\n - PYTHONPATH=/commandjobs\n - TERM=xterm-256color\n env_file:\n - ../.env\n\n # Mount entire project into docker container under /repo\n volumes:\n - ..:/commandjobs\n\n # Use host network mode (may require changes depending on Docker environment)\n network_mode: host\n\n tty: true # Allocate a pseudo-TTY\n stdin_open: true # Keep STDIN open\n\n working_dir: /commandjobs\n entrypoint: [\"sh\", \"/commandjobs/docker/docker-entrypoint.sh\"]" }, { "path": "docker/docker-entrypoint.sh", "content": "#!/bin/bash\nset -e # Exit immediately if any command fails\n\necho \"Starting the application...\"\n\necho \">>> Installing dependencies...\"\npip3 install -r config/requirements.txt || echo \"Error, could not install requirements.txt $?\"\n\necho \">>> Running database migrations...\"\n# Loop through every .py in src/migrations, sorted by filename\nfor migration in src/migrations/*.py; do\n echo \"----> Applying $(basename \"$migration\")\"\n python3 \"$migration\"\ndone\n\necho \">>> Launching application...\"\nexec python3 src/menu.py || echo \"Python script exited with error code $?\"\n\necho \"Application has terminated.\"\n" }, { "path": "job_scraper/__init__.py", "content": "" }, { "path": "job_scraper/hacker_news/__init__.py", "content": "" }, { "path": "job_scraper/hacker_news/scraper.py", "content": "import requests\nfrom bs4 import BeautifulSoup\nimport sqlite3\n\n# Define a new exception for interrupting scraping\nclass ScrapingInterrupt(Exception):\n pass\n\nclass HNScraper:\n def __init__(self, db_path='job_listings.db'):\n self.db_path = db_path\n # Define the base URL for Ask HN: Who's hiring\n self.base_url = 'https://news.ycombinator.com/item?id=45438503&p=1'\n self.new_entries_count = 0 # Initialize counter for new entries\n\n def save_to_database(self, original_text, original_html, source, external_id):\n \"\"\"Save a job listing to the SQLite database.\"\"\"\n from datetime import datetime\n \n conn = sqlite3.connect(self.db_path)\n conn.execute(\"PRAGMA journal_mode=WAL;\")\n c = conn.cursor()\n \n # Get current timestamp\n scraped_at = datetime.now().isoformat()\n \n # Use INSERT OR IGNORE to skip existing records with the same external_id\n c.execute(\"INSERT OR IGNORE INTO job_listings (original_text, original_html, source, external_id, scraped_at) VALUES (?, ?, ?, ?, ?)\",\n (original_text, original_html, source, external_id, scraped_at))\n conn.commit()\n conn.close()\n return c.rowcount > 0 # True if the listing was inserted\n\n def scrape_hn_jobs(self, start_url, stdscr, update_func=None, done_event=None, result_queue=None):\n \"\"\"Scrape job listings from Hacker News and save them to the database.\"\"\"\n url = start_url\n update_func(f\"Scraping: {start_url}\")\n while url:\n try:\n response = requests.get(url, timeout=10)\n soup = BeautifulSoup(response.text, 'html.parser')\n\n comments = soup.find_all('tr', class_='athing comtr')\n for comment in comments:\n ind_cell = comment.find('td', class_='ind')\n img = ind_cell.find('img') if ind_cell else None\n if img and img.get('width') == \"0\": # Top-level comment\n job_description = comment.find('div', class_='commtext c00')\n if job_description:\n original_text = job_description.text\n original_html = job_description.prettify()\n # Extract the external_id from the comment element\n comment_id = comment.get('id')\n external_id = f\"https://news.ycombinator.com/item?id={comment_id}\"\n source = \"Hacker News\"\n inserted = self.save_to_database(original_text, original_html, source, external_id)\n\n if inserted: # if the row was inserted\n self.new_entries_count += 1 # Increment the new entries count\n # Check for updates and interrupts\n if update_func:\n update_func(original_text[:100]) # Call the update function with truncated text\n if update_func:\n update_func(f\"Scraping: {source}\")\n\n more_link = soup.find('a', class_='morelink')\n if more_link:\n url = 'https://news.ycombinator.com/' + more_link['href']\n if update_func:\n update_func(f\"Page complete, loading next... {self.new_entries_count} listings added so far\")\n else:\n url = None\n\n except requests.exceptions.Timeout as e:\n if update_func:\n update_func(\"Request timed out. Try again later.\")\n break\n \n except requests.exceptions.RequestException as e:\n if update_func:\n update_func(f\"Request failed: {str(e)}\")\n break\n\n # Handle user interrupts\n except ScrapingInterrupt:\n if update_func:\n update_func(f\"Scraping interrupted by user. {self.new_entries_count} new listings added\")\n break\n\n if update_func:\n # Put the result into the queue\n result_queue.put(self.new_entries_count)\n if done_event:\n done_event.set() # Set the event to signal that scraping is done\n\nif __name__ == \"__main__\":\n db_path = 'job_listings.db'\n scraper = HNScraper(db_path)\n start_url = 'https://news.ycombinator.com/item?id=45438503&p=1'\n scraper.scrape_hn_jobs(start_url)\n" }, { "path": "job_scraper/scraper_selectors/__init__.py", "content": "" }, { "path": "job_scraper/scraper_selectors/workday_selectors.py", "content": "from enum import StrEnum\n\n\nclass WorkDaySelectors(StrEnum):\n JOB_LISTING_XPATH = '//li[@class=\"css-1q2dra3\"]'\n JOB_TITLE_XPATH = './/h3/a'\n JOB_ID_XPATH = './/ul[@data-automation-id=\"subtitle\"]/li'\n POSTED_ON_XAPTH = './/dd[@class=\"css-129m7dg\"][preceding-sibling::dt[contains(text(),\"posted on\")]]'\n JOB_DESCRIPTION_XPATH = '//div[@data-automation-id=\"jobPostingDescription\"]'\n NEXT_PAGE_XPATH = \"//button[@data-uxi-element-id='next']\"" }, { "path": "job_scraper/utils.py", "content": "def get_workday_company_urls() -> dict:\n urls = {\n 'NVIDIA': 'https://nvidia.wd5.myworkdayjobs.com/NVIDIAExternalCareerSite?jobFamilyGroup=0c40f6bd1d8f10ae43ffaefd46dc7e78',\n 'SALESFORCE': 'https://salesforce.wd12.myworkdayjobs.com/en-US/External_Career_Site/details/Lead-Marketing-Cloud-Solution-Engineer_JR268932?jobFamilyGroup=14fa3452ec7c1011f90d0002a2100000',\n 'RED_HAT': 'https://redhat.wd5.myworkdayjobs.com/Jobs',\n 'CROWDSTRIKE': 'https://crowdstrike.wd5.myworkdayjobs.com/crowdstrikecareers'\n }\n return urls\n\ndef get_workday_post_time_range() -> list[str]:\n return ['posted today', 'posted yesterday', 'posted 2 days ago', 'posted 3 days ago',\n 'posted 4 days ago', 'posted 5 days ago', 'posted 6 days ago', 'posted 7 days ago']\n\n" }, { "path": "job_scraper/waas/__init__.py", "content": "" }, { "path": "job_scraper/waas/work_startup_scraper.py", "content": "import sqlite3\nimport requests\nfrom bs4 import BeautifulSoup\nimport json\n\nclass ScrapingInterrupt(Exception):\n pass\n\nclass WorkStartupScraper:\n\n def __init__(self, db_path='job_listings.db'):\n self.db_path = db_path\n # Define the base URL for Ask HN: Who's hiring\n self.base_url = 'https://www.workatastartup.com/jobs'\n self.new_entries_count = 0 # Initialize counter for new entries\n\n def get_company_links(self):\n response = requests.get(self.base_url)\n soup = BeautifulSoup(response.content, 'html.parser')\n company_links_set = set()\n company_links = []\n \n for a in soup.select('a[target=\"company\"]'):\n company_url = a['href']\n if company_url not in company_links_set:\n company_links.append(company_url)\n company_links_set.add(company_url)\n \n return company_links\n\n\n def get_job_links(self, company_url):\n \n # Fetch the HTML content from the URL\n response = requests.get(company_url)\n soup = BeautifulSoup(response.content, 'html.parser')\n\n # Find all elements with a data-page attribute\n data_page_elements = soup.find_all(attrs={\"data-page\": True})\n\n # Initialize a list to store matching links\n job_links = []\n\n # Find the div with the data-page attribute\n div = soup.find('div', {'data-page': True})\n if div:\n # Extract the JSON-like content from the data-page attribute\n data_page_content = div['data-page']\n \n # Parse the JSON content\n data = json.loads(data_page_content)\n \n # Extract job links\n for job in data['props']['rawCompany']['jobs']:\n job_link = job['show_path']\n job_links.append(job_link)\n \n return job_links\n\n\n def get_job_details(self, job_url):\n response = requests.get(job_url)\n soup = BeautifulSoup(response.content, 'html.parser')\n\n # Find the \"About the role\" section and extract content until \"How you'll contribute\"\n about_section = soup.find(string=\"About the role\")\n if about_section:\n # Find the parent element of \"About the role\"\n about_div = about_section.find_parent('div')\n if about_div:\n # Extract content between \"About the role\" and \"How you'll contribute\"\n extracted_content = []\n for sibling in about_div.next_siblings:\n if sibling.name == 'div' and sibling.find(string=\"How you'll contribute\"):\n break\n extracted_content.append(str(sibling))\n\n # Join the extracted content\n extracted_content_str = ''.join(extracted_content).strip()\n\n # Get original text and HTML\n original_text = BeautifulSoup(extracted_content_str, 'html.parser').get_text(strip=True)\n original_html = extracted_content_str\n\n # Extract external ID from job URL\n external_id = job_url\n source = \"Work at a startup\"\n\n return {\n 'original_text': original_text,\n 'original_html': original_html,\n 'source': source,\n 'external_id': external_id\n }\n else:\n print(f\"No parent element found for 'About the role' in {job_url}\")\n else:\n print(f\"'About the role' section not found in {job_url}\")\n return None\n\n def scrape_jobs(self, stdscr, update_func=None, done_event=None, result_queue=None):\n \"\"\"Scrape job listings from Work at a Startup and save them to the database.\"\"\"\n jobs_list = []\n update_func(f\"Scraping: {self.base_url}\")\n try: \n company_links = self.get_company_links()\n count = 0\n flag1 = False\n flag2 = False\n flag3 = False\n for company_link in company_links:\n count += 1\n job_links = self.get_job_links(company_link)\n for job_link in job_links:\n job_details = self.get_job_details(job_link)\n if job_details:\n jobs_list.append(job_details)\n if update_func:\n update_func(f\"Scraping: {company_link}\")\n # Updates the progress of the scraping\n if count / len(company_links)>= 0.25 and not flag1:\n update_func(\"Scraping: 25% of companies completed\")\n flag1 = True\n elif count / len(company_links)>= 0.5 and not flag2:\n update_func(\"Scraping: 50% of companies completed\")\n flag2 = True\n elif count / len(company_links)>= 0.75:\n update_func(\"Scraping: 75% of companies completed\")\n flag3 = True\n \n for job in jobs_list:\n inserted= self.save_to_database(job['original_text'], job['original_html'], job['source'], job['external_id'])\n if inserted:\n self.new_entries_count += 1\n \n if job==jobs_list[-1]:\n if done_event:\n result_queue.put(self.new_entries_count)\n done_event.set() # Set the event to signal that scraping is done\n \n except requests.exceptions.Timeout as e:\n if update_func:\n update_func(\"Request timed out. Try again later.\")\n \n except requests.exceptions.RequestException as e:\n if update_func:\n update_func(f\"Request failed: {str(e)}\")\n\n # Handle user interrupts\n except ScrapingInterrupt:\n if update_func:\n update_func(f\"Scraping interrupted by user. {self.new_entries_count} new listings added\")\n\n\n def save_to_database(self, original_text, original_html, source, external_id):\n \"\"\"Save a job listing to the SQLite database.\"\"\"\n from datetime import datetime\n \n conn = sqlite3.connect(self.db_path)\n conn.execute(\"PRAGMA journal_mode=WAL;\")\n c = conn.cursor()\n \n # Get current timestamp\n scraped_at = datetime.now().isoformat()\n \n # Use INSERT OR IGNORE to skip existing records with the same external_id\n c.execute(\"INSERT OR IGNORE INTO job_listings (original_text, original_html, source, external_id, scraped_at) VALUES (?, ?, ?, ?, ?)\",\n (original_text, original_html, source, external_id, scraped_at))\n conn.commit()\n conn.close()\n return c.rowcount > 0 # True if the listing was inserted\n" }, { "path": "job_scraper/workday/__init__.py", "content": "" }, { "path": "job_scraper/workday/scraper.py", "content": "import sqlite3\nimport time\nfrom selenium import webdriver\nfrom selenium.common.exceptions import TimeoutException, StaleElementReferenceException\nfrom webdriver_manager.chrome import ChromeDriverManager\nfrom selenium.webdriver.chrome.service import Service\nfrom selenium.webdriver.chrome.options import Options\nfrom selenium.webdriver.common.by import By\nfrom selenium.webdriver.support import expected_conditions as EC\nfrom selenium.webdriver.support.ui import WebDriverWait\n\nfrom job_scraper.scraper_selectors.workday_selectors import WorkDaySelectors\nfrom job_scraper.utils import get_workday_post_time_range, get_workday_company_urls\n\n\nclass WorkdayScraper:\n def __init__(self, db_path='job_listings.db', update_func=None, done_event=None, result_queue=None):\n self.db_path = db_path\n self.driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()), options=self.get_selenium_configs())\n self.one_week_span_text = get_workday_post_time_range()\n self.company_urls = get_workday_company_urls()\n self.new_entries_count = 0\n self.done_event = done_event\n self.result_queue = result_queue\n self.update_func = update_func\n self.job_listings = []\n\n @staticmethod\n def get_selenium_configs() -> Options:\n chrome_options = Options()\n chrome_options.add_argument(\"--headless\")\n chrome_options.add_argument(\"--no-sandbox\")\n chrome_options.add_argument(\"--disable-dev-shm-usage\")\n chrome_options.add_argument(\"--disable-gpu\")\n return chrome_options\n\n def save_to_database(self, original_text, original_html, source, external_id):\n from datetime import datetime\n \n conn = sqlite3.connect(self.db_path)\n conn.execute(\"PRAGMA journal_mode=WAL;\")\n c = conn.cursor()\n \n # Get current timestamp\n scraped_at = datetime.now().isoformat()\n \n c.execute(\"INSERT OR IGNORE INTO job_listings (original_text, original_html, source, external_id, scraped_at) VALUES (?, ?, ?, ?, ?)\",\n (original_text, original_html, source, external_id, scraped_at))\n conn.commit()\n conn.close()\n return c.rowcount > 0\n\n def save_new_job_listing(self, job_description, job_description_html, job_url, job_id):\n if not job_description:\n return\n if not job_description_html:\n return\n if not job_url:\n return\n if not job_id:\n return\n self.job_listings.append({\n 'original_text': job_description,\n 'original_html': job_description_html,\n 'source': job_url,\n 'external_id': job_id\n })\n\n def save_job_listings_to_db(self):\n for job in self.job_listings:\n inserted = self.save_to_database(\n job['original_text'],\n job['original_html'],\n job['source'],\n job['external_id']\n )\n if inserted:\n self.new_entries_count += 1\n if self.done_event:\n self.result_queue.put(self.new_entries_count)\n self.done_event.set()\n\n def scrape(self):\n self.update_func(f\"Scraping Workday companies:\\t{\", \".join(self.company_urls.keys())}\")\n\n for company_name, company_url in self.company_urls.items():\n self.driver.get(company_url)\n wait = WebDriverWait(self.driver, 10)\n\n posted_this_week = True\n while posted_this_week:\n try:\n wait.until(EC.presence_of_element_located((By.XPATH, WorkDaySelectors.JOB_LISTING_XPATH)))\n except TimeoutException:\n self.update_func(\"Job Listing Element not found. Try again later\")\n break\n\n job_elements = self.driver.find_elements(By.XPATH, WorkDaySelectors.JOB_LISTING_XPATH)\n for job_element in job_elements:\n try:\n self.update_func(f\"Scraping {company_name}: {self.driver.current_url}\")\n job_title_element = job_element.find_element(By.XPATH, WorkDaySelectors.JOB_TITLE_XPATH)\n job_id_element = job_element.find_element(By.XPATH, WorkDaySelectors.JOB_ID_XPATH)\n job_id = job_id_element.text\n posted_on_element = job_element.find_element(By.XPATH, WorkDaySelectors.POSTED_ON_XAPTH)\n posted_on = posted_on_element.text\n\n if posted_on.lower() in self.one_week_span_text:\n job_url = job_title_element.get_attribute('href')\n job_title_element.click()\n job_description_element = wait.until(\n EC.presence_of_element_located((By.XPATH, WorkDaySelectors.JOB_DESCRIPTION_XPATH))\n )\n job_description = job_description_element.text\n job_description_html = job_description_element.get_attribute(\"innerHTML\")\n self.save_new_job_listing(job_description, job_description_html, job_url, job_id)\n else:\n posted_this_week = False\n break\n except StaleElementReferenceException:\n continue\n\n if not posted_this_week:\n break\n\n try:\n next_page_button = wait.until(\n EC.element_to_be_clickable((By.XPATH, WorkDaySelectors.NEXT_PAGE_XPATH))\n )\n next_page_button.click()\n except TimeoutException:\n self.update_func(\"TimeoutException. Please try again later!\")\n break\n\n self.save_job_listings_to_db()\n self.update_func(\"Scraping completed for all companies.\")\n" }, { "path": "src/__init__.py", "content": "" }, { "path": "src/database_manager.py", "content": "import sqlite3\nimport asyncio\n\nclass DatabaseManager:\n def __init__(self, db_path):\n self.conn = sqlite3.connect(db_path)\n self.conn.execute(\"PRAGMA journal_mode=WAL;\")\n self.cursor = self.conn.cursor()\n self.initialize_db()\n\n def initialize_db(self):\n self.cursor.execute('''\n CREATE TABLE IF NOT EXISTS job_listings (\n id INTEGER PRIMARY KEY AUTOINCREMENT,\n original_text TEXT,\n original_html TEXT,\n source TEXT,\n external_id TEXT UNIQUE\n )\n ''')\n self.conn.commit()\n self.cursor.execute('''\n CREATE TABLE IF NOT EXISTS gpt_interactions (\n id INTEGER PRIMARY KEY,\n job_id INTEGER,\n prompt TEXT,\n answer TEXT\n )\n ''')\n self.conn.commit()\n\n def fetch_job_listings(self, listings_per_batch):\n # The LIMIT here is effectively throttling GPT usage\n # every time the AI processing runs,\n # it only checks {listings_per_batch} listings\n # 10 by default\n listings_per_batch = listings_per_batch or 10\n query = f\"\"\"\n SELECT jl.id, jl.original_text, jl.original_html\n FROM job_listings jl\n LEFT JOIN gpt_interactions gi ON jl.id = gi.job_id\n WHERE gi.job_id IS NULL LIMIT {listings_per_batch}\n \"\"\"\n self.cursor.execute(query)\n return self.cursor.fetchall()\n \n def fetch_processed_listings_count(self):\n query = \"SELECT COUNT(id) FROM gpt_interactions\"\n self.cursor.execute(query)\n result = self.cursor.fetchone() # Fetch the first row of the result set\n if result:\n return result[0] # Return the first element of the tuple, which is the count\n else:\n return 0 # Return 0 if no rows are found, for safety\n \n def fetch_applied_listings_count(self):\n \"\"\"Return the total number of listings the user has marked as applied.\"\"\"\n query = \"SELECT COUNT(*) FROM applications WHERE status = 'Open'\"\n self.cursor.execute(query)\n result = self.cursor.fetchone()\n return result[0] if result else 0\n\n\n def save_gpt_interaction(self, job_id, prompt, answer):\n self.cursor.execute(\"INSERT INTO gpt_interactions (job_id, prompt, answer) VALUES (?, ?, ?)\", (job_id, prompt, answer))\n self.conn.commit()\n\n def close(self):\n self.conn.close()\n" }, { "path": "src/display_all_jobs.py", "content": "import locale\nimport sqlite3\nimport curses\nimport textwrap\nimport logging\nimport json\nfrom datetime import datetime\nfrom display_applications import ApplicationsDisplay\n\nlocale.setlocale(locale.LC_ALL, '')\n\nclass AllJobsDisplay:\n def __init__(self, stdscr, db_path):\n self.stdscr = stdscr\n self.db_path = db_path\n self.highlighted_row_index = 0\n self.current_page = 1\n self.total_pages = 0\n self.rows_per_page = 3\n self.search_term = \"\"\n logging.basicConfig(filename='all_jobs_display.log', level=logging.DEBUG)\n\n def log(self, message):\n \"\"\"Log a message for debugging.\"\"\"\n logging.debug(message)\n \n def format_scraped_date(self, scraped_at):\n \"\"\"Format scraped_at timestamp for display.\"\"\"\n try:\n if scraped_at:\n # Parse the ISO timestamp and format for display\n dt = datetime.fromisoformat(scraped_at)\n return dt.strftime(\"%Y-%m-%d\")\n return \"Unknown\"\n except (ValueError, TypeError):\n return \"Unknown\"\n \n def get_search_filters(self):\n \"\"\"Build additional WHERE conditions for search filtering.\"\"\"\n if not self.search_term:\n return \"\"\n \n # Search in company name, summary, job description, and available positions\n search_conditions = [\n f\"lower(json_extract(gi.answer, '$.company_name')) LIKE '%{self.search_term.lower()}%'\",\n f\"lower(json_extract(gi.answer, '$.small_summary')) LIKE '%{self.search_term.lower()}%'\", \n f\"lower(jl.original_text) LIKE '%{self.search_term.lower()}%'\",\n f\"lower(json_extract(gi.answer, '$.available_positions')) LIKE '%{self.search_term.lower()}%'\"\n ]\n \n return \" AND (\" + \" OR \".join(search_conditions) + \")\"\n \n def prompt_search(self):\n \"\"\"Prompt user for search term and update search filters.\"\"\"\n max_y, max_x = self.stdscr.getmaxyx()\n \n # Create input window\n input_win = curses.newwin(3, max_x - 4, max_y - 5, 2)\n input_win.box()\n input_win.addstr(1, 2, f\"Search (current: '{self.search_term}'): \")\n input_win.refresh()\n \n # Enable echo and get input\n curses.echo()\n curses.curs_set(1) # Show cursor\n \n # Get user input\n try:\n search_input = input_win.getstr(1, len(f\"Search (current: '{self.search_term}'): \") + 2, 50).decode('utf-8')\n self.search_term = search_input.strip()\n except:\n pass # Handle any input errors\n finally:\n curses.noecho()\n curses.curs_set(0) # Hide cursor\n \n # Reset pagination\n self.current_page = 1\n self.highlighted_row_index = 0\n \n # Clear the input window\n input_win.clear()\n input_win.refresh()\n del input_win\n\n def fetch_total_entries(self):\n try:\n conn = sqlite3.connect(self.db_path)\n cur = conn.cursor()\n search_filters = self.get_search_filters()\n cur.execute(f\"\"\"\n SELECT COUNT(gi.job_id)\n FROM gpt_interactions gi\n JOIN job_listings jl ON gi.job_id = jl.id\n WHERE json_valid(gi.answer) = 1\n AND (jl.discarded IS NULL OR jl.discarded = 0)\n AND (jl.applied IS NULL OR jl.applied = 0){search_filters}\n \"\"\")\n total_entries = cur.fetchone()[0]\n conn.close()\n return total_entries\n except (sqlite3.OperationalError, sqlite3.DatabaseError):\n return 0\n\n def fetch_job(self, offset=None):\n if offset is None:\n offset = (self.current_page - 1) * self.rows_per_page + self.highlighted_row_index\n try:\n conn = sqlite3.connect(self.db_path)\n cur = conn.cursor()\n search_filters = self.get_search_filters()\n \n query = f\"\"\"\n SELECT\n json_extract(gi.answer, '$.company_name') AS company_name,\n json_extract(gi.answer, '$.available_positions') AS available_positions,\n json_extract(gi.answer, '$.small_summary') AS summary,\n json_extract(gi.answer, '$.fit_for_resume') AS fit_for_resume,\n json_extract(gi.answer, '$.fit_justification') AS fit_justification,\n gi.job_id,\n jl.original_text,\n jl.external_id,\n jl.scraped_at\n FROM\n gpt_interactions gi\n JOIN\n job_listings jl ON gi.job_id = jl.id\n WHERE\n json_valid(gi.answer) = 1\n AND (jl.discarded IS NULL OR jl.discarded = 0)\n AND (jl.applied IS NULL OR jl.applied = 0){search_filters}\n ORDER BY jl.scraped_at DESC, jl.id DESC\n LIMIT 1 OFFSET {offset}\n \"\"\"\n \n self.log(f\"Executing query: {query}\") # Log the query\n cur.execute(query)\n data = cur.fetchone()\n self.log(f\"Fetched 1 row\") # Log the number of results\n conn.close()\n return data\n except (sqlite3.OperationalError, sqlite3.DatabaseError):\n return None\n\n def fetch_data(self, page_num):\n offset = (page_num - 1) * self.rows_per_page\n try:\n conn = sqlite3.connect(self.db_path)\n cur = conn.cursor()\n search_filters = self.get_search_filters()\n \n query = f\"\"\"\n SELECT\n json_extract(gi.answer, '$.company_name') AS company_name,\n json_extract(gi.answer, '$.available_positions') AS available_positions,\n json_extract(gi.answer, '$.small_summary') AS summary,\n json_extract(gi.answer, '$.fit_for_resume') AS fit_for_resume,\n json_extract(gi.answer, '$.fit_justification') AS fit_justification,\n gi.job_id,\n jl.original_text,\n jl.external_id,\n jl.scraped_at\n FROM\n gpt_interactions gi\n JOIN\n job_listings jl ON gi.job_id = jl.id\n WHERE\n json_valid(gi.answer) = 1\n AND (jl.discarded IS NULL OR jl.discarded = 0)\n AND (jl.applied IS NULL OR jl.applied = 0){search_filters}\n ORDER BY jl.scraped_at DESC, jl.id DESC\n LIMIT {self.rows_per_page} OFFSET {offset}\n \"\"\"\n \n self.log(f\"Executing query: {query}\") # Log the query\n cur.execute(query)\n data = cur.fetchall()\n self.log(f\"Fetched {len(data)} rows\") # Log the number of results\n conn.close()\n return data\n except (sqlite3.OperationalError, sqlite3.DatabaseError):\n return None\n\n def draw_page(self, current_page):\n max_y, max_x = self.stdscr.getmaxyx()\n data = self.fetch_data(page_num=current_page)\n\n # Column widths for processed jobs\n column_widths = {\n \"Company\": 15,\n \"Position\": 25,\n \"Summary\": 50,\n \"Why?\": 30 # Fit justification\n }\n\n self.stdscr.clear()\n header = \" \".join(title.center(column_widths[title]) for title in column_widths.keys())\n self.stdscr.attron(curses.color_pair(4))\n self.stdscr.addstr(0, 0, header)\n self.stdscr.attroff(curses.color_pair(4))\n\n y_offset = 2 # Start below the header\n\n for idx, listing in enumerate(data):\n if idx == self.highlighted_row_index:\n self.stdscr.attron(curses.color_pair(3))\n\n max_height_wrapped_text = 1\n for i, key in enumerate(column_widths.keys()):\n if key == \"Company\":\n field = listing[0] # company_name from AI analysis\n elif key == \"Position\":\n # Parse JSON positions and extract titles\n try:\n positions = json.loads(listing[1]) or []\n titles = [pos.get(\"position\") for pos in positions\n if isinstance(pos.get(\"position\"), str)]\n field = \", \".join(titles) if titles else \"Various\"\n except (json.JSONDecodeError, TypeError):\n field = \"Various\"\n elif key == \"Summary\":\n field = listing[2] # small_summary from AI analysis\n elif key == \"Why?\":\n # Show fit status and brief justification\n fit_status = listing[3] if listing[3] else \"Unknown\"\n justification = listing[4] if listing[4] else \"\"\n field = f\"{fit_status}: {justification[:100]}\" if justification else fit_status\n \n width = column_widths[key]\n \n # For the 'Company' column, add scraped date underneath\n if key == \"Company\":\n scraped_at = listing[8] if len(listing) > 8 else None\n formatted_date = self.format_scraped_date(scraped_at)\n field = f\"{field}\\n({formatted_date})\"\n \n # This part takes a field content and wraps it in width\n wrapped_text = textwrap.wrap(str(field), width=width)\n for j, line in enumerate(wrapped_text):\n line_pos = sum(column_widths[title] for title in list(column_widths.keys())[:i]) + i * 3\n if line_pos + width <= max_x and y_offset + j < max_y - 1:\n self.stdscr.addstr(y_offset + j, line_pos, line.ljust(width))\n \n if j > max_height_wrapped_text:\n max_height_wrapped_text = j\n \n y_offset += max_height_wrapped_text + 2\n\n if y_offset >= max_y - 3: # Check if we've reached the end of the screen\n break # Stop drawing if there's no more space on the screen\n\n if idx == self.highlighted_row_index:\n self.stdscr.attroff(curses.color_pair(3))\n\n # --- footer line: pagination + controls ---\n footer_y = max_y - 2\n\n # 1) Draw pagination (flush-left)\n search_status = f\" (filtered: '{self.search_term}')\" if self.search_term else \"\"\n pagination = f\"Page {self.current_page} of {self.total_pages} ({self.total_entries} job listings{search_status} 📋)\"\n self.stdscr.attron(curses.color_pair(5))\n self.stdscr.addstr(footer_y, 0, pagination.ljust(max_x))\n self.stdscr.attroff(curses.color_pair(5))\n\n # 2) Prepare controls text\n controls_text = \"[↑↓] Move [←→ ] Page [Enter] View [d] Discard [a] Apply [s] Search [c] Clear [q] Back\"\n\n # 3) Clear the next line so no overlap\n self.stdscr.move(footer_y + 1, 0)\n self.stdscr.clrtoeol()\n\n # 4) Draw controls (same left alignment)\n self.stdscr.attron(curses.color_pair(7))\n self.stdscr.addstr(footer_y + 1, 0, controls_text[: max_x - 1])\n self.stdscr.attroff(curses.color_pair(7))\n\n self.stdscr.refresh()\n\n def draw_table(self):\n self.total_entries = self.fetch_total_entries()\n self.total_pages = (self.total_entries + self.rows_per_page - 1) // self.rows_per_page\n\n self.draw_page(self.current_page)\n\n while True:\n key = self.stdscr.getch()\n if key == curses.KEY_DOWN:\n self.highlighted_row_index = min(self.highlighted_row_index + 1, self.rows_per_page - 1)\n self.draw_page(self.current_page)\n elif key == curses.KEY_UP:\n self.highlighted_row_index = max(0, self.highlighted_row_index - 1)\n self.draw_page(self.current_page)\n elif key == curses.KEY_RIGHT:\n if self.current_page < self.total_pages:\n self.current_page += 1\n self.highlighted_row_index = 0 # Reset highlighted row for the new page\n self.draw_page(self.current_page)\n elif key == curses.KEY_LEFT:\n if self.current_page > 1:\n self.current_page -= 1\n self.highlighted_row_index = 0 # Reset highlighted row for the new page\n self.draw_page(self.current_page)\n elif key in [curses.KEY_ENTER, 10, 13]:\n self.show_job_detail(self.highlighted_row_index + (self.current_page - 1) * self.rows_per_page)\n self.draw_page(self.current_page) # Redraw the table after returning from the detail view\n elif key == ord('d'):\n # Discard current job\n job = self.fetch_job(self.highlighted_row_index + (self.current_page - 1) * self.rows_per_page)\n if job:\n self.discard_listing(job[5]) # job[5] = job_id\n self.total_entries = self.fetch_total_entries()\n self.total_pages = (self.total_entries + self.rows_per_page - 1) // self.rows_per_page\n self.draw_page(self.current_page)\n elif key == ord('s'):\n # Search functionality\n self.prompt_search()\n self.total_entries = self.fetch_total_entries()\n self.total_pages = (self.total_entries + self.rows_per_page - 1) // self.rows_per_page\n self.draw_page(self.current_page)\n elif key == ord('c'):\n # Clear search\n if self.search_term:\n self.search_term = \"\"\n self.current_page = 1\n self.highlighted_row_index = 0\n self.total_entries = self.fetch_total_entries()\n self.total_pages = (self.total_entries + self.rows_per_page - 1) // self.rows_per_page\n self.draw_page(self.current_page)\n elif key == ord('q'):\n break # Exit the table view\n elif key == ord('a'):\n # Apply to current job\n job = self.fetch_job(self.highlighted_row_index + (self.current_page - 1) * self.rows_per_page)\n if job:\n self.apply_to_listing(job[5]) # job[5] = job_id\n # Show post-apply dialog\n choice = self.show_post_apply_dialog()\n if choice == 'a':\n # Go to applications view\n apps = ApplicationsDisplay(self.stdscr, self.db_path)\n apps.draw_board()\n return\n # If 'q', just return to table view\n self.total_entries = self.fetch_total_entries()\n self.total_pages = (self.total_entries + self.rows_per_page - 1) // self.rows_per_page\n self.draw_page(self.current_page)\n\n def discard_listing(self, job_id):\n try:\n conn = sqlite3.connect(self.db_path)\n cur = conn.cursor()\n cur.execute(\"UPDATE job_listings SET discarded = 1 WHERE id = ?\", (job_id,))\n conn.commit()\n conn.close()\n self.log(f\"Discarded job {job_id}\")\n except Exception as e:\n self.log(f\"Error discarding job {job_id}: {e}\")\n\n def apply_to_listing(self, job_id):\n try:\n conn = sqlite3.connect(self.db_path)\n conn.execute(\"PRAGMA journal_mode=WAL;\")\n cur = conn.cursor()\n\n # 1) mark the listing itself as applied\n from datetime import date\n today = date.today().isoformat() # e.g. \"2025-05-14\"\n cur.execute(\"\"\"\n UPDATE job_listings\n SET applied = 1,\n applied_date = ?\n WHERE id = ?\n \"\"\", (today, job_id))\n\n # 2) upsert into applications\n cur.execute(\"SELECT id FROM applications WHERE job_id = ?\", (job_id,))\n row = cur.fetchone()\n if row:\n application_id = row[0]\n cur.execute(\"\"\"\n UPDATE applications\n SET status = 'Open',\n created_at = ?,\n updated_at = ?\n WHERE id = ?\n \"\"\", (today, today, application_id))\n else:\n cur.execute(\"\"\"\n INSERT INTO applications (job_id, status, created_at, updated_at)\n VALUES (?, 'Open', ?, ?)\n \"\"\", (job_id, today, today))\n\n conn.commit()\n conn.close()\n\n self.log(f\"Applied to job {job_id} (and created application record)\")\n except Exception as e:\n self.log(f\"Error marking job {job_id} as applied: {e}\")\n\n def show_job_detail(self, job_index):\n self.total_entries = self.fetch_total_entries() # Get total number of entries for cycling\n\n # Enter a loop to allow cycling through job details\n while True:\n job = self.fetch_job(job_index) # Fetch job details\n if not job:\n return # If no job is found, simply return\n if job:\n self.stdscr.clear()\n\n # Screen dimensions\n max_y, max_x = self.stdscr.getmaxyx()\n\n # Set maximum content width\n content_width = min(76, max_x)\n start_col = max(0, (max_x - content_width) // 2) # Calculate start position for centered text\n\n y_offset = 1 # Start from the second row for better visibility\n # Show: Company, Position, Summary, Why it's a fit, Job Description, External Link\n company_name = job[0]\n positions_json = job[1]\n try:\n positions = json.loads(positions_json) or []\n position_titles = [pos.get(\"position\") for pos in positions if isinstance(pos.get(\"position\"), str)]\n position_text = \", \".join(position_titles) if position_titles else \"Various\"\n except (json.JSONDecodeError, TypeError):\n position_text = \"Various\"\n \n details = [company_name, position_text, job[2], job[4], job[7], job[6]] # company, positions, summary, fit_justification, external_id, original_text\n headers = [\"Company\", \"Position\", \"Summary\", \"Why it's a good fit\", \"External Link\", \"Job Description\"]\n \n for idx, detail in enumerate(details):\n self.log(f'{idx} {detail}')\n header = headers[idx]\n\n # Calculate the position for left-aligned headers within the content area\n header_lines = textwrap.wrap(header, content_width)\n\n # Header with background\n self.stdscr.attron(curses.color_pair(4))\n header_start_col = max(0, start_col - 2) # Ensure we don't go negative\n header_line = f' {header_lines[0]} '\n if y_offset < max_y - 1 and header_start_col + len(header_line) < max_x:\n self.stdscr.addstr(y_offset, header_start_col, header_line)\n header_width = len(header_line)\n y_offset += 1\n self.stdscr.attroff(curses.color_pair(4))\n \n y_offset += 1\n \n # Detail text - special handling for job description to prevent overflow\n detail_text = detail if detail is not None else \"\"\n \n if header == \"External Link\":\n # Display external link with underline formatting, consistent with other sections\n detail_lines = textwrap.wrap(detail_text, content_width)\n for line in detail_lines:\n if y_offset < max_y - 3: # Check to avoid writing beyond the screen\n detail_start_col = max(start_col, (max_x - len(line)) // 2) # Center detail text\n self.stdscr.addstr(y_offset, detail_start_col, line, curses.A_UNDERLINE)\n y_offset += 1\n else:\n break\n elif header == \"Job Description\":\n # Limit job description to prevent screen overflow\n remaining_lines = max_y - y_offset - 4 # Leave space for controls\n detail_lines = textwrap.wrap(detail_text, content_width)\n \n # Show only as many lines as will fit, with truncation indicator\n if len(detail_lines) > remaining_lines:\n display_lines = detail_lines[:remaining_lines-1]\n display_lines.append(\"... [See full description on External Link above]\")\n else:\n display_lines = detail_lines\n \n for line in display_lines:\n if y_offset < max_y - 3: # Leave more space for controls\n detail_start_col = max(start_col, (max_x - len(line)) // 2) # Center detail text\n self.stdscr.addstr(y_offset, detail_start_col, line)\n y_offset += 1\n else:\n break\n else:\n # Normal processing for other sections\n detail_lines = textwrap.wrap(detail_text, content_width)\n for line in detail_lines:\n if y_offset < max_y - 3: # Check to avoid writing beyond the screen\n detail_start_col = max(start_col, (max_x - len(line)) // 2) # Center detail text\n self.stdscr.addstr(y_offset, detail_start_col, line)\n y_offset += 1\n else:\n break\n y_offset += 1 # Extra space between sections\n\n self.stdscr.refresh()\n # Ensure getch() waits for input by disabling nodelay mode\n self.stdscr.nodelay(False)\n\n while True:\n # Draw control hints at the bottom center\n controls = \"[← ] Prev [→ ] Next [q] Back [a] Apply\"\n self.stdscr.attron(curses.color_pair(7))\n control_x = max(0, (max_x - len(controls)) // 2)\n control_y = max_y - 2\n if control_y >= 0 and control_x + len(controls) < max_x:\n self.stdscr.addstr(control_y, control_x, controls)\n self.stdscr.attroff(curses.color_pair(7))\n self.stdscr.refresh()\n \n ch = self.stdscr.getch()\n if ch == ord('q'):\n return # Quit the detail view\n elif ch == curses.KEY_LEFT:\n job_index = (job_index - 1) % self.total_entries # Move to the previous job or wrap around\n break # Break the inner loop to refresh the job detail view with the new index\n elif ch == curses.KEY_RIGHT:\n job_index = (job_index + 1) % self.total_entries # Move to the next job or wrap around\n break # Break the inner loop to refresh the job detail view with the new index\n elif ch == ord('a'):\n # Apply directly from detail view\n job_id = job[5]\n self.apply_to_listing(job_id)\n # Show post-apply dialog\n choice = self.show_post_apply_dialog()\n if choice == 'a':\n # Go to applications view\n apps = ApplicationsDisplay(self.stdscr, self.db_path)\n apps.draw_board()\n return\n # If 'q', just return to detail view\n break\n\n def show_post_apply_dialog(self):\n \"\"\"\n Display a centered dialog offering [q] Keep browsing or [a] Go to applications.\n Returns 'q' or 'a'.\n \"\"\"\n max_y, max_x = self.stdscr.getmaxyx()\n text = \"[q] Keep browsing [a] Go to applications?\"\n width = len(text) + 4\n height = 3\n start_y = (max_y - height) // 2\n start_x = (max_x - width) // 2\n\n win = curses.newwin(height, width, start_y, start_x)\n win.box()\n win.attron(curses.color_pair(7))\n win.addstr(1, 2, text)\n win.attroff(curses.color_pair(7))\n win.refresh()\n\n # Immediately listen for q or a (no Enter required)\n while True:\n ch = win.getch()\n if ch in (ord('q'), ord('a')):\n break\n\n # Clear dialog and refresh underlying screen\n win.clear()\n self.stdscr.touchwin()\n self.stdscr.refresh()\n return chr(ch)" }, { "path": "src/display_applications.py", "content": "import locale\nimport curses\nimport base64\nimport sys\nimport sqlite3\nimport textwrap\nimport json\nimport datetime\n\n# make sure we’re in a UTF-8 locale so curses can handle wide chars:\nlocale.setlocale(locale.LC_ALL, '')\n\nclass ApplicationsDisplay:\n def __init__(self, stdscr, db_path):\n self.stdscr = stdscr\n self.db_path = db_path\n self.cursor = 0\n # Pane state\n self.active_pane = 'applications' # or 'notes'\n self.note_cursor = 0\n # Data\n self.applications = [] # (application_id, job_id, company, applied_date, status)\n self.notes = [] # (id, note, created_at)\n self.job_detail = None\n self.show_finalized_only = False\n\n def fetch_applications(self):\n \"\"\"\n Load self.applications = [\n (application_id, job_id, company_name, applied_date, status, last_activity), ...\n ], ordered by last_activity DESC.\n \"\"\"\n conn = sqlite3.connect(self.db_path)\n conn.execute(\"PRAGMA journal_mode=WAL;\")\n cur = conn.cursor()\n\n base_query = \"\"\"\n SELECT\n a.id AS application_id,\n a.job_id AS job_id,\n json_extract(gi.answer, '$.company_name') AS company_name,\n a.created_at AS applied_date,\n a.status AS status,\n COALESCE(\n (\n SELECT MAX(created_at)\n FROM application_notes\n WHERE application_id = a.id\n ),\n a.created_at\n ) AS last_activity\n FROM applications AS a\n JOIN gpt_interactions AS gi\n ON gi.job_id = a.job_id\n \"\"\"\n if not self.show_finalized_only:\n base_query += \" WHERE a.status = 'Open'\"\n else:\n base_query += \" WHERE a.status <> 'Open'\"\n\n base_query += \" ORDER BY last_activity DESC\"\n\n cur.execute(base_query)\n self.applications = cur.fetchall()\n conn.close()\n\n def fetch_notes(self, application_id):\n \"\"\"\n Load self.notes = [(id, note, created_at), ...] for the given application_id.\n \"\"\"\n conn = sqlite3.connect(self.db_path)\n cur = conn.cursor()\n cur.execute(\n \"SELECT id, note, created_at FROM application_notes WHERE application_id = ? ORDER BY created_at DESC\",\n (application_id,)\n )\n self.notes = cur.fetchall()\n conn.close()\n # clamp cursor\n if self.note_cursor >= len(self.notes):\n self.note_cursor = max(0, len(self.notes) - 1)\n\n def fetch_job_detail(self, job_id):\n \"\"\"\n Return a dict with positions_list, Summary, How to Apply, and Listing Link.\n \"\"\"\n conn = sqlite3.connect(self.db_path)\n conn.execute(\"PRAGMA journal_mode=WAL;\")\n cur = conn.cursor()\n cur.execute(\n \"\"\"\n SELECT\n json_extract(gi.answer, '$.available_positions'),\n json_extract(gi.answer, '$.small_summary'),\n json_extract(gi.answer, '$.how_to_apply'),\n jl.external_id\n FROM gpt_interactions gi\n JOIN job_listings jl ON gi.job_id = jl.id\n WHERE jl.id = ?\n \"\"\", (job_id,)\n )\n row = cur.fetchone()\n conn.close()\n detail = {\"positions_list\": [], \"Summary\": \"\", \"How to Apply\": \"\", \"Listing Link\": \"\"}\n if not row:\n return detail\n raw_positions, summary, apply, link = row\n try:\n detail[\"positions_list\"] = json.loads(raw_positions) or []\n except Exception:\n detail[\"positions_list\"] = []\n detail[\"Summary\"] = summary or \"\"\n detail[\"How to Apply\"] = apply or \"\"\n detail[\"Listing Link\"] = link or \"\"\n return detail\n\n def delete_note(self, note_id):\n \"\"\"\n Prompt for confirmation, delete if confirmed.\n \"\"\"\n h, w = self.stdscr.getmaxyx()\n prompt = \"Delete this note? [y/N]: \"\n self.stdscr.attron(curses.color_pair(5))\n self.stdscr.addstr(h - 3, 2, prompt)\n self.stdscr.attroff(curses.color_pair(5))\n self.stdscr.refresh()\n\n curses.echo()\n choice = self.stdscr.getstr(h - 3, 3 + len(prompt), 1).decode('utf-8').lower()\n curses.noecho()\n # clear prompt line\n self.stdscr.move(h - 3, 0)\n self.stdscr.clrtoeol()\n self.stdscr.refresh()\n\n if choice == 'y':\n conn = sqlite3.connect(self.db_path)\n cur = conn.cursor()\n cur.execute(\"DELETE FROM application_notes WHERE id = ?\", (note_id,))\n conn.commit()\n conn.close()\n\n def add_note(self, application_id, job_id):\n \"\"\"\n Multi-line note entry with visible cursor.\n Save with Ctrl-G, cancel with Ctrl-D.\n \"\"\"\n # ─── show cursor ────────────────────────────────────────────\n curses.curs_set(1)\n\n # ─── compute box dims ───────────────────────────────────────\n h, w = self.stdscr.getmaxyx()\n box_h, box_w = h - 6, w - 8\n start_y, start_x = 3, 4\n\n # ─── draw outer border & title ─────────────────────────────\n win = curses.newwin(box_h, box_w, start_y, start_x)\n win.keypad(True)\n win.box()\n win.addstr(0, 2, \" Enter note ([Ctrl-G] Save / [Ctrl-D] Cancel) \")\n win.refresh()\n\n # ─── create a pad big enough for huge pastes ───────────────\n pad_h = max(10000, box_h * 20)\n # allow up to 2048 columns before running out of space\n pad_w = max(2048, box_w - 2)\n pad = curses.newpad(pad_h, pad_w)\n pad.scrollok(True)\n pad.idlok(True)\n pad.keypad(True)\n\n pad_row = 0\n cur_y, cur_x = 0, 0\n saved = False\n\n # ─── edit loop ─────────────────────────────────────────────\n while True:\n # redraw border & title\n win.box()\n win.addstr(0, 2, \" Enter note ([Ctrl-G] Save / [Ctrl-D] Cancel) \")\n win.refresh()\n\n # show the pad slice\n pad.refresh(\n pad_row, 0,\n start_y + 1, start_x + 1,\n start_y + box_h - 2, start_x + box_w - 2\n )\n\n # use get_wch to receive wide characters properly:\n try:\n ch = pad.get_wch()\n except curses.error:\n continue\n\n # Ctrl-G → save, Ctrl-D → cancel\n if ch == '\\x07':\n saved = True\n break\n if ch == '\\x04':\n saved = False\n break\n\n # ── printable wide‐char (string) ─────────────────────────\n if isinstance(ch, str):\n if ch == '\\n':\n cur_y += 1\n cur_x = 0\n else:\n try:\n pad.addstr(cur_y, cur_x, ch)\n except curses.error:\n pass\n cur_x += 1\n else:\n # it's an integer key‐code: arrows, backspace, etc.\n if ch == curses.KEY_UP and pad_row > 0:\n pad_row -= 1\n elif ch == curses.KEY_DOWN and cur_y - pad_row >= box_h - 2:\n pad_row += 1\n elif ch == curses.KEY_LEFT and cur_x > 0:\n cur_x -= 1\n elif ch == curses.KEY_RIGHT:\n cur_x += 1\n elif ch in (curses.KEY_BACKSPACE, 127):\n if cur_x > 0:\n cur_x -= 1\n try: pad.delch(cur_y, cur_x)\n except curses.error: pass\n elif cur_y > 0:\n cur_y -= 1\n line = pad.instr(cur_y, 0, pad_w).decode('utf-8').rstrip('\\x00')\n cur_x = len(line)\n # ignore other int‐codes\n\n # ─── auto-scroll vertically ───────────────────────────────\n if cur_y - pad_row > box_h - 3:\n pad_row = cur_y - (box_h - 3)\n elif cur_y < pad_row:\n pad_row = cur_y\n pad_row = max(0, min(pad_row, pad_h - (box_h - 2)))\n\n # ─── move the hardware cursor ─────────────────────────────\n real_y = start_y + 1 + (cur_y - pad_row)\n real_x = start_x + 1 + cur_x\n curses.setsyx(real_y, real_x)\n curses.doupdate()\n\n # ─── hide cursor & clear any leftover input ────────────────\n curses.curs_set(0)\n # try the built-in flush…\n curses.flushinp()\n # …and as a fallback, nodelay‐drain stdscr\n self.stdscr.nodelay(True)\n while True:\n if self.stdscr.getch() == curses.ERR:\n break\n self.stdscr.nodelay(False)\n\n if not saved:\n return # cancelled\n\n # ─── grab all lines from the pad ───────────────────────────\n lines = []\n for y in range(cur_y + 1):\n raw = pad.instr(y, 0, pad_w).decode('utf-8').rstrip('\\x00')\n lines.append(raw.rstrip())\n note_text = \"\\n\".join(lines).strip()\n if not note_text:\n return\n\n # ─── persist to DB ────────────────────────────────────────\n conn = sqlite3.connect(self.db_path)\n conn.execute(\"PRAGMA journal_mode=WAL;\")\n cur = conn.cursor()\n\n if application_id is None:\n now = datetime.datetime.utcnow().isoformat()\n cur.execute(\n \"INSERT INTO applications (job_id, status, created_at, updated_at) \"\n \"VALUES (?, 'Open', ?, ?)\",\n (job_id, now, now),\n )\n application_id = cur.lastrowid\n\n cur.execute(\n \"INSERT INTO application_notes (application_id, note) VALUES (?, ?)\",\n (application_id, note_text)\n )\n conn.commit()\n conn.close()\n\n def view_note(self, note_text):\n \"\"\"\n Display a read-only note with a one-cell margin inside the box.\n Ctrl-K copies to clipboard.\n \"\"\"\n full_text = note_text\n curses.curs_set(0)\n h, w = self.stdscr.getmaxyx()\n box_h, box_w = h - 6, w - 8\n start_y, start_x = 3, 4\n\n # outer frame\n win = curses.newwin(box_h, box_w, start_y, start_x)\n win.keypad(True)\n\n # —— prepare wrapped lines using the *inner* width (box_w - 4) ——\n inner_w = box_w - 4 # leave one‐cell on left + right\n lines = []\n for paragraph in note_text.split('\\n'):\n wrapped = textwrap.wrap(paragraph, inner_w)\n lines.extend(wrapped if wrapped else [''])\n pad_h = max(len(lines), box_h - 4)\n pad = curses.newpad(pad_h, inner_w)\n\n for idx, line in enumerate(lines):\n try:\n pad.addnstr(idx, 0, line, inner_w)\n except curses.error:\n pass\n\n pad_pos = 0\n title = \" View note: [↑↓] Scroll | [q/Esc] Close | [k] Copy to clipboard \"\n\n # compute the *inner* viewport coordinates\n top = start_y + 1\n left = start_x + 1\n bottom = start_y + box_h - 2\n right = start_x + box_w - 2\n\n while True:\n # redraw frame and title\n win.erase()\n win.box()\n win.addstr(0, 2, title)\n win.refresh()\n\n # refresh the pad inside the 1-cell margin\n pad.refresh(pad_pos, 0,\n top + 1, # shift down one for margin\n left + 1, # shift right one for margin\n bottom - 1, # shift up one for margin\n right - 1) # shift left one for margin\n\n ch = win.getch()\n if ch in (ord('q'), 27):\n break\n elif ch == curses.KEY_UP and pad_pos > 0:\n pad_pos -= 1\n elif ch == curses.KEY_DOWN and pad_pos < len(lines) - (box_h - 4):\n pad_pos += 1\n\n elif ch == ord('k'): # lowercase “k” to copy\n try:\n # 1) base64-encode the full text\n b64 = base64.b64encode(full_text.encode('utf-8')).decode('ascii')\n # 2) build OSC52 sequence (c = clipboard)\n seq = f\"\\033]52;c;{b64}\\a\"\n\n # 3) temporarily end curses so we can write raw escapes\n curses.def_prog_mode()\n curses.endwin()\n\n # 4) send the sequence to the terminal\n sys.stdout.write(seq)\n sys.stdout.flush()\n\n # 5) resume curses\n curses.reset_prog_mode()\n curses.doupdate()\n curses.curs_set(0)\n\n # 6) flash confirmation on the last interior line\n h, w = self.stdscr.getmaxyx()\n prompt_row = h - 3\n msg = \"Copied note to clipboard\"\n\n self.stdscr.attron(curses.color_pair(5))\n self.stdscr.addstr(prompt_row, 5, msg)\n self.stdscr.attroff(curses.color_pair(5))\n self.stdscr.refresh()\n\n curses.napms(1000)\n\n # clear that prompt line\n self.stdscr.move(prompt_row, 0)\n self.stdscr.clrtoeol()\n self.stdscr.refresh()\n except Exception:\n # (if something really weird happens)\n msg = \"Copy failed\"\n win.addstr(box_h - 2, 2, msg, curses.A_BOLD)\n win.refresh()\n curses.napms(1000)\n win.addstr(box_h - 2, 2, \" \" * len(msg))\n win.refresh()\n\n # when we break out, draw_board will redraw the main screen\n\n def finalize(self, application_id, job_id):\n # unchanged\n curses.echo()\n prompt_row = curses.LINES - 4\n self.stdscr.attron(curses.color_pair(5))\n prompt_txt = \" 👉 Finalize reason ([h] Hired / [r] Rejected / [a] Abandoned / [k] Keep Open):\"\n self.stdscr.addstr(prompt_row, 0, prompt_txt)\n self.stdscr.attroff(curses.color_pair(5))\n choice = self.stdscr.getkey().lower()\n curses.noecho()\n mapping = {'h': 'Hired', 'r': 'Rejected', 'a': 'Abandoned'}\n if choice not in mapping:\n return\n status = mapping[choice]\n now = datetime.datetime.now().isoformat(sep=' ', timespec='seconds')\n conn = sqlite3.connect(self.db_path)\n conn.execute(\"PRAGMA journal_mode=WAL;\")\n cur = conn.cursor()\n if application_id is not None:\n cur.execute(\"UPDATE applications SET status=?,updated_at=? WHERE id=?\", (status, now, application_id))\n else:\n cur.execute(\n \"INSERT INTO applications (job_id,status,created_at,updated_at) VALUES (?,?,?,?)\",\n (job_id, status, now, now)\n )\n application_id = cur.lastrowid\n cur.execute(\n \"INSERT INTO application_notes (application_id,note) VALUES (?,?)\",\n (application_id, f\"FINALIZED: {status}\")\n )\n conn.commit()\n conn.close()\n # Refresh applications list and adjust cursor\n self.fetch_applications()\n if self.cursor >= len(self.applications) and len(self.applications) > 0:\n self.cursor = len(self.applications) - 1\n elif len(self.applications) == 0:\n self.cursor = 0\n\n def draw_board(self):\n while True:\n self.stdscr.clear()\n h, w = self.stdscr.getmaxyx()\n left_w, mid_w = w // 4, w // 3\n right_w = w - left_w - mid_w - 4\n # Header\n titles = [\"Company\".center(left_w), \"Notes\".center(mid_w), \"Details\".center(right_w)]\n header_line = \" \".join(titles)\n self.stdscr.attron(curses.color_pair(4))\n self.stdscr.addstr(0, 0, header_line[:w])\n self.stdscr.attroff(curses.color_pair(4))\n base_y = 2\n # Load data\n self.fetch_applications()\n # Left pane\n for idx, (app_id, job_id, company, adate, status, last_activity) in enumerate(self.applications):\n y = base_y + idx\n label = f\" {company} ({adate.split(' ')[0]}) [{status[0]}] \"\n attr = curses.A_REVERSE if self.active_pane == 'applications' and idx == self.cursor else curses.A_NORMAL\n self.stdscr.addnstr(y, 0, label, left_w - 1, attr)\n # Middle pane (Notes)\n if self.applications and self.cursor < len(self.applications):\n application_id, job_id, *_ = self.applications[self.cursor]\n self.fetch_notes(application_id)\n note_y = base_y\n for idx, (nid, note, ts) in enumerate(self.notes):\n display = f\"{ts.split(' ')[0]}: {note.replace('\\n',' ')[:mid_w-6]}\"\n y = note_y + idx\n if y < h - 4:\n attr = curses.A_REVERSE if self.active_pane == 'notes' and idx == self.note_cursor else curses.A_NORMAL\n self.stdscr.addnstr(y, left_w + 2, display, mid_w - 2, attr)\n hint = \"[n] add note\"\n if self.active_pane == 'notes':\n hint += \" [d] delete note [Enter] view note\"\n self.stdscr.addstr(min(h - 5, note_y + len(self.notes) + 1), left_w + 2, hint, curses.A_DIM)\n # Right pane (Details)\n x0, y0 = left_w + mid_w + 4, base_y\n detail = self.fetch_job_detail(job_id)\n # Available Positions\n self.stdscr.addstr(y0, x0, \"Available Positions:\", curses.A_BOLD)\n y0 += 1\n for p in detail[\"positions_list\"]:\n for wrapped in textwrap.wrap(f\"{p.get('position','')} — {p.get('link','')}\", right_w - 1):\n if y0 < h - 4:\n self.stdscr.addstr(y0, x0, wrapped)\n y0 += 1\n y0 += 1\n # Summary\n self.stdscr.addstr(y0, x0, \"Summary:\", curses.A_BOLD)\n y0 += 1\n for wrapped in textwrap.wrap(detail[\"Summary\"], right_w - 1):\n if y0 < h - 4:\n self.stdscr.addstr(y0, x0, wrapped)\n y0 += 1\n y0 += 1\n # How to Apply\n self.stdscr.addstr(y0, x0, \"How to Apply:\", curses.A_BOLD)\n y0 += 1\n for wrapped in textwrap.wrap(detail[\"How to Apply\"], right_w - 1):\n if y0 < h - 4:\n self.stdscr.addstr(y0, x0, wrapped)\n y0 += 1\n y0 += 1\n # Listing Link\n self.stdscr.addstr(y0, x0, \"Listing Link:\", curses.A_BOLD)\n y0 += 1\n for wrapped in textwrap.wrap(detail[\"Listing Link\"], right_w - 1):\n if y0 < h - 4:\n self.stdscr.addstr(y0, x0, wrapped)\n y0 += 1\n # Help line\n help_txt = \"[←→ ] Switch pane [↑↓] Move [space] Toggle Finalized [n] Note [f] Finalize [q] Back\"\n sx = max(0, (w - len(help_txt)) // 2)\n self.stdscr.attron(curses.color_pair(7))\n self.stdscr.addnstr(h - 2, sx, help_txt, len(help_txt))\n self.stdscr.attroff(curses.color_pair(7))\n self.stdscr.refresh()\n # Key handling\n c = self.stdscr.getch()\n # Pane switching\n if c == curses.KEY_RIGHT and self.applications:\n self.active_pane = 'notes' if self.active_pane == 'applications' else 'applications'\n elif c == curses.KEY_LEFT:\n self.active_pane = 'applications'\n # Within applications pane\n elif self.active_pane == 'applications':\n if c == curses.KEY_UP and self.cursor > 0:\n self.cursor -= 1\n self.note_cursor = 0\n elif c == curses.KEY_DOWN and self.cursor < len(self.applications) - 1:\n self.cursor += 1\n self.note_cursor = 0\n elif c == ord(' '):\n self.show_finalized_only = not self.show_finalized_only\n self.cursor = 0\n elif c == ord('n'):\n self.add_note(*self.applications[self.cursor][:2])\n elif c == ord('f'):\n self.finalize(*self.applications[self.cursor][:2])\n elif c in (ord('q'), 27):\n break\n # Within notes pane\n elif self.active_pane == 'notes':\n if c == curses.KEY_UP and self.note_cursor > 0:\n self.note_cursor -= 1\n elif c == curses.KEY_DOWN and self.note_cursor < len(self.notes) - 1:\n self.note_cursor += 1\n elif c == ord('d') and self.notes:\n nid = self.notes[self.note_cursor][0]; self.delete_note(nid)\n elif c in (ord('\\n'), curses.KEY_ENTER) and self.notes:\n note_text = self.notes[self.note_cursor][1]\n self.view_note(note_text)\n elif c == ord('n'):\n self.add_note(*self.applications[self.cursor][:2])\n elif c == ord('f'):\n self.finalize(*self.applications[self.cursor][:2])\n elif c in (ord('q'), 27):\n self.active_pane = 'applications'\n" }, { "path": "src/display_matching_table.py", "content": "import locale\nimport sqlite3\nimport curses\nimport textwrap\nimport logging\nimport json\nfrom datetime import date, datetime\nfrom display_applications import ApplicationsDisplay\n\nlocale.setlocale(locale.LC_ALL, '')\n\nclass MatchingTableDisplay:\n def __init__(self, stdscr, db_path):\n self.stdscr = stdscr\n self.db_path = db_path\n self.highlighted_row_index = 0\n self.current_page = 1\n self.total_pages = 0\n self.rows_per_page = 3\n self.search_term = \"\"\n logging.basicConfig(filename='matching_table_display.log', level=logging.DEBUG)\n \n self.good_match_filters = '''\n json_valid(gi.answer) = 1\n AND json_extract(gi.answer, '$.fit_for_resume') = 'Yes'\n AND json_extract(gi.answer, '$.remote_positions') = 'Yes'\n AND json_extract(gi.answer, '$.hiring_in_us') <> 'No'\n AND (jl.discarded IS NULL OR jl.discarded = 0)\n AND (jl.applied IS NULL OR jl.applied = 0)\n '''\n\n def log(self, message):\n \"\"\"Log a message for debugging.\"\"\"\n logging.debug(message)\n \n def format_scraped_date(self, scraped_at):\n \"\"\"Format scraped_at timestamp for display.\"\"\"\n try:\n if scraped_at:\n # Parse the ISO timestamp and format for display\n dt = datetime.fromisoformat(scraped_at)\n return dt.strftime(\"%Y-%m-%d\")\n return \"Unknown\"\n except (ValueError, TypeError):\n return \"Unknown\"\n \n def get_search_filters(self):\n \"\"\"Build additional WHERE conditions for search filtering.\"\"\"\n if not self.search_term:\n return \"\"\n \n # Search in company name, summary, job description, and available positions\n search_conditions = [\n f\"lower(json_extract(gi.answer, '$.company_name')) LIKE '%{self.search_term.lower()}%'\",\n f\"lower(json_extract(gi.answer, '$.small_summary')) LIKE '%{self.search_term.lower()}%'\", \n f\"lower(jl.original_text) LIKE '%{self.search_term.lower()}%'\",\n f\"lower(json_extract(gi.answer, '$.available_positions')) LIKE '%{self.search_term.lower()}%'\"\n ]\n \n return \" AND (\" + \" OR \".join(search_conditions) + \")\"\n \n def prompt_search(self):\n \"\"\"Prompt user for search term and update search filters.\"\"\"\n max_y, max_x = self.stdscr.getmaxyx()\n \n # Create input window\n input_win = curses.newwin(3, max_x - 4, max_y - 5, 2)\n input_win.box()\n input_win.addstr(1, 2, f\"Search (current: '{self.search_term}'): \")\n input_win.refresh()\n \n # Enable echo and get input\n curses.echo()\n curses.curs_set(1) # Show cursor\n \n # Get user input\n try:\n search_input = input_win.getstr(1, len(f\"Search (current: '{self.search_term}'): \") + 2, 50).decode('utf-8')\n self.search_term = search_input.strip()\n except:\n pass # Handle any input errors\n finally:\n curses.noecho()\n curses.curs_set(0) # Hide cursor\n \n # Reset pagination\n self.current_page = 1\n self.highlighted_row_index = 0\n \n # Clear the input window\n input_win.clear()\n input_win.refresh()\n del input_win\n\n def fetch_total_entries(self):\n try:\n conn = sqlite3.connect(self.db_path)\n cur = conn.cursor()\n search_filters = self.get_search_filters()\n cur.execute(f\"\"\"\n SELECT COUNT(gi.job_id)\n FROM gpt_interactions gi\n JOIN job_listings jl ON gi.job_id = jl.id\n WHERE {self.good_match_filters}{search_filters}\n \"\"\")\n total_entries = cur.fetchone()[0]\n conn.close()\n return total_entries\n except (sqlite3.OperationalError, sqlite3.DatabaseError):\n return 0\n\n def fetch_job(self, offset=None):\n if offset is None:\n offset = (self.current_page - 1) * self.rows_per_page + self.highlighted_row_index\n try:\n conn = sqlite3.connect(self.db_path)\n cur = conn.cursor()\n query = f\"\"\"\n SELECT\n json_extract(gi.answer, '$.company_name') AS company_name,\n json_extract(gi.answer, '$.available_positions') AS available_positions,\n json_extract(gi.answer, '$.small_summary') AS summary,\n json_extract(gi.answer, '$.fit_for_resume') AS fit_for_resume,\n json_extract(gi.answer, '$.fit_justification') AS fit_justification,\n json_extract(gi.answer, '$.how_to_apply') AS how_to_apply,\n json_extract(gi.answer, '$.remote_positions') AS remote_positions,\n json_extract(gi.answer, '$.hiring_in_us') AS hiring_in_us,\n gi.job_id,\n jl.original_text,\n jl.external_id,\n jl.scraped_at\n FROM\n gpt_interactions gi\n JOIN\n job_listings jl ON gi.job_id = jl.id\n WHERE\n {self.good_match_filters}{self.get_search_filters()}\n ORDER BY jl.scraped_at DESC, jl.id DESC\n LIMIT 1 OFFSET {offset}\n \"\"\"\n self.log(f\"Executing query: {query}\") # Log the query\n cur.execute(query)\n data = cur.fetchone()\n self.log(f\"Fetched {len(data)} rows\") # Log the number of results\n conn.close()\n return data\n except (sqlite3.OperationalError, sqlite3.DatabaseError):\n return None\n\n def fetch_data(self, page_num):\n offset = (page_num - 1) * self.rows_per_page\n try:\n conn = sqlite3.connect(self.db_path)\n cur = conn.cursor()\n query = f\"\"\"\n SELECT\n json_extract(gi.answer, '$.company_name') AS company_name,\n json_extract(gi.answer, '$.available_positions') AS available_positions,\n json_extract(gi.answer, '$.small_summary') AS summary,\n json_extract(gi.answer, '$.fit_for_resume') AS fit_for_resume,\n json_extract(gi.answer, '$.fit_justification') AS fit_justification,\n json_extract(gi.answer, '$.how_to_apply') AS how_to_apply,\n json_extract(gi.answer, '$.remote_positions') AS remote_positions,\n json_extract(gi.answer, '$.hiring_in_us') AS hiring_in_us,\n gi.job_id,\n jl.original_text,\n jl.scraped_at\n FROM\n gpt_interactions gi\n JOIN\n job_listings jl ON gi.job_id = jl.id\n WHERE\n {self.good_match_filters}{self.get_search_filters()}\n ORDER BY jl.scraped_at DESC, jl.id DESC\n LIMIT {self.rows_per_page} OFFSET {offset}\n \"\"\"\n self.log(f\"Executing query: {query}\") # Log the query\n cur.execute(query)\n data = cur.fetchall()\n self.log(f\"Fetched {len(data)} rows\") # Log the number of results\n conn.close()\n return data\n except (sqlite3.OperationalError, sqlite3.DatabaseError):\n return None\n\n\n def draw_page(self, current_page):\n max_y, max_x = self.stdscr.getmaxyx()\n data = self.fetch_data(page_num=current_page)\n\n # Adjusted column widths\n column_widths = {\n \"Company\": 15,\n \"Position\": 20, # Assign 1/4 screen width to Position for JSON data\n \"Summary\": 40, # Summary could be long, so assign 1/4 screen width\n \"Good Fit?\": 10,\n \"Why?\": 30,\n \"How to Apply?\": 20\n }\n\n self.stdscr.clear()\n header = \" \".join(title.center(column_widths[title]) for title in column_widths.keys())\n self.stdscr.attron(curses.color_pair(4))\n self.stdscr.addstr(0, 0, header)\n self.stdscr.attroff(curses.color_pair(4))\n\n y_offset = 2 # Start below the header\n\n for idx, listing in enumerate(data):\n if idx == self.highlighted_row_index:\n self.stdscr.attron(curses.color_pair(3))\n\n max_height_wrapped_text = 1\n for i, key in enumerate(column_widths.keys()):\n field = listing[i]\n width = column_widths[key]\n\n # Parse JSON for the 'Position' column and extract position titles\n if key == \"Position\":\n try:\n positions = json.loads(field) or []\n # keep only those with a real string for \"position\"\n titles = [pos.get(\"position\") for pos in positions\n if isinstance(pos.get(\"position\"), str)]\n field = \", \".join(titles) if titles else \"\"\n except (json.JSONDecodeError, TypeError):\n # JSON was bad, or field was None\n field = \"Invalid data\"\n \n # For the 'Company' column, add scraped date underneath\n if key == \"Company\":\n # listing has scraped_at as the last field (index 10 in fetch_data, index 11 in fetch_job)\n scraped_at = listing[10] if len(listing) > 10 else None\n formatted_date = self.format_scraped_date(scraped_at)\n field = f\"{field}\\n({formatted_date})\"\n \n # This part takes a field content and wraps it in width\n # then it loops through it line by line, and \n wrapped_text = textwrap.wrap(str(field), width=width)\n for j, line in enumerate(wrapped_text):\n line_pos = sum(column_widths[title] for title in list(column_widths.keys())[:i]) + i * 3\n # if line_pos + len(line) < max_x and y_offset + j < max_y:\n if line_pos + width <= max_x and y_offset + j < max_y - 1:\n self.stdscr.addstr(y_offset + j, line_pos, line.ljust(width))\n \n if j > max_height_wrapped_text:\n max_height_wrapped_text = j\n \n y_offset += max_height_wrapped_text + 2\n\n if y_offset >= max_y - 3: # Check if we've reached the end of the screen\n break # Stop drawing if there's no more space on the screen\n\n if idx == self.highlighted_row_index:\n self.stdscr.attroff(curses.color_pair(3))\n\n # Pagination info (this section is duplicated below and can be removed)\n # pagination_info = f\"Page {self.current_page} of {self.total_pages} ({self.total_entries} great matches for your resume 😁)\"\n # self.stdscr.attron(curses.color_pair(5))\n # self.stdscr.addstr(max_y - 2, 0, pagination_info)\n # self.stdscr.attroff(curses.color_pair(5))\n\n # --- new controls hint bar ---\n # --- footer line: pagination + controls ---\n footer_y = max_y - 2\n\n # 1) Draw pagination (flush-left)\n search_status = f\" (filtered: '{self.search_term}')\" if self.search_term else \"\"\n pagination = f\"Page {self.current_page} of {self.total_pages} ({self.total_entries} great matches{search_status} 😁)\"\n self.stdscr.attron(curses.color_pair(5))\n self.stdscr.addstr(footer_y, 0, pagination.ljust(max_x))\n self.stdscr.attroff(curses.color_pair(5))\n\n # 2) Prepare controls text\n controls_text = \"[↑↓] Move [←→ ] Page [Enter] View [d] Discard [a] Apply [s] Search [c] Clear [q] Back\"\n\n # 3) Clear the next line so no overlap\n self.stdscr.move(footer_y + 1, 0)\n self.stdscr.clrtoeol()\n\n # 4) Draw controls (same left alignment)\n self.stdscr.attron(curses.color_pair(7))\n self.stdscr.addstr(footer_y + 1, 0, controls_text[: max_x - 1])\n self.stdscr.attroff(curses.color_pair(7))\n\n self.stdscr.refresh()\n\n\n def draw_table(self):\n self.total_entries = self.fetch_total_entries()\n self.total_pages = (self.total_entries + self.rows_per_page - 1) // self.rows_per_page\n\n self.draw_page(self.current_page)\n\n while True:\n key = self.stdscr.getch()\n if key == curses.KEY_DOWN:\n self.highlighted_row_index = min(self.highlighted_row_index + 1, self.rows_per_page - 1)\n self.draw_page(self.current_page)\n elif key == curses.KEY_UP:\n self.highlighted_row_index = max(0, self.highlighted_row_index - 1)\n self.draw_page(self.current_page)\n elif key == curses.KEY_RIGHT:\n if self.current_page < self.total_pages:\n self.current_page += 1\n self.highlighted_row_index = 0 # Reset highlighted row for the new page\n self.draw_page(self.current_page)\n elif key == curses.KEY_LEFT:\n if self.current_page > 1:\n self.current_page -= 1\n self.highlighted_row_index = 0 # Reset highlighted row for the new page\n self.draw_page(self.current_page)\n elif key in [curses.KEY_ENTER, 10, 13]:\n self.show_job_detail(self.highlighted_row_index + (self.current_page - 1) * self.rows_per_page)\n self.draw_page(self.current_page) # Redraw the table after returning from the detail view\n elif key == ord('d'):\n # Discard current job\n job = self.fetch_job(self.highlighted_row_index + (self.current_page - 1) * self.rows_per_page)\n if job:\n self.discard_listing(job[8]) # job[8] = job_id\n self.total_entries = self.fetch_total_entries()\n self.total_pages = (self.total_entries + self.rows_per_page - 1) // self.rows_per_page\n # self.highlighted_row_index = 0\n self.draw_page(self.current_page)\n elif key == ord('s'):\n # Search functionality\n self.prompt_search()\n self.total_entries = self.fetch_total_entries()\n self.total_pages = (self.total_entries + self.rows_per_page - 1) // self.rows_per_page\n self.draw_page(self.current_page)\n elif key == ord('c'):\n # Clear search\n if self.search_term:\n self.search_term = \"\"\n self.current_page = 1\n self.highlighted_row_index = 0\n self.total_entries = self.fetch_total_entries()\n self.total_pages = (self.total_entries + self.rows_per_page - 1) // self.rows_per_page\n self.draw_page(self.current_page)\n elif key == ord('q'):\n # Clear search when exiting to avoid confusing menu display\n self.search_term = \"\"\n break # Exit the table view\n elif key == ord('a'):\n # Apply to current job\n job = self.fetch_job(self.highlighted_row_index + (self.current_page - 1) * self.rows_per_page)\n if job:\n self.apply_to_listing(job[8]) # job[8] = job_id\n # Show post-apply dialog\n choice = self.show_post_apply_dialog()\n if choice == 'a':\n # Clear search when navigating to applications\n self.search_term = \"\"\n # Go to applications view\n apps = ApplicationsDisplay(self.stdscr, self.db_path)\n apps.draw_board()\n return\n # If 'q', just return to table view\n self.total_entries = self.fetch_total_entries()\n self.total_pages = (self.total_entries + self.rows_per_page - 1) // self.rows_per_page\n # self.highlighted_row_index = 0\n self.draw_page(self.current_page)\n\n def discard_listing(self, job_id):\n try:\n conn = sqlite3.connect(self.db_path)\n cur = conn.cursor()\n cur.execute(\"UPDATE job_listings SET discarded = 1 WHERE id = ?\", (job_id,))\n conn.commit()\n conn.close()\n self.log(f\"Discarded job {job_id}\")\n except Exception as e:\n self.log(f\"Error discarding job {job_id}: {e}\")\n\n def apply_to_listing(self, job_id):\n try:\n conn = sqlite3.connect(self.db_path)\n conn.execute(\"PRAGMA journal_mode=WAL;\")\n cur = conn.cursor()\n\n # 1) mark the listing itself as applied\n today = date.today().isoformat() # e.g. \"2025-05-14\"\n cur.execute(\"\"\"\n UPDATE job_listings\n SET applied = 1,\n applied_date = ?\n WHERE id = ?\n \"\"\", (today, job_id))\n\n # 2) upsert into applications\n # - if an application already exists, just refresh its timestamps/status\n # - otherwise insert a new one\n cur.execute(\"SELECT id FROM applications WHERE job_id = ?\", (job_id,))\n row = cur.fetchone()\n if row:\n application_id = row[0]\n cur.execute(\"\"\"\n UPDATE applications\n SET status = 'Open',\n created_at = ?, -- in case you want created_at to match apply date\n updated_at = ?\n WHERE id = ?\n \"\"\", (today, today, application_id))\n else:\n cur.execute(\"\"\"\n INSERT INTO applications (job_id, status, created_at, updated_at)\n VALUES (?, 'Open', ?, ?)\n \"\"\", (job_id, today, today))\n\n conn.commit()\n conn.close()\n\n self.log(f\"Applied to job {job_id} (and created application record)\")\n except Exception as e:\n self.log(f\"Error marking job {job_id} as applied: {e}\")\n\n\n def show_job_detail(self, job_index):\n self.total_entries = self.fetch_total_entries() # Get total number of entries for cycling\n\n # Enter a loop to allow cycling through job details\n while True:\n job = self.fetch_job(job_index) # Fetch job details\n if not job:\n return # If no job is found, simply return\n if job:\n self.stdscr.clear()\n\n # Screen dimensions\n max_y, max_x = self.stdscr.getmaxyx()\n\n # Set maximum content width\n content_width = min(76, max_x)\n start_col = max(0, (max_x - content_width) // 2) # Calculate start position for centered text\n\n y_offset = 1 # Start from the second row for better visibility\n for idx, detail in enumerate([job[0], job[1], job[4], job[5], job[9]]):\n self.log(f'{idx} {detail}')\n header = [\"Company\", \"Position\", \"Why it's a good fit\", \"How to Apply\", \"Job Description\"][idx]\n \n if header == \"Position\":\n try:\n # if detail is None or \"null\", coerce to empty list\n positions = json.loads(detail) or []\n # only keep real strings\n titles = [\n p.get(\"position\")\n for p in positions\n if isinstance(p.get(\"position\"), str)\n ]\n detail = \", \".join(titles)\n except (json.JSONDecodeError, TypeError):\n # fall back to blank if we can’t parse\n detail = \"\"\n\n # Calculate the position for left-aligned headers within the content area\n header_lines = textwrap.wrap(header, content_width)\n\n # Header with background\n self.stdscr.attron(curses.color_pair(4))\n header_start_col = start_col # Align left within the content width\n header_line = f' {header_lines[0]} '\n self.stdscr.addstr(y_offset, header_start_col - 2, header_line)\n header_width = len(header_line)\n y_offset += 1\n self.stdscr.attroff(curses.color_pair(4))\n if header == \"Job Description\":\n y_offset -= 1\n link_text = job[10] if job[10] is not None else \"\"\n link_lines = textwrap.wrap(link_text, content_width)\n for idx, line in enumerate(link_lines):\n start_on = start_col\n if idx == 0:\n start_on += header_width + 1\n # Underline the text of the link\n self.stdscr.addstr(y_offset, start_on, line, curses.A_UNDERLINE)\n y_offset += 1\n y_offset += 1\n \n # Detail text\n # avoid passing None to wrap()\n detail_text = detail if detail is not None else \"\"\n detail_lines = textwrap.wrap(detail_text, content_width)\n for line in detail_lines:\n if y_offset < max_y - 1: # Check to avoid writing beyond the screen\n detail_start_col = max(start_col, (max_x - len(line)) // 2) # Center detail text\n self.stdscr.addstr(y_offset, detail_start_col, line)\n y_offset += 1\n y_offset += 1 # Extra space between sections\n\n self.stdscr.refresh()\n # Ensure getch() waits for input by disabling nodelay mode\n self.stdscr.nodelay(False)\n\n while True:\n # Draw control hints at the bottom center\n controls = \"[← ] Prev [→ ] Next [q] Back [a] Apply\"\n self.stdscr.attron(curses.color_pair(7))\n self.stdscr.addstr(max_y - 2,\n max(0, (max_x - len(controls)) // 2),\n controls)\n self.stdscr.attroff(curses.color_pair(7))\n self.stdscr.refresh()\n \n ch = self.stdscr.getch()\n if ch == ord('q'):\n # Clear search when exiting detail view to avoid confusing menu display\n self.search_term = \"\"\n return # Quit the detail view\n elif ch == curses.KEY_LEFT:\n job_index = (job_index - 1) % self.total_entries # Move to the previous job or wrap around\n break # Break the inner loop to refresh the job detail view with the new index\n elif ch == curses.KEY_RIGHT:\n job_index = (job_index + 1) % self.total_entries # Move to the next job or wrap around\n break # Break the inner loop to refresh the job detail view with the new index\n elif ch == ord('a'):\n # Apply directly from detail view\n job_id = job[8] # adjust index if needed\n self.apply_to_listing(job_id)\n # Show post-apply dialog\n choice = self.show_post_apply_dialog()\n if choice == 'a':\n # Clear search when navigating to applications\n self.search_term = \"\"\n # Go to applications view\n apps = ApplicationsDisplay(self.stdscr, self.db_path)\n apps.draw_board()\n return\n # If 'q', just return to detail view\n break\n\n def show_post_apply_dialog(self):\n \"\"\"\n Display a centered dialog offering [q] Keep browsing or [a] Go to applications.\n Returns 'q' or 'a'.\n \"\"\"\n max_y, max_x = self.stdscr.getmaxyx()\n text = \"[q] Keep browsing [a] Go to applications?\"\n width = len(text) + 4\n height = 3\n start_y = (max_y - height) // 2\n start_x = (max_x - width) // 2\n\n win = curses.newwin(height, width, start_y, start_x)\n win.box()\n win.attron(curses.color_pair(7))\n win.addstr(1, 2, text)\n win.attroff(curses.color_pair(7))\n win.refresh()\n\n # Immediately listen for q or a (no Enter required)\n while True:\n ch = win.getch()\n if ch in (ord('q'), ord('a')):\n break\n\n # Clear dialog and refresh underlying screen\n win.clear()\n self.stdscr.touchwin()\n self.stdscr.refresh()\n return chr(ch)" }, { "path": "src/display_table.py", "content": "# display_table.py\nimport sqlite3\nimport curses\nimport textwrap\n\ndef fetch_data(db_path):\n try:\n conn = sqlite3.connect(db_path)\n cur = conn.cursor()\n cur.execute(\"SELECT original_text, external_id FROM job_listings LIMIT 5\")\n data = cur.fetchall()\n conn.close()\n return data\n except (sqlite3.OperationalError, sqlite3.DatabaseError):\n return None\n\ndef draw_table(stdscr, db_path):\n curses.init_pair(3, curses.COLOR_WHITE, curses.COLOR_BLUE) # Highlight color\n data = fetch_data(db_path)\n max_y, max_x = stdscr.getmaxyx()\n max_table_width = min(120, max_x - 4) # Adjusted for padding and separators\n text_col_width = 78 # Adjusted for spacing between cells\n source_col_width = 18 # Adjusted for spacing\n\n if not data:\n stdscr.addstr(0, 0, \"No data found or database is missing.\")\n stdscr.refresh()\n stdscr.getch()\n return\n\n highlighted_row_index = 0\n offset = 0\n\n while True:\n stdscr.clear()\n row_num = 2 # Starting row for data\n\n for idx, (original_text, source) in enumerate(data[offset:]):\n wrapped_text = textwrap.wrap(original_text[:80], width=text_col_width)\n wrapped_source = textwrap.wrap(source, width=source_col_width)\n\n row_height = max(len(wrapped_text), len(wrapped_source))\n for i in range(row_height):\n text_line = wrapped_text[i] if i < len(wrapped_text) else \"\"\n source_line = wrapped_source[i] if i < len(wrapped_source) else \"\"\n # Construct the line with spacing between cells\n line = f\"{text_line.ljust(text_col_width)} | {source_line.ljust(source_col_width)}\"\n\n if idx + offset == highlighted_row_index:\n stdscr.attron(curses.color_pair(3))\n stdscr.addstr(row_num, 1, line) # Adjusted to start from column 1 for padding\n stdscr.attroff(curses.color_pair(3))\n else:\n stdscr.addstr(row_num, 1, line)\n\n row_num += 1\n\n # Draw a horizontal separator line after each row\n stdscr.addstr(row_num, 1, '-' * (text_col_width + source_col_width + 3)) # '+3' for cell spacing and separator\n row_num += 1 # Increment row_num to account for the separator line\n\n if row_num >= max_y - 1:\n break\n\n stdscr.refresh()\n\n # Key handling for scrolling and quitting\n key = stdscr.getch()\n if key == curses.KEY_DOWN and highlighted_row_index < len(data) - 1:\n highlighted_row_index += 1\n if row_num >= max_y - 1 and offset < len(data) - (max_y - 2):\n offset += 1 # Scroll down\n elif key == curses.KEY_UP and highlighted_row_index > 0:\n highlighted_row_index -= 1\n if highlighted_row_index < offset:\n offset -= 1 # Scroll up\n elif key == ord('q'):\n break # Quit the table view\n\n" }, { "path": "src/gpt_processor.py", "content": "import asyncio\nimport os\nimport json\n\nfrom openai import AsyncOpenAI\nfrom dotenv import load_dotenv\n\nclass GPTProcessor:\n def __init__(self, db_manager, api_key):\n # Load environment variables\n load_dotenv()\n self.db_manager = db_manager\n self.client = AsyncOpenAI(api_key=api_key)\n self.log_file = 'gpt_processor.log' # Log file path\n self.listings_per_batch = os.getenv('COMMANDJOBS_LISTINGS_PER_BATCH')\n if self.listings_per_batch is None:\n raise ValueError(f\"COMMANDJOBS_LISTINGS_PER_BATCH is not set; exiting.\")\n\n def log(self, message):\n \"\"\"Append a message to the log file.\"\"\"\n with open(self.log_file, 'a') as f:\n f.write(f\"{message}\\n\")\n\n async def process_job_listings_with_gpt(self, resume_path, update_ui_callback):\n update_ui_callback(f\"Getting job listings\")\n resume = self.read_resume_from_file(resume_path)\n job_listings = self.db_manager.fetch_job_listings(self.listings_per_batch)\n update_ui_callback(f\"Processing {len(job_listings)} listings with AI. Please wait...\")\n self.log(f\"Creating tasks for {len(job_listings)} job listings\")\n tasks = [self.process_single_listing(job_id, job_text, job_html, resume, update_ui_callback) for job_id, job_text, job_html in job_listings]\n self.log(f\"About to 'gather' {len(tasks)} tasks\")\n # Letting the exceptions bubble up to MenuApp\n await asyncio.gather(*tasks)\n\n async def process_single_listing(self, job_id, job_text, job_html, resume, update_ui_callback):\n prompt = self.generate_prompt(job_text, job_html, resume)\n self.log(f\"Prompt: {prompt}\") # Log the prompt\n if not prompt: # Check if prompt is None or empty\n raise ValueError(\"Prompt is None or empty, skipping GPT request.\")\n \n answer_dict = {}\n # Letting bubble up the potential exceptions from\n # the two lines below, up to process_job_listings_with_gpt\n answer = await self.get_gpt_response(prompt)\n self.db_manager.save_gpt_interaction(job_id, prompt, answer) \n \n # Attempt to load the JSON string into a Python dictionary\n try:\n answer_dict = json.loads(answer)\n # Show a little preview of the processed jobs\n update_ui_callback(f\"Processed {answer_dict['company_name']} / {answer_dict['small_summary'][:50]}\")\n except json.JSONDecodeError:\n self.log(f\"Invalid JSON format: {answer}\")\n \n self.log(f\"Processed job_id: {job_id}\")\n\n def read_resume_from_file(self, file_path):\n try:\n with open(file_path, 'r') as file:\n return file.read()\n except FileNotFoundError:\n return \"Resume file not found.\"\n\n def generate_prompt(self, job_text, job_html, resume):\n # Similar to the original prompt creation logic\n # Ensure to return the formatted prompt string\n # output_format = \"\"\"{\n # \"small_summary\": \"Wine and Open Source developers for C-language systems programming\",\n # \"company_name\": \"CodeWeavers\",\n # \"available_positions\": [\n # {\n # \"position\": \"Wine and General Open Source Developers\",\n # \"link\": \"https://www.codeweavers.com/about/jobs\"\n # }\n # ],\n # \"tech_stack_description\": \"C-language systems programming\",\n # \"use_rails\": \"No\",\n # \"use_python\": \"No\",\n # \"remote_positions\": \"Yes\",\n # \"hiring_in_us\": \"Yes\",\n # \"how_to_apply\": \"Apply through our website, here is the link: https://www.codeweavers.com/about/jobs\",\n # \"back_ground_with_priority\": null,\n # \"fit_for_resume\": \"No\",\n # \"fit_justification\": \"The position is for Wine and Open Source developers, neither of which the resume has experience with. The job is remote in the US\"\n # }\"\"\"\n output_format_str = os.getenv('COMMANDJOBS_OUTPUT_FORMAT')\n self.log(f\"output_format_str: {output_format_str}\") \n # Convert the escaped newlines back to actual newline characters\n output_format = output_format_str.encode().decode('unicode_escape')\n # self.log(f\"output_format: {output_format}\") \n roles = os.getenv('COMMANDJOBS_ROLE')\n job_requirement_exclusions=os.getenv('COMMANDJOBS_EXCLUSIONS')\n # self.log(f\"job_requirement_exclusions: {job_requirement_exclusions}\")\n ideal_job_questions_template = os.getenv('COMMANDJOBS_IDEAL_JOB_QUESTIONS')\n prompt_template = os.getenv('COMMANDJOBS_PROMPT')\n\n # Perform the interpolation\n ideal_job_questions = ideal_job_questions_template.format(job_requirement_exclusions=job_requirement_exclusions)\n prompt = prompt_template.format(job_html=job_html, resume=resume, roles=roles, ideal_job_questions=ideal_job_questions, output_format=output_format)\n\n return prompt\n\n async def get_gpt_response(self, prompt):\n response = await self.client.chat.completions.create(\n messages=[{\"role\": \"user\", \"content\": prompt}],\n model=os.getenv('OPENAI_GPT_MODEL'),\n )\n self.log(f\"response.choices: {response.choices}\")\n return response.choices[0].message.content\n\n" }, { "path": "src/menu.py", "content": "import curses\nimport os\nimport time\nfrom job_scraper.hacker_news.scraper import HNScraper\nfrom display_table import draw_table\nfrom database_manager import DatabaseManager\nfrom display_matching_table import MatchingTableDisplay\nfrom display_applications import ApplicationsDisplay\nfrom display_all_jobs import AllJobsDisplay\nfrom gpt_processor import GPTProcessor\n\nimport asyncio\nimport sqlite3\nimport logging\nimport threading\nfrom queue import Queue\nfrom dotenv import load_dotenv\n\nfrom job_scraper.workday.scraper import WorkdayScraper\nfrom job_scraper.waas.work_startup_scraper import WorkStartupScraper\n\nDB_PATH='job_listings.db'\n\nclass MenuApp:\n def __init__(self, stdscr, logger):\n # Load environment variables\n load_dotenv()\n required_values = (\n \"OPENAI_API_KEY\",\n \"OPENAI_GPT_MODEL\",\n \"BASE_RESUME_PATH\",\n \"HN_START_URL\",\n \"COMMANDJOBS_LISTINGS_PER_BATCH\",\n )\n for required_value in required_values:\n if not os.getenv(required_value):\n error_message = f'''\n {required_value} env variable is not set; Please check the documentation at\n https://github.com/nicobrenner/commandjobs?tab=readme-ov-file#configuration\n '''\n raise ValueError(error_message)\n\n self.scraping_done_event = threading.Event() # Event to signal scraping completion\n self.logger = logger\n self.stdscr = stdscr\n self.setup_ncurses()\n self.db_path = DB_PATH\n self.db_manager = DatabaseManager(self.db_path) # Specify the path\n self.gpt_processor = GPTProcessor(self.db_manager, os.getenv('OPENAI_API_KEY'))\n self.resume_path = os.getenv('BASE_RESUME_PATH')\n self.table_display = MatchingTableDisplay(self.stdscr, self.db_path)\n self.all_jobs_display = AllJobsDisplay(self.stdscr, self.db_path)\n self.total_ai_job_recommendations = self.table_display.fetch_total_entries()\n self.update_processed_listings_count()\n self.total_listings = self.get_total_listings()\n env_limit = 0 if os.getenv('COMMANDJOBS_LISTINGS_PER_BATCH') is None else os.getenv('COMMANDJOBS_LISTINGS_PER_BATCH')\n self.listings_per_request = max(int(env_limit), 10)\n\n resume_menu = \"📄 Create resume (just paste it here once)\"\n find_best_matches_menu = \"🧠 Find best matches with AI (Create your resume first)\"\n resume_str = self.read_resume_from_file()\n if len(resume_str) > 0:\n resume_menu = \"📄 Edit resume\"\n find_best_matches_menu = f\"🧠 Find best matches for resume with AI (will check {self.listings_per_request} listings at a time)\"\n\n total_processed = f'{self.processed_listings_count} processed with AI so far'\n db_menu_item = f\"💾 Navigate jobs in local db ({self.total_listings} listings, {total_processed})\"\n ai_recommendations_menu = \"😅 No job matches for your resume yet\"\n if self.total_ai_job_recommendations > 0:\n ai_recommendations_menu = f\"✅ {self.total_ai_job_recommendations} recommended listings, out of {total_processed}\"\n \n # Fetch applied-listings count\n applied_count = self.db_manager.fetch_applied_listings_count()\n applications_menu = f\"📋 Applications ({applied_count})\"\n\n self.menu_items = [\n applications_menu, # 1 <-- moved up\n ai_recommendations_menu, # 2 <-- moved up\n find_best_matches_menu, # 3 <-- moved up\n \"🕸 Scrape \\\"Ask HN: Who's hiring?\\\"\", # 4\n \"🕸 Scrape \\\"Work at a Startup jobs\\\"\", # 5\n \"🕸 Scrape \\\"Workday\\\"\", # 6\n resume_menu, # 0\n db_menu_item # 7 <-- moved down\n ]\n self.current_row = 0\n self.display_splash_screen()\n self.run()\n\n def update_processed_listings_count(self):\n self.processed_listings_count = self.db_manager.fetch_processed_listings_count()\n\n async def process_with_gpt(self):\n exit_message = 'Processing completed successfully'\n try:\n self.logger.debug('Calling: self.gpt_processor.process_job_listings_with_gpt')\n await self.gpt_processor.process_job_listings_with_gpt(self.resume_path, update_ui_callback=self.update_status_bar)\n except Exception as e:\n self.logger.exception(\"Failed to process listings with GPT: %s\", str(e))\n exit_message = f'Failed to process listings with GPT: {str(e)}'\n finally:\n new_count = self.table_display.fetch_total_entries()\n if new_count > self.total_ai_job_recommendations:\n count_diff = new_count - self.total_ai_job_recommendations\n exit_message = f'Processing completed successfully. {count_diff} new matches found ({new_count} total)'\n else:\n exit_message = f'Processing completed successfully. No new matches found ({new_count} total)'\n\n return exit_message\n\n\n def read_resume_from_file(self):\n try:\n with open(self.resume_path, 'r') as file:\n return file.read()\n except FileNotFoundError:\n return ''\n\n def setup_ncurses(self):\n curses.curs_set(0) # Turn off cursor visibility\n self.stdscr.keypad(True) # Enable keypad mode\n curses.start_color()\n curses.init_pair(1, curses.COLOR_BLACK, curses.COLOR_CYAN)\n curses.init_pair(2, curses.COLOR_BLACK, curses.COLOR_WHITE)\n curses.init_pair(3, curses.COLOR_WHITE, curses.COLOR_BLUE) # Highlight color\n curses.init_pair(4, curses.COLOR_BLACK, curses.COLOR_WHITE) # Highlight headers color\n curses.init_pair(5, curses.COLOR_WHITE, curses.COLOR_MAGENTA) # Highlight headers color\n curses.init_pair(6, curses.COLOR_RED, curses.COLOR_BLACK) # Highlight headers color\n curses.init_pair(7, curses.COLOR_GREEN, curses.COLOR_BLACK)\n curses.init_pair(8, curses.COLOR_YELLOW, curses.COLOR_BLACK)\n curses.init_pair(9, curses.COLOR_BLUE, curses.COLOR_BLACK)\n curses.init_pair(10, curses.COLOR_MAGENTA, curses.COLOR_BLACK)\n curses.init_pair(11, curses.COLOR_RED, curses.COLOR_BLACK)\n\n def display_splash_screen(self):\n splash_text = [\n \" ██████╗ ██████╗ ███╗ ███╗███╗ ███╗ █████╗ ███╗ ██╗██████╗ \",\n \"██╔════╝██╔═══██╗████╗ ████║████╗ ████║██╔══██╗████╗ ██║██╔══██╗\",\n \"██║ ██║ ██║██╔████╔██║██╔████╔██║███████║██╔██╗ ██║██║ ██║\",\n \"██║ ██║ ██║██║╚██╔╝██║██║╚██╔╝██║██╔══██║██║╚██╗██║██║ ██║\",\n \"╚██████╗╚██████╔╝██║ ╚═╝ ██║██║ ╚═╝ ██║██║ ██║██║ ╚████║██████╔╝\",\n \"╚═════╝ ╚═════╝ ╚═╝ ╚═╝╚═╝ ╚═╝╚═╝ ╚═╝╚═╝ ╚═══╝╚═════╝ \",\n \" \",\n \" ██╗ ██████╗ ██████╗ ███████╗ \",\n \" ██║██╔═══██╗██╔══██╗██╔════╝ \",\n \" ██║██║ ██║██████╔╝███████╗ \",\n \" ██ ██║██║ ██║██╔══██╗╚════██║ \",\n \" ╚█████╔╝╚██████╔╝██████╔╝███████║ \",\n \" ╚════╝ ╚═════╝ ╚═════╝ ╚══════╝ \",\n\n ]\n self.stdscr.clear()\n max_y, max_x = self.stdscr.getmaxyx()\n\n # Repeat base animation 3 times\n for i in range(0, 3):\n # Loop through color pairs 7 to 11\n # defined inside setup_ncurses()\n for color in range(7, 12):\n self.stdscr.attron(curses.color_pair(color))\n for i, line in enumerate(splash_text):\n # Calculate the starting position for each line to be centered\n start_x = max(0, (max_x - len(line)) // 2)\n self.stdscr.addstr(i + (max_y - len(splash_text)) // 2, start_x, line)\n self.stdscr.refresh()\n # 100ms per color\n curses.napms(100)\n self.stdscr.attroff(curses.color_pair(color))\n self.stdscr.clear()\n self.stdscr.refresh()\n\n def draw_title(self, title=\"Command Jobs\"):\n max_y, max_x = self.stdscr.getmaxyx()\n title_x = max(0, (max_x - len(title)) // 2)\n self.stdscr.attron(curses.A_BOLD)\n self.stdscr.addstr(0, title_x, title)\n self.stdscr.attroff(curses.A_BOLD)\n self.stdscr.addstr(1, 0, \"-\" * max_x)\n\n def draw_menu(self):\n # Draw title and menu items\n self.draw_title()\n h, w = self.stdscr.getmaxyx()\n for idx, item in enumerate(self.menu_items):\n x = w // 2 - len(item) // 2\n y = h // 2 - len(self.menu_items) // 2 + idx\n if idx == self.current_row:\n self.stdscr.attron(curses.color_pair(1))\n self.stdscr.addstr(y, x, item)\n self.stdscr.attroff(curses.color_pair(1))\n else:\n self.stdscr.addstr(y, x, item)\n\n # --- Centered controls hint line ---\n controls = \"[↑↓] Select [Enter] Go into [q] Quit to terminal\"\n # place it two rows up from bottom\n hint_y = h - 2\n hint_x = max(0, (w - len(controls)) // 2)\n self.stdscr.attron(curses.color_pair(7))\n self.stdscr.addstr(hint_y, hint_x, controls[:w - hint_x - 1])\n self.stdscr.attroff(curses.color_pair(7))\n\n self.stdscr.refresh()\n\n def run(self):\n while True:\n self.draw_menu()\n key = self.stdscr.getch()\n self.handle_keypress(key)\n\n def handle_keypress(self, key):\n if key == curses.KEY_UP:\n self.current_row = max(0, self.current_row - 1)\n elif key == curses.KEY_DOWN:\n self.current_row = min(len(self.menu_items) - 1, self.current_row + 1)\n elif key in [curses.KEY_ENTER, 10, 13]:\n self.execute_menu_action()\n elif key == ord('q'):\n exit()\n\n def update_menu_items(self):\n # Update the total and processed listings count\n self.total_listings = self.get_total_listings()\n self.total_ai_job_recommendations = self.table_display.fetch_total_entries()\n self.update_processed_listings_count()\n\n # Update the resume option\n resume_menu = \"📄 Create resume (just paste it here once)\"\n find_best_matches_menu = \"🧠 Find best matches with AI (Create your resume first)\"\n resume_str = self.read_resume_from_file()\n if len(resume_str) > 0:\n resume_menu = \"📄 Edit resume\"\n find_best_matches_menu = f\"🧠 Find best matches for resume with AI (will check {self.listings_per_request} listings at a time)\"\n\n # Update menu items with the new counts\n total_processed = f'{self.processed_listings_count} processed with AI so far'\n db_menu_item = f\"💾 Navigate jobs in local db ({self.total_listings} listings, {total_processed})\"\n ai_recommendations_menu = \"😅 No job matches for your resume yet\"\n if self.total_ai_job_recommendations > 0:\n ai_recommendations_menu = f\"✅ {self.total_ai_job_recommendations} recommended listings, out of {total_processed}\"\n\n # Update the Applications counter\n applied_count = self.db_manager.fetch_applied_listings_count()\n applications_menu = f\"📋 Applications ({applied_count})\"\n\n # Update the relevant menu items\n # -----------------------------------------------\n # refresh the *same* slots used in self.menu_items\n # 0 📋 Applications\n # 1 ✅ Recommended\n # 2 🧠 Find best matches\n # 3 🕸 Scrape HN ← leave untouched!\n # 4 🕸 Scrape W@S\n # 5 🕸 Scrape Workday\n # 6 📄 Resume ← update this one\n # 7 💾 Navigate DB\n # -----------------------------------------------\n self.menu_items[0] = applications_menu\n self.menu_items[1] = ai_recommendations_menu\n self.menu_items[2] = find_best_matches_menu\n self.menu_items[6] = resume_menu # ← was 3\n self.menu_items[7] = db_menu_item\n\n # Redraw the menu to reflect the updated items\n self.draw_menu()\n\n\n # Menu options, the number map to the self.menu_items array\n # eg. first option (0): self.menu_items[0] = resume_menu\n # = \"Create or replace base resume\"\n def execute_menu_action(self):\n exit_message = ''\n if self.current_row == 0: # 📋 Applications\n self.app_display = ApplicationsDisplay(self.stdscr, self.db_path)\n self.app_display.draw_board()\n\n elif self.current_row == 1: # ✅ Recommended listings\n self.table_display.draw_table()\n\n elif self.current_row == 2: # 🧠 Find best matches\n exit_message = asyncio.run(self.process_with_gpt())\n\n elif self.current_row == 3: # 🕸 Scrape “Ask HN”\n self.start_scraping_with_status_updates()\n\n elif self.current_row == 4: # 🕸 Scrape “Work at a Startup”\n self.start_scraping_WaaS_with_status_updates()\n\n elif self.current_row == 5: # 🕸 Scrape “Workday”\n self.start_scraping_workday_with_status_updates()\n\n elif self.current_row == 6: # 📄 Resume\n exit_message = self.manage_resume(self.stdscr)\n\n elif self.current_row == 7: # 💾 Navigate DB\n self.all_jobs_display.draw_table()\n\n # redraw status / menu after the action\n self.stdscr.clear()\n self.update_menu_items()\n if exit_message != '':\n self.update_status_bar(exit_message)\n\n def display_text_with_scrolling(self, header, lines):\n curses.noecho()\n max_y, max_x = self.stdscr.getmaxyx()\n offset = 0 # How much we've scrolled\n resume_updated = False\n new_lines = ''\n\n while True:\n self.stdscr.clear()\n self.draw_title() # Call draw_title as a method of the class\n # Draw the sticky header below the title\n self.stdscr.attron(curses.color_pair(2)) # Apply color pair for white background\n self.stdscr.addstr(2, 0, header + \" \" * (max_x - len(header))) # Extend background to full width\n self.stdscr.attroff(curses.color_pair(2)) # Turn off color pair\n\n for i, line in enumerate(lines[offset:offset+max_y-5]):\n self.stdscr.addstr(i+3, 0, line.strip())\n\n key = self.stdscr.getch()\n if key in [ord('q'), ord('Q')]:\n break\n elif key == curses.KEY_DOWN:\n if offset < len(lines) - max_y + 2:\n offset += 1\n elif key == curses.KEY_UP:\n if offset > 0:\n offset -= 1\n elif key in [ord('r'), ord('R')]:\n new_lines = self.capture_text_with_scrolling()\n if len(new_lines) > 0:\n resume_updated = new_lines != lines\n break\n\n return resume_updated\n\n def get_total_listings(self):\n \"\"\"Return the total number of job listings in the database.\"\"\"\n conn = sqlite3.connect(self.db_path)\n conn.execute(\"PRAGMA journal_mode=WAL;\")\n cur = conn.cursor()\n cur.execute(\"SELECT COUNT(*) FROM job_listings\")\n total = cur.fetchone()[0]\n conn.close()\n return total\n\n def manage_resume(self, stdscr):\n curses.echo()\n resume_path = os.getenv('BASE_RESUME_PATH')\n\n resume_updated = False\n exit_message = 'Resume not updated'\n\n if os.path.exists(resume_path):\n with open(resume_path, 'r') as file:\n lines = file.readlines()\n\n header = \"Base Resume (Press 'q' to go back, 'r' to replace):\" # Use a separator for clarity\n resume_updated = self.display_text_with_scrolling(header, lines)\n else:\n resume_updated = self.capture_text_with_scrolling()\n\n if resume_updated:\n exit_message = f\"Resume saved to {self.resume_path}\"\n\n return exit_message\n\n def update_status_bar(self, text):\n max_y, max_x = self.stdscr.getmaxyx()\n # Ensure the status text will not overflow the screen width\n status_text = text[:max_x - 3]\n try:\n # Clear the previous status bar content\n self.stdscr.move(max_y - 1, 0)\n self.stdscr.clrtoeol()\n # Write the new status bar content\n self.stdscr.addstr(max_y - 1, 0, status_text, curses.color_pair(2))\n self.stdscr.refresh()\n except curses.error:\n pass # Ignore the error or handle it as needed\n\n def start_scraping_with_status_updates(self):\n # Create a queue to receive the result from the scraping thread\n result_queue = Queue()\n # Pass self.update_status_bar as the update function to HNScraper\n self.scraper = HNScraper(self.db_path) # Initialize the scraper\n start_url = os.getenv('HN_START_URL') # Starting URL\n scraping_thread = threading.Thread(target=self.scraper.scrape_hn_jobs, args=(\n start_url, self.stdscr, self.update_status_bar, self.scraping_done_event, result_queue))\n scraping_thread.start()\n # Call this method after the scraping is done\n self.scraping_done_event.wait() # Wait for the event to be set by the scraping thread\n # Retrieve the result from the queue\n new_listings_count = result_queue.get() # This will block until the result is available\n self.update_status_bar(f\"Scraping completed {new_listings_count} new listings added\")\n self.scraping_done_event.clear() # Clear the event for the next scraping operation\n\n\n\n def start_scraping_WaaS_with_status_updates(self):\n result_queue= Queue()\n self.scraper = WorkStartupScraper(self.db_path)\n scraping_thread = threading.Thread(target=self.scraper.scrape_jobs, args=(self.stdscr, self.update_status_bar, self.scraping_done_event, result_queue))\n scraping_thread.start()\n self.scraping_done_event.wait()\n new_listings_count = result_queue.get()\n self.update_status_bar(f\"Scraping of Waas completed {new_listings_count} new listings added\")\n self.scraping_done_event.clear()\n time.sleep(3)\n self.stdscr.clear()\n\n def start_scraping_workday_with_status_updates(self):\n result_queue= Queue()\n self.scraper = WorkdayScraper(self.db_path, self.update_status_bar, self.scraping_done_event, result_queue)\n scraping_thread = threading.Thread(target=self.scraper.scrape)\n scraping_thread.start()\n self.scraping_done_event.wait()\n new_listings_count = result_queue.get()\n self.update_status_bar(f\"Scraping of Workday completed: {new_listings_count} new listings added\")\n self.scraping_done_event.clear()\n time.sleep(3)\n self.stdscr.clear()\n\n\n # Despite the name of the method, this currently\n # is not handling scrolling 😅\n\n # It directs the user to paste text into the terminal\n # When Esc is pressed, captures the input and returns it\n def capture_text_with_scrolling(self):\n directions = \"Paste your resume text, then Press the 'Esc' key to finish and save\"\n curses.curs_set(1) # Show cursor\n self.stdscr.keypad(True) # Enable keypad mode\n curses.noecho() # Don't echo keypresses\n curses.raw() # Raw mode - get all inputs\n self.stdscr.clear() # Clear the screen\n self.stdscr.scrollok(True) # Enable scrolling in the window\n\n text = []\n y, x = 0, 0 # Initial position\n max_y, max_x = self.stdscr.getmaxyx()\n\n # This loop \"listens\" for keyboard input\n while True:\n self.stdscr.addstr(0, 0, directions, curses.A_REVERSE)\n try:\n char = self.stdscr.get_wch() # Get character or key press\n except AttributeError:\n # To be able to handle utf8, we need ncurses to have\n # the stdscr.get_wch() method available\n self.stdscr.addstr(0, 0, \"Error, app needs stdscr.get_wch() method\", curses.A_REVERSE)\n\n return ''\n\n if char == '\\x1b': # Escape key pressed\n break\n elif char == '\\n': # Handle newline\n text.append('\\n')\n y += 1\n x = 0\n if y >= max_y - 1:\n self.stdscr.scroll(1)\n y -= 1\n elif isinstance(char, str): # Regular character input\n if x >= max_x - 1: # Move to next line if at the end\n y += 1\n x = 0\n if y >= max_y - 1:\n self.stdscr.scroll(1)\n y -= 1\n text.append(char)\n try:\n self.stdscr.addstr(y, x, char)\n except curses.error:\n pass # Ignore errors potentially caused by edge cases in window size\n x += 1\n self.stdscr.refresh()\n\n input_lines = ''.join(text)\n if text != []:\n with open(self.resume_path, 'w') as file:\n file.writelines(input_lines)\n\n curses.curs_set(0) # hide cursor again\n\n return input_lines\n\n# Ensure logging is configured to write to a file or standard output\nlogging.basicConfig(filename='application.log', level=logging.DEBUG, format='%(asctime)s %(levelname)s %(name)s %(message)s')\nlogger = logging.getLogger(__name__)\n\ndef main(stdscr):\n global logger\n app = MenuApp(stdscr, logger)\n app.run() # Ensuring app.run is called to start the application loop\n\nif __name__ == \"__main__\":\n curses.wrapper(main)\n\n" }, { "path": "src/migrations/000_create_initial_tables.py", "content": "# src/migrations/000_create_initial_tables.py\n\nimport sqlite3\nimport os\n\n# two levels up from this file's folder, then job_listings.db:\nDB_PATH = os.path.abspath(\n os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, 'job_listings.db')\n)\n\ndef table_exists(cursor, table_name):\n cursor.execute(\n \"SELECT name FROM sqlite_master \"\n \"WHERE type='table' AND name=?\",\n (table_name,)\n )\n return cursor.fetchone() is not None\n\ndef main():\n conn = sqlite3.connect(DB_PATH)\n cursor = conn.cursor()\n\n # 1) job_listings\n if not table_exists(cursor, 'job_listings'):\n print(\"Creating table job_listings…\")\n cursor.execute('''\n CREATE TABLE job_listings (\n id INTEGER PRIMARY KEY AUTOINCREMENT,\n original_text TEXT,\n original_html TEXT,\n source TEXT,\n external_id TEXT UNIQUE\n )\n ''')\n\n # 2) gpt_interactions\n if not table_exists(cursor, 'gpt_interactions'):\n print(\"Creating table gpt_interactions…\")\n cursor.execute('''\n CREATE TABLE gpt_interactions (\n id INTEGER PRIMARY KEY,\n job_id INTEGER,\n prompt TEXT,\n answer TEXT\n )\n ''')\n\n conn.commit()\n conn.close()\n print(\"000_create_initial_tables.py completed.\")\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "src/migrations/001_add_discarded_applied.py", "content": "# src/migrations/001_add_discarded_applied.py\n\nimport sqlite3\n\nDB_PATH = 'job_listings.db' # <-- adjust if you use a different path\n\ndef column_exists(cursor, table_name, column_name):\n cursor.execute(f\"PRAGMA table_info({table_name})\")\n return any(col[1] == column_name for col in cursor.fetchall())\n\ndef main():\n conn = sqlite3.connect(DB_PATH)\n cursor = conn.cursor()\n\n if not column_exists(cursor, 'job_listings', 'discarded'):\n print(\"Adding 'discarded' column...\")\n cursor.execute(\"ALTER TABLE job_listings ADD COLUMN discarded INTEGER DEFAULT 0\")\n\n if not column_exists(cursor, 'job_listings', 'applied'):\n print(\"Adding 'applied' column...\")\n cursor.execute(\"ALTER TABLE job_listings ADD COLUMN applied INTEGER DEFAULT 0\")\n\n conn.commit()\n conn.close()\n print(\"Migration completed.\")\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "src/migrations/002_create_application_notes.py", "content": "# src/migrations/002_create_application_notes.py\n\nimport sqlite3\n\nDB = 'job_listings.db'\n\ndef column_exists(cur, table, column):\n cur.execute(f\"PRAGMA table_info({table})\")\n return any(r[1] == column for r in cur.fetchall())\n\ndef main():\n conn = sqlite3.connect(DB)\n cur = conn.cursor()\n\n # Ensure applied column exists\n if not column_exists(cur, 'job_listings', 'applied'):\n cur.execute(\"ALTER TABLE job_listings ADD COLUMN applied INTEGER DEFAULT 0\")\n\n # Create notes table\n cur.execute('''\n CREATE TABLE IF NOT EXISTS application_notes (\n id INTEGER PRIMARY KEY AUTOINCREMENT,\n job_id INTEGER NOT NULL,\n note TEXT NOT NULL,\n created_at DATETIME DEFAULT CURRENT_TIMESTAMP\n )\n ''')\n\n conn.commit()\n conn.close()\n print(\"✔️ Migration 002 complete\")\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "src/migrations/003_add_applied_date.py", "content": "# src/migrations/003_add_applied_date.py\nimport sqlite3\n\nDB = 'job_listings.db'\n\ndef column_exists(cur, table, column):\n cur.execute(f\"PRAGMA table_info({table})\")\n return any(r[1] == column for r in cur.fetchall())\n\ndef main():\n conn = sqlite3.connect(DB)\n cur = conn.cursor()\n if not column_exists(cur, 'job_listings', 'applied_date'):\n print(\"Adding applied_date column…\")\n # store date in ISO YYYY-MM-DD\n cur.execute(\"ALTER TABLE job_listings ADD COLUMN applied_date TEXT\")\n conn.commit()\n conn.close()\n print(\"✔️ Migration 003 complete\")\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "src/migrations/004_migrate_applications_table.py", "content": "# src/migrations/004_migrate_applications_table.py\n\nimport sqlite3\nimport os\nimport sys\n\nDB = 'job_listings.db'\n\ndef table_exists(cur, name):\n cur.execute(\"SELECT name FROM sqlite_master WHERE type='table' AND name=?\", (name,))\n return cur.fetchone() is not None\n\ndef column_list(cur, table):\n cur.execute(f\"PRAGMA table_info({table})\")\n return [row[1] for row in cur.fetchall()]\n\ndef main():\n if not os.path.exists(DB):\n print(f\"Error: database file not found at {DB}\", file=sys.stderr)\n sys.exit(1)\n\n conn = sqlite3.connect(DB)\n cur = conn.cursor()\n\n # If we've already migrated (i.e. new schema is present), skip\n if table_exists(cur, 'applications') and 'application_id' in column_list(cur, 'application_notes'):\n print(\"✔️ Migration 004 already applied, skipping.\")\n return\n\n try:\n print(\"Running Migration 004…\")\n conn.executescript(\"\"\"\n PRAGMA foreign_keys = OFF;\n\n CREATE TABLE IF NOT EXISTS applications (\n id INTEGER PRIMARY KEY AUTOINCREMENT,\n job_id INTEGER NOT NULL,\n status TEXT NOT NULL DEFAULT 'Open',\n created_at TEXT NOT NULL DEFAULT (datetime('now')),\n updated_at TEXT NOT NULL DEFAULT (datetime('now')),\n FOREIGN KEY(job_id) REFERENCES job_listings(id)\n );\n\n INSERT OR IGNORE INTO applications (job_id, status, created_at, updated_at)\n SELECT id, 'Open', applied_date, applied_date\n FROM job_listings\n WHERE applied = 1\n AND applied_date IS NOT NULL;\n\n ALTER TABLE application_notes RENAME TO _old_notes;\n\n CREATE TABLE IF NOT EXISTS application_notes (\n id INTEGER PRIMARY KEY AUTOINCREMENT,\n application_id INTEGER NOT NULL,\n note TEXT NOT NULL,\n created_at DATETIME DEFAULT CURRENT_TIMESTAMP,\n FOREIGN KEY(application_id) REFERENCES applications(id)\n );\n\n INSERT INTO application_notes (application_id, note, created_at)\n SELECT a.id, n.note, n.created_at\n FROM _old_notes AS n\n JOIN applications AS a\n ON n.job_id = a.job_id;\n\n DROP TABLE _old_notes;\n\n PRAGMA foreign_keys = ON;\n \"\"\")\n conn.commit()\n print(\"✅ Migration 004 complete!\")\n except Exception as e:\n conn.rollback()\n print(\"❌ Migration 004 failed:\", e, file=sys.stderr)\n sys.exit(1)\n finally:\n conn.close()\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "src/migrations/005_migrate_old_notes.py", "content": "# src/migrations/005_replace_notes_table.py\n\nimport sqlite3\nimport sys\nimport os\n\nDB = 'job_listings.db'\n\ndef table_exists(cur, name):\n cur.execute(\"SELECT name FROM sqlite_master WHERE type='table' AND name=?\", (name,))\n return cur.fetchone() is not None\n\ndef main():\n if not os.path.exists(DB):\n print(f\"Error: database file not found at {DB}\", file=sys.stderr)\n sys.exit(1)\n\n conn = sqlite3.connect(DB)\n cur = conn.cursor()\n\n try:\n # only run if _old_notes exists\n if table_exists(cur, '_old_notes'):\n # drop the empty new notes table\n if table_exists(cur, 'application_notes'):\n print(\"Dropping empty application_notes…\")\n cur.execute(\"DROP TABLE application_notes\")\n\n # rename the old one into place\n print(\"Renaming _old_notes → application_notes…\")\n cur.execute(\"ALTER TABLE _old_notes RENAME TO application_notes\")\n\n # make sure the schema is what we expect (you can add more PRAGMAs here)\n conn.commit()\n print(\"✅ Migration 005 complete\")\n else:\n print(\"⚠️ _old_notes not found, skipping migration 005\")\n except Exception as e:\n conn.rollback()\n print(\"❌ Migration 005 failed:\", e, file=sys.stderr)\n sys.exit(1)\n finally:\n conn.close()\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "src/migrations/006_unique_applications_job_id.py", "content": "# src/migrations/006_unique_applications_job_id.py\n\nimport sqlite3\nimport sys\nimport os\n\nDB_PATH = 'job_listings.db'\n\ndef main(db_path):\n if not os.path.exists(db_path):\n print(f\"Error: database file not found at {db_path}\", file=sys.stderr)\n sys.exit(1)\n\n conn = sqlite3.connect(db_path)\n try:\n cur = conn.cursor()\n cur.executescript(\"\"\"\nPRAGMA foreign_keys = OFF;\nBEGIN;\n\n/*\n 1) Create a fresh table with the exact same schema as `applications`,\n except named `applications_new`. We explicitly include the `id` column\n so we can re-insert with the same primary keys.\n*/\nCREATE TABLE IF NOT EXISTS applications_new (\n id INTEGER PRIMARY KEY, -- we'll preserve the old PK\n job_id INTEGER NOT NULL UNIQUE,\n status TEXT NOT NULL DEFAULT 'Open',\n created_at TEXT NOT NULL,\n updated_at TEXT NOT NULL,\n FOREIGN KEY(job_id) REFERENCES job_listings(id)\n);\n\n/*\n 2) Copy in exactly one row per job_id, picking the *earliest* created_at.\n By selecting MIN(id) per job_id, we also pick its original PK.\n*/\nINSERT OR IGNORE INTO applications_new (id, job_id, status, created_at, updated_at)\n SELECT\n id,\n job_id,\n status,\n created_at,\n updated_at\n FROM applications\n WHERE id IN (\n SELECT MIN(id) -- pick the very first row inserted for each job_id\n FROM applications\n GROUP BY job_id\n );\n\n/*\n 3) Drop the old table and swap in the new one\n*/\nDROP TABLE applications;\nALTER TABLE applications_new RENAME TO applications;\n\nCOMMIT;\nPRAGMA foreign_keys = ON;\n\"\"\")\n conn.commit()\n print(\"✔️ Migration 006 complete: duplicates removed, original IDs preserved\")\n except Exception as e:\n conn.rollback()\n print(\"❌ Migration 006 failed:\", e, file=sys.stderr)\n sys.exit(1)\n finally:\n conn.close()\n\nif __name__ == \"__main__\":\n main(DB_PATH)\n" }, { "path": "src/migrations/007_add_scraped_at_timestamp.py", "content": "# src/migrations/007_add_scraped_at_timestamp.py\n\nimport sqlite3\nimport sys\nimport os\n\nDB_PATH = 'job_listings.db'\n\ndef main(db_path):\n if not os.path.exists(db_path):\n print(f\"Error: database file not found at {db_path}\", file=sys.stderr)\n sys.exit(1)\n\n conn = sqlite3.connect(db_path)\n try:\n cur = conn.cursor()\n \n # Check if the column already exists\n cur.execute(\"PRAGMA table_info(job_listings)\")\n columns = [column[1] for column in cur.fetchall()]\n \n if 'scraped_at' not in columns:\n # Add the scraped_at column\n cur.execute(\"ALTER TABLE job_listings ADD COLUMN scraped_at TEXT\")\n \n # Set a default timestamp for existing entries (Jan 1 2025)\n default_timestamp = \"2025-01-01T00:00:00\"\n cur.execute(\"UPDATE job_listings SET scraped_at = ? WHERE scraped_at IS NULL\", (default_timestamp,))\n \n conn.commit()\n print(\"✔️ Migration 007 complete: added scraped_at timestamp column\")\n else:\n print(\"✔️ Migration 007 already applied, skipping.\")\n \n except Exception as e:\n conn.rollback()\n print(\"❌ Migration 007 failed:\", e, file=sys.stderr)\n sys.exit(1)\n finally:\n conn.close()\n\nif __name__ == \"__main__\":\n main(DB_PATH)" }, { "path": "src/test_menu.py", "content": "import os\nimport unittest\nfrom unittest.mock import patch, MagicMock\nfrom menu import MenuApp\n\nDB_PATH='test_db.db'\n\nclass TestManageResume(unittest.TestCase):\n @patch('menu.curses')\n @patch('menu.os.getenv')\n \n def test_manage_resume(self, mock_getenv, mock_curses):\n # Mock environment variables\n mock_getenv.side_effect = lambda x: {'OPENAI_API_KEY': 'test_key', 'BASE_RESUME_PATH': 'temp_base_resume.txt', 'HN_START_URL': 'test_url', 'COMMANDJOBS_LISTINGS_PER_BATCH': '10', 'OPENAI_GPT_MODEL': 'gpt-3.5'}.get(x, None)\n \n # Mock stdscr object\n mock_stdscr = MagicMock()\n mock_curses.initscr.return_value = mock_stdscr\n mock_stdscr.getmaxyx.return_value = (100, 40) # Example values for a terminal size\n\n # Use config/base_resume.sample as the test resume \n test_resume_text = ''\n with open('config/base_resume.sample', 'r') as file:\n test_resume_text = file.read()\n\n # This is testing when the resume file doesn't exist\n # Remove test resume file, to make sure it doesn't exist\n temp_test_resume_path = os.getenv('BASE_RESUME_PATH')\n if os.path.exists(temp_test_resume_path):\n os.remove(temp_test_resume_path)\n \n # Mock user input sequence for getch and get_wch \n # And then paste the resume text + Esc ('\\x1b'), to save the resume\n mock_stdscr.get_wch.side_effect = list(test_resume_text) + ['\\x1b']\n \n # Initialize Menu with mocked stdscr and logger\n logger = MagicMock()\n with patch.object(MenuApp, 'run', return_value=None):\n menu = MenuApp(mock_stdscr, logger)\n \n # Simulate calling capture_text_with_scrolling\n exit_message = menu.manage_resume(mock_stdscr)\n \n # Verify we got a success message\n self.assertEqual(exit_message, f'Resume saved to {temp_test_resume_path}')\n \n # Verify the text was saved to base_resume.txt\n with open(temp_test_resume_path, 'r') as file:\n saved_text = file.read()\n\n self.assertEqual(saved_text, test_resume_text)\n \n # Remove temp test resume file\n if os.path.exists(temp_test_resume_path):\n os.remove(temp_test_resume_path)\n \n temp_test_db_path = DB_PATH\n if os.path.exists(temp_test_db_path):\n os.remove(temp_test_db_path)\n\nif __name__ == '__main__':\n unittest.main()\n" }, { "path": "src/truncate_tables.py", "content": "import sqlite3\n\n# Replace 'job_listings.db' with the correct path to your database file\ndb_path = 'job_listings.db'\n\ndef truncate_tables(database_path):\n # Connect to the SQLite database\n conn = sqlite3.connect(database_path)\n cursor = conn.cursor()\n\n # SQL commands to truncate tables\n truncate_gpt_interactions = \"DELETE FROM gpt_interactions;\"\n # truncate_job_listings = \"DELETE FROM job_listings;\"\n\n try:\n # Execute the SQL commands\n cursor.execute(truncate_gpt_interactions)\n # cursor.execute(truncate_job_listings)\n\n # Commit the changes\n conn.commit()\n print(\"Tables truncated successfully.\")\n\n except sqlite3.Error as e:\n print(f\"An error occurred: {e}\")\n\n finally:\n # Close the connection\n conn.close()\n\n# Call the function to truncate tables\ntruncate_tables(db_path)\n" }, { "path": "tests/__init__.py", "content": "" }, { "path": "tests/test_workday_scraper.py", "content": "import pytest\nfrom selenium import webdriver\nfrom selenium.webdriver.common.by import By\nfrom selenium.webdriver.chrome.service import Service\nfrom selenium.webdriver.support.ui import WebDriverWait\nfrom selenium.webdriver.support import expected_conditions as EC\nfrom webdriver_manager.chrome import ChromeDriverManager\nfrom selenium.webdriver.chrome.options import Options\nfrom selenium.common.exceptions import TimeoutException\n\nfrom job_scraper.scraper_selectors.workday_selectors import WorkDaySelectors\nfrom job_scraper.utils import get_workday_company_urls\n\n\n@pytest.fixture(scope=\"module\")\ndef selenium_driver():\n chrome_options = Options()\n chrome_options.add_argument(\"--headless\")\n chrome_options.add_argument(\"--no-sandbox\")\n chrome_options.add_argument(\"--disable-dev-shm-usage\")\n chrome_options.add_argument(\"--disable-gpu\")\n driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()), options=chrome_options)\n yield driver\n driver.quit()\n\n@pytest.mark.parametrize(\n \"company_name, url\", list(get_workday_company_urls().items())\n)\ndef test_job_listing_xpath_present(selenium_driver, company_name, url):\n selenium_driver.get(url)\n wait = WebDriverWait(selenium_driver, 10)\n\n try:\n wait.until(EC.presence_of_element_located((By.XPATH, WorkDaySelectors.JOB_LISTING_XPATH)))\n except TimeoutException:\n pytest.fail(f\"FAIL: JOB_LISTING_XPATH not found for {company_name}\")" } ]