Repository: sertalpbilal/FPL-Optimization-Tools Branch: main Commit: 7c438b5cf0eb Files: 26 Total size: 181.4 KB Directory structure: gitextract_uuzo_4bk/ ├── .gitignore ├── .pre-commit-config.yaml ├── LICENSE ├── README.md ├── data/ │ ├── README.md │ ├── binary_fixtures.md │ ├── comprehensive_settings.json │ ├── images/ │ │ └── .gitkeep │ ├── results/ │ │ └── .gitkeep │ ├── team.json.sample │ └── user_settings.json ├── dev/ │ ├── data_parser.py │ ├── solver.py │ └── visualization.py ├── paths.py ├── pyproject.toml ├── run/ │ ├── binary_file_generator.py │ ├── run_parallel.py │ ├── sensitivity.py │ ├── simulations.py │ ├── solve.py │ └── tmp/ │ ├── .gitignore │ └── .gitkeep ├── tests/ │ ├── __init__.py │ └── test_options_parsing.py └── utils.py ================================================ FILE CONTENTS ================================================ ================================================ FILE: .gitignore ================================================ *.txt *.mps *.sol *.cmd *.code-workspace **/__pycache__/ output/* archive/* solver/* */.ipynb_checkpoints/* *.log *.sol *.mps *.opt *.csv *.png _test*.* .cache/ ================================================ FILE: .pre-commit-config.yaml ================================================ repos: - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.12.7 hooks: - id: ruff-format args: [--line-length=150] - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.5.0 hooks: - id: trailing-whitespace args: [--markdown-linebreak-ext=md] - id: check-json - id: fix-byte-order-marker - id: mixed-line-ending args: [--fix=lf] ================================================ FILE: LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. * License This project is dual-licensed under the Apache License 2.0 and a Commercial License. * Apache License 2.0 You may use this project under the terms of the Apache License 2.0. * Commercial License For commercial use, please contact info@fploptimized.com to obtain a commercial license. * Contributor License Agreement By contributing to this project, you agree that your contributions can be licensed under both the Apache License 2.0 and the Commercial License. ================================================ FILE: README.md ================================================ # FPL Optimization Tools This repository provides a set of tools for solving deterministic **Fantasy Premier League (FPL)** optimization problems. The Python code uses **`pandas`** for data management, **`sasoptpy`** for building the optimization model, and **HiGHS** via **`highspy`** to solve the model. It allows users to: - Automatically select the best FPL squad based on the given projection data and solver settings. - Customize squad constraints, formation rules, transfer strategies, and more. - Modify data sources and parameters to suit personal models or preferences. ## 🔧 Installation ### 1. Install `uv` `uv` handles **both Python installation and dependency management**, so you **do not need to install Python separately**. **Windows (PowerShell)** Open PowerShell and run: ```powershell powershell -ExecutionPolicy ByPass -c "irm https://astral.sh/uv/install.ps1 | iex" ``` **macOS / Linux** ```bash wget -qO- https://astral.sh/uv/install.sh | sh ``` Restart your terminal after installation, then verify: ```bash uv --version ``` --- ### 2. Install Git **Windows** Download from [git-scm.com](https://git-scm.com/download/win) and accept all default installation options. **macOS** Git is usually pre-installed. If not, run: ```bash brew install git ``` ### 3. Clone the Repository Open a terminal (search for *Command Prompt* in Windows) and run: ```bash cd Documents git clone https://github.com/solioanalytics/open-fpl-solver.git cd open-fpl-solver ``` ### 4. Install Dependencies (and Python) ```bash uv sync ``` ## 🚀 Running the Optimizer ### 1. Add Projection Data Place your projections file (e.g., `solio.csv`) in the `data/` folder. ### 2. Configure Data Source If you are not using the default data source, update the `datasource` field in `data/user_settings.json` to match your CSV file name. Example: if you are using a file named `projections.csv`, the settings file should contain: ```json "datasource": "projections" ``` ### 3. Edit Settings Edit any desired settings in `comprehensive_settings.json` or `user_settings.json`. - The majority of useful settings for most people will be in `user_settings.json`. - `comprehensive_settings.json` provides a wider range of options that will be used as defaults unless altered in `user_settings.json`. Details of what each setting does can be found in the `.md` file in the `/data/` folder. ### 4. Run the Solver ```bash cd run uv run python solve.py ``` ## 🎥 Videos There is a YouTube playlist [here](https://www.youtube.com/playlist?list=PLrIyJJU8_viOags1yudB_wyafRuTNs1Ed) by Sertalp, showing the early stages of this tool, explaining how it was built, and discussing ideas around optimization with a focus on FPL. ## 🌍 Browser-based optimization There is also a browser-based version of the optimizer that doesn't require the download or installation of anything to your device, and works on mobile. It is hosted in a google colab notebook that can be found [here](https://colab.research.google.com/drive/1fwYcG28zpIOJf7R8yx31bDL_kJG1JRLu). Simply follow the instructions on that page to run the optimizer. ## 🛠️ Issues If you have issues, feel free to open an issue on GitHub and I will get back to you as soon as possible. Alternatively, you can email me at **chris.musson@hotmail.com**. ================================================ FILE: data/README.md ================================================ ## Setting Explanations This file documents all configurable settings for the FPL solver. Settings are organized by complexity: - **[User-Friendly Settings](#user-friendly-settings)** – Essential options for most users (see `user_settings.json`) - **[Advanced Settings](#advanced-settings)** – Fine-tuning options (see `comprehensive_settings.json`) - **[Complete Reference](#complete-reference)** – Full alphabetical listing of all settings --- ## User-Friendly Settings These are the core settings most users will interact with. You'll find them in `user_settings.json`. ### Planning Horizon - `horizon`: length of the planning horizon (number of gameweeks to optimize) - Example: `"horizon": 4` (plan 4 gameweeks ahead) ### Decay & Valuation - `decay_base`: value assigned to decay rate of expected points (discounts future GWs) - Example: `"decay_base": 0.9` (10% discount per GW) - `ft_value_list`: values of rolling free transfers in different states - Example: `"ft_value_list": {"2": 2, "3": 1.6, "4": 1.3, "5": 1.1}` assigns value 2.0 for rolling from 1FT to 2FTs, 1.6 for rolling from 2FTs to 3FTs, etc. ### Data Source & Team - `datasource`: specifies which projection CSV to use or `"mixed"` for multiple sources - Example: `"datasource": "fplreview"` or `"solio"` - `team_data`: how to provide team data (`"id"` for team_id, `"json"` for inline JSON, or default uses `team.json`) - Example: `"team_data": "id"` - `team_id`: your FPL team ID (requires `team_data: "id"`) - Example: `"team_id": 2211381` ### Player Pool Filtering - `xmin_lb`: drop players below this many expected minutes across the horizon - Example: `"xmin_lb": 300` (drop players with <300 expected minutes) - `ev_per_price_cutoff`: drop players below this percentile of expected value per price - Example: `"ev_per_price_cutoff": 30` (drop bottom 30%) - `keep_top_ev_percent`: force the filtering to always keep the top n% of players by total expected value, even if they would have been otherwise filtered out by other steps - Example: `"keep_top_ev_percent": 5` (avoid filtering out players in the top 5% by total EV) ### Player Constraints - `banned`: list of player IDs to exclude from entire horizon - Example: `"banned": []` - `locked`: list of player IDs to always keep in squad - Example: `"locked": [430]` (always include player 430 (Haaland)) ### Transfer Constraints - `no_transfer_last_gws`: number of gameweeks at end where transfers are banned - Example: `"no_transfer_last_gws": 0` ### Chips - `use_wc`: list of gameweeks to use Wildcard - Example: `"use_wc": []` or `"use_wc": [15]` - `use_bb`: list of gameweeks to use Bench Boost - Example: `"use_bb": [25]` - `use_fh`: list of gameweeks to use Free Hit - Example: `"use_fh": []` - `use_tc`: list of gameweeks to use Triple Captain - Example: `"use_tc": []` ### Output - `verbose`: whether to print solver progress to screen - Example: `"verbose": true` --- ## Advanced Settings These settings provide fine-grained control over the optimization. Most users won't need to adjust these. See `comprehensive_settings.json` for default values. ### Scoring & Objective Function - `ft_value`: value (in points) assigned to having one extra free transfer - Example: `"ft_value": 1.5` - `bench_weights`: weights for each bench position's expected points (0=subGK, 1=sub1, 2=sub2, 3=sub3) - Example: `"bench_weights": {"0": 0.03, "1": 0.21, "2": 0.06, "3": 0.002}` - `vcap_weight`: weight for vice-captain points in the objective function - Example: `"vcap_weight": 0.1` - `itb_value`: value (in points) assigned to having 1.0 extra budget in the bank - Example: `"itb_value": 0.08` - `itb_loss_per_transfer`: reduction in ITB value per scheduled transfer in future (tries to give some budget flexibility for future gameweeks) - Example: `"itb_loss_per_transfer": 0.05` - `ft_use_penalty`: penalty applied when a free transfer is used (prevents trivial scheduled transfers) - Example: `"ft_use_penalty": 0.2` ### Transfer & Hit Management - `no_future_transfer`: if `true`, disable planning transfers beyond current gameweek - Example: `"no_future_transfer": false` - `no_transfer_by_position`: list of positions to ban transfers in/out. Valid: `["G", "D", "M", "F"]` - Example: `"no_transfer_by_position": ["G", "D"]` - `force_ft_state_lb`: list of `[GW, minimum_FTs]` pairs to force minimum FTs in specific gameweeks - Example: `"force_ft_state_lb": [[4, 3], [7, 2]]` ensures at least 3 FTs in GW4 and 2 FTs in GW7 - `force_ft_state_ub`: list of `[GW, maximum_FTs]` pairs to force maximum FTs in specific gameweeks - Example: `"force_ft_state_ub": [[4, 4], [7, 3]]` ensures at most 4 FTs in GW4 and 3 FTs in GW7 - `num_transfers`: fixed number of transfers for this gameweek (optional override) - Example: `"num_transfers": null` - `hit_limit`: maximum total hits allowed across entire horizon - Example: `"hit_limit": null` (no limit) - `weekly_hit_limit`: maximum hits allowed in a single gameweek - Example: `"weekly_hit_limit": 0` - `hit_cost`: points deducted per hit (default 4) - Example: `"hit_cost": 4` - `future_transfer_limit`: upper bound on total transfers made in future gameweeks - Example: `"future_transfer_limit": 5` - `no_transfer_gws`: list of gameweek numbers where transfers are not allowed - Example: `"no_transfer_gws": []` - `transfer_itb_buffer`: minimum ITB (in the bank) to maintain if any transfer is planned (for robustness) - Example: `"transfer_itb_buffer": null` or `0.1` to leave £0.1m in bank - `booked_transfers`: pre-scheduled transfers for future gameweeks - Format: `[{"gw": 5, "transfer_in": 427}, {"gw": 7, "transfer_out": 427}]` (buy player 427 on GW5, sell on GW7) - `only_booked_transfers`: if `true`, next GW can only use booked transfers - Example: `"only_booked_transfers": false` - `no_trs_except_wc`: if `true`, prevent transfers except via Wildcard - Example: `"no_trs_except_wc": false` ### Player Management (Advanced) - `banned_next_gw`: list of player IDs to ban from next gameweek, or `[ID, GW]` to ban for specific GW - Example: `"banned_next_gw": [100, [200, 32]]` bans player ID 100 next GW, and player ID 200 for GW32 only - `locked_next_gw`: list of player IDs to force into next gameweek's squad (supports per-GW like `banned_next_gw`) - Example: `"locked_next_gw": []` - `keep`: list of player IDs that will not be kept throughout the player filtering process, even if they would otherwise be filtered out. - Example: `"keep": []` - `price_changes`: list of `[player_ID, price_change]` pairs to simulate price changes (in £0.1m increments) - Example: `"price_changes": [[311, 1], [351, -1]]` simulates player 311 up £0.1m, player 351 down £0.1m - `pick_prices`: force players at specific price points by position - Example: `"pick_prices": {"G": "", "D": "", "M": "8", "F": "11.5,11.5"}` ### Randomization - `randomized`: if `true`, add random noise to expected values for varied solutions - Example: `"randomized": false` - `randomization_seed`: seed for reproducible random noise (null = different seed each time) - Example: `"randomization_seed": null` - `randomization_strength`: multiplier for the random noise (default 1.0) - Example: `"randomization_strength": 1.0` ### Chip Management - `chip_limits`: maximum count for each chip type (note: this does not need to be edited if using `use_fh`, `use_wc` etc. ) - Example: `"chip_limits": {"bb": 0, "wc": 0, "fh": 0, "tc": 0}` - `no_chip_gws`: list of gameweeks where no chips can be used - Example: `"no_chip_gws": []` - `allowed_chip_gws`: dictionary of chip types to lists of allowed gameweeks - Example: `"allowed_chip_gws": {"wc": [25, 27], "fh": [30, 31]}` - `forced_chip_gws`: dictionary of chip types to lists of gameweeks where chip MUST be used - Example: `"forced_chip_gws": {"wc": [], "bb": [], "fh": [], "tc": []}` - `preseason`: special flag for GW1 solving where team data is not important - Example: `"preseason": false` ### Lineup Constraints - `no_opposing_play`: controls opposing-play logic - `true` = no two players can play each other in same GW - `false` = no constraint - `"penalty"` = penalize each opposing-play instance - Example: `"no_opposing_play": false` - `opposing_play_group`: scope of opposing-play constraint - `"all"` = no opposing players at all - `"position"` = only offense vs defense (not M/F vs each other or D/G vs each other) - Example: `"opposing_play_group": "position"` - `opposing_play_penalty`: penalty deducted per opposing-play when using `"penalty"` mode - Example: `"opposing_play_penalty": 0.5` - `max_defenders_per_team`: maximum defenders + goalkeepers from single team (default 3) - Example: `"max_defenders_per_team": 3` - `double_defense_pick`: forces solver to use either 0 or 2+ defenders/GKs from each team - Example: `"double_defense_pick": false` - `no_gk_rotation_after`: gameweek after which to lock to single goalkeeper (no rotation) - Example: `"no_gk_rotation_after": null` ### Solution Variants - `num_iterations`: number of alternative solutions to generate - Example: `"num_iterations": 1` - `iteration_criteria`: rule for what makes each iteration "different" - Options: `"this_gw_transfer_in"`, `"this_gw_transfer_out"`, `"this_gw_transfer_in_out"`, `"chip_gws"`, `"target_gws_transfer_in"`, `"this_gw_lineup"` - Example: `"iteration_criteria": "this_gw_transfer_in_out"` - `iteration_difference`: number of players that must differ (only for `"this_gw_lineup"` criteria) - Example: `"iteration_difference": 1` - `iteration_target`: list of gameweeks to target for iteration changes (used with `"target_gws_transfer_in"`) - Example: `"iteration_target": []` ### Data Sources - `data_weights`: weight percentage for each data source when using `"datasource": "mixed"` - Example: `"data_weights": {"solio": 1, "review": 1}` - `export_data`: option to export mixed data as CSV - Example: `"export_data": "mixed.csv"` - `report_decay_base`: list of decay bases to compute and report for the solve - Example: `"report_decay_base": [0.85, 1.0, 1.017]` ### Team Data Options - `team_json`: supply team JSON inline (requires `"team_data": "json"`) - Can be run as: `uv run python solve.py --team_json '{"picks": [{...}]}'` - Example: `"team_json": null` - `override_next_gw`: override the starting gameweek for planning horizon - Example: `"override_next_gw": null` ### Solver Behavior - `secs`: time limit for solver in seconds - Example: `"secs": 600` (10 minutes) - `gap`: relative optimality gap (0.0–1.0). Solver stops when within this gap of optimal. Set to 0 for proven optimality - Example: `"gap": 0` (solve to optimality) - `delete_tmp`: if `true`, delete temporary solver files after solving - Example: `"delete_tmp": true` - `single_solve`: internal flag for single solve mode - Example: `"single_solve": true` - `solver`: which solver to use (e.g., `"highs"`) - Example: `"solver": "highs"` ### Output & Exports - `export_image`: if `true`, generate and export lineup visualizations - Example: `"export_image": false` - `solve_name`: name for the solve (used in output filenames) - Example: `"solve_name": "regular"` - `print_result_table`: if `true`, print result table to console - Example: `"print_result_table": true` - `print_decay_metrics`: if `true`, print decay metric analysis to console - Example: `"print_decay_metrics": false` - `print_transfer_chip_summary`: if `true`, print transfer/chip summary per gameweek - Example: `"print_transfer_chip_summary": true` - `print_squads`: if `true`, print full lineup and bench for each gameweek - Example: `"print_squads": true` - `dataframe_format`: [tabulate](https://pypi.org/project/tabulate/) format for printing tables - Examples: `"plain"`, `"rounded_grid"`, `"fancy_outline"` - `hide_transfers`: if `true`, hide transfer details in result table - Example: `"hide_transfers": false` - `solutions_file`: if provided (filepath ending in `.csv`), save all solutions to this file - Example: `"solutions_file": ""` - `save_squads`: if `true` and `solutions_file` is set, include lineup/bench info per gameweek - Example: `"save_squads": true` - `solutions_file_player_type`: whether solutions file contains player `"id"` or `"name"` - Example: `"solutions_file_player_type": "name"` ### Binary Files (Advanced) - `binary_file_weights`: configure binary file names and weights - Example: `"binary_file_weights": {"binary_1.csv": 0.6, "binary_2.csv": 0.3, "binary_3.csv": 0.1}` - `generate_binary_files`: if `true`, generate binary files based on fixture settings - Example: `"generate_binary_files": false` --- ## Complete Reference For a complete listing of all settings and their default values, see [`comprehensive_settings.json`](comprehensive_settings.json). | Setting | Type | User-Friendly? | |----------------|------|----------------| | [`allowed_chip_gws`](#chip-management) | dict | ❌ | | [`banned`](#player-constraints) | list | ✅ | | [`banned_next_gw`](#player-management-advanced) | list | ❌ | | [`bench_weights`](#scoring--objective-function) | dict | ❌ | | [`binary_file_weights`](#binary-files-advanced) | dict | ❌ | | [`booked_transfers`](#transfer--hit-management) | list | ❌ | | [`chip_limits`](#chip-management) | dict | ❌ | | [`data_weights`](#data-sources) | dict | ❌ | | [`datasource`](#data-source--team) | string | ✅ | | [`dataframe_format`](#output--exports) | string | ❌ | | [`decay_base`](#decay--valuation) | float | ✅ | | [`delete_tmp`](#solver-behavior) | bool | ❌ | | [`double_defense_pick`](#lineup-constraints) | bool | ❌ | | [`ev_per_price_cutoff`](#player-pool-filtering) | int | ✅ | | [`export_data`](#data-sources) | string | ❌ | | [`export_image`](#output--exports) | bool | ❌ | | [`force_ft_state_lb`](#transfer--hit-management) | list | ❌ | | [`force_ft_state_ub`](#transfer--hit-management) | list | ❌ | | [`forced_chip_gws`](#chip-management) | dict | ❌ | | [`ft_use_penalty`](#scoring--objective-function) | float | ❌ | | [`ft_value`](#scoring--objective-function) | float | ❌ | | [`ft_value_list`](#decay--valuation) | dict | ✅ | | [`future_transfer_limit`](#transfer--hit-management) | int | ❌ | | [`gap`](#solver-behavior) | float | ❌ | | [`generate_binary_files`](#binary-files-advanced) | bool | ❌ | | [`hide_transfers`](#output--exports) | bool | ❌ | | [`hit_cost`](#transfer--hit-management) | int | ❌ | | [`hit_limit`](#transfer--hit-management) | int | ❌ | | [`horizon`](#planning-horizon) | int | ✅ | | [`itb_loss_per_transfer`](#scoring--objective-function) | float | ❌ | | [`itb_value`](#scoring--objective-function) | float | ❌ | | [`iteration_criteria`](#solution-variants) | string | ❌ | | [`iteration_difference`](#solution-variants) | int | ❌ | | [`iteration_target`](#solution-variants) | list | ❌ | | [`keep`](#player-management-advanced) | list | ❌ | | [`keep_top_ev_percent`](#player-pool-filtering) | int | ✅ | | [`locked`](#player-constraints) | list | ✅ | | [`locked_next_gw`](#player-management-advanced) | list | ❌ | | [`max_defenders_per_team`](#lineup-constraints) | int | ❌ | | [`no_chip_gws`](#chip-management) | list | ❌ | | [`no_future_transfer`](#transfer--hit-management) | bool | ❌ | | [`no_gk_rotation_after`](#lineup-constraints) | int | ❌ | | [`no_opposing_play`](#lineup-constraints) | bool/str | ❌ | | [`no_transfer_by_position`](#transfer--hit-management) | list | ❌ | | [`no_transfer_gws`](#transfer--hit-management) | list | ❌ | | [`no_transfer_last_gws`](#transfer-constraints) | int | ✅ | | [`no_trs_except_wc`](#transfer--hit-management) | bool | ❌ | | [`num_iterations`](#solution-variants) | int | ❌ | | [`num_transfers`](#transfer--hit-management) | int | ❌ | | [`only_booked_transfers`](#transfer--hit-management) | bool | ❌ | | [`opposing_play_group`](#lineup-constraints) | string | ❌ | | [`opposing_play_penalty`](#lineup-constraints) | float | ❌ | | [`override_next_gw`](#team-data-options) | int | ❌ | | [`pick_prices`](#player-management-advanced) | dict | ❌ | | [`preseason`](#chip-management) | bool | ❌ | | [`price_changes`](#player-management-advanced) | list | ❌ | | [`print_decay_metrics`](#output--exports) | bool | ❌ | | [`print_result_table`](#output--exports) | bool | ❌ | | [`print_squads`](#output--exports) | bool | ❌ | | [`print_transfer_chip_summary`](#output--exports) | bool | ❌ | | [`randomization_seed`](#randomization) | int/null | ❌ | | [`randomization_strength`](#randomization) | float | ❌ | | [`randomized`](#randomization) | bool | ❌ | | [`report_decay_base`](#data-sources) | list | ❌ | | [`secs`](#solver-behavior) | int | ❌ | | [`single_solve`](#solver-behavior) | bool | ❌ | | [`solver`](#solver-behavior) | string | ❌ | | [`solutions_file`](#output--exports) | string | ❌ | | [`solutions_file_player_type`](#output--exports) | string | ❌ | | [`solve_name`](#output--exports) | string | ❌ | | [`team_data`](#data-source--team) | string | ✅ | | [`team_id`](#data-source--team) | int | ✅ | | [`team_json`](#team-data-options) | object | ❌ | | [`transfer_itb_buffer`](#transfer--hit-management) | float | ❌ | | [`use_bb`](#chips) | list | ✅ | | [`use_fh`](#chips) | list | ✅ | | [`use_tc`](#chips) | list | ✅ | | [`use_wc`](#chips) | list | ✅ | | [`vcap_weight`](#scoring--objective-function) | float | ❌ | | [`verbose`](#output) | bool | ✅ | | [`xmin_lb`](#player-pool-filtering) | int | ✅ | ================================================ FILE: data/binary_fixtures.md ================================================ ## Binary Files This file contains instructions for generating binary files for when there are fixture uncertainties, for use with `simulations.py`. ### Instructions 1. Set the fixtures in your data source so there are no blank or double gameweeks, i.e. each team should have the fixtures as originally scheduled in their respective GW. Download that CSV and save it as `data/original.csv`. 2. Configure the settings in your `data/user_settings.json` file. **(a) binary_fixture_settings** - Create the settings that describe which fixtures are being moved in each binary file. For each team, the dictionary contains `{a:b}` pairs, where `a` represents the gameweek the expected points will be taken from, and `b` represents the gameweek where the expected points will be put into. In the example configuration provided below and looking at `binary1.csv`, expected points will be moved from GW33 to GW34 for Bournemounth, Man Utd, Man City, and Aston Villa, and from GW36 to GW34 for Newcastle and Ipswich. Note that the team names here should match with the team names in your data source. **(b) binary_file_weights** - Set the weighting for each possible binary file. In the example below, 60% of simulations will use `binary1.csv`, 30% will use `binary2.csv`, and 10% will use `binary3.csv`. **(c) generate_binary_files** - Set this to `true` if you want to generate the binary files. This is only a setting because if you run multiple sets of simulations, there is sometimes no need to regenerate the files, so this setting can just be set to `false`. 3. Run `simulations.py` and when it asks whether you want to use binaries, respond with `y`. ### Example JSON Settings ```json { "generate_binary_files": true, "binary_file_weights": { "binary1.csv": 0.6, "binary2.csv": 0.3, "binary3.csv": 0.1 }, "binary_fixture_settings": { "binary1.csv": { "Bournemouth": { "33": "34" }, "Man Utd": { "33": "34" }, "Man City": { "33": "34" }, "Aston Villa": { "33": "34" }, "Newcastle": { "36": "34" }, "Ipswich": { "36": "34" } }, "binary2.csv": { "Bournemouth": { "33": "34", "36": "37" }, "Man Utd": { "33": "34" }, "Man City": { "33": "34", "36": "37" }, "Aston Villa": { "33": "34" }, "Newcastle": { "36": "34" }, "Ipswich": { "36": "34" } }, "binary3.csv": { "Man City": { "33": "34" }, "Aston Villa": { "33": "34" }, "Newcastle": { "36": "34" }, "Ipswich": { "36": "34" }, "Arsenal": { "36": "34" }, "Crystal Palace": { "36": "34" } } } } ``` ================================================ FILE: data/comprehensive_settings.json ================================================ { "horizon": 8, "decay_base": 0.9, "ft_value": 1.5, "ft_value_list": { "2": 2, "3": 1.6, "4": 1.3, "5": 1.1 }, "bench_weights": { "0": 0.03, "1": 0.21, "2": 0.06, "3": 0.002 }, "vcap_weight": 0.1, "ft_use_penalty": 0.2, "itb_value": 0.08, "itb_loss_per_transfer": 0, "no_future_transfer": false, "no_transfer_last_gws": 2, "no_transfer_by_position": [], "force_ft_state_lb": [], "force_ft_state_ub": [], "randomized": false, "randomization_seed": null, "randomization_strength": 1.0, "xmin_lb": 300, "ev_per_price_cutoff": 30, "keep_top_ev_percent": 5, "banned": [], "banned_next_gw": [], "locked": [], "locked_next_gw": [], "price_changes": [], "keep": [], "delete_tmp": true, "single_solve": true, "solver": "highs", "secs": 600, "gap": 0, "num_transfers": null, "hit_limit": null, "weekly_hit_limit": 0, "hit_cost": 4, "use_wc": [], "use_bb": [], "use_fh": [], "use_tc": [], "chip_limits": { "bb": 0, "wc": 0, "fh": 0, "tc": 0 }, "no_chip_gws": [], "allowed_chip_gws": { "bb": [], "wc": [], "fh": [], "tc": [] }, "forced_chip_gws": { "bb": [], "wc": [], "fh": [], "tc": [] }, "future_transfer_limit": null, "no_transfer_gws": [], "booked_transfers": [], "only_booked_transfers": false, "no_trs_except_wc": false, "preseason": false, "no_opposing_play": false, "opposing_play_group": "position", "opposing_play_penalty": 0.5, "pick_prices": { "G": "", "D": "", "M": "", "F": "" }, "no_gk_rotation_after": null, "max_defenders_per_team": 3, "double_defense_pick": false, "transfer_itb_buffer": null, "num_iterations": 1, "iteration_criteria": "this_gw_transfer_in_out", "iteration_difference": 1, "iteration_target": [], "report_decay_base": [ 0.85, 1.0, 1.017 ], "datasource": "solio", "data_weights": { "solio": 1, "review": 1 }, "export_data": "mixed.csv", "team_data": "json", "team_id": null, "team_json": null, "export_image": false, "solve_name": "regular", "override_next_gw": null, "generate_binary_files": false, "binary_file_weights": {}, "binary_fixture_settings": {}, "verbose": true, "print_result_table": true, "print_decay_metrics": false, "print_transfer_chip_summary": true, "print_squads": true, "dataframe_format": "plain", "hide_transfers": false, "solutions_file": "", "save_squads": true, "solutions_file_player_type": "name" } ================================================ FILE: data/images/.gitkeep ================================================ ================================================ FILE: data/results/.gitkeep ================================================ ================================================ FILE: data/team.json.sample ================================================ { "picks": [ { "element": 307, "position": 1, "selling_price": 55, "multiplier": 1, "purchase_price": 55, "is_captain": false, "is_vice_captain": false }, { "element": 284, "position": 2, "selling_price": 70, "multiplier": 1, "purchase_price": 70, "is_captain": false, "is_vice_captain": false }, { "element": 8, "position": 3, "selling_price": 50, "multiplier": 1, "purchase_price": 50, "is_captain": false, "is_vice_captain": false }, { "element": 429, "position": 4, "selling_price": 50, "multiplier": 1, "purchase_price": 50, "is_captain": false, "is_vice_captain": false }, { "element": 309, "position": 5, "selling_price": 60, "multiplier": 1, "purchase_price": 60, "is_captain": false, "is_vice_captain": true }, { "element": 33, "position": 6, "selling_price": 50, "multiplier": 1, "purchase_price": 50, "is_captain": false, "is_vice_captain": false }, { "element": 301, "position": 7, "selling_price": 120, "multiplier": 1, "purchase_price": 120, "is_captain": false, "is_vice_captain": false }, { "element": 142, "position": 8, "selling_price": 80, "multiplier": 2, "purchase_price": 80, "is_captain": true, "is_vice_captain": false }, { "element": 383, "position": 9, "selling_price": 45, "multiplier": 1, "purchase_price": 45, "is_captain": false, "is_vice_captain": false }, { "element": 66, "position": 10, "selling_price": 60, "multiplier": 1, "purchase_price": 60, "is_captain": false, "is_vice_captain": false }, { "element": 427, "position": 11, "selling_price": 115, "multiplier": 1, "purchase_price": 115, "is_captain": false, "is_vice_captain": false }, { "element": 289, "position": 12, "selling_price": 40, "multiplier": 0, "purchase_price": 40, "is_captain": false, "is_vice_captain": false }, { "element": 13, "position": 13, "selling_price": 80, "multiplier": 0, "purchase_price": 80, "is_captain": false, "is_vice_captain": false }, { "element": 116, "position": 14, "selling_price": 55, "multiplier": 0, "purchase_price": 55, "is_captain": false, "is_vice_captain": false }, { "element": 476, "position": 15, "selling_price": 70, "multiplier": 0, "purchase_price": 70, "is_captain": false, "is_vice_captain": false } ], "chips": [ { "status_for_entry": "available", "played_by_entry": [], "name": "bboost", "number": 1, "start_event": 1, "stop_event": 38, "chip_type": "team" }, { "status_for_entry": "available", "played_by_entry": [], "name": "3xc", "number": 1, "start_event": 1, "stop_event": 38, "chip_type": "team" } ], "transfers": { "cost": 4, "status": "unlimited", "limit": null, "made": 0, "bank": 0, "value": 1000 } } ================================================ FILE: data/user_settings.json ================================================ { "datasource": "solio", "team_data": "id", "team_id": null, "horizon": 8, "no_transfer_last_gws": 2, "decay_base": 0.9, "ft_value_list": { "2": 2, "3": 1.6, "4": 1.3, "5": 1.1 }, "xmin_lb": 300, "ev_per_price_cutoff": 30, "keep_top_ev_percent": 5, "banned": [], "locked": [], "use_wc": [], "use_bb": [], "use_fh": [], "use_tc": [], "verbose": true } ================================================ FILE: dev/data_parser.py ================================================ import csv import os import sys from unicodedata import combining, normalize import numpy as np import pandas as pd import requests from fuzzywuzzy import fuzz from paths import DATA_DIR from utils import cached_request def read_data(options, source=None): source = options.get("datasource") weights = options.get("data_weights") list_of_files = [x for x in os.listdir(DATA_DIR) if x.endswith(".csv")] if not source: try: latest_file = max(list_of_files, key=os.path.getctime) print(f"No source specified, using most recent projection file: {latest_file}") return pd.read_csv(latest_file) except Exception: print("Cannot find projection data in /data/. Upload it to /data/ and make sure it is a .csv file") sys.exit(0) if source == "mixed": return read_mixed(options, weights) if f"{source}.csv" not in list_of_files: raise FileNotFoundError(f"Data file {source}.csv not found in /data/. Please upload it there and try again.") for reader in [read_mikkel, read_solio, read_fplreview]: try: return reader(options) except Exception: # print(f"{reader.__name__} failed: {e}") continue raise RuntimeError("All data readers failed.") def read_solio(options): # TODO: implement more complex solio data parsing when additional data is added to csv filepath = options.get("data_path", DATA_DIR / f"{options['datasource']}.csv") return pd.read_csv(filepath, encoding="utf-8") def read_fplreview(options): filepath = options.get("data_path", DATA_DIR / f"{options['datasource']}.csv") return pd.read_csv(filepath, encoding="utf-8") def read_mikkel(options): output_file = "mikkel_cleaned.csv" input_file = options.get("data_path", DATA_DIR / f"{options['datasource']}.csv") convert_mikkel_to_review(input_file, output_file=output_file) return pd.read_csv(DATA_DIR / f"{output_file}", encoding="utf-8") def read_mixed(options, weights): # Get each source separately and mix with given weights all_data = [] for name, weight in weights.items(): if weight == 0: continue options["datasource"] = name df = read_data(options) # drop players without data first_gw_col = None for col in df.columns: if "_Pts" in col: first_gw_col = col break # drop missing ones df = df[~df[first_gw_col].isnull()].copy() for col in df.columns: if "_Pts" in col: df[col.split("_")[0] + "_weight"] = weight all_data.append(df) for i, d in enumerate(all_data): # d["ID"] = d["ID"].astype(np.int64) for col in d.columns: if "_xMins" in col: d[col] = pd.to_numeric(d[col], errors="coerce").fillna(0).astype(int) all_data[i] = d # Update EV by weight new_data = [] # for d, w in zip(data, data_weights): for d in all_data: pts_columns = [i for i in d if "_Pts" in i] min_columns = [i for i in d if "_xMins" in i] weights_cols = [i.split("_")[0] + "_weight" for i in pts_columns] # d[pts_columns] = d[pts_columns].multiply(d[weights_cols], axis='index') d[pts_columns] = pd.DataFrame(d[pts_columns].values * d[weights_cols].values, columns=d[pts_columns].columns, index=d[pts_columns].index) weights_cols = [i.split("_")[0] + "_weight" for i in min_columns] d[min_columns] = pd.DataFrame(d[min_columns].values * d[weights_cols].values, columns=d[min_columns].columns, index=d[min_columns].index) new_data.append(d.copy()) combined_data = pd.concat(new_data, ignore_index=True) combined_data = combined_data.copy() combined_data["real_id"] = combined_data["ID"] combined_data = combined_data.reset_index(drop=True) key_dict = {} for i in combined_data.columns.to_list(): if "_weight" in i: # weight column key_dict[i] = "sum" elif "_xMins" in i: key_dict[i] = "sum" elif "_Pts" in i: key_dict[i] = "sum" else: key_dict[i] = "first" # key_dict = {i: 'first' if ("_x" not in i and "_P" not in i) else 'median' for i in main_keys} grouped_data = combined_data.groupby("real_id").agg(key_dict) final_data = grouped_data[grouped_data["ID"] != 0].copy() # adjust by weight sum for each player for c in final_data.columns: if "_Pts" in c or "_xMins" in c: gw = c.split("_")[0] final_data[c] = final_data[c] / final_data[gw + "_weight"] # Find missing players and add them fpl_data = cached_request("https://fantasy.premierleague.com/api/bootstrap-static/") players = fpl_data["elements"] existing_ids = final_data["ID"].tolist() element_type_dict = {1: "G", 2: "D", 3: "M", 4: "F"} teams = fpl_data["teams"] team_code_dict = {i["code"]: i for i in teams} missing_players = [] for p in players: if p["id"] in existing_ids: continue missing_players.append( { "fpl_id": p["id"], "ID": p["id"], "real_id": p["id"], "team": "", "Name": p["web_name"], "Pos": element_type_dict[p["element_type"]], "Value": p["now_cost"] / 10, "Team": team_code_dict[p["team_code"]]["name"], "Missing": 1, } ) final_data = pd.concat([final_data, pd.DataFrame(missing_players)]).fillna(0) final_data.to_csv(DATA_DIR / "mixed.csv", index=False, encoding="utf-8", float_format="%.2f") return final_data # To remove accents in names def fix_name_dialect(name): new_name = "".join([c for c in normalize("NFKD", name) if not combining(c)]) return new_name.replace("Ø", "O").replace("ø", "o").replace("ã", "a") def get_best_score(r): return max(r["wn_score"], r["cn_score"]) # To add FPL ID column to Mikkel's data and clean empty rows def fix_mikkel(file_address): for enc in ["utf-8", "latin-1"]: try: with open(file_address, encoding=enc, errors="replace") as f: # Use csv.Sniffer to detect delimiter, either , or ; dialect = csv.Sniffer().sniff(f.readline(), delimiters=",;") df = pd.read_csv(file_address, encoding=enc, sep=dialect.delimiter) break except Exception: continue fpl_data = cached_request("https://fantasy.premierleague.com/api/bootstrap-static/") players = fpl_data["elements"] mikkel_team_dict = { "BHA": "BRI", "CRY": "CPL", "NFO": "NOT", "WHU": "WHM", } teams = fpl_data["teams"] for t in teams: t["mikkel_short"] = mikkel_team_dict.get(t["short_name"], t["short_name"]) df = df.rename(columns={x: x.strip() for x in df.columns}) df["BCV_clean"] = df["BCV"].astype(str).str.replace(r"\((.*)\)", "-\\1", regex=True).astype(str).str.strip() df["BCV_numeric"] = pd.to_numeric(df["BCV_clean"], errors="coerce") df = df.loc[df["BCV_numeric"] != -1] df_cleaned = df.loc[~((df["Player"] == "0") | (df["No."].isnull()) | (df["BCV_numeric"].isnull()))].copy() df_cleaned["Clean_Name"] = df_cleaned["Player"].apply(fix_name_dialect) # df_cleaned["Team"] = df_cleaned["Team"].map(mikkel_team_dict, na_action="ignore") df_cleaned["Position"] = df_cleaned["Position"].replace({"GK": "G"}) df_cleaned = df_cleaned.dropna(subset=["Team"]) element_type_dict = {1: "G", 2: "D", 3: "M", 4: "F"} team_code_dict = {i["code"]: i for i in teams} player_names = [ { "id": e["id"], "web_name": e["web_name"], "combined": e["first_name"] + " " + e["second_name"], "team": team_code_dict[e["team_code"]]["mikkel_short"], "position": element_type_dict[e["element_type"]], } for e in players ] for target in player_names: target["wn"] = fix_name_dialect(target["web_name"]) target["cn"] = fix_name_dialect(target["combined"]) entries = [] for player in df_cleaned.iloc: possible_matches = [i for i in player_names if i["team"] == player["Team"] and i["position"] == player["Position"]] for target in possible_matches: p = player["Clean_Name"] target["wn_score"] = fuzz.token_set_ratio(p, target["wn"]) target["cn_score"] = fuzz.token_set_ratio(p, target["cn"]) best_match = max(possible_matches, key=get_best_score) entries.append({"player_input": player["Player"], "team_input": player["Team"], "position_input": player["Position"], **best_match}) entries_df = pd.DataFrame(entries) entries_df["score"] = entries_df[["wn_score", "cn_score"]].max(axis=1) entries_df["name_team"] = entries_df["player_input"] + " @ " + entries_df["team_input"] entry_dict = entries_df.set_index("name_team")["id"].to_dict() fpl_name_dict = entries_df.set_index("id")["web_name"].to_dict() score_dict = entries_df.set_index("name_team")["score"].to_dict() df_cleaned["name_team"] = df_cleaned["Player"] + " @ " + df_cleaned["Team"] df_cleaned["FPL ID"] = df_cleaned["name_team"].map(entry_dict) df_cleaned["fpl_name"] = df_cleaned["FPL ID"].map(fpl_name_dict) df_cleaned["score"] = df_cleaned["name_team"].map(score_dict) # Check for duplicate IDs duplicate_rows = df_cleaned["FPL ID"].duplicated(keep=False) if len(df_cleaned[duplicate_rows]) > 0: print("WARNING: There are players with duplicate IDs, lowest name match accuracy (score) will be dropped") print(df_cleaned[duplicate_rows][["Player", "fpl_name", "score"]].head()) df_cleaned = df_cleaned.sort_values(by=["score"], ascending=False) df_cleaned = df_cleaned.loc[~df_cleaned["FPL ID"].duplicated(keep="first")].sort_index() existing_ids = df_cleaned["FPL ID"].tolist() missing_players = [] for p in players: if p["id"] in existing_ids: continue missing_players.append( { "Position": element_type_dict[p["element_type"]], "Player": p["web_name"], "Price": p["now_cost"] / 10, "FPL ID": p["id"], "Weighted minutes": 0, "Missing": 1, } ) return pd.concat([df_cleaned, pd.DataFrame(missing_players)]).fillna(0) # To convert cleaned Mikkel data into Review format def convert_mikkel_to_review(target, output_file): # Read and add ID column df = fix_mikkel(target) static_url = "https://fantasy.premierleague.com/api/bootstrap-static/" fpl_data = cached_request(static_url) teams = fpl_data["teams"] new_names = {i: i.strip() for i in df.columns} df = df.rename(columns=new_names) df["Price"] = pd.to_numeric(df["Price"], errors="coerce") df["Weighted minutes"] = df["Weighted minutes"].fillna(90) df["ID"] = df["FPL ID"].astype(int) pos_fix = {"GK": "G"} df["Pos"] = df["Position"] df["Pos"] = df["Pos"].map(pos_fix).fillna(df["Pos"]) df.loc[df["Pos"].isin(["G", "D"]), "Weighted minutes"] = "90" gws = [] for i in df.columns: try: int(i) df[f"{i}_Pts"] = df[i].str.strip().replace({"-": 0}).astype(float) df[f"{i}_xMins"] = df["Weighted minutes"].str.strip().replace({"-": 0}).astype(float).replace({np.nan: 0}) gws.append(i) except Exception: continue df["Name"] = df["Player"] df["Value"] = df["Price"] df_final = df[["ID", "Name", "Pos", "Value"] + [f"{gw}_{tag}" for gw in gws for tag in ["Pts", "xMins"]]].copy() elements_data = fpl_data["elements"] player_ids = [i["id"] for i in elements_data] player_names = {i["id"]: i["web_name"] for i in elements_data} player_pos = {i["id"]: i["element_type"] for i in elements_data} player_price = {i["id"]: i["now_cost"] / 10 for i in elements_data} pos_no = {1: "G", 2: "D", 3: "M", 4: "F"} values = [] existing_players = df_final["ID"].to_list() for i in player_ids: if i not in existing_players: entry = { "ID": i, "Name": player_names[i], "Pos": pos_no[player_pos[i]], "Value": player_price[i], **{f"{gw}_{tag}": 0 for gw in gws for tag in ["Pts", "xMins"]}, } values.append(entry) team_data = teams team_dict = {i["code"]: i["name"] for i in team_data} player_teams = {i["id"]: team_dict[i["team_code"]] for i in elements_data} # Add missing players # df_final = pd.concat([df_final, pd.DataFrame(values, columns=df_final.columns)], ignore_index=True) df_final["Team"] = df_final["ID"].map(player_teams) df_final["fpl_id"] = df_final["ID"] df_final["Name"] = df_final["ID"].replace(player_names) df_final = df_final.set_index("fpl_id") df_final.to_csv(DATA_DIR / output_file, index=False, encoding="utf-8", float_format="%.2f") # convert_mikkel_to_review("../data/TransferAlgorithm.csv") ================================================ FILE: dev/solver.py ================================================ import os import subprocess import threading import time import warnings from collections import Counter from pathlib import Path import highspy import numpy as np import pandas as pd import sasoptpy as so from dev.data_parser import read_data from utils import cached_request, get_random_id warnings.filterwarnings("ignore", category=FutureWarning, module="sasoptpy") BINARY_THRESHOLD = 0.5 # threshold value for evaluating binary variables BASE_URL = "https://fantasy.premierleague.com/api" IS_COLAB = "COLAB_GPU" in os.environ SQUAD_SIZE = 15 LINEUP_SIZE = 11 MAX_GAMEWEEK = 38 MAX_PLAYERS_PER_TEAM = 3 AFCON_GW = 16 def generate_team_json(team_id, options): static_url = f"{BASE_URL}/bootstrap-static/" static = cached_request(static_url) element_to_type_dict = {x["id"]: x["element_type"] for x in static["elements"]} next_gw = next(x for x in static["events"] if x["is_next"])["id"] start_prices = {x["id"]: x["now_cost"] - x["cost_change_start"] for x in static["elements"]} transfers_url = f"{BASE_URL}/entry/{team_id}/transfers/" transfers = cached_request(transfers_url)[::-1] history_url = f"{BASE_URL}/entry/{team_id}/history/" history = cached_request(history_url) chips = history["chips"] fh_gws = [x["event"] for x in chips if x["name"] == "freehit"] wc_gws = [x["event"] for x in chips if x["name"] == "wildcard"] # find out the first gameweek that the user played in - don't assume gw1 first_gw = history["current"][0]["event"] first_gw_url = f"{BASE_URL}/entry/{team_id}/event/{first_gw}/picks/" first_gw_data = cached_request(first_gw_url) # squad will remain an ID:puchase_price map throughout iteration over transfers # once they have been iterated through, can then add on the current selling price squad = {x["element"]: start_prices[x["element"]] for x in first_gw_data["picks"]} itb = 1000 - sum(squad.values()) for t in transfers: if t["event"] in fh_gws: continue itb += t["element_out_cost"] itb -= t["element_in_cost"] if t["element_in"]: squad[t["element_in"]] = t["element_in_cost"] if t["element_out"]: del squad[t["element_out"]] fts = calculate_fts(transfers, first_gw, next_gw, fh_gws, wc_gws) my_data = {"chips": chips, "picks": [], "team_id": team_id, "transfers": {"bank": itb, "limit": fts, "made": 0}} for player_id, purchase_price in squad.items(): now_cost = next(x for x in static["elements"] if x["id"] == player_id)["now_cost"] diff = now_cost - purchase_price if diff > 0: selling_price = purchase_price + diff // 2 else: selling_price = now_cost my_data["picks"].append( {"element": player_id, "purchase_price": purchase_price, "selling_price": selling_price, "element_type": element_to_type_dict[player_id]} ) return my_data def calculate_fts(transfers, first_gw, next_gw, fh_gws, wc_gws): n_transfers = dict.fromkeys(range(2, next_gw), 0) for t in transfers: n_transfers[t["event"]] += 1 fts = dict.fromkeys(range(first_gw + 1, next_gw + 1), 0) fts[first_gw + 1] = 1 for i in range(first_gw + 2, next_gw + 1): if i == AFCON_GW: fts[i] = 5 continue if (i - 1) in fh_gws: fts[i] = fts[i - 1] continue if i - 1 in wc_gws: fts[i] = fts[i - 1] continue fts[i] = fts[i - 1] fts[i] -= n_transfers[i - 1] fts[i] = max(fts[i], 0) fts[i] += 1 fts[i] = min(fts[i], 5) return fts[next_gw] def prep_data(my_data, options): fpl_data = cached_request("https://fantasy.premierleague.com/api/bootstrap-static/") valid_ids = [x["id"] for x in fpl_data["elements"]] for pid, change in options.get("price_changes", []): if pid not in valid_ids: continue player = next(x for x in fpl_data["elements"] if x["id"] == pid) player["now_cost"] += change if options.get("override_next_gw", None): gw = int(options["override_next_gw"]) else: gw = 0 for e in fpl_data["events"]: if e["is_next"]: gw = e["id"] break horizon = options.get("horizon", 3) element_data = pd.DataFrame(fpl_data["elements"]) team_data = pd.DataFrame(fpl_data["teams"]) elements_team = pd.merge(element_data, team_data, left_on="team", right_on="id") element_to_team = {x["id"]: x["team"] for x in fpl_data["elements"]} # dict mapping element to team id max_players_from_team = Counter([element_to_team[x["element"]] for x in my_data["picks"]]).most_common(1)[0][1] if my_data["picks"] else 3 data = read_data(options) merged_data = pd.merge(elements_team, data, left_on="id_x", right_on="ID") merged_data.set_index(["id_x"], inplace=True) # Drop duplicates merged_data = merged_data.drop_duplicates(subset=["ID"], keep="first") # Check if data exists for week in range(gw, min(39, gw + horizon)): if f"{week}_Pts" not in data.keys(): raise ValueError(f"{week}_Pts is not inside prediction data - change your horizon or update your prediction data") original_keys = merged_data.columns.to_list() keys = [k for k in original_keys if "_Pts" in k] min_keys = [k for k in original_keys if "_xMins" in k] merged_data["total_ev"] = merged_data[keys].sum(axis=1) merged_data["total_min"] = merged_data[min_keys].sum(axis=1) merged_data.sort_values(by=["total_ev"], ascending=[False], inplace=True) locked_next_gw = [int(i[0]) if isinstance(i, list) else int(i) for i in options.get("locked_next_gw", [])] safe_players_due_price = [] for pos, vals in options.get("pick_prices", {}).items(): if vals is None or vals == "": continue price_vals = [float(i) for i in vals.split(",")] pp = merged_data[(merged_data["Pos"] == pos) & ((merged_data["now_cost"] / 10).isin(price_vals))]["ID"].to_list() safe_players_due_price += pp # Filter players by total EV cutoff = merged_data["total_ev"].quantile((100 - options.get("keep_top_ev_percent", 10)) / 100) safe_players_due_ev = merged_data[(merged_data["total_ev"] > cutoff)]["ID"].tolist() initial_squad = [int(i["element"]) for i in my_data["picks"]] safe_players = initial_squad + options.get("locked", []) + options.get("keep", []) + locked_next_gw + safe_players_due_price + safe_players_due_ev for bt in options.get("booked_transfers", []): if bt.get("transfer_in"): safe_players.append(bt["transfer_in"]) if bt.get("transfer_out"): safe_players.append(bt["transfer_out"]) # Filter players by xMin xmin_lb = options.get("xmin_lb", 100) num_players_before = len(merged_data) merged_data = merged_data[(merged_data["total_min"] >= xmin_lb) | (merged_data["ID"].isin(safe_players))].copy() # Filter by ev per price ev_per_price_cutoff = options.get("ev_per_price_cutoff", 0) if ev_per_price_cutoff != 0: cutoff = (merged_data["total_ev"] / merged_data["now_cost"]).quantile(ev_per_price_cutoff / 100) merged_data = merged_data[(merged_data["total_ev"] / merged_data["now_cost"] > cutoff) | (merged_data["ID"].isin(safe_players))].copy() num_players_after = len(merged_data) print(f"Filtered player pool from {num_players_before} to {num_players_after} players") if options.get("randomized", False): rng = np.random.default_rng(seed=options.get("randomization_seed")) gws = list(range(gw, min(39, gw + horizon))) for w in gws: noise = merged_data[f"{w}_Pts"] * (92 - merged_data[f"{w}_xMins"]) / 134 * rng.standard_normal(size=len(merged_data)) merged_data[f"{w}_Pts"] = merged_data[f"{w}_Pts"] + noise * options.get("randomization_strength", 1) type_data = pd.DataFrame(fpl_data["element_types"]).set_index(["id"]) buy_price = (merged_data["now_cost"] / 10).to_dict() sell_price = {i["element"]: i["selling_price"] / 10 for i in my_data["picks"]} price_modified_players = [] preseason = options.get("preseason", False) if not preseason: for i in my_data["picks"]: if buy_price[i["element"]] != sell_price[i["element"]]: price_modified_players.append(i["element"]) print(f"Added player {i['element']} to list, buy price {buy_price[i['element']]} sell price {sell_price[i['element']]}") itb = my_data["transfers"]["bank"] / 10 ft_base = None if my_data["transfers"]["limit"] is None: ft = 1 ft_base = 1 else: ft = my_data["transfers"]["limit"] - my_data["transfers"]["made"] ft_base = my_data["transfers"]["limit"] ft = max(ft, 0) # If wildcard is active, then you have: "status_for_entry": "active" under my_data['chips'] # can only pass the check when using "team_data": "json" for c in my_data["chips"]: if c["name"] == "wildcard" and c.get("status_for_entry", "") == "active": options["use_wc"] = [gw] if options["chip_limits"]["wc"] == 0: options["chip_limits"]["wc"] = 1 break # Fixture info team_code_dict = team_data.set_index("id")["name"].to_dict() fixture_data = cached_request("https://fantasy.premierleague.com/api/fixtures/") fixtures = [{"gw": f["event"], "home": team_code_dict[f["team_h"]], "away": team_code_dict[f["team_a"]]} for f in fixture_data] return { "merged_data": merged_data, "team_data": team_data, "my_data": my_data, "type_data": type_data, "next_gw": gw, "initial_squad": initial_squad, "sell_price": sell_price, "buy_price": buy_price, "price_modified_players": price_modified_players, "itb": itb, "ft": ft, "ft_base": ft_base, "fixtures": fixtures, "max_players_from_team": max_players_from_team, } def solve_multi_period_fpl(data, options): """ Solves multi-objective FPL problem with transfers Parameters ---------- data: dict Pre-processed data for the problem definition options: dict User controlled values for the problem instance """ print( "This solver is free for personal, educational, or non-commercial use under the " "Apache License 2.0. Commercial entities must obtain a Commercial License before " "accessing, viewing, or using the code for any commercial purposes. Unauthorized " "access or use by commercial entities without a valid commercial license is strictly " "prohibited. To obtain a commercial license, please contact us at " "info@fploptimized.com." ) try: commit_hash = subprocess.check_output(["git", "rev-parse", "--short", "HEAD"], stderr=subprocess.DEVNULL).decode("utf-8").strip() commit_count = subprocess.check_output(["git", "rev-list", "--count", "HEAD"], stderr=subprocess.DEVNULL).decode("utf-8").strip() version = f"{commit_count} - {commit_hash}" print(f"Version: {version}") except Exception: pass # Arguments problem_id = get_random_id(5) horizon = options.get("horizon", 3) objective = options.get("objective", "decay") decay_base = options.get("decay_base", 0.84) bench_weights = options.get("bench_weights", {0: 0.03, 1: 0.21, 2: 0.06, 3: 0.002}) bench_weights = {int(key): value for (key, value) in bench_weights.items()} # wc_limit = options.get('wc_limit', 0) ft_value = options.get("ft_value", 1.5) ft_value_list = options.get("ft_value_list", {}) # ft_gw_value = {} ft_use_penalty = options.get("ft_use_penalty", None) itb_value = options.get("itb_value", 0.08) initial_ft = max(0, data.get("ft", 1)) ft_base = data.get("ft_base", 1) chip_limits = options.get("chip_limits", {}) allowed_chip_gws = options.get("allowed_chip_gws", {}) forced_chip_gws = options.get("forced_chip_gws", {}) booked_transfers = options.get("booked_transfers", []) preseason = options.get("preseason", False) itb_loss_per_transfer = options.get("itb_loss_per_transfer", None) if itb_loss_per_transfer is None: itb_loss_per_transfer = 0 weekly_hit_limit = options.get("weekly_hit_limit", None) # Data problem_name = f"mp_h{horizon}_regular" if objective == "regular" else f"mp_h{horizon}_o{objective[0]}_d{decay_base}" merged_data = data["merged_data"] team_data = data["team_data"] type_data = data["type_data"] next_gw = data["next_gw"] initial_squad = data["initial_squad"] itb = data["itb"] fixtures = data["fixtures"] if preseason: itb = 100 threshold_gw = 2 else: threshold_gw = next_gw # Sets players = merged_data.index.to_list() el_types = type_data.index.to_list() teams = team_data["name"].to_list() last_gw = next_gw + horizon - 1 if last_gw > MAX_GAMEWEEK: last_gw = MAX_GAMEWEEK horizon = MAX_GAMEWEEK + 1 - next_gw gws = list(range(next_gw, last_gw + 1)) all_gw = [next_gw - 1, *gws] order = [0, 1, 2, 3] price_modified_players = data["price_modified_players"] ft_states = [0, 1, 2, 3, 4, 5] # Model model = so.Model(name=problem_name) # Variables squad = model.add_variables(players, all_gw, name="squad", vartype=so.binary) squad_fh = model.add_variables(players, gws, name="squad_fh", vartype=so.binary) lineup = model.add_variables(players, gws, name="lineup", vartype=so.binary) captain = model.add_variables(players, gws, name="captain", vartype=so.binary) vicecap = model.add_variables(players, gws, name="vicecap", vartype=so.binary) bench = model.add_variables(players, gws, order, name="bench", vartype=so.binary) transfer_in = model.add_variables(players, gws, name="transfer_in", vartype=so.binary) transfer_out_first = model.add_variables(price_modified_players, gws, name="tr_out_first", vartype=so.binary) transfer_out_regular = model.add_variables(players, gws, name="tr_out_reg", vartype=so.binary) transfer_out = { (p, w): transfer_out_regular[p, w] + (transfer_out_first[p, w] if p in price_modified_players else 0) for p in players for w in gws } in_the_bank = model.add_variables(all_gw, name="itb", vartype=so.continuous, lb=0) fts = model.add_variables(all_gw, name="ft", vartype=so.integer, lb=0, ub=5) ft_above_ub = model.add_variables(gws, name="ft_above", vartype=so.binary) ft_below_lb = model.add_variables(gws, name="ft_below", vartype=so.binary) fts_state = model.add_variables(gws, ft_states, name="ft_state", vartype=so.binary) penalized_transfers = model.add_variables(gws, name="pt", vartype=so.integer, lb=0) aux = model.add_variables(gws, name="aux", vartype=so.binary) transfer_count = model.add_variables(gws, name="trc", vartype=so.integer, lb=0, ub=SQUAD_SIZE) use_wc = model.add_variables(gws, name="use_wc", vartype=so.binary) use_bb = model.add_variables(gws, name="use_bb", vartype=so.binary) use_fh = model.add_variables(gws, name="use_fh", vartype=so.binary) use_tc = model.add_variables(players, gws, name="use_tc", vartype=so.binary) # Dictionaries lineup_type_count = {(t, w): so.expr_sum(lineup[p, w] for p in players if merged_data.loc[p, "element_type"] == t) for t in el_types for w in gws} squad_type_count = {(t, w): so.expr_sum(squad[p, w] for p in players if merged_data.loc[p, "element_type"] == t) for t in el_types for w in gws} squad_fh_type_count = { (t, w): so.expr_sum(squad_fh[p, w] for p in players if merged_data.loc[p, "element_type"] == t) for t in el_types for w in gws } player_type = merged_data["element_type"].to_dict() # player_price = (merged_data['now_cost'] / 10).to_dict() sell_price = data["sell_price"] buy_price = data["buy_price"] sold_amount = { w: ( so.expr_sum(sell_price[p] * transfer_out_first[p, w] for p in price_modified_players) + so.expr_sum(buy_price[p] * transfer_out_regular[p, w] for p in players) ) for w in gws } fh_sell_price = {p: sell_price[p] if p in price_modified_players else buy_price[p] for p in players} bought_amount = {w: so.expr_sum(buy_price[p] * transfer_in[p, w] for p in players) for w in gws} points_player_week = {(p, w): merged_data.loc[p, f"{w}_Pts"] for p in players for w in gws} minutes_player_week = {(p, w): merged_data.loc[p, f"{w}_xMins"] for p in players for w in gws} player_team = {p: merged_data.loc[p, "name"] for p in players} squad_count = {w: so.expr_sum(squad[p, w] for p in players) for w in gws} squad_fh_count = {w: so.expr_sum(squad_fh[p, w] for p in players) for w in gws} num_transfers = {w: so.expr_sum(transfer_out[p, w] for p in players) for w in gws} transfer_diff = {w: num_transfers[w] - fts[w] - SQUAD_SIZE * use_wc[w] for w in gws} use_tc_gw = {w: so.expr_sum(use_tc[p, w] for p in players) for w in gws} # Initial conditions model.add_constraints((squad[p, next_gw - 1] == 1 for p in initial_squad), name="initial_squad_players") model.add_constraints((squad[p, next_gw - 1] == 0 for p in players if p not in initial_squad), name="initial_squad_others") model.add_constraint(in_the_bank[next_gw - 1] == itb, name="initial_itb") model.add_constraint(fts[next_gw] == initial_ft * (1 - use_wc[next_gw]) + ft_base * use_wc[next_gw], name="initial_ft") model.add_constraints((fts[w] >= 1 for w in gws if w > next_gw), name="future_ft_limit") # Constraints model.add_constraints((squad_count[w] == SQUAD_SIZE for w in gws), name="squad_count") model.add_constraints((squad_fh_count[w] == SQUAD_SIZE * use_fh[w] for w in gws), name="squad_fh_count") model.add_constraints( (so.expr_sum(lineup[p, w] for p in players) == LINEUP_SIZE + (SQUAD_SIZE - LINEUP_SIZE) * use_bb[w] for w in gws), name="lineup_count" ) model.add_constraints((so.expr_sum(bench[p, w, 0] for p in players if player_type[p] == 1) == 1 - use_bb[w] for w in gws), name="bench_gk") model.add_constraints((so.expr_sum(bench[p, w, o] for p in players) == 1 - use_bb[w] for w in gws for o in [1, 2, 3]), name="bench_count") model.add_constraints((so.expr_sum(captain[p, w] for p in players) == 1 for w in gws), name="captain_count") model.add_constraints((so.expr_sum(vicecap[p, w] for p in players) == 1 for w in gws), name="vicecap_count") model.add_constraints((lineup[p, w] <= squad[p, w] + use_fh[w] for p in players for w in gws), name="lineup_squad_rel") model.add_constraints((bench[p, w, o] <= squad[p, w] + use_fh[w] for p in players for w in gws for o in order), name="bench_squad_rel") model.add_constraints((lineup[p, w] <= squad_fh[p, w] + 1 - use_fh[w] for p in players for w in gws), name="lineup_squad_fh_rel") model.add_constraints((bench[p, w, o] <= squad_fh[p, w] + 1 - use_fh[w] for p in players for w in gws for o in order), name="bench_squad_fh_rel") model.add_constraints((captain[p, w] <= lineup[p, w] for p in players for w in gws), name="captain_lineup_rel") model.add_constraints((vicecap[p, w] <= lineup[p, w] for p in players for w in gws), name="vicecap_lineup_rel") model.add_constraints((captain[p, w] + vicecap[p, w] <= 1 for p in players for w in gws), name="cap_vc_rel") model.add_constraints((lineup[p, w] + so.expr_sum(bench[p, w, o] for o in order) <= 1 for p in players for w in gws), name="lineup_bench_rel") model.add_constraints((lineup_type_count[t, w] >= type_data.loc[t, "squad_min_play"] for t in el_types for w in gws), name="valid_formation_lb") model.add_constraints( (lineup_type_count[t, w] <= type_data.loc[t, "squad_max_play"] + use_bb[w] for t in el_types for w in gws), name="valid_formation_ub" ) model.add_constraints((squad_type_count[t, w] == type_data.loc[t, "squad_select"] for t in el_types for w in gws), name="valid_squad") model.add_constraints( (squad_fh_type_count[t, w] == type_data.loc[t, "squad_select"] * use_fh[w] for t in el_types for w in gws), name="valid_squad_fh" ) # special case where user's current squad has too many players from the same team # only works for 4 players from same team at the moment if data["max_players_from_team"] > MAX_PLAYERS_PER_TEAM: no_transfer = model.add_variables(gws, vartype=so.binary, name="no_transfer") model.add_constraints((transfer_count[w] <= SQUAD_SIZE * (1 - no_transfer[w]) for w in gws), name="no_transfer_1") model.add_constraints((transfer_count[w] >= 1 - SQUAD_SIZE * no_transfer[w] for w in gws), name="no_transfer_2") model.add_constraints( (so.expr_sum(squad[p, w] for p in players if player_team[p] == t) <= MAX_PLAYERS_PER_TEAM + no_transfer[w] for t in teams for w in gws), name="team_limit", ) else: # normal case where user has a valid squad model.add_constraints( (so.expr_sum(squad[p, w] for p in players if player_team[p] == t) <= MAX_PLAYERS_PER_TEAM for t in teams for w in all_gw), name="team_limit", ) model.add_constraints( (so.expr_sum(squad_fh[p, w] for p in players if player_team[p] == t) <= MAX_PLAYERS_PER_TEAM * use_fh[w] for t in teams for w in gws), name="team_limit_fh", ) ## Transfer constraints model.add_constraints( (squad[p, w] == squad[p, w - 1] + transfer_in[p, w] - transfer_out[p, w] for p in players for w in gws), name="squad_transfer_rel" ) model.add_constraints( ( in_the_bank[w] == in_the_bank[w - 1] + sold_amount[w] - bought_amount[w] - (transfer_count[w] * itb_loss_per_transfer if w > next_gw else 0) for w in gws ), name="cont_budget", ) model.add_constraints( ( so.expr_sum(fh_sell_price[p] * squad[p, w - 1] for p in players) + in_the_bank[w - 1] >= so.expr_sum(fh_sell_price[p] * squad_fh[p, w] for p in players) for w in gws ), name="fh_budget", ) model.add_constraints((transfer_in[p, w] <= 1 - use_fh[w] for p in players for w in gws), name="no_tr_in_fh") model.add_constraints((transfer_out[p, w] <= 1 - use_fh[w] for p in players for w in gws), name="no_tr_out_fh") ## Free transfer constraints # 2024-2025 variation: min 1 / max 5 / roll over WC & FH # raw_gw_ft = {w: fts[w] - transfer_count[w] + 1 - use_wc[w] - use_fh[w] for w in gws} # 2056-26 afcon variation: always have 5 ft in gw16 no matter what raw_gw_ft = {w: fts[w] - transfer_count[w] + (5 if w == AFCON_GW - 1 else 1) - use_wc[w] - use_fh[w] for w in gws} m = 20 # big m for bounding constraints, picked 20 because nobody will ever get to 20 ft in a solve # FT_BELOW_LB AND FT_ABOVE_UB LOGIC # ft_above_ub[w] == 1 <=> raw_gw_ft[w] > 5 model.add_constraints((raw_gw_ft[w] >= 6 - m * (1 - ft_above_ub[w]) for w in gws), name="ft_above_ub_lb") model.add_constraints((raw_gw_ft[w] <= 5 + m * ft_above_ub[w] for w in gws), name="ft_above_ub_ub") # ft_below_lb[w] == 1 <=> raw_gw_ft[w] <= 0 model.add_constraints((raw_gw_ft[w] <= 0 + m * (1 - ft_below_lb[w]) for w in gws), name="ft_below_lb_ub") model.add_constraints((raw_gw_ft[w] >= 1 - m * ft_below_lb[w] for w in gws), name="ft_below_lb_lb") # FREE TRANSFER LOGIC # raw_gw_ft[w] > 5 => fts[w+1] = 5 model.add_constraints((fts[w + 1] <= 5 + m * (1 - ft_above_ub[w]) for w in gws if w + 1 in gws), name="ft_cap_upper_ub") model.add_constraints((fts[w + 1] >= 5 - m * (1 - ft_above_ub[w]) for w in gws if w + 1 in gws), name="ft_cap_upper_lb") # raw_gw_ft[w] < 0 => fts[w+1] = 1 model.add_constraints((fts[w + 1] <= 1 + m * (1 - ft_below_lb[w]) for w in gws if w + 1 in gws), name="ft_cap_lower_ub") model.add_constraints((fts[w + 1] >= 1 - m * (1 - ft_below_lb[w]) for w in gws if w + 1 in gws), name="ft_cap_lower_lb") # 0 <= raw_gw_ft <= 5 => fts[w+1] = raw_gw_ft[w] model.add_constraints((fts[w + 1] - raw_gw_ft[w] <= m * (ft_above_ub[w] + ft_below_lb[w]) for w in gws if w + 1 in gws), name="ft_inrange_ub") model.add_constraints((raw_gw_ft[w] - fts[w + 1] <= m * (ft_above_ub[w] + ft_below_lb[w]) for w in gws if w + 1 in gws), name="ft_inrange_lb") model.add_constraints((fts[w] == so.expr_sum(fts_state[w, s] * s for s in ft_states) for w in gws), name="ftsc1") model.add_constraints((so.expr_sum(fts_state[w, s] for s in ft_states) == 1 for w in gws), name="ftsc2") if preseason and threshold_gw in gws: model.add_constraint(fts[threshold_gw] == 1, name="ps_initial_ft") model.add_constraints((penalized_transfers[w] >= transfer_diff[w] for w in gws), name="pen_transfer_rel") ## Chip constraints model.add_constraints((use_wc[w] + use_fh[w] + use_bb[w] + use_tc_gw[w] <= 1 for w in gws), name="single_chip") model.add_constraints((aux[w] <= 1 - use_wc[w - 1] for w in gws if w > next_gw), name="ft_after_wc") model.add_constraints((aux[w] <= 1 - use_fh[w - 1] for w in gws if w > next_gw), name="ft_after_fh") model.add_constraints((use_tc[p, w] <= captain[p, w] for p in players for w in gws), name="tc_cap_rel") wc = options.get("use_wc", []) if len(wc) > 0: model.add_constraints((use_wc[w] == 1 for w in wc), name="force_wc") chip_limits["wc"] = len(wc) bb = options.get("use_bb", []) if len(bb) > 0: model.add_constraints((use_bb[w] == 1 for w in bb), name="force_bb") chip_limits["bb"] = len(bb) fh = options.get("use_fh", []) if len(fh) > 0: model.add_constraints((use_fh[w] == 1 for w in fh), name="force_fh") chip_limits["fh"] = len(fh) tc = options.get("use_tc", []) if len(tc) > 0: model.add_constraints((use_tc_gw[w] == 1 for w in tc), name="force_tc") chip_limits["tc"] = len(tc) if len(allowed_chip_gws.get("wc", [])) > 0: gws_banned = [w for w in gws if w not in allowed_chip_gws["wc"]] model.add_constraints((use_wc[w] == 0 for w in gws_banned), name="banned_wc_gws") chip_limits["wc"] = 1 if len(allowed_chip_gws.get("fh", [])) > 0: gws_banned = [w for w in gws if w not in allowed_chip_gws["fh"]] model.add_constraints((use_fh[w] == 0 for w in gws_banned), name="banned_fh_gws") chip_limits["fh"] = 1 if len(allowed_chip_gws.get("bb", [])) > 0: gws_banned = [w for w in gws if w not in allowed_chip_gws["bb"]] model.add_constraints((use_bb[w] == 0 for w in gws_banned), name="banned_bb_gws") chip_limits["bb"] = 1 if len(allowed_chip_gws.get("tc", [])) > 0: gws_banned = [w for w in gws if w not in allowed_chip_gws["tc"]] model.add_constraints((use_tc_gw[w] == 0 for w in gws_banned), name="banned_tc_gws") chip_limits["tc"] = 1 if len(forced_chip_gws.get("wc", [])) > 0: model.add_constraint(so.expr_sum(use_wc[w] for w in forced_chip_gws["wc"]) == 1, name="force_wc_gw") chip_limits["wc"] = 1 if len(forced_chip_gws.get("fh", [])) > 0: model.add_constraint(so.expr_sum(use_fh[w] for w in forced_chip_gws["fh"]) == 1, name="force_fh_gw") chip_limits["fh"] = 1 if len(forced_chip_gws.get("bb", [])) > 0: model.add_constraint(so.expr_sum(use_bb[w] for w in forced_chip_gws["bb"]) == 1, name="force_bb_gw") chip_limits["bb"] = 1 if len(forced_chip_gws.get("tc", [])) > 0: model.add_constraint(so.expr_sum(use_tc_gw[w] for w in forced_chip_gws["tc"]) == 1, name="force_tc_gw") chip_limits["tc"] = 1 model.add_constraint(so.expr_sum(use_wc[w] for w in gws) <= chip_limits.get("wc", 0), name="use_wc_limit") model.add_constraint(so.expr_sum(use_bb[w] for w in gws) <= chip_limits.get("bb", 0), name="use_bb_limit") model.add_constraint(so.expr_sum(use_fh[w] for w in gws) <= chip_limits.get("fh", 0), name="use_fh_limit") model.add_constraint(so.expr_sum(use_tc_gw[w] for w in gws) <= chip_limits.get("tc", 0), name="use_tc_limit") model.add_constraints((squad_fh[p, w] <= use_fh[w] for p in players for w in gws), name="fh_squad_logic") ## Multiple-sell fix model.add_constraints( (transfer_out_first[p, w] + transfer_out_regular[p, w] <= 1 for p in price_modified_players for w in gws), name="multi_sell_1" ) model.add_constraints( ( horizon * so.expr_sum(transfer_out_first[p, w] for w in gws if w <= wbar) >= so.expr_sum(transfer_out_regular[p, w] for w in gws if w >= wbar) for p in price_modified_players for wbar in gws ), name="multi_sell_2", ) model.add_constraints((so.expr_sum(transfer_out_first[p, w] for w in gws) <= 1 for p in price_modified_players), name="multi_sell_3") ## Transfer in/out fix model.add_constraints((transfer_in[p, w] + transfer_out[p, w] <= 1 for p in players for w in gws), name="tr_in_out_limit") ## Tr Count Constraints ft_penalty = dict.fromkeys(gws, 0) model.add_constraints((transfer_count[w] >= num_transfers[w] - SQUAD_SIZE * use_wc[w] for w in gws), name="trc_lb") model.add_constraints((transfer_count[w] <= num_transfers[w] for w in gws), name="trc_ub1") model.add_constraints((transfer_count[w] <= SQUAD_SIZE * (1 - use_wc[w]) for w in gws), name="trc_ub2") if ft_use_penalty is not None: ft_penalty = {w: ft_use_penalty * transfer_count[w] for w in gws} ## Optional constraints if options.get("banned", None): print("OC - Banned") banned_players = options["banned"] model.add_constraints((so.expr_sum(squad[p, w] for w in gws) == 0 for p in banned_players if p in players), name="ban_player") model.add_constraints((so.expr_sum(squad_fh[p, w] for w in gws) == 0 for p in banned_players if p in players), name="ban_player_fh") if options.get("banned_next_gw", None): print("OC - Banned Next GW") banned_in_gw = [(x, gws[0]) if isinstance(x, int) else tuple(x) for x in options["banned_next_gw"]] model.add_constraints((squad[p0, p1] == 0 for (p0, p1) in banned_in_gw if p0 in players), name="ban_player_specified_gw") model.add_constraints((squad_fh[p0, p1] == 0 for (p0, p1) in banned_in_gw if p0 in players), name="ban_player_specified_gw_fh") if options.get("locked", None): print("OC - Locked") locked_players = options["locked"] model.add_constraints((squad[p, w] + squad_fh[p, w] == 1 for p in locked_players for w in gws), name="lock_player") if options.get("locked_next_gw", None): print("OC - Locked Next GW") locked_in_gw = [(x, gws[0]) if isinstance(x, int) else tuple(x) for x in options["locked_next_gw"]] model.add_constraints((squad[p0, p1] == 1 for (p0, p1) in locked_in_gw), name="lock_player_specified_gw") if options.get("no_future_transfer", None): print("OC - No Future Tr") model.add_constraint( so.expr_sum(transfer_in[p, w] for p in players for w in gws if w > next_gw and w != options.get("use_wc")) == 0, name="no_future_transfer" ) if options.get("no_transfer_last_gws", None): print("OC - No TR last GWs") no_tr_gws = options["no_transfer_last_gws"] if horizon > no_tr_gws: model.add_constraints( (so.expr_sum(transfer_in[p, w] for p in players) <= SQUAD_SIZE * use_wc[w] for w in gws if w > last_gw - no_tr_gws), name="tr_ban_gws" ) if options.get("num_transfers", None) is not None: print("OC - Num Transfers") model.add_constraint(so.expr_sum(transfer_in[p, next_gw] for p in players) == options["num_transfers"], name="tr_limit") if options.get("hit_limit", None): print("OC - Hit Limit") model.add_constraint(so.expr_sum(penalized_transfers[w] for w in gws) <= int(options["hit_limit"]), name="horizon_hit_limit") if options.get("weekly_hit_limit") is not None: weekly_hit_limit = int(options.get("weekly_hit_limit")) model.add_constraints((penalized_transfers[w] <= weekly_hit_limit for w in gws), name="gw_hit_lim") # if options.get("ft_custom_value", None) is not None: # ft_custom_value = {int(key): value for (key, value) in options.get('ft_custom_value', {}).items()} # ft_gw_value = {**{gw: ft_value for gw in gws}, **ft_custom_value} if options.get("future_transfer_limit", None): print("OC - Future TR Limit") model.add_constraint( so.expr_sum(transfer_in[p, w] for p in players for w in gws if w > next_gw and w not in options.get("use_wc", [])) <= options["future_transfer_limit"], name="future_tr_limit", ) if options.get("no_transfer_gws", None): print("OC - No TR GWs") if len(options["no_transfer_gws"]) > 0: model.add_constraint(so.expr_sum(transfer_in[p, w] for p in players for w in options["no_transfer_gws"]) == 0, name="banned_gws_for_tr") if options.get("no_transfer_by_position", None): print("OC - No TR by position") if len(options["no_transfer_by_position"]) > 0: # ignore w=1 as you must transfer in a full squad model.add_constraints( ( transfer_in[p, w] <= use_wc[w] for p in players for w in gws if w > 1 if merged_data.loc[p, "Pos"] in options["no_transfer_by_position"] ), name="no_tr_by_pos", ) max_defs_per_team = options.get("max_defenders_per_team", 3) if max_defs_per_team < MAX_PLAYERS_PER_TEAM: # only add constraints if necessary model.add_constraints( ( so.expr_sum(squad[p, w] for p in players if player_team[p] == t and merged_data.loc[p, "Pos"] in {"G", "D"}) <= max_defs_per_team for t in teams for w in gws ), name="defenders_per_team_limit", ) model.add_constraints( ( so.expr_sum(squad_fh[p, w] for p in players if player_team[p] == t and merged_data.loc[p, "Pos"] in {"G", "D"}) <= max_defs_per_team * use_fh[w] for t in teams for w in gws ), name="defenders_per_team_limit_fh", ) for booked_transfer in booked_transfers: print("OC - Booked TRs") transfer_gw = booked_transfer.get("gw", None) if transfer_gw is None: continue player_in = booked_transfer.get("transfer_in", None) player_out = booked_transfer.get("transfer_out", None) if player_in is not None: model.add_constraint(transfer_in[player_in, transfer_gw] == 1, name=f"booked_transfer_in_{transfer_gw}_{player_in}") if player_out is not None: model.add_constraint(transfer_out[player_out, transfer_gw] == 1, name=f"booked_transfer_out_{transfer_gw}_{player_out}") cp_penalty = {} if options.get("no_opposing_play") is True: print("OC - No Opposing Play") gw_opp_teams = { w: [(f["home"], f["away"]) for f in fixtures if f["gw"] == w] + [(f["away"], f["home"]) for f in fixtures if f["gw"] == w] for w in gws } for gw in gws: [i for i in fixtures if i["gw"] == gw] if options.get("opposing_play_group", "all") == "all": opposing_players = [(p1, p2) for p1 in players for p2 in players if (player_team[p1], player_team[p2]) in gw_opp_teams[gw]] model.add_constraints((lineup[p1, gw] + lineup[p2, gw] <= 1 for (p1, p2) in opposing_players), name=f"no_opp_{gw}") elif options.get("opposing_play_group") == "position": opposing_positions = [ (1, 3), (1, 4), (2, 3), (2, 4), (3, 1), (4, 1), (3, 2), (4, 2), ] # gk vs mid, gk vs fwd, def vs mid, def vs fwd opposing_players = [ (p1, p2) for p1 in players for p2 in players if (player_team[p1], player_team[p2]) in gw_opp_teams[gw] and (player_type[p1], player_type[p2]) in opposing_positions ] model.add_constraints((lineup[p1, gw] + lineup[p2, gw] <= 1 for (p1, p2) in opposing_players), name=f"no_opp_{gw}") elif options.get("no_opposing_play") == "penalty": print("OC - Penalty Opposing Play") gw_opp_teams = { w: [(f["home"], f["away"]) for f in fixtures if f["gw"] == w] + [(f["away"], f["home"]) for f in fixtures if f["gw"] == w] for w in gws } if options.get("opposing_play_group") == "all": cp_list = [ (p1, p2, w) for p1 in players for p2 in players for w in gws if (player_team[p1], player_team[p2]) in gw_opp_teams[w] and minutes_player_week[p1, w] > 0 and minutes_player_week[p2, w] > 0 ] elif options.get("opposing_play_group", "position") == "position": opposing_positions = [(1, 3), (1, 4), (2, 3), (2, 4), (3, 1), (4, 1), (3, 2), (4, 2)] cp_list = [ (p1, p2, w) for p1 in players for p2 in players for w in gws if (player_team[p1], player_team[p2]) in gw_opp_teams[w] and minutes_player_week[p1, w] > 0 and minutes_player_week[p2, w] > 0 and (player_type[p1], player_type[p2]) in opposing_positions ] cp_pen_var = model.add_variables(cp_list, name="cp_v", vartype=so.binary) opposing_play_penalty = options.get("opposing_play_penalty", 0.5) cp_penalty = {w: opposing_play_penalty * so.expr_sum(cp_pen_var[p1, p2, w1] for (p1, p2, w1) in cp_list if w1 == w) for w in gws} model.add_constraints((lineup[p1, w] + lineup[p2, w] <= 1 + cp_pen_var[p1, p2, w] for (p1, p2, w) in cp_list), name="cp1") model.add_constraints((cp_pen_var[p1, p2, w] <= lineup[p1, w] for (p1, p2, w) in cp_list), name="cp2") model.add_constraints((cp_pen_var[p1, p2, w] <= lineup[p2, w] for (p1, p2, w) in cp_list), name="cp3") if options.get("double_defense_pick") is True: print("OC - Double Defense Pick") team_players = {t: [p for p in players if player_team[p] == t] for t in teams} gk_df_players = {t: [p for p in team_players[t] if player_type[p] in [1, 2]] for t in teams} weekly_sum = {(t, w): so.expr_sum(lineup[p, w] for p in gk_df_players[t]) for t in teams for w in gws} def_aux = model.add_variables(teams, gws, vartype=so.binary, name="daux") model.add_constraints((weekly_sum[t, w] <= 3 * def_aux[t, w] for t in teams for w in gws), name="dauxc1") model.add_constraints((weekly_sum[t, w] >= 2 - 3 * (1 - def_aux[t, w]) for t in teams for w in gws), name="dauxc2") if options.get("transfer_itb_buffer"): buffer_amount = float(options["transfer_itb_buffer"]) gw_with_tr = model.add_variables(gws, name="gw_with_tr", vartype=so.binary) model.add_constraints((SQUAD_SIZE * gw_with_tr[w] >= num_transfers[w] for w in gws), name="gw_with_tr_lb") model.add_constraints((gw_with_tr[w] <= num_transfers[w] for w in gws), name="gw_with_tr_ub") model.add_constraints((in_the_bank[w] >= buffer_amount * gw_with_tr[w] for w in gws), name="buffer_con") if options.get("pick_prices", None) not in [None, {"G": "", "D": "", "M": "", "F": ""}]: print("OC - Pick Prices") buffer = 0.2 price_choices = options["pick_prices"] for pos, val in price_choices.items(): if val == "": continue price_points = [float(i) for i in val.split(",")] value_dict = {i: price_points.count(i) for i in set(price_points)} con_iter = 0 for key, count in value_dict.items(): target_players = [ p for p in players if merged_data.loc[p, "Pos"] == pos and buy_price[p] >= key - buffer and buy_price[p] <= key + buffer ] model.add_constraints((so.expr_sum(squad[p, w] for p in target_players) >= count for w in gws), name=f"price_point_{pos}_{con_iter}") con_iter += 1 if options.get("no_gk_rotation_after", None): print("OC - No GK rotation") target_gw = int(options["no_gk_rotation_after"]) players_gk = [p for p in players if player_type[p] == 1] model.add_constraints( (lineup[p, w] >= lineup[p, target_gw] - use_fh[w] for p in players_gk for w in gws if w > target_gw), name="fixed_lineup_gk" ) if len(options.get("no_chip_gws", [])) > 0: print("OC - No Chip GWs") no_chip_gws = options["no_chip_gws"] model.add_constraint(so.expr_sum(use_bb[w] + use_wc[w] + use_fh[w] for w in no_chip_gws) == 0, name="no_chip_gws") if options.get("only_booked_transfers") is True: print("OC - Only Booked Transfers") forced_in = [] forced_out = [] for bt in options.get("booked_transfers", []): if bt["gw"] == next_gw: if bt.get("transfer_in") is not None: forced_in.append(bt["transfer_in"]) if bt.get("transfer_out") is not None: forced_out.append(bt["transfer_out"]) in_players = {(p): 1 if p in forced_in else 0 for p in players} out_players = {(p): 1 if p in forced_out else 0 for p in players} model.add_constraints((transfer_in[p, next_gw] == in_players[p] for p in players), name="fix_tgw_tr_in") model.add_constraints((transfer_out[p, next_gw] == out_players[p] for p in players), name="fix_tgw_tr_out") # if options.get('have_2ft_in_gws', None) is not None: # for gw in options['have_2ft_in_gws']: # model.add_constraint(fts[gw] == 2, name=f'have_2ft_{gw}') if options.get("force_ft_state_lb", None): print("OC - Force FT LB") for gw, ft_pos in options["force_ft_state_lb"]: model.add_constraint(fts[gw] >= ft_pos, name=f"cft_lb_{gw}") if options.get("force_ft_state_ub", None): print("OC - Force FT UB") for gw, ft_pos in options["force_ft_state_ub"]: model.add_constraint(fts[gw] <= ft_pos, name=f"cft_ub_{gw}") if options.get("no_trs_except_wc", False) is True: print("OC - No TRS except WC") model.add_constraints((num_transfers[w] <= SQUAD_SIZE * use_wc[w] for w in gws), name="wc_trs_only") # FT gain ft_state_value = {} for s in ft_states: ft_state_value[s] = ft_state_value.get(s - 1, 0) + ft_value_list.get(str(s), ft_value) # print(f"Using FT state values of {ft_state_value}") print(f"Using FT values of {ft_value_list}") gw_ft_value = {w: so.expr_sum(ft_state_value[s] * fts_state[w, s] for s in ft_states) for w in gws} gw_ft_gain = {w: gw_ft_value[w] - gw_ft_value.get(w - 1, 0) for w in gws} # Objectives hit_cost = options.get("hit_cost", 4) vcap_weight = options.get("vcap_weight", 0.1) gw_xp = { w: so.expr_sum( points_player_week[p, w] * ( lineup[p, w] + captain[p, w] + vcap_weight * vicecap[p, w] + use_tc[p, w] + so.expr_sum(bench_weights[o] * bench[p, w, o] for o in order) ) for p in players ) for w in gws } gw_total = { w: gw_xp[w] - hit_cost * penalized_transfers[w] + gw_ft_gain[w] - ft_penalty[w] + itb_value * in_the_bank[w] - cp_penalty.get(w, 0) for w in gws } if objective == "regular": total_xp = so.expr_sum(gw_total[w] for w in gws) model.set_objective(-total_xp, sense="N", name="total_regular_xp") else: decay_objective = so.expr_sum(gw_total[w] * pow(decay_base, w - next_gw) for w in gws) model.set_objective(-decay_objective, sense="N", name="total_decay_xp") report_decay_base = options.get("report_decay_base", []) decay_metrics = {i: so.expr_sum(gw_total[w] * pow(i, w - next_gw) for w in gws) for i in report_decay_base} num_iterations = options.get("num_iterations", 1) iteration_criteria = options.get("iteration_criteria", "this_gw_transfer_in") # fix for multiple iterations when free-hitting next gameweek if iteration_criteria in {"this_gw_transfer_in", "this_gw_transfer_in_out"} and next_gw in options.get("use_fh", []): iteration_criteria = "this_gw_lineup" solutions = [] for iteration in range(num_iterations): mps_file_name = f"tmp/{problem_name}_{problem_id}_{iteration}.mps" sol_file_name = f"tmp/{problem_name}_{problem_id}_{iteration}_sol.txt" opt_file_name = f"tmp/{problem_name}_{problem_id}_{iteration}.opt" tmp_folder = Path() / "tmp" tmp_folder.mkdir(exist_ok=True, parents=True) model.export_mps(mps_file_name) print(f"Exported model with name: {problem_name}_{problem_id}_{iteration}") if options.get("export_debug", False): with open("debug.sas", "w") as file: file.write(model.to_optmodel()) # use_cmd = options.get("use_cmd", False) solver = options.get("solver", "highs") if solver.lower() == "highs": # Use highspy Python interface instead of command line secs = options.get("secs", 20 * 60) presolve = options.get("presolve", "on") gap = options.get("gap", 0) random_seed = options.get("random_seed", 0) verbose = options.get("verbose", False) solver_instance = highspy.Highs() solver_instance.readModel(str(mps_file_name)) solver_instance.setOptionValue("parallel", "on") solver_instance.setOptionValue("random_seed", random_seed) solver_instance.setOptionValue("presolve", presolve) solver_instance.setOptionValue("time_limit", secs) solver_instance.setOptionValue("mip_rel_gap", gap) solver_instance.setOptionValue("log_to_console", verbose) solver_instance.run() solution = solver_instance.getSolution() values = list(solution.col_value) for idx, v in enumerate(model.get_variables()): v.set_value(values[idx]) elif solver == "gurobi": use_cmd = options.get("use_cmd", False) gap = options.get("gap", 0) sol_file_name = sol_file_name.replace("_sol", "").replace("txt", "sol") command = f"gurobi_cl MIPGap={gap} ResultFile={sol_file_name} {mps_file_name}" if use_cmd: os.system(command) else: def print_output(process): while True: output = process.stdout.readline() if "Solving report" in output: time.sleep(2) process.kill() elif output == "" and process.poll() is not None: break elif output: print(output.strip()) process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) output_thread = threading.Thread(target=print_output, args=(process,)) output_thread.start() output_thread.join() # Parsing with open(sol_file_name) as f: for v in model.get_variables(): v.set_value(0) for line in f: if line[0] == "#": continue if line == "": break words = line.split() v = model.get_variable(words[0]) try: if v.get_type() == so.INT: v.set_value(round(float(words[1]))) elif v.get_type() == so.BIN: v.set_value(round(float(words[1]))) elif v.get_type() == so.CONT: v.set_value(round(float(words[1]), 3)) except Exception: print("Error", words[0], line) # DataFrame generation picks = [] for w in gws: for p in players: if squad[p, w].get_value() + squad_fh[p, w].get_value() + transfer_out[p, w].get_value() > BINARY_THRESHOLD: lp = merged_data.loc[p] is_captain = 1 if captain[p, w].get_value() > BINARY_THRESHOLD else 0 is_squad = ( 1 if (use_fh[w].get_value() < BINARY_THRESHOLD and squad[p, w].get_value() > BINARY_THRESHOLD) or (use_fh[w].get_value() > BINARY_THRESHOLD and squad_fh[p, w].get_value() > BINARY_THRESHOLD) else 0 ) is_lineup = 1 if lineup[p, w].get_value() > BINARY_THRESHOLD else 0 is_vice = 1 if vicecap[p, w].get_value() > BINARY_THRESHOLD else 0 is_tc = 1 if use_tc[p, w].get_value() > BINARY_THRESHOLD else 0 is_transfer_in = 1 if transfer_in[p, w].get_value() > BINARY_THRESHOLD else 0 is_transfer_out = 1 if transfer_out[p, w].get_value() > BINARY_THRESHOLD else 0 bench_value = -1 for o in order: if bench[p, w, o].get_value() > BINARY_THRESHOLD: bench_value = o position = type_data.loc[lp["element_type"], "singular_name_short"] player_buy_price = 0 if not is_transfer_in else buy_price[p] player_sell_price = ( 0 if not is_transfer_out else ( sell_price[p] if p in price_modified_players and transfer_out_first[p, w].get_value() > BINARY_THRESHOLD else buy_price[p] ) ) multiplier = 1 * (is_lineup == 1) + 1 * (is_captain == 1) + 1 * (is_tc == 1) xp_cont = points_player_week[p, w] * multiplier # chip if use_wc[w].get_value() > BINARY_THRESHOLD: chip_text = "WC" elif use_fh[w].get_value() > BINARY_THRESHOLD: chip_text = "FH" elif use_bb[w].get_value() > BINARY_THRESHOLD: chip_text = "BB" elif use_tc[p, w].get_value() > BINARY_THRESHOLD: chip_text = "TC" else: chip_text = "" picks.append( { "id": p, "week": w, "name": lp["web_name"], "pos": position, "type": lp["element_type"], "team": lp["name"], "buy_price": player_buy_price, "sell_price": player_sell_price, "xP": round(points_player_week[p, w], 2), "xMin": minutes_player_week[p, w], "squad": is_squad, "lineup": is_lineup, "bench": bench_value, "captain": is_captain, "vicecaptain": is_vice, "transfer_in": is_transfer_in, "transfer_out": is_transfer_out, "multiplier": multiplier, "xp_cont": xp_cont, "chip": chip_text, "iter": iteration, "ft": fts[w].get_value(), "transfer_count": num_transfers[w].get_value(), } ) picks_df = pd.DataFrame(picks).sort_values(by=["week", "lineup", "type", "xP"], ascending=[True, False, True, True]) total_xp = so.expr_sum((lineup[p, w] + captain[p, w]) * points_player_week[p, w] for p in players for w in gws).get_value() picks_df.sort_values(by=["week", "squad", "lineup", "bench", "type"], ascending=[True, False, False, True, True], inplace=True) # Writing summary summary_of_actions = "" move_summary = {"chip": [], "buy": [], "sell": []} # collect statistics statistics = {} for w in all_gw: if w == all_gw[0]: statistics[int(w)] = {"itb": in_the_bank[w].get_value(), "ft": fts[w].get_value()} continue summary_of_actions += f"** GW {w}:\n" chip_decision = ( ("WC" if use_wc[w].get_value() > BINARY_THRESHOLD else "") + ("FH" if use_fh[w].get_value() > BINARY_THRESHOLD else "") + ("BB" if use_bb[w].get_value() > BINARY_THRESHOLD else "") + ("TC" if use_tc_gw[w].get_value() > BINARY_THRESHOLD else "") ) if chip_decision != "": summary_of_actions += "CHIP " + chip_decision + "\n" move_summary["chip"].append(chip_decision + str(w)) summary_of_actions += ( f"ITB={round(in_the_bank[w - 1].get_value(), 1)}->{round(in_the_bank[w].get_value(), 1)}, " f"FT={round(fts[w].get_value())}, " f"PT={round(penalized_transfers[w].get_value())}, " f"NT={round(num_transfers[w].get_value())}\n" ) for p in players: if transfer_in[p, w].get_value() > BINARY_THRESHOLD: summary_of_actions += f"Buy {p} - {merged_data['web_name'][p]}\n" if w == next_gw: move_summary["buy"].append(merged_data["web_name"][p]) for p in players: if transfer_out[p, w].get_value() > BINARY_THRESHOLD: summary_of_actions += f"Sell {p} - {merged_data['web_name'][p]}\n" if w == next_gw: move_summary["sell"].append(merged_data["web_name"][p]) picks_df[picks_df["week"] == w] lineup_players = picks_df[(picks_df["week"] == w) & (picks_df["lineup"] == 1)] bench_players = picks_df[(picks_df["week"] == w) & (picks_df["bench"] >= 0)] # captain_name = picks_df[(picks_df['week'] == w) & (picks_df['captain'] == 1)].iloc[0]['name'] # vicecap_name = picks_df[(picks_df['week'] == w) & (picks_df['vicecaptain'] == 1)].iloc[0]['name'] summary_of_actions += "\nLineup: \n" def get_display(row): return f"{row['name']} ({row['xP']}{', C' if row['captain'] == 1 else ''}{', V' if row['vicecaptain'] == 1 else ''})" for typ in [1, 2, 3, 4]: type_players = lineup_players[lineup_players["type"] == typ] entries = type_players.apply(get_display, axis=1) summary_of_actions += "\t" + ", ".join(entries.tolist()) + "\n" summary_of_actions += "Bench: \n\t" + ", ".join(bench_players.apply(get_display, axis=1)) + "\n" summary_of_actions += "Lineup xPts: " + str(round(lineup_players["xp_cont"].sum(), 2)) + "\n" if w != max(gws): summary_of_actions += "\n\n" statistics[int(w)] = { "itb": in_the_bank[w].get_value(), "ft": fts[w].get_value(), "pt": penalized_transfers[w].get_value(), "nt": num_transfers[w].get_value(), "xP": lineup_players["xp_cont"].sum(), "obj": round(gw_total[w].get_value(), 2), "chip": chip_decision if chip_decision != "" else None, } if options.get("delete_tmp", True): time.sleep(0.1) try: try: os.unlink(mps_file_name) except Exception: pass try: os.unlink(sol_file_name) except Exception: pass try: os.unlink(opt_file_name) except Exception: pass except Exception: print("Could not delete temporary files") def format_decisions(items): return ", ".join(items) if items else "-" buy_decisions = format_decisions(move_summary["buy"]) sell_decisions = format_decisions(move_summary["sell"]) chip_decisions = format_decisions(move_summary["chip"]) if options.get("hide_transfers"): buy_decisions = sell_decisions = "-" # Add current solution to a list, and add a new cut solutions.append( { "iter": iteration, "model": model, "picks": picks_df, "total_xp": total_xp, "summary": summary_of_actions, "statistics": statistics, "buy": buy_decisions, "sell": sell_decisions, "chip": chip_decisions, "score": -model.get_objective_value(), "decay_metrics": {key: value.get_value() for key, value in decay_metrics.items()}, } ) if num_iterations == 1: return solutions iter_diff = options.get("iteration_difference", 1) if iteration_criteria == "this_gw_transfer_in": actions = so.expr_sum( 1 - transfer_in[p, next_gw] for p in players if transfer_in[p, next_gw].get_value() > BINARY_THRESHOLD ) + so.expr_sum(transfer_in[p, next_gw] for p in players if transfer_in[p, next_gw].get_value() < BINARY_THRESHOLD) model.add_constraint(actions >= 1, name=f"cutoff_{iteration}") elif iteration_criteria == "this_gw_transfer_out": actions = so.expr_sum( 1 - transfer_out[p, next_gw] for p in players if transfer_out[p, next_gw].get_value() > BINARY_THRESHOLD ) + so.expr_sum(transfer_out[p, next_gw] for p in players if transfer_out[p, next_gw].get_value() < BINARY_THRESHOLD) model.add_constraint(actions >= 1, name=f"cutoff_{iteration}") elif iteration_criteria == "this_gw_transfer_in_out": actions = ( so.expr_sum(1 - transfer_in[p, next_gw] for p in players if transfer_in[p, next_gw].get_value() > BINARY_THRESHOLD) + so.expr_sum(transfer_in[p, next_gw] for p in players if transfer_in[p, next_gw].get_value() < BINARY_THRESHOLD) + so.expr_sum(1 - transfer_out[p, next_gw] for p in players if transfer_out[p, next_gw].get_value() > BINARY_THRESHOLD) + so.expr_sum(transfer_out[p, next_gw] for p in players if transfer_out[p, next_gw].get_value() < BINARY_THRESHOLD) ) model.add_constraint(actions >= 1, name=f"cutoff_{iteration}") elif iteration_criteria == "chip_gws": actions = ( so.expr_sum(1 - use_wc[w] for w in gws if use_wc[w].get_value() > BINARY_THRESHOLD) + so.expr_sum(use_wc[w] for w in gws if use_wc[w].get_value() < BINARY_THRESHOLD) + so.expr_sum(1 - use_bb[w] for w in gws if use_bb[w].get_value() > BINARY_THRESHOLD) + so.expr_sum(use_bb[w] for w in gws if use_bb[w].get_value() < BINARY_THRESHOLD) + so.expr_sum(1 - use_fh[w] for w in gws if use_fh[w].get_value() > BINARY_THRESHOLD) + so.expr_sum(use_fh[w] for w in gws if use_fh[w].get_value() < BINARY_THRESHOLD) ) model.add_constraint(actions >= 1, name=f"cutoff_{iteration}") elif iteration_criteria == "target_gws_transfer_in": target_gws = options.get("iteration_target", [next_gw]) transferred_players = [[p, w] for p in players for w in target_gws if transfer_in[p, w].get_value() > BINARY_THRESHOLD] remaining_players = [[p, w] for p in players for w in target_gws if transfer_in[p, w].get_value() < BINARY_THRESHOLD] actions = so.expr_sum(1 - transfer_in[p, w] for [p, w] in transferred_players) + so.expr_sum( transfer_in[p, w] for [p, w] in remaining_players ) model.add_constraint(actions >= 1, name=f"cutoff_{iteration}") elif iteration_criteria == "this_gw_lineup": selected_lineup = [p for p in players if lineup[p, next_gw].get_value() > BINARY_THRESHOLD] model.add_constraint( so.expr_sum(lineup[p, next_gw] for p in selected_lineup) <= len(selected_lineup) - iter_diff, name=f"cutoff_{iteration}" ) return solutions ================================================ FILE: dev/visualization.py ================================================ import os import matplotlib.path as mpath import matplotlib.pyplot as plt import pandas as pd from matplotlib import patches from paths import DATA_DIR HIT_COST = 4 # Spacing and sizing BOX_HEIGHT = 0.9 BOX_WIDTH = 9 PLAYER_SPACING = 1.2 PLAYER_NAME_FONT_SIZE = 11 STATS_FONT_SIZE = 9 GAMEWEEK_SPACING = 14 POSITION_BORDER_WIDTH = 0.12 CAPTAIN_BORDER_WIDTH = 0.2 CHIP_BACKGROUND_ZORDERS = { "FH": -1.0, "WC": -5.0, "BB": -5.0, "TC": -5.0, } # color scheme CAPTAIN_COLOR = "#ffd700" VICE_CAPTAIN_COLOR = "#c0c0c0" BG_COLOR = "#0f0f0f" CELL_BG_COLOR = "#1e1e1e" BENCH_BG_COLOR = "#2a2a2a" TEXT_COLOR = "#ffffff" STATS_COLOR = "#b0b0b0" CHIP_BACKGROUND_COLOR = "#1a1a1a" # Position constants POSITIONS = ["GKP", "DEF", "MID", "FWD"] POSITION_COLORS = {"GKP": "#8b5cf6", "DEF": "#3b82f6", "MID": "#f59e0b", "FWD": "#ef4444"} BASE_Y = 16 def calculate_bezier(x_start, x_end, y_start, y_end): """ Calculates a bezier curve using the 4 given points. These are used to draw the lines signifying transfers between gameweeks. """ x_control1 = x_start + (x_end - x_start) * 0.3 x_control2 = x_start + (x_end - x_start) * 0.7 y_control1 = y_start + (y_end - y_start) * 0.02 y_control2 = y_start + (y_end - y_start) * 0.98 path_data = [ ((x_start, y_start), mpath.Path.MOVETO), ((x_control1, y_control1), mpath.Path.CURVE4), ((x_control2, y_control2), mpath.Path.CURVE4), ((x_end, y_end), mpath.Path.CURVE4), ] return patches.PathPatch( mpath.Path(*zip(*path_data, strict=True)), facecolor="none", edgecolor="#60a5fa", alpha=0.8, linewidth=1.5, zorder=-3.0, ) def calculate_player_cells(gw_idx, player_idx, player): y_pos = BASE_Y - player_idx * PLAYER_SPACING data = [] # base cell data.append( patches.Rectangle( (gw_idx * GAMEWEEK_SPACING - BOX_WIDTH / 2, y_pos - BOX_HEIGHT / 2), BOX_WIDTH, BOX_HEIGHT, facecolor=CELL_BG_COLOR if player["lineup"] else BENCH_BG_COLOR, edgecolor="none", ) ) # position border data.append( patches.Rectangle( (gw_idx * GAMEWEEK_SPACING - BOX_WIDTH / 2, y_pos - BOX_HEIGHT / 2 - POSITION_BORDER_WIDTH), BOX_WIDTH, POSITION_BORDER_WIDTH, facecolor=POSITION_COLORS[player["pos"]], edgecolor="none", ) ) # captain border if player["captain"] == 1 and gw_idx > 0: data.append( patches.Rectangle( (gw_idx * GAMEWEEK_SPACING - BOX_WIDTH / 2, y_pos - BOX_HEIGHT / 2), CAPTAIN_BORDER_WIDTH, BOX_HEIGHT, facecolor=CAPTAIN_COLOR, edgecolor="none", ) ) # vice captain border elif player["vicecaptain"] == 1 and gw_idx > 0: data.append( patches.Rectangle( (gw_idx * GAMEWEEK_SPACING - BOX_WIDTH / 2, y_pos - BOX_HEIGHT / 2), CAPTAIN_BORDER_WIDTH, BOX_HEIGHT, facecolor=VICE_CAPTAIN_COLOR, edgecolor="none", ) ) return data def _setup_figure_and_data(picks, current_squad): """Setup the matplotlib figure and prepare data for visualization.""" df = pd.DataFrame(picks) df_squad = df[df["squad"] == 1] df_base = df[df["week"] == min(df["week"])] gameweeks = sorted(df_squad["week"].unique()) # Handle preseason scenario (earliest gameweek is 1) if min(gameweeks) == 1: base_week = None # No base team in preseason else: base_week = min(gameweeks) - 1 fh_week = df.loc[df["chip"] == "FH"].iloc[0]["week"] if len(df.loc[df["chip"] == "FH"]) > 0 else None fig, ax = plt.subplots(figsize=(26, 14)) ax.set_facecolor(BG_COLOR) fig.patch.set_facecolor(BG_COLOR) return fig, ax, df, df_squad, df_base, gameweeks, base_week, fh_week def _get_week_players(week, base_week, df_base, df_squad, current_squad): """Get players for a specific gameweek.""" if base_week is not None and week == base_week: gw_players = df_base[df_base["id"].isin(current_squad)] gw_players.loc[:, "lineup"] = 1 else: gw_players = df_squad[df_squad["week"] == week] return gw_players def _add_week_header(ax, gw_idx, week, base_week, gw_players): """Add gameweek header and chip information.""" if base_week is not None and week == base_week: ax.text(gw_idx * GAMEWEEK_SPACING, BASE_Y + 1.2, "Base", color=TEXT_COLOR, fontsize=13, ha="center", weight="bold") else: ax.text(gw_idx * GAMEWEEK_SPACING, BASE_Y + 1.2, f"GW{week}", color=TEXT_COLOR, fontsize=13, ha="center", weight="bold") if "chip" in gw_players.columns and not gw_players["chip"].isna().all(): try: chip = gw_players.loc[gw_players["chip"] != ""]["chip"].iloc[0] except Exception: chip = gw_players["chip"].iloc[0] if pd.notna(chip): ax.text(gw_idx * GAMEWEEK_SPACING, BASE_Y + 0.8, chip, color="#fbbf24", fontsize=11, ha="center", weight="bold") def _add_player_cells(ax, gw_idx, gw_players, week, player_indexes): """Add player cells for starting XI and bench.""" starting_xi = gw_players[gw_players["lineup"] == 1].sort_values(["type", "xP"], ascending=[True, False]).reset_index() bench = gw_players[gw_players["lineup"] == 0].sort_values(["type", "xP"], ascending=[True, False]).reset_index() bench.index = bench.index + 11 player_indexes[week] = {} # Starting XI for player_idx, player in starting_xi.iterrows(): y_pos = BASE_Y - player_idx * PLAYER_SPACING player_indexes[week][player["id"]] = (y_pos, player["pos"]) cells = calculate_player_cells(gw_idx, player_idx, player) for cell in cells: ax.add_patch(cell) text_pos = (gw_idx * GAMEWEEK_SPACING, y_pos + 0.2) ax.text(*text_pos, player["name"], color=TEXT_COLOR, ha="center", va="center", fontsize=PLAYER_NAME_FONT_SIZE, weight="medium") # Check if this is not the base week by looking at the data structure if "xP" in player and "xMin" in player: stats_text = f"{player['xP']:.1f} xPts • {int(player['xMin'])} xMin" ax.text(gw_idx * GAMEWEEK_SPACING, y_pos - 0.25, stats_text, color=STATS_COLOR, ha="center", va="center", fontsize=STATS_FONT_SIZE) # Bench for player_idx, player in bench.iterrows(): y_pos = BASE_Y - player_idx * PLAYER_SPACING player_indexes[week][player["id"]] = (BASE_Y - player_idx * PLAYER_SPACING, player["pos"]) cells = calculate_player_cells(gw_idx, player_idx, player) for cell in cells: ax.add_patch(cell) text_pos = (gw_idx * GAMEWEEK_SPACING, y_pos + 0.2) ax.text(*text_pos, player["name"], color=TEXT_COLOR, ha="center", va="center", fontsize=PLAYER_NAME_FONT_SIZE, weight="medium") stats_text = f"{player['xP']:.1f} xPts • {int(player['xMin'])} xMin" ax.text(gw_idx * GAMEWEEK_SPACING, y_pos - 0.25, stats_text, color=STATS_COLOR, ha="center", va="center", fontsize=STATS_FONT_SIZE) def _add_transfers(ax, gw_idx, week, picks, player_indexes): """Add transfer lines between gameweeks.""" # Calculate fh_week from picks data fh_week = picks.loc[picks["chip"] == "FH"].iloc[0]["week"] if len(picks.loc[picks["chip"] == "FH"]) > 0 else None # Get previous week from player_indexes keys prev_weeks = [w for w in player_indexes.keys() if w < week] prev_week_int = max(prev_weeks) if prev_weeks else week - 1 transfers_in = picks.loc[(picks["week"] == week) & (picks["transfer_in"] == 1)] transfers_out = picks.loc[(picks["week"] == week) & (picks["transfer_out"] == 1)] for pos in POSITIONS: players_out = transfers_out.loc[transfers_out["pos"] == pos].to_dict(orient="records") players_in = transfers_in.loc[transfers_in["pos"] == pos].to_dict(orient="records") if week == 1: # don't draw any lines continue for player_out, player_in in zip(players_out, players_in, strict=True): skip_fh = int(prev_week_int == fh_week) if fh_week else 0 x_start = (gw_idx - 1 - skip_fh) * GAMEWEEK_SPACING + BOX_WIDTH / 2 x_end = gw_idx * GAMEWEEK_SPACING - BOX_WIDTH / 2 y_start = player_indexes[prev_week_int - skip_fh][player_out["id"]][0] y_end = player_indexes[week][player_in["id"]][0] ax.add_patch(calculate_bezier(x_start, x_end, y_start, y_end)) def _add_gameweek_statistics(ax, gw_idx, week, statistics, player_idx): """Add gameweek statistics below the squad.""" # Determine base week from statistics keys base_week = min(statistics.keys()) if statistics else week if week == base_week: return stats_y = BASE_Y - (player_idx + 0.5) * PLAYER_SPACING ax.text( gw_idx * GAMEWEEK_SPACING, stats_y - 0.5, f"{statistics[int(week)]['xP']:.2f} xPts", color=TEXT_COLOR, fontsize=11, ha="center", weight="medium", ) if week > 1: itb_text = f"{statistics[week - 1]['itb']:.1f} → {statistics[week]['itb']:.1f}" else: itb_text = f"{statistics[week]['itb']:.1f}" ax.text( gw_idx * GAMEWEEK_SPACING, stats_y - 0.9, f"ITB: {itb_text}", color=STATS_COLOR, fontsize=9, ha="center", ) if week > 1 and statistics[week]["chip"] not in ["FH", "WC"]: fts_available = round(statistics[week]["ft"]) transfer_str = f"FTs: {round(statistics[week]['nt'])}/{fts_available}" if statistics[week]["pt"] > 0: transfer_str += f" (-{statistics[week]['pt'] * HIT_COST})" ax.text(gw_idx * GAMEWEEK_SPACING, stats_y - 1.3, transfer_str, color=STATS_COLOR, fontsize=9, ha="center") def _add_chip_backgrounds(ax, df, base_week, bottom_limit, top_limit): """Add background rectangles for chip gameweeks.""" chip_weeks = dict(df.loc[df["chip"] != ""][["week", "chip"]].drop_duplicates().values) for gw, chip in chip_weeks.items(): # Handle preseason scenario (no base week) if base_week is not None: x_center = (gw - base_week) * GAMEWEEK_SPACING else: x_center = (gw - 1) * GAMEWEEK_SPACING # Use GW1 as reference in preseason rect = patches.FancyBboxPatch( (x_center - GAMEWEEK_SPACING / 2, bottom_limit), GAMEWEEK_SPACING, top_limit - bottom_limit, edgecolor="none", facecolor=CHIP_BACKGROUND_COLOR, zorder=CHIP_BACKGROUND_ZORDERS[chip], boxstyle=patches.BoxStyle("Round", pad=-0.3, rounding_size=2), alpha=0.85, ) ax.add_patch(rect) def create_squad_timeline(current_squad, statistics, picks, filename): """Create a timeline visualization of squad changes across gameweeks.""" fig, ax, df, df_squad, df_base, gameweeks, base_week, fh_week = _setup_figure_and_data(picks, current_squad) player_indexes = {} # Handle preseason scenario (no base week) if base_week is not None: display_weeks = [base_week, *gameweeks] else: display_weeks = gameweeks for gw_idx, week in enumerate(display_weeks): gw_players = _get_week_players(week, base_week, df_base, df_squad, current_squad) _add_week_header(ax, gw_idx, week, base_week, gw_players) _add_player_cells(ax, gw_idx, gw_players, week, player_indexes) _add_transfers(ax, gw_idx, week, picks, player_indexes) _add_gameweek_statistics(ax, gw_idx, week, statistics, len(gw_players) - 1) # Set plot limits and styling total_width = (len(display_weeks) - 1) * GAMEWEEK_SPACING + BOX_WIDTH ax.set_xlim(-6, total_width + 2) bottom_limit = BASE_Y - (len(gw_players) + 1.5) * PLAYER_SPACING top_limit = BASE_Y + 2.8 ax.set_ylim(bottom_limit, top_limit) ax.axis("off") plt.title(filename, color=TEXT_COLOR, fontsize=14, weight="bold", pad=20) _add_chip_backgrounds(ax, df, base_week, bottom_limit, top_limit) # Ensure the images directory exists os.makedirs(DATA_DIR / "images", exist_ok=True) plt.savefig(DATA_DIR / "images" / f"{filename}.png", bbox_inches="tight", facecolor=BG_COLOR) plt.close() ================================================ FILE: paths.py ================================================ from pathlib import Path PROJECT_ROOT = Path(__file__).parent DATA_DIR = PROJECT_ROOT / "data" RUN_DIR = PROJECT_ROOT / "run" DEV_DIR = PROJECT_ROOT / "dev" ================================================ FILE: pyproject.toml ================================================ [project] name = "fpl-optimization-tools" version = "0.1.0" description = "Fantasy Premier League optimization tools" authors = [ {name = "Sertalp Bilal"} ] maintainers = [ {name = "Chris Musson", email = "chris.musson@hotmail.com"} ] readme = "README.md" requires-python = ">=3.12,<3.14" dependencies = [ "pandas", "numpy", "sasoptpy>=1.0.5a0", "fuzzywuzzy", "python-Levenshtein", "matplotlib", "highspy>=1.11.0", "tabulate>=0.9.0", ] [project.optional-dependencies] dev = [ "pytest", "ruff", ] [build-system] requires = ["hatchling"] build-backend = "hatchling.build" [tool.hatch.build.targets.wheel] packages = ["src", "run", "tests", "dev"] [tool.ruff] line-length = 150 target-version = "py313" [tool.ruff.lint] select = ["E", "F", "I", "N", "W", "B", "A", "C4", "UP", "PL", "RUF"] fixable = ["E501"] [tool.ruff.lint.per-file-ignores] "__init__.py" = ["F401"] "tests/**/*" = ["PLR2004"] "run/solve.py" = ["PLR0915", "PLR0912"] "dev/solver.py" = ["PLR0915", "PLR0912"] [tool.ruff.lint.isort] known-first-party = ["src", "run", "tests"] ================================================ FILE: run/binary_file_generator.py ================================================ import pandas as pd from paths import DATA_DIR def generate_binary_files(file_path, fixtures_json): # Iterate through each binary file entry in config for file_name, fixtures in fixtures_json.items(): # Load original fixture CSV file df = pd.read_csv(file_path) for team, binary_fix in fixtures.items(): # Apply changes only to rows where the Team column matches the specified team team_mask = df["Team"] == team for orig_gw, new_gw in binary_fix.items(): new_gw_pts_col = f"{new_gw}_Pts" new_gw_xmins_col = f"{new_gw}_xMins" orig_gw_pts_col = f"{orig_gw}_Pts" orig_gw_xmins_col = f"{orig_gw}_xMins" if not new_gw: df.loc[team_mask, orig_gw_pts_col] = 0 df.loc[team_mask, orig_gw_xmins_col] = 0 # Ensure relevant columns exist in the dataframe if all(col in df.columns for col in [new_gw_pts_col, new_gw_xmins_col, orig_gw_pts_col, orig_gw_xmins_col]): # Convert columns to numeric values for the rows we are updating df.loc[team_mask, new_gw_pts_col] = pd.to_numeric(df.loc[team_mask, new_gw_pts_col], errors="coerce") df.loc[team_mask, new_gw_xmins_col] = pd.to_numeric(df.loc[team_mask, new_gw_xmins_col], errors="coerce") df.loc[team_mask, orig_gw_pts_col] = pd.to_numeric(df.loc[team_mask, orig_gw_pts_col], errors="coerce") df.loc[team_mask, orig_gw_xmins_col] = pd.to_numeric(df.loc[team_mask, orig_gw_xmins_col], errors="coerce") # Update target GW xPts by adding xPts from original GW df.loc[team_mask, new_gw_pts_col] += df.loc[team_mask, orig_gw_pts_col] # Use target GW xMins df.loc[team_mask, new_gw_xmins_col] = df.loc[team_mask, new_gw_xmins_col] # Zero out key_value_Pts and key_value_xMins df.loc[team_mask, [orig_gw_pts_col, orig_gw_xmins_col]] = 0 # Save the updated CSV file df.to_csv(DATA_DIR / file_name, index=False) print(f"Generated: {file_name}") ================================================ FILE: run/run_parallel.py ================================================ import os from concurrent.futures import ProcessPoolExecutor import pandas as pd from solve import solve_regular from utils import get_dict_combinations def run_parallel_solves(chip_combinations, max_workers=None): if not max_workers: max_workers = os.cpu_count() - 2 # these are added just to reduce the output, you can remove them or put any settings you want here options = { "verbose": False, "print_result_table": False, "print_decay_metrics": False, "print_transfer_chip_summary": False, "print_squads": False, } args = [] for combination in chip_combinations: args.append({**options, **combination}) # Use ProcessPoolExecutor to run commands in parallel with ProcessPoolExecutor(max_workers=max_workers) as executor: results = list(executor.map(solve_regular, args)) df = pd.concat(results).sort_values(by="score", ascending=False).reset_index(drop=True) df = df.drop("iter", axis=1) print(df) # you can save the results to a csv file if you want to, by uncommenting the line below df.to_csv("chip_solve.csv", encoding="utf-8", index=False) if __name__ == "__main__": # edit the gameweeks you want to have chips available in here. # in this example it means it will run solves for 11 chips combinations: # no chips, bb1, bb2, fh2, fh3, fh4, bb1fh2, bb1fh3, bb1fh4, bb2fh3, bb2fh4 # note that this is the 3 bb options multiplied by the 4 fh options, minus the invalid combination bb2fh2 chip_gameweeks = { "use_bb": [None, 1, 2], "use_wc": [], "use_fh": [None, 2, 3, 4], "use_tc": [], } combinations = get_dict_combinations(chip_gameweeks) run_parallel_solves(combinations) ================================================ FILE: run/sensitivity.py ================================================ import argparse from collections import Counter from pathlib import Path import pandas as pd from paths import DATA_DIR ITER_SCORING = {0: 10, 1: 9, 2: 8} def get_user_inputs(options=None): """Get user inputs for sensitivity analysis.""" if options is None: options = {} gw = options.get("gw") situation = options.get("situation") if gw is not None and situation is not None: all_gws = "n" else: all_gws = options.get("all_gws") if all_gws is None: all_gws = input("Do you want to display a summary of buys and sells for all GWs? (y/n) ").strip().lower() print() return all_gws, gw, situation def process_all_gameweeks(): """Process and display results for all gameweeks.""" directory = DATA_DIR / "results" buys = [] sells = [] move = [] no_plans = 0 gameweeks = set() # Read CSV files and process plans for filename in Path(directory).glob("*.csv"): plan = pd.read_csv(filename) plan = plan.loc[(plan["squad"] == 1) | (plan["transfer_out"] == 1)] plan = plan.sort_values(by=["week", "iter", "pos", "id"]) iteration = plan.iloc[0]["iter"] if not plan.empty else 0 gameweeks.update(plan["week"].unique()) for week in gameweeks: if plan[(plan["week"] == week) & (plan["transfer_in"] == 1)]["name"].to_list() == []: buys.append({"move": "No transfer", "iter": iteration, "week": week}) sells.append({"move": "No transfer", "iter": iteration, "week": week}) move.append({"move": "No transfer", "iter": iteration, "week": week}) else: buy_list = plan[(plan["week"] == week) & (plan["transfer_in"] == 1)]["name"].to_list() for buy in buy_list: buys.append({"move": buy, "iter": iteration, "week": week}) sell_list = plan[(plan["week"] == week) & (plan["transfer_out"] == 1)]["name"].to_list() for sell in sell_list: sells.append({"move": sell, "iter": iteration, "week": week}) move.append( { "move": sell + " -> " + ", ".join(buy_list), "iter": iteration, "week": week, } ) no_plans += 1 return buys, sells, move, no_plans def print_pivot_tables_all_gws(buy_df, sell_df, no_plans): """Print pivot tables for all gameweeks analysis.""" show_top_n = input("Show top N results (y/n)? ").strip().lower() if show_top_n == "y": top_n = int(input("What do you want to use as N? ")) def print_pivots(df, title): print(f"{title}:") # Create the pivot table with counts df_counts = df.pivot_table(index="move", columns="week", aggfunc="size", fill_value=0) # Calculate percentages df_percentages = df_counts.divide(no_plans).multiply(100) # Map percentages to display with 0 decimals, hide 0% df_percentages = df_percentages.map(lambda x: f"{x:.0f}%" if x != 0 else "") # Store numeric version for sorting and total calculations df_percentages_numeric = df_counts.divide(no_plans).multiply(100) # Add 'Total' column summing over all weeks df_percentages["Total"] = df_percentages_numeric.sum(axis=1) df_percentages["Total"] = pd.to_numeric(df_percentages["Total"]) # Sort by 'Total' column df_percentages.sort_values(by="Total", ascending=False, inplace=True) # Apply the top N filter if requested if show_top_n == "y": df_percentages = df_percentages.head(top_n) # Format 'Total' column as percentages df_percentages["Total"] = df_percentages["Total"].map(lambda x: f"{x:.0f}%") # Print the result print(df_percentages) print() # Display the results print() print(f"Number of plans: {no_plans}") print() # Print Buy and Sell pivots print_pivots(buy_df, "Buy") print_pivots(sell_df, "Sell") def process_single_gameweek(gw, situation): """Process and display results for a single gameweek.""" directory = DATA_DIR / "results" if situation == "n": return process_regular_transfers(gw, directory) elif situation == "y": return process_wildcard_transfers(gw, directory) else: print("Invalid input, please enter 'y' for a wildcard or 'n' for a regular transfer plan.") return None def process_regular_transfers(gw, directory): """Process regular transfer analysis for a specific gameweek.""" print(f"Processing for GW {gw} without wildcard.") buys = [] sells = [] move = [] no_plans = 0 for filename in Path(directory).glob("*.csv"): plan = pd.read_csv(filename) plan = plan.loc[(plan["squad"] == 1) | (plan["transfer_out"] == 1)] plan = plan.sort_values(by=["week", "iter", "pos", "id"]) try: iteration = plan.iloc[0]["iter"] except Exception: iteration = 0 if plan[(plan["week"] == gw) & (plan["transfer_in"] == 1)]["name"].to_list() == []: buys.append({"move": "No transfer", "iter": iteration}) sells.append({"move": "No transfer", "iter": iteration}) move.append({"move": "No transfer", "iter": iteration}) else: buy_list = plan[(plan["week"] == gw) & (plan["transfer_in"] == 1)]["name"].to_list() buy = ", ".join(buy_list) buys.append({"move": buy, "iter": iteration}) sell_list = plan[(plan["week"] == gw) & (plan["transfer_out"] == 1)]["name"].to_list() sell = ", ".join(sell_list) sells.append({"move": sell, "iter": iteration}) move.append({"move": sell + " -> " + buy, "iter": iteration}) no_plans += 1 print() print(f"Number of plans: {no_plans}") print() return create_regular_transfer_pivots(buys, sells, move, no_plans) def create_regular_transfer_pivots(buys, sells, move, no_plans): """Create and display pivot tables for regular transfers.""" buy_df = pd.DataFrame(buys) buy_pivot = buy_df.pivot_table(index="move", columns="iter", aggfunc="size", fill_value=0) iters = sorted(buy_df["iter"].unique()) buy_pivot["PSB"] = buy_pivot.loc[:, iters].sum(axis=1) / buy_pivot.sum().sum() buy_pivot["PSB"] = buy_pivot["PSB"].apply(lambda x: f"{x:.0%}") buy_pivot["Score"] = buy_pivot.apply(lambda r: sum(r[i] * ITER_SCORING.get(i, 0) for i in iters), axis=1) buy_pivot.sort_values(by="Score", ascending=False, inplace=True) sell_df = pd.DataFrame(sells) sell_pivot = sell_df.pivot_table(index="move", columns="iter", aggfunc="size", fill_value=0) iters = sorted(sell_df["iter"].unique()) sell_pivot["PSB"] = sell_pivot.loc[:, iters].sum(axis=1) / sell_pivot.sum().sum() sell_pivot["PSB"] = sell_pivot["PSB"].apply(lambda x: f"{x:.0%}") sell_pivot["Score"] = sell_pivot.apply(lambda r: sum(r[i] * ITER_SCORING.get(i, 0) for i in iters), axis=1) sell_pivot.sort_values(by="Score", ascending=False, inplace=True) move_df = pd.DataFrame(move) move_pivot = move_df.pivot_table(index="move", columns="iter", aggfunc="size", fill_value=0) iters = sorted(move_df["iter"].unique()) move_pivot["PSB"] = move_pivot.loc[:, iters].sum(axis=1) / move_pivot.sum().sum() move_pivot["PSB"] = move_pivot["PSB"].apply(lambda x: f"{x:.0%}") move_pivot["Score"] = move_pivot.apply(lambda r: sum(r[i] * ITER_SCORING.get(i, 0) for i in iters), axis=1) move_pivot.sort_values(by="Score", ascending=False, inplace=True) # Set the display options for wider column width pd.set_option("display.max_colwidth", None) # Ask once for filtering choice at the beginning show_top_n = input("Show top N results (y/n)? ").strip().lower() if show_top_n == "y": top_n = int(input("What do you want to use as N? ")) print() # Apply the top N filter if requested if show_top_n == "y": buy_pivot = buy_pivot.head(top_n) sell_pivot = sell_pivot.head(top_n) move_pivot = move_pivot.head(top_n) print("Buy:") print(buy_pivot) print() print("Sell:") print(sell_pivot) print() print("Move:") print(move_pivot) print() return {"buy_pivot": buy_pivot, "sell_pivot": sell_pivot, "move_pivot": move_pivot} def process_wildcard_transfers(gw, directory): """Process wildcard transfer analysis for a specific gameweek.""" print(f"Processing for GW {gw} with wildcard.") goalkeepers = [] defenders = [] midfielders = [] forwards = [] no_plans = 0 for filename in Path(directory).glob("*.csv"): plan = pd.read_csv(filename) plan = plan.loc[(plan["squad"] == 1) | (plan["transfer_out"] == 1)] # Goalkeepers list of tuples (name, lineup status) goalkeepers += ( plan[(plan["week"] == gw) & (plan["pos"] == "GKP") & (plan["transfer_out"] != 1)][["name", "lineup"]] .apply(lambda x: (x["name"], 1 if x["lineup"] == 1 else 0), axis=1) .to_list() ) # Defenders list of tuples (name, lineup status) defenders += ( plan[(plan["week"] == gw) & (plan["pos"] == "DEF") & (plan["transfer_out"] != 1)][["name", "lineup"]] .apply(lambda x: (x["name"], 1 if x["lineup"] == 1 else 0), axis=1) .to_list() ) # Midfielders list of tuples (name, lineup status) midfielders += ( plan[(plan["week"] == gw) & (plan["pos"] == "MID") & (plan["transfer_out"] != 1)][["name", "lineup"]] .apply(lambda x: (x["name"], 1 if x["lineup"] == 1 else 0), axis=1) .to_list() ) # Forwards list of tuples (name, lineup status) forwards += ( plan[(plan["week"] == gw) & (plan["pos"] == "FWD") & (plan["transfer_out"] != 1)][["name", "lineup"]] .apply(lambda x: (x["name"], 1 if x["lineup"] == 1 else 0), axis=1) .to_list() ) no_plans += 1 print() print(f"Number of plans: {no_plans}") print() return create_wildcard_pivots(goalkeepers, defenders, midfielders, forwards, no_plans) def calculate_counts(player_list): """Calculate total counts and lineup counts for players.""" total_count = Counter([name for name, lineup in player_list]) lineup_count = Counter([name for name, lineup in player_list if lineup == 1]) # Convert to DataFrame total_df = pd.DataFrame(total_count.items(), columns=["player", "PSB"]) lineup_df = pd.DataFrame(lineup_count.items(), columns=["player", "Lineup"]) # Merge both DataFrames on player name merged_df = pd.merge(total_df, lineup_df, on="player", how="left").fillna(0) return merged_df def calculate_percentage(df, no_plans): """Convert counts to percentages and sort by PSB.""" # Sort by PSB before converting to percentages df = df.sort_values(by="PSB", ascending=False).reset_index(drop=True) df["#_PSB"] = df["PSB"].astype(int) df["#_Lineup"] = df["Lineup"].astype(int) # Convert to percentage df["PSB"] = ["{:.0%}".format(df["PSB"][x] / no_plans) for x in range(df.shape[0])] df["Lineup"] = ["{:.0%}".format(df["Lineup"][x] / no_plans) for x in range(df.shape[0])] return df def print_dataframe(df, title, use_color=False, psb_threshold=0.05): """Print DataFrame with aligned columns and optional color grading.""" print(f"{title}:") # Sort the DataFrame by PSB_count in descending order df = df.sort_values(by="#_PSB", ascending=False).reset_index(drop=True) # Define the max length for each column for proper alignment max_name_len = df["player"].str.len().max() max_psb_len = 8 max_lineup_len = 8 max_psb_count_len = max(8, df["#_PSB"].astype(str).str.len().max()) max_lineup_count_len = max(8, df["#_Lineup"].astype(str).str.len().max()) # Print the headers first with fixed width formatting header_format = ( f"{'player':<{max_name_len}} " f"{'PSB':<{max_psb_len}} " f"{'Lineup':<{max_lineup_len}} " f"{'#_PSB':<{max_psb_count_len}} " f"{'#_Lineup':<{max_lineup_count_len}}" ) print(header_format) # Ensure PSB and Lineup are strings and handle any non-string values df["PSB"] = df["PSB"].astype(str) df["Lineup"] = df["Lineup"].astype(str) # Normalize values for PSB and Lineup to range [0, 1] try: df["PSB_normalized"] = df["PSB"].str.extract(r"(\d+)%")[0].astype(float) / 100 df["Lineup_normalized"] = df["Lineup"].str.extract(r"(\d+)%")[0].astype(float) / 100 except Exception as e: print(f"Error normalizing data: {e}") return # Filter out players with PSB less than 5% df = df[df["PSB_normalized"] >= psb_threshold] # Calculate the maximum normalized values for the current DataFrame max_normalized_psb = df["PSB_normalized"].max() if not df.empty else 1 max_normalized_lineup = df["Lineup_normalized"].max() if not df.empty else 1 # Print each row with calculated widths and optional color grading for _, row in df.iterrows(): if use_color: # Calculate brightness for PSB based on its maximum value brightness_psb = int(200 * (row["PSB_normalized"] / max_normalized_psb)) if max_normalized_psb > 0 else 200 # Calculate brightness for Lineup based on its maximum value brightness_lineup = int(200 * (row["Lineup_normalized"] / max_normalized_lineup)) if max_normalized_lineup > 0 else 200 # Define colors for both PSB and Lineup color_psb = f"\033[38;2;0;{brightness_psb};{255 - brightness_psb}m" # Blue to Green gradient for PSB color_lineup = f"\033[38;2;0;{brightness_lineup};{255 - brightness_lineup}m" # Blue to Green gradient for Lineup else: # No color formatting if use_color is False color_psb = color_lineup = "" # Print each row with or without color player_part = f"{row['player']:<{max_name_len}}" psb_part = f"{color_psb}{row['PSB']:<{max_psb_len}}\033[0m" lineup_part = f"{color_lineup}{row['Lineup']:<{max_lineup_len}}\033[0m" psb_count_part = f"{color_psb}{row['#_PSB']:<{max_psb_count_len}}\033[0m" lineup_count_part = f"{color_lineup}{row['#_Lineup']:<{max_lineup_count_len}}\033[0m" print(f"{player_part} {psb_part} {lineup_part} {psb_count_part} {lineup_count_part}") print() # Add an empty line for separation between tables def create_wildcard_pivots(goalkeepers, defenders, midfielders, forwards, no_plans): """Create and display pivot tables for wildcard transfers.""" # Calculate for each position keepers = calculate_counts(goalkeepers) defs = calculate_counts(defenders) mids = calculate_counts(midfielders) fwds = calculate_counts(forwards) # Calculate percentages and sort for each position keepers = calculate_percentage(keepers, no_plans) defs = calculate_percentage(defs, no_plans) mids = calculate_percentage(mids, no_plans) fwds = calculate_percentage(fwds, no_plans) # Print sorted DataFrames for each position with proper alignment print_dataframe(keepers, "Goalkeepers") print_dataframe(defs, "Defenders") print_dataframe(mids, "Midfielders") print_dataframe(fwds, "Forwards") return {"keepers": keepers, "defs": defs, "mids": mids, "fwds": fwds} def read_sensitivity(options=None): """Main function to read and process sensitivity analysis.""" all_gws, gw, situation = get_user_inputs(options) # If all_gws is 'y', gw and wildcard are not needed if all_gws == "y": print("Processing all gameweeks.") buys, sells, move, no_plans = process_all_gameweeks() buy_df = pd.DataFrame(buys) sell_df = pd.DataFrame(sells) print_pivot_tables_all_gws(buy_df, sell_df, no_plans) # If all_gws is 'n', use gw and situation or ask for them else: if gw is None: gw = int(input("What GW are you assessing? ")) if situation is None: situation = input("Is this a wildcard or preseason (GW1) solve? (y/n) ").strip().lower() return process_single_gameweek(gw, situation) if __name__ == "__main__": import argparse try: parser = argparse.ArgumentParser(description="Summarize sensitivity analysis results") parser.add_argument( "--all_gws", choices=["Y", "y", "N", "n"], help="'y' if you want to display all gameweeks, 'n' otherwise", ) parser.add_argument("--gw", type=int, help="Numeric value for 'gw'") parser.add_argument( "--wildcard", choices=["Y", "y", "N", "n"], help="'y' if using wildcard, 'n' otherwise", ) args = parser.parse_args() # Prepare options for read_sensitivity based on arguments options = {} # Handle command-line arguments if args.all_gws: options["all_gws"] = args.all_gws.strip().lower() if args.gw is not None: options["gw"] = args.gw if args.wildcard: options["situation"] = args.wildcard.strip().lower() # If no command-line arguments are provided, prompt for input if not any(vars(args).values()): print("No command-line arguments provided, prompting for input... use command line arguments if you want to skip questions") print() read_sensitivity() # Prompt for input elif "all_gws" in options or ("gw" in options and "situation" in options): read_sensitivity(options) else: print("Error: You must specify either --all_gws or both --gw and --wildcard.") except Exception as e: print(f"Error occurred: {e}") print("Falling back to user input mode.") # Fallback to prompt user for input read_sensitivity() ================================================ FILE: run/simulations.py ================================================ import argparse import json import time from concurrent.futures import ProcessPoolExecutor from binary_file_generator import generate_binary_files from solve import solve_regular from paths import DATA_DIR from utils import load_settings def get_user_input(): print("Remember to delete results folder before running simulations") runs = int(input("How many simulations would you like to run? ")) processes = int(input("How many processes you want to run in parallel? ")) use_binaries = input("Use binaries (y or n)? ") return runs, processes, use_binaries def get_options_from_args(options): runs = options.get("count", 1) processes = options.get("processes", 1) use_binaries = options.get("use_binaries", "n") return runs, processes, use_binaries def setup_binary_files(): settings = load_settings() source = settings.get("datasource", {}) if settings.get("generate_binary_files"): print("Generating binary files") binary_fixture_settings = settings.get("binary_fixture_settings", {}) if not binary_fixture_settings: raise ValueError("Your `binary_fixture_settings` setting is empty!") file_path = DATA_DIR / f"{source}.csv" generate_binary_files(file_path, binary_fixture_settings) return settings def run_simulations_with_binaries(runs, processes, options): """Run simulations using binary files""" print("Using binary config for simulations") settings = setup_binary_files() weights = settings.get("binary_file_weights", {}) total_weights = sum(weights.values()) for binary, weight in weights.items(): scaled_weight = weight / total_weights print(f"Binary file {binary} weight scaled from {weight} to {scaled_weight:.2f}") weighted_runs = round(scaled_weight * runs) print(f"Running {weighted_runs} simulations for binary file {binary}") start = time.time() runtime_options = options.get("runtime_options", {}) all_jobs = [{"run_no": str(i + 1), "randomized": True, "datasource": binary.rstrip(".csv"), **runtime_options} for i in range(weighted_runs)] with ProcessPoolExecutor(max_workers=processes) as executor: list(executor.map(solve_regular, all_jobs)) print(f"\nTotal time taken is {(time.time() - start) / 60:.2f} minutes") def run_simulations_standard(runs, processes, options): start = time.time() runtime_options = options.get("runtime_options", {}) all_jobs = [{"run_no": str(i + 1), "randomized": True, **runtime_options} for i in range(runs)] with ProcessPoolExecutor(max_workers=processes) as executor: list(executor.map(solve_regular, all_jobs)) print(f"\nTotal time taken is {(time.time() - start) / 60:.2f} minutes") def run_sensitivity(options=None): if options is None or "count" not in options: runs, processes, use_binaries = get_user_input() else: runs, processes, use_binaries = get_options_from_args(options) # if use_binaries is set, loop through binary_files dict in settings # and set number of sim run for each binary based on provided weights if use_binaries.lower() == "y": run_simulations_with_binaries(runs, processes, options) else: run_simulations_standard(runs, processes, options) def parse_unknown_arguments(unknown): """Parse unknown command line arguments and convert them to runtime options""" runtime_options = {} i = 0 while i < len(unknown): if unknown[i].startswith("--"): key = unknown[i][2:] # Remove -- prefix if i + 1 < len(unknown) and not unknown[i + 1].startswith("--"): value = unknown[i + 1] if value.isdigit(): runtime_options[key] = int(value) else: try: runtime_options[key] = float(value) except ValueError: if value[0] in "[{": try: runtime_options[key] = json.loads(value) except json.JSONDecodeError: runtime_options[key] = json.loads(value.replace("'", '"')) else: runtime_options[key] = value i += 2 else: runtime_options[key] = True i += 1 else: i += 1 return runtime_options if __name__ == "__main__": try: parser = argparse.ArgumentParser(description="Run sensitivity analysis") parser.add_argument("--no", type=int, help="Number of runs") parser.add_argument("--parallel", type=int, help="Number of parallel runs") parser.add_argument("--use_binaries", type=str, help="Do you want to use binaries? (y/n)") # Parse known arguments first args, unknown = parser.parse_known_args() options = {} if args.no: options["count"] = args.no if args.parallel: options["processes"] = args.parallel if args.use_binaries: options["use_binaries"] = args.use_binaries options["runtime_options"] = parse_unknown_arguments(unknown) except Exception: options = None # Clear command line arguments to prevent them from being passed to solve_regular import sys sys.argv = [sys.argv[0]] run_sensitivity(options) ================================================ FILE: run/solve.py ================================================ import argparse import csv import datetime import json import os import subprocess import sys import textwrap import time import pandas as pd import requests from tabulate import tabulate from dev.solver import generate_team_json, prep_data, solve_multi_period_fpl from dev.visualization import create_squad_timeline from paths import DATA_DIR from utils import cached_request, get_random_id, load_config_files, load_settings IS_COLAB = "COLAB_GPU" in os.environ BINARY_THRESHOLD = 0.5 def is_latest_version(): try: # Get the current branch name branch = subprocess.check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"], stderr=subprocess.DEVNULL, text=True).strip() # Fetch the latest updates from the remote subprocess.run(["git", "fetch"], check=True, stderr=subprocess.DEVNULL) # Check if there are commits in the remote branch not in the local branch updates = subprocess.check_output(["git", "rev-list", f"HEAD..origin/{branch}"], stderr=subprocess.DEVNULL, text=True).strip() if updates: print("Your repository is not up-to-date. Please pull the latest changes.") return False else: print("Your repository is up-to-date.") return True except subprocess.CalledProcessError: print("Error: Could not check the repository status.") return False def solve_regular(runtime_options=None): # if not IS_COLAB: # print("Checking for updates...") # is_latest_version() # Create a base parser first for the --config argument # remaining_args is all the command line args that aren't --config base_parser = argparse.ArgumentParser(add_help=False) base_parser.add_argument("--config", type=str, help="Path to one or more configuration files (semicolon-delimited)") base_args, remaining_args = base_parser.parse_known_args() # Load base configuration file options = load_settings() # Load and merge additional configuration files if specified if base_args.config: config_options = load_config_files(base_args.config) options.update(config_options) # Override base config with additional configs # Create the full parser with all configuration options parser = argparse.ArgumentParser(parents=[base_parser]) for key, value in options.items(): if value is None or isinstance(value, list | dict): parser.add_argument(f"--{key}", default=value) continue parser.add_argument(f"--{key}", type=type(value), default=value) # Parse remaining arguments, which will take highest priority args = vars(parser.parse_args(remaining_args)) # this code block is to look at command line arguments (read as a string) and determine what type # they should be when there is no default argument type set by the code above for key, value in args.items(): if key not in options: continue if value == options[key]: # skip anything that hasn't been edited by command line argument continue if options[key] is None or isinstance(options[key], list | dict): if value.isdigit(): args[key] = int(value) continue try: args[key] = float(value) continue except ValueError: pass if value[0] in "[{": try: args[key] = json.loads(value) continue except json.JSONDecodeError: args[key] = json.loads(value.replace("'", '"')) continue finally: pass print(f"Problem with CL argument: {key}. Original value: {options[key]}, New value: {value}") cli_options = {k: v for k, v in args.items() if v is not None and k != "config"} # Update options with CLI arguments (highest priority) options.update(cli_options) if runtime_options is not None: options.update(runtime_options) if options.get("preseason"): my_data = {"picks": [], "chips": [], "transfers": {"limit": None, "cost": 4, "bank": 1000, "value": 0}} elif options.get("team_data", "json").lower() == "id": team_id = options.get("team_id", None) if team_id is None: print("You must supply your team_id in data/user_settings.json") sys.exit(0) my_data = generate_team_json(team_id, options) elif options.get("team_json"): my_data = json.loads(options["team_json"]) else: try: with open(DATA_DIR / "team.json") as f: my_data = json.load(f) except FileNotFoundError: msg = """ team.json file not found in the data folder. You must either: 1. Download your team data from https://fantasy.premierleague.com/api/my-team/YOUR-TEAM-ID/ and either a) save it inside the data folder with the filename 'team.json' or b) supply it to the "team_json" option in user_settings.json 2. Set "team_data" in user_settings to "ID", and set the "team_id" value to your team's ID """ print(textwrap.dedent(msg)) sys.exit(0) if price_changes := options.get("price_changes", []): my_squad_ids = [x["element"] for x in my_data["picks"]] fpl_data = cached_request("https://fantasy.premierleague.com/api/bootstrap-static/") elements = fpl_data["elements"] current_prices = {x["id"]: x["now_cost"] for x in elements if x["id"] in my_squad_ids} for pid, change in price_changes: if pid not in my_squad_ids: continue new_price = current_prices[pid] + change player = next(x for x in my_data["picks"] if x["element"] == pid) if player["purchase_price"] >= new_price: player["selling_price"] = new_price else: player["selling_price"] = player["purchase_price"] + (new_price - player["purchase_price"]) // 2 data = prep_data(my_data, options) response = solve_multi_period_fpl(data, options) run_id = get_random_id(5) options["run_id"] = run_id for i, result in enumerate(response): if options.get("print_squads"): print(f"\n\nSolution {i + 1}") print(textwrap.indent(result["summary"], " ")) total_xp = sum(gw_stats.get("xP", 0) for _, gw_stats in result["statistics"].items()) print(f"Total xPts over the horizon: {total_xp:.2f}\n") iteration = result["iter"] time_now = datetime.datetime.now() stamp = time_now.strftime("%Y-%m-%d_%H-%M-%S") source = options.get("datasource") filename = f"{source}_{stamp}_{run_id}_{iteration}" if not os.path.exists(DATA_DIR / "results/"): os.mkdir(DATA_DIR / "results/") result["picks"].to_csv(DATA_DIR / "results" / f"{filename}.csv", index=False) if options.get("export_image", 0) and not IS_COLAB: create_squad_timeline( current_squad=data["initial_squad"], statistics=result["statistics"], picks=result["picks"], filename=filename, ) result_table = pd.DataFrame(response) result_table = result_table.sort_values(by="score", ascending=False) result_table = result_table[["iter", "sell", "buy", "chip", "score"]] dataframe_format = options.get("dataframe_format", "plain") if options.get("print_decay_metrics"): # print decay metrics if len(options.get("report_decay_base", [])) > 0: try: print("\nDecay Metrics") metrics_df = pd.DataFrame([{"iter": result["iter"], **result["decay_metrics"]} for result in response]) print(tabulate(metrics_df, headers="keys", tablefmt=dataframe_format, showindex=False, floatfmt=".2f")) except Exception: pass if options.get("print_transfer_chip_summary"): print("\n\n\nTransfer Overview") for result in response: print_transfer_chip_summary(result, options) if options.get("print_result_table"): # print result table print(f"\n\nResult{'s' if len(response) > 1 else ''}") print(tabulate(result_table, headers="keys", tablefmt=dataframe_format, showindex=False, floatfmt=".2f")) print("\n\n") if solutions_file := options.get("solutions_file"): for result in response: write_line_to_file(solutions_file, result, options) return result_table def print_transfer_chip_summary(result, options): picks = result["picks"] gws = picks["week"].unique() print(f"\nSolution {result['iter'] + 1}") for gw in sorted(gws): chip_text = "" line_text = "" chip = picks.loc[(picks["week"] == gw) & (picks["chip"] != "")] if not chip.empty: chip_text = chip.iloc[0]["chip"] line_text += f"({chip_text}) " sell_text = ", ".join(picks[(picks["week"] == gw) & (picks["transfer_out"] == 1)]["name"].to_list()) buy_text = ", ".join(picks[(picks["week"] == gw) & (picks["transfer_in"] == 1)]["name"].to_list()) if sell_text != "" or buy_text != "": line_text += sell_text + " -> " + buy_text elif chip_text == "FH": line_text += "" else: line_text += "Roll" print(f"\tGW{gw}: {line_text}") def write_line_to_file(filename, result, options): t = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) gw = min(result["picks"]["week"]) score = round(result["score"], 3) picks = result["picks"] run_id = options["run_id"] iteration = result["iter"] team_id = options.get("team_id") datasource = options.get("datasource") chips = [",".join(map(str, options.get(x, []))) for x in ["use_wc", "use_bb", "use_fh", "use_tc"]] squad = picks.loc[(picks["week"] == gw) & ((picks["lineup"] == 1) | (picks["bench"] >= 0))].sort_values( by=["lineup", "bench", "type"], ascending=[False, True, True] ) sells = picks.loc[(picks["week"] == gw) & (picks["transfer_out"] == 1)] buys = picks.loc[(picks["week"] == gw) & (picks["transfer_in"] == 1)] cap = picks.loc[(picks["week"] == gw) & (picks["captain"] > BINARY_THRESHOLD)].iloc[0] vcap = picks.loc[(picks["week"] == gw) & (picks["vicecaptain"] > BINARY_THRESHOLD)].iloc[0] if options.get("solutions_file_player_type", "name") == "name": squad = squad["name"].to_list() sell_text = ",".join(sells["name"].to_list()) buy_text = ",".join(buys["name"].to_list()) cap = cap["name"] vcap = vcap["name"] else: squad = squad["id"].astype(int).to_list() sell_text = ", ".join(sells["id"].astype(str).to_list()) buy_text = ", ".join(buys["id"].astype(str).to_list()) cap = cap["id"].astype(int) vcap = vcap["id"].astype(int) headers = [ "run_id", "iter", "user_id", "datasource", "wc", "bb", "fh", "tc", *[f"p{i}" for i in range(1, 16)], "cap", "vcap", "sell", "buy", "score", "datetime", ] data = [run_id, iteration, team_id, datasource, *chips, *squad, cap, vcap, sell_text, buy_text, score, t] if options.get("save_squads", False): headers.append("summary") data.append(result["summary"]) if not os.path.exists(filename): with open(filename, "w", newline="") as f: writer = csv.writer(f) writer.writerow(headers) with open(filename, "a", newline="", encoding="utf-8") as f: writer = csv.writer(f) writer.writerow(data) # Link to FPL.Team # get_fplteam_link(options, response) def get_fplteam_link(options, response): print("\nYou can see the solutions on a planner using the following FPL.Team links:") team_id = options.get("team_id", 1) if options.get("team_id") is None: print("(Do not forget to add your team ID to user_settings.json file to get a custom link.)") url_base = f"https://fpl.team/plan/{team_id}/?" for result in response: result_url = url_base picks = result["picks"] gws = picks["week"].unique() for gw in gws: lineup_players = ",".join(picks[(picks["week"] == gw) & (picks["lineup"] > BINARY_THRESHOLD)]["id"].astype(str).to_list()) bench_players = ",".join(picks[(picks["week"] == gw) & (picks["bench"] > -BINARY_THRESHOLD)]["id"].astype(str).to_list()) cap = picks[(picks["week"] == gw) & (picks["captain"] > BINARY_THRESHOLD)].iloc[0]["id"] vcap = picks[(picks["week"] == gw) & (picks["vicecaptain"] > BINARY_THRESHOLD)].iloc[0]["id"] chip = picks[picks["week"] == gw].iloc[0]["chip"] sold_players = ( picks[(picks["week"] == gw) & (picks["transfer_out"] > BINARY_THRESHOLD)].sort_values(by="type")["id"].astype(str).to_list() ) bought_players = ( picks[(picks["week"] == gw) & (picks["transfer_in"] > BINARY_THRESHOLD)].sort_values(by="type")["id"].astype(str).to_list() ) if gw == 1: sold_players = [] bought_players = [] tr_string = ";".join([f"{i},{j}" for (i, j) in zip(sold_players, bought_players, strict=False)]) if tr_string == "": tr_string = ";" sub_text = "" if gw == 1: sub_text = ";" else: prev_lineup = ( picks[(picks["week"] == gw - 1) & (picks["lineup"] > BINARY_THRESHOLD)].sort_values(by="type")["id"].astype(str).to_list() ) now_bench = picks[(picks["week"] == gw) & (picks["bench"] > -BINARY_THRESHOLD)].sort_values(by="type")["id"].astype(str).to_list() lineup_to_bench = [i for i in prev_lineup if i in now_bench] prev_bench = ( picks[(picks["week"] == gw - 1) & (picks["bench"] > -BINARY_THRESHOLD)].sort_values(by="type")["id"].astype(str).to_list() ) now_lineup = picks[(picks["week"] == gw) & (picks["lineup"] > BINARY_THRESHOLD)].sort_values(by="type")["id"].astype(str).to_list() bench_to_lineup = [i for i in prev_bench if i in now_lineup] sub_text = ";".join([f"{i},{j}" for (i, j) in zip(lineup_to_bench, bench_to_lineup, strict=False)]) if sub_text == "": sub_text = ";" gw_params = ( f"lineup{gw}={lineup_players}&bench{gw}={bench_players}&cap{gw}={cap}&vcap{gw}={vcap}" f"&chip{gw}={chip}&transfers{gw}={tr_string}&subs{gw}={sub_text}&opt=true" ) result_url += ("" if gw == gws[0] else "&") + gw_params print(f"Solution {result['iter'] + 1}: {result_url}") if __name__ == "__main__": solve_regular() ================================================ FILE: run/tmp/.gitignore ================================================ *.txt ================================================ FILE: run/tmp/.gitkeep ================================================ ================================================ FILE: tests/__init__.py ================================================ ================================================ FILE: tests/test_options_parsing.py ================================================ import argparse import json import sys import tempfile from pathlib import Path import pytest from run.solve_regular import load_config_files @pytest.fixture def temp_config_files(): """Create temporary config files for testing.""" with tempfile.TemporaryDirectory() as tmpdir: base_config = {"solve_name": "test", "horizon": 5, "iterations": 100} base_path = Path(tmpdir) / "base_config.json" with open(base_path, "w") as f: json.dump(base_config, f) override_config = {"iterations": 200, "export_image": True} override_path = Path(tmpdir) / "override_config.json" with open(override_path, "w") as f: json.dump(override_config, f) yield {"base_path": str(base_path), "override_path": str(override_path)} def test_load_single_config(temp_config_files): """Test loading a single configuration file.""" config = load_config_files(temp_config_files["base_path"]) assert config["solve_name"] == "test" assert config["horizon"] == 5 assert config["iterations"] == 100 def test_load_multiple_configs(temp_config_files): """Test loading multiple configuration files with overrides.""" config_paths = f"{temp_config_files['base_path']};{temp_config_files['override_path']}" config = load_config_files(config_paths) assert config["solve_name"] == "test" assert config["horizon"] == 5 assert config["iterations"] == 200 assert config["export_image"] def test_load_nonexistent_config(): """Test handling of nonexistent configuration file.""" config = load_config_files("nonexistent.json") assert config == {} def test_load_invalid_json(tmp_path): """Test handling of invalid JSON configuration file.""" invalid_config = tmp_path / "invalid.json" invalid_config.write_text("{invalid json") config = load_config_files(str(invalid_config)) assert config == {} def test_empty_config_path(): """Test handling of empty configuration path.""" config = load_config_files("") assert config == {} def test_semicolon_only_config_path(): """Test handling of config path with only semicolons.""" config = load_config_files(";;") assert config == {} @pytest.fixture def mock_argv(monkeypatch): """Fixture to temporarily replace sys.argv""" def _mock_argv(args): monkeypatch.setattr(sys.argv, args) return _mock_argv def create_arg_parser(): """Helper function to create argument parser similar to solve_regular.py""" base_parser = argparse.ArgumentParser(add_help=False) base_parser.add_argument("--config", type=str, help="Path to one or more configuration files (semicolon-delimited)") return base_parser def test_cli_no_config_argument(): """Test CLI parsing with no config argument.""" parser = create_arg_parser() args = parser.parse_known_args(["--other-arg", "value"])[0] assert args.config is None def test_cli_single_config(): """Test CLI parsing with single config path.""" parser = create_arg_parser() args = parser.parse_known_args(["--config", "path/to/config.json"])[0] assert args.config == "path/to/config.json" def test_cli_multiple_configs(): """Test CLI parsing with multiple semicolon-separated config paths.""" parser = create_arg_parser() args = parser.parse_known_args(["--config", "config1.json;config2.json"])[0] assert args.config == "config1.json;config2.json" def test_config_priority_order(temp_config_files, monkeypatch): """Test that configuration priority is respected: base -> override configs -> CLI args""" base_parser = argparse.ArgumentParser(add_help=False) base_parser.add_argument("--config", type=str) base_config = {"solve_name": "base", "horizon": 5, "iterations": 100} override_config = {"horizon": 7, "export_image": True} with tempfile.TemporaryDirectory() as tmpdir: base_path = Path(tmpdir) / "base.json" with open(base_path, "w") as f: json.dump(base_config, f) override_path = Path(tmpdir) / "override.json" with open(override_path, "w") as f: json.dump(override_config, f) test_args = ["script.py", "--config", f"{base_path};{override_path}", "--horizon", "10"] monkeypatch.setattr(sys, "argv", test_args) base_args, remaining = base_parser.parse_known_args() options = base_config.copy() if base_args.config: config_options = load_config_files(base_args.config) options.update(config_options) parser = argparse.ArgumentParser(parents=[base_parser]) for key, value in options.items(): if not isinstance(value, list | dict): parser.add_argument(f"--{key}", type=type(value), default=value) args = parser.parse_args(remaining) cli_options = {k: v for k, v in vars(args).items() if v is not None and k != "config"} options.update(cli_options) assert options["solve_name"] == "base" assert options["horizon"] == 10 assert options["iterations"] == 100 assert options["export_image"] def test_partial_cli_override(temp_config_files, monkeypatch): """Test that CLI arguments only override specified values""" base_parser = argparse.ArgumentParser(add_help=False) base_parser.add_argument("--config", type=str) with tempfile.TemporaryDirectory() as tmpdir: config = {"solve_name": "test", "horizon": 5, "iterations": 100} config_path = Path(tmpdir) / "config.json" with open(config_path, "w") as f: json.dump(config, f) test_args = ["script.py", "--config", str(config_path), "--horizon", "7"] monkeypatch.setattr(sys, "argv", test_args) base_args, remaining = base_parser.parse_known_args() options = load_config_files(base_args.config) parser = argparse.ArgumentParser(parents=[base_parser]) for key, value in options.items(): if not isinstance(value, list | dict): parser.add_argument(f"--{key}", type=type(value), default=value) args = parser.parse_args(remaining) cli_options = {k: v for k, v in vars(args).items() if v is not None and k != "config"} options.update(cli_options) assert options["solve_name"] == "test" assert options["horizon"] == 7 assert options["iterations"] == 100 # Unchanged from config ================================================ FILE: utils.py ================================================ import json import random import string import time from itertools import product from pathlib import Path import requests from paths import DATA_DIR # Cache configuration CACHE_DIR = Path(__file__).parent / ".cache" CACHE_FILE = CACHE_DIR / "http_cache.json" CACHE_EXPIRATION = 300 def load_settings(): with open(DATA_DIR / "comprehensive_settings.json") as f: options = json.load(f) with open(DATA_DIR / "user_settings.json") as f: options = {**options, **json.load(f)} return options def get_random_id(n): return "".join(random.choice(string.ascii_letters + string.digits) for _ in range(n)) def xmin_to_prob(xmin, sub_on=0.5, sub_off=0.3): start = min(max((xmin - 25 * sub_on) / (90 * (1 - sub_off) + 65 * sub_off - 25 * sub_on), 0.001), 0.999) return start + (1 - start) * sub_on def get_dict_combinations(my_dict): keys = my_dict.keys() for key in keys: if my_dict[key] is None or len(my_dict[key]) == 0: my_dict[key] = [None] all_combs = [dict(zip(my_dict.keys(), values, strict=False)) for values in product(*my_dict.values())] feasible_combs = [] for comb in all_combs: c_values = [i for i in comb.values() if i is not None] if len(c_values) == len(set(c_values)): feasible_combs.append({k: [v] for k, v in comb.items() if v is not None}) # else we have a duplicate return feasible_combs def load_config_files(config_paths): """ Load and merge multiple configuration files. Files are merged in order, with later files overriding earlier ones. """ merged_config = {} if not config_paths: return merged_config paths = config_paths.split(";") for path in paths: stripped_path = path.strip() if not path: continue try: with open(stripped_path) as f: config = json.load(f) merged_config.update(config) except FileNotFoundError: print(f"Warning: Configuration file {stripped_path} not found") except json.JSONDecodeError: print(f"Warning: Configuration file {stripped_path} is not valid JSON") return merged_config def cached_request(url): """ Fetch data from URL with caching support. Returns cached data if available and not expired (< 24 hours old). Otherwise fetches fresh data, updates cache, and returns the data. Args: url: The URL to fetch data from Returns: dict: JSON response from the URL """ # Create cache directory if it doesn't exist CACHE_DIR.mkdir(exist_ok=True) # Load existing cache cache = {} if CACHE_FILE.exists(): try: with open(CACHE_FILE) as f: cache = json.load(f) except (json.JSONDecodeError, IOError): # If cache is corrupted, start fresh cache = {} # Check if URL is in cache and not expired current_time = time.time() if url in cache: cached_entry = cache[url] timestamp = cached_entry.get("timestamp", 0) if current_time - timestamp < CACHE_EXPIRATION: # Cache is still valid return cached_entry["data"] # Cache miss or expired - fetch fresh data try: response = requests.get(url) response.raise_for_status() data = response.json() # Update cache cache[url] = { "data": data, "timestamp": current_time } # Save cache to file with open(CACHE_FILE, "w") as f: json.dump(cache, f, indent=2) return data except requests.RequestException as e: # If network request fails and we have expired cache, return it anyway if url in cache: print(f"Warning: Failed to fetch {url}, using expired cache. Error: {e}") return cache[url]["data"] raise