[ { "path": ".gitignore", "content": "# Byte-compiled / optimized / DLL files\n__pycache__/\n*.py[cod]\n*$py.class\n\n# C extensions\n*.so\n\n# Distribution / packaging\n.Python\nbuild/\ndevelop-eggs/\ndist/\ndownloads/\neggs/\n.eggs/\nlib/\nlib64/\nparts/\nsdist/\nvar/\nwheels/\npip-wheel-metadata/\nshare/python-wheels/\n*.egg-info/\n.installed.cfg\n*.egg\nMANIFEST\n\n# PyInstaller\n# Usually these files are written by a python script from a template\n# before PyInstaller builds the exe, so as to inject date/other infos into it.\n*.manifest\n*.spec\n\n# Installer logs\npip-log.txt\npip-delete-this-directory.txt\n\n# Unit test / coverage reports\nhtmlcov/\n.tox/\n.nox/\n.coverage\n.coverage.*\n.cache\nnosetests.xml\ncoverage.xml\n*.cover\n*.py,cover\n.hypothesis/\n.pytest_cache/\n\n# Translations\n*.mo\n*.pot\n\n# Django stuff:\n*.log\nlocal_settings.py\ndb.sqlite3\ndb.sqlite3-journal\n\n# Flask stuff:\ninstance/\n.webassets-cache\n\n# Scrapy stuff:\n.scrapy\n\n# Sphinx documentation\ndocs/_build/\n\n# PyBuilder\ntarget/\n\n# Jupyter Notebook\n.ipynb_checkpoints\n\n# IPython\nprofile_default/\nipython_config.py\n\n# pyenv\n.python-version\n\n# pipenv\n# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.\n# However, in case of collaboration, if having platform-specific dependencies or dependencies\n# having no cross-platform support, pipenv may install dependencies that don't work, or not\n# install all needed dependencies.\n#Pipfile.lock\n\n# PEP 582; used by e.g. github.com/David-OConnor/pyflow\n__pypackages__/\n\n# Celery stuff\ncelerybeat-schedule\ncelerybeat.pid\n\n# SageMath parsed files\n*.sage.py\n\n# Environments\n.env\n.venv\nenv/\nvenv/\nENV/\nenv.bak/\nvenv.bak/\n\n# Spyder project settings\n.spyderproject\n.spyproject\n\n# Rope project settings\n.ropeproject\n\n# mkdocs documentation\n/site\n\n# mypy\n.mypy_cache/\n.dmypy.json\ndmypy.json\n\n# Pyre type checker\n.pyre/\n\n# Custom GitIgnore\n*.code-workspace\n*.out\nlogs/\nout/\ndataset/\n.vscode/" }, { "path": "LICENSE", "content": "MIT License\n\nCopyright (c) 2020 Vijay Prakash Dwivedi, Xavier Bresson\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n" }, { "path": "README.md", "content": "# SAN\n\nImplementation of Spectral Attention Networks, a powerful GNN that leverages key principles from spectral graph theory to enable full graph attention.\n\n![full_method](https://user-images.githubusercontent.com/47570400/119883871-046aa280-befe-11eb-9063-108f4fe1a123.png)\n\n# Overview\n\n* ```nets``` contains the Node, Edge and no LPE architectures implemented with PyTorch.\n* ```layers``` contains the multi-headed attention employed by the Main Graph Transformer implemented in DGL.\n* ```train``` contains methods to train the models.\n* ```data``` contains dataset classes and various methods used in precomputation.\n* ```configs``` contains the various parameters used in the ablation and SOTA comparison studies.\n* ```misc``` contains scripts from https://github.com/graphdeeplearning/graphtransformer to download datasets and setup environments.\n* ```scripts``` contains scripts to reproduce ablation and SOTA comparison results. See ```scripts/reproduce.md``` for details.\n\n\n" }, { "path": "configs/CLUSTER/ablation/full/1e-1/node", "content": "{\n \"gpu\": {\n \"use\": true,\n \"id\": 0\n },\n \n \"model\": \"GraphTransformer\",\n \"dataset\": \"SBM_CLUSTER\",\n \n \"out_dir\": \"out/SBM_CLUSTER/full/1e-1/node\",\n \n \"params\": {\n \"seed\": 41,\n \"epochs\": 1000,\n \"batch_size\": 16,\n \"init_lr\": 0.0005,\n \"lr_reduce_factor\": 0.5,\n \"lr_schedule_patience\": 10,\n \"min_lr\": 1e-5,\n \"weight_decay\": 0.0,\n \"print_epoch_interval\": 5,\n \"max_time\": 24\n },\n \n \"net_params\": {\n \"full_graph\": true,\n \"gamma\": 1e-1,\n \"m\": 10,\n \n \"LPE\": \"node\",\n \"LPE_layers\": 1,\n \"LPE_dim\": 16,\n \"LPE_n_heads\": 4,\n\n \"GT_layers\": 16,\n \"GT_hidden_dim\": 48,\n \"GT_out_dim\": 48,\n \"GT_n_heads\": 8,\n\n \"residual\": true,\n \"readout\": \"mean\",\n \"in_feat_dropout\": 0.0,\n \"dropout\": 0.0,\n \"layer_norm\": false,\n \"batch_norm\": true\n }\n}\n" }, { "path": "configs/CLUSTER/ablation/full/1e-2/node", "content": "{\n \"gpu\": {\n \"use\": true,\n \"id\": 0\n },\n \n \"model\": \"GraphTransformer\",\n \"dataset\": \"SBM_CLUSTER\",\n \n \"out_dir\": \"out/SBM_CLUSTER/full/1e-2/node\",\n \n \"params\": {\n \"seed\": 41,\n \"epochs\": 1000,\n \"batch_size\": 16,\n \"init_lr\": 0.0005,\n \"lr_reduce_factor\": 0.5,\n \"lr_schedule_patience\": 10,\n \"min_lr\": 1e-5,\n \"weight_decay\": 0.0,\n \"print_epoch_interval\": 5,\n \"max_time\": 24\n },\n \n \"net_params\": {\n \"full_graph\": true,\n \"gamma\": 1e-2,\n \"m\": 10,\n \n \"LPE\": \"node\",\n \"LPE_layers\": 1,\n \"LPE_dim\": 16,\n \"LPE_n_heads\": 4,\n\n \"GT_layers\": 16,\n \"GT_hidden_dim\": 48,\n \"GT_out_dim\": 48,\n \"GT_n_heads\": 8,\n\n \"residual\": true,\n \"readout\": \"mean\",\n \"in_feat_dropout\": 0.0,\n \"dropout\": 0.0,\n \"layer_norm\": false,\n \"batch_norm\": true\n }\n}\n" }, { "path": "configs/CLUSTER/ablation/full/1e-3/node", "content": "{\n \"gpu\": {\n \"use\": true,\n \"id\": 0\n },\n \n \"model\": \"GraphTransformer\",\n \"dataset\": \"SBM_CLUSTER\",\n \n \"out_dir\": \"out/SBM_CLUSTER/full/1e-3/node\",\n \n \"params\": {\n \"seed\": 41,\n \"epochs\": 1000,\n \"batch_size\": 16,\n \"init_lr\": 0.0005,\n \"lr_reduce_factor\": 0.5,\n \"lr_schedule_patience\": 10,\n \"min_lr\": 1e-5,\n \"weight_decay\": 0.0,\n \"print_epoch_interval\": 5,\n \"max_time\": 24\n },\n \n \"net_params\": {\n \"full_graph\": true,\n \"gamma\": 1e-3,\n \"m\": 10,\n \n \"LPE\": \"node\",\n \"LPE_layers\": 1,\n \"LPE_dim\": 16,\n \"LPE_n_heads\": 4,\n\n \"GT_layers\": 16,\n \"GT_hidden_dim\": 48,\n \"GT_out_dim\": 48,\n \"GT_n_heads\": 8,\n\n \"residual\": true,\n \"readout\": \"mean\",\n \"in_feat_dropout\": 0.0,\n \"dropout\": 0.0,\n \"layer_norm\": false,\n \"batch_norm\": true\n }\n}\n" }, { "path": "configs/CLUSTER/ablation/full/1e-4/node", "content": "{\n \"gpu\": {\n \"use\": true,\n \"id\": 0\n },\n \n \"model\": \"GraphTransformer\",\n \"dataset\": \"SBM_CLUSTER\",\n \n \"out_dir\": \"out/SBM_CLUSTER/full/1e-4/node\",\n \n \"params\": {\n \"seed\": 41,\n \"epochs\": 1000,\n \"batch_size\": 16,\n \"init_lr\": 0.0005,\n \"lr_reduce_factor\": 0.5,\n \"lr_schedule_patience\": 10,\n \"min_lr\": 1e-5,\n \"weight_decay\": 0.0,\n \"print_epoch_interval\": 5,\n \"max_time\": 24\n },\n \n \"net_params\": {\n \"full_graph\": true,\n \"gamma\": 1e-4,\n \"m\": 10,\n \n \"LPE\": \"node\",\n \"LPE_layers\": 1,\n \"LPE_dim\": 16,\n \"LPE_n_heads\": 4,\n\n \"GT_layers\": 16,\n \"GT_hidden_dim\": 48,\n \"GT_out_dim\": 48,\n \"GT_n_heads\": 8,\n\n \"residual\": true,\n \"readout\": \"mean\",\n \"in_feat_dropout\": 0.0,\n \"dropout\": 0.0,\n \"layer_norm\": false,\n \"batch_norm\": true\n }\n}\n" }, { "path": "configs/CLUSTER/ablation/full/1e-5/node", "content": "{\n \"gpu\": {\n \"use\": true,\n \"id\": 0\n },\n \n \"model\": \"GraphTransformer\",\n \"dataset\": \"SBM_CLUSTER\",\n \n \"out_dir\": \"out/SBM_CLUSTER/full/1e-5/node\",\n \n \"params\": {\n \"seed\": 41,\n \"epochs\": 1000,\n \"batch_size\": 16,\n \"init_lr\": 0.0005,\n \"lr_reduce_factor\": 0.5,\n \"lr_schedule_patience\": 10,\n \"min_lr\": 1e-5,\n \"weight_decay\": 0.0,\n \"print_epoch_interval\": 5,\n \"max_time\": 24\n },\n \n \"net_params\": {\n \"full_graph\": true,\n \"gamma\": 1e-5,\n \"m\": 10,\n \n \"LPE\": \"node\",\n \"LPE_layers\": 1,\n \"LPE_dim\": 16,\n \"LPE_n_heads\": 4,\n\n \"GT_layers\": 16,\n \"GT_hidden_dim\": 48,\n \"GT_out_dim\": 48,\n \"GT_n_heads\": 8,\n\n \"residual\": true,\n \"readout\": \"mean\",\n \"in_feat_dropout\": 0.0,\n \"dropout\": 0.0,\n \"layer_norm\": false,\n \"batch_norm\": true\n }\n}\n" }, { "path": "configs/CLUSTER/ablation/full/1e-6/node", "content": "{\n \"gpu\": {\n \"use\": true,\n \"id\": 0\n },\n \n \"model\": \"GraphTransformer\",\n \"dataset\": \"SBM_CLUSTER\",\n \n \"out_dir\": \"out/SBM_CLUSTER/full/1e-6/node\",\n \n \"params\": {\n \"seed\": 41,\n \"epochs\": 1000,\n \"batch_size\": 16,\n \"init_lr\": 0.0005,\n \"lr_reduce_factor\": 0.5,\n \"lr_schedule_patience\": 10,\n \"min_lr\": 1e-5,\n \"weight_decay\": 0.0,\n \"print_epoch_interval\": 5,\n \"max_time\": 24\n },\n \n \"net_params\": {\n \"full_graph\": true,\n \"gamma\": 1e-6,\n \"m\": 10,\n \n \"LPE\": \"node\",\n \"LPE_layers\": 1,\n \"LPE_dim\": 16,\n \"LPE_n_heads\": 4,\n\n \"GT_layers\": 16,\n \"GT_hidden_dim\": 48,\n \"GT_out_dim\": 48,\n \"GT_n_heads\": 8,\n\n \"residual\": true,\n \"readout\": \"mean\",\n \"in_feat_dropout\": 0.0,\n \"dropout\": 0.0,\n \"layer_norm\": false,\n \"batch_norm\": true\n }\n}\n" }, { "path": "configs/CLUSTER/ablation/full/1e-7/node", "content": "{\n \"gpu\": {\n \"use\": true,\n \"id\": 0\n },\n \n \"model\": \"GraphTransformer\",\n \"dataset\": \"SBM_CLUSTER\",\n \n \"out_dir\": \"out/SBM_CLUSTER/full/1e-7/node\",\n \n \"params\": {\n \"seed\": 41,\n \"epochs\": 1000,\n \"batch_size\": 16,\n \"init_lr\": 0.0005,\n \"lr_reduce_factor\": 0.5,\n \"lr_schedule_patience\": 10,\n \"min_lr\": 1e-5,\n \"weight_decay\": 0.0,\n \"print_epoch_interval\": 5,\n \"max_time\": 24\n },\n \n \"net_params\": {\n \"full_graph\": true,\n \"gamma\": 1e-7,\n \"m\": 10,\n \n \"LPE\": \"node\",\n \"LPE_layers\": 1,\n \"LPE_dim\": 16,\n \"LPE_n_heads\": 4,\n\n \"GT_layers\": 16,\n \"GT_hidden_dim\": 48,\n \"GT_out_dim\": 48,\n \"GT_n_heads\": 8,\n\n \"residual\": true,\n \"readout\": \"mean\",\n \"in_feat_dropout\": 0.0,\n \"dropout\": 0.0,\n \"layer_norm\": false,\n \"batch_norm\": true\n }\n}\n" }, { "path": "configs/CLUSTER/ablation/full/1e-8/node", "content": "{\n \"gpu\": {\n \"use\": true,\n \"id\": 0\n },\n \n \"model\": \"GraphTransformer\",\n \"dataset\": \"SBM_CLUSTER\",\n \n \"out_dir\": \"out/SBM_CLUSTER/full/1e-8/node\",\n \n \"params\": {\n \"seed\": 41,\n \"epochs\": 1000,\n \"batch_size\": 16,\n \"init_lr\": 0.0005,\n \"lr_reduce_factor\": 0.5,\n \"lr_schedule_patience\": 10,\n \"min_lr\": 1e-5,\n \"weight_decay\": 0.0,\n \"print_epoch_interval\": 5,\n \"max_time\": 24\n },\n \n \"net_params\": {\n \"full_graph\": true,\n \"gamma\": 1e-8,\n \"m\": 10,\n \n \"LPE\": \"node\",\n \"LPE_layers\": 1,\n \"LPE_dim\": 16,\n \"LPE_n_heads\": 4,\n\n \"GT_layers\": 16,\n \"GT_hidden_dim\": 48,\n \"GT_out_dim\": 48,\n \"GT_n_heads\": 8,\n\n \"residual\": true,\n \"readout\": \"mean\",\n \"in_feat_dropout\": 0.0,\n \"dropout\": 0.0,\n \"layer_norm\": false,\n \"batch_norm\": true\n }\n}\n" }, { "path": "configs/CLUSTER/ablation/full/none", "content": "{\n \"gpu\": {\n \"use\": true,\n \"id\": 0\n },\n \n \"model\": \"GraphTransformer\",\n \"dataset\": \"SBM_CLUSTER\",\n \n \"out_dir\": \"out/SBM_CLUSTER/full/none\",\n \n \"params\": {\n \"seed\": 41,\n \"epochs\": 1000,\n \"batch_size\": 16,\n \"init_lr\": 0.0005,\n \"lr_reduce_factor\": 0.5,\n \"lr_schedule_patience\": 10,\n \"min_lr\": 1e-5,\n \"weight_decay\": 0.0,\n \"print_epoch_interval\": 5,\n \"max_time\": 24\n },\n \n \"net_params\": {\n \"full_graph\": true,\n \"gamma\": 1e-1,\n \"m\": 10,\n \n \"LPE\": \"none\",\n \"LPE_layers\": 1,\n \"LPE_dim\": 16,\n \"LPE_n_heads\": 4,\n\n \"GT_layers\": 16,\n \"GT_hidden_dim\": 48,\n \"GT_out_dim\": 48,\n \"GT_n_heads\": 8,\n\n \"residual\": true,\n \"readout\": \"mean\",\n \"in_feat_dropout\": 0.0,\n \"dropout\": 0.0,\n \"layer_norm\": false,\n \"batch_norm\": true\n }\n}\n" }, { "path": "configs/CLUSTER/ablation/sparse/node", "content": "{\n \"gpu\": {\n \"use\": true,\n \"id\": 0\n },\n \n \"model\": \"GraphTransformer\",\n \"dataset\": \"SBM_CLUSTER\",\n \n \"out_dir\": \"out/SBM_CLUSTER/sparse/node\",\n \n \"params\": {\n \"seed\": 41,\n \"epochs\": 1000,\n \"batch_size\": 16,\n \"init_lr\": 0.0005,\n \"lr_reduce_factor\": 0.5,\n \"lr_schedule_patience\": 10,\n \"min_lr\": 1e-5,\n \"weight_decay\": 0.0,\n \"print_epoch_interval\": 5,\n \"max_time\": 24\n },\n \n \"net_params\": {\n \"full_graph\": false,\n \"gamma\": 1e-1,\n \"m\": 10,\n \n \"LPE\": \"node\",\n \"LPE_layers\": 1,\n \"LPE_dim\": 16,\n \"LPE_n_heads\": 4,\n\n \"GT_layers\": 16,\n \"GT_hidden_dim\": 56,\n \"GT_out_dim\": 56,\n \"GT_n_heads\": 8,\n\n \"residual\": true,\n \"readout\": \"mean\",\n \"in_feat_dropout\": 0.0,\n \"dropout\": 0.0,\n \"layer_norm\": false,\n \"batch_norm\": true\n }\n}\n" }, { "path": "configs/CLUSTER/ablation/sparse/none", "content": "{\n \"gpu\": {\n \"use\": true,\n \"id\": 0\n },\n \n \"model\": \"GraphTransformer\",\n \"dataset\": \"SBM_CLUSTER\",\n \n \"out_dir\": \"out/SBM_CLUSTER/sparse/none\",\n \n \"params\": {\n \"seed\": 41,\n \"epochs\": 1000,\n \"batch_size\": 16,\n \"init_lr\": 0.0005,\n \"lr_reduce_factor\": 0.5,\n \"lr_schedule_patience\": 10,\n \"min_lr\": 1e-5,\n \"weight_decay\": 0.0,\n \"print_epoch_interval\": 5,\n \"max_time\": 24\n },\n \n \"net_params\": {\n \"full_graph\": false,\n \"gamma\": 1e-1,\n \"m\": 10,\n \n \"LPE\": \"none\",\n \"LPE_layers\": 1,\n \"LPE_dim\": 16,\n \"LPE_n_heads\": 4,\n\n \"GT_layers\": 16,\n \"GT_hidden_dim\": 56,\n \"GT_out_dim\": 56,\n \"GT_n_heads\": 8,\n\n \"residual\": true,\n \"readout\": \"mean\",\n \"in_feat_dropout\": 0.0,\n \"dropout\": 0.0,\n \"layer_norm\": false,\n \"batch_norm\": true\n }\n}\n" }, { "path": "configs/CLUSTER/optimized", "content": "{\n \"gpu\": {\n \"use\": true,\n \"id\": 0\n },\n \n \"model\": \"GraphTransformer\",\n \"dataset\": \"SBM_CLUSTER\",\n \n \"out_dir\": \"out/SBM_CLUSTER/full/1e-1/node\",\n \n \"params\": {\n \"seed\": 41,\n \"epochs\": 1000,\n \"batch_size\": 16,\n \"init_lr\": 0.0005,\n \"lr_reduce_factor\": 0.5,\n \"lr_schedule_patience\": 10,\n \"min_lr\": 1e-5,\n \"weight_decay\": 0.0,\n \"print_epoch_interval\": 5,\n \"max_time\": 24\n },\n \n \"net_params\": {\n \"full_graph\": true,\n \"gamma\": 1e-1,\n \"m\": 10,\n \n \"LPE\": \"node\",\n \"LPE_layers\": 1,\n \"LPE_dim\": 16,\n \"LPE_n_heads\": 4,\n\n \"GT_layers\": 16,\n \"GT_hidden_dim\": 48,\n \"GT_out_dim\": 48,\n \"GT_n_heads\": 8,\n\n \"residual\": true,\n \"readout\": \"mean\",\n \"in_feat_dropout\": 0.0,\n \"dropout\": 0.0,\n \"layer_norm\": false,\n \"batch_norm\": true\n }\n}\n" }, { "path": "configs/MOLHIV/ablation/full/1e-3/node", "content": "{\n \"gpu\": {\n \"use\": true,\n \"id\": 0\n },\n\n \"model\": \"GraphTransformer\",\n \"dataset\": \"MOL-HIV\",\n\n \"out_dir\": \"out/MOLHIV/full/1e-3/node/\",\n\n \"params\": {\n \"seed\": 41,\n \"epochs\": 1000,\n \"batch_size\": 128,\n \"init_lr\": 0.0001,\n \"lr_reduce_factor\": 0.5,\n \"lr_schedule_patience\": 20,\n \"min_lr\": 1e-5,\n \"weight_decay\": 0,\n \"print_epoch_interval\": 5,\n \"max_time\": 24\n },\n\n \"net_params\": {\n \"full_graph\": true,\n \"gamma\": 1e-3,\n \"m\": 10,\n\n \"LPE\": \"node\",\n \"LPE_layers\": 2,\n \"LPE_dim\": 16,\n \"LPE_n_heads\": 4,\n\n \"GT_layers\": 6,\n \"GT_hidden_dim\": 72,\n \"GT_out_dim\": 72,\n \"GT_n_heads\": 4,\n\n \"residual\": true,\n \"readout\": \"sum\",\n \"in_feat_dropout\": 0.0,\n \"dropout\": 0.03,\n \"layer_norm\": false,\n \"batch_norm\": true\n }\n}\n" }, { "path": "configs/MOLHIV/ablation/full/1e-4/node", "content": "{\n \"gpu\": {\n \"use\": true,\n \"id\": 0\n },\n\n \"model\": \"GraphTransformer\",\n \"dataset\": \"MOL-HIV\",\n\n \"out_dir\": \"out/MOLHIV/full/1e-4/node/\",\n\n \"params\": {\n \"seed\": 41,\n \"epochs\": 1000,\n \"batch_size\": 128,\n \"init_lr\": 0.0001,\n \"lr_reduce_factor\": 0.5,\n \"lr_schedule_patience\": 20,\n \"min_lr\": 1e-5,\n \"weight_decay\": 0,\n \"print_epoch_interval\": 5,\n \"max_time\": 24\n },\n\n \"net_params\": {\n \"full_graph\": true,\n \"gamma\": 1e-4,\n \"m\": 10,\n\n \"LPE\": \"node\",\n \"LPE_layers\": 2,\n \"LPE_dim\": 16,\n \"LPE_n_heads\": 4,\n\n \"GT_layers\": 6,\n \"GT_hidden_dim\": 72,\n \"GT_out_dim\": 72,\n \"GT_n_heads\": 4,\n\n \"residual\": true,\n \"readout\": \"sum\",\n \"in_feat_dropout\": 0.0,\n \"dropout\": 0.03,\n \"layer_norm\": false,\n \"batch_norm\": true\n }\n}\n" }, { "path": "configs/MOLHIV/ablation/full/1e-5/node", "content": "{\n \"gpu\": {\n \"use\": true,\n \"id\": 0\n },\n\n \"model\": \"GraphTransformer\",\n \"dataset\": \"MOL-HIV\",\n\n \"out_dir\": \"out/MOLHIV/full/1e-5/node/\",\n\n \"params\": {\n \"seed\": 41,\n \"epochs\": 1000,\n \"batch_size\": 128,\n \"init_lr\": 0.0001,\n \"lr_reduce_factor\": 0.5,\n \"lr_schedule_patience\": 20,\n \"min_lr\": 1e-5,\n \"weight_decay\": 0,\n \"print_epoch_interval\": 5,\n \"max_time\": 24\n },\n\n \"net_params\": {\n \"full_graph\": true,\n \"gamma\": 1e-5,\n \"m\": 10,\n\n \"LPE\": \"node\",\n \"LPE_layers\": 2,\n \"LPE_dim\": 16,\n \"LPE_n_heads\": 4,\n\n \"GT_layers\": 6,\n \"GT_hidden_dim\": 72,\n \"GT_out_dim\": 72,\n \"GT_n_heads\": 4,\n\n \"residual\": true,\n \"readout\": \"sum\",\n \"in_feat_dropout\": 0.0,\n \"dropout\": 0.03,\n \"layer_norm\": false,\n \"batch_norm\": true\n }\n}\n" }, { "path": "configs/MOLHIV/ablation/full/1e-6/node", "content": "{\n \"gpu\": {\n \"use\": true,\n \"id\": 0\n },\n\n \"model\": \"GraphTransformer\",\n \"dataset\": \"MOL-HIV\",\n\n \"out_dir\": \"out/MOLHIV/full/1e-6/node/\",\n\n \"params\": {\n \"seed\": 41,\n \"epochs\": 1000,\n \"batch_size\": 128,\n \"init_lr\": 0.0001,\n \"lr_reduce_factor\": 0.5,\n \"lr_schedule_patience\": 20,\n \"min_lr\": 1e-5,\n \"weight_decay\": 0,\n \"print_epoch_interval\": 5,\n \"max_time\": 24\n },\n\n \"net_params\": {\n \"full_graph\": true,\n \"gamma\": 1e-6,\n \"m\": 10,\n\n \"LPE\": \"node\",\n \"LPE_layers\": 2,\n \"LPE_dim\": 16,\n \"LPE_n_heads\": 4,\n\n \"GT_layers\": 6,\n \"GT_hidden_dim\": 72,\n \"GT_out_dim\": 72,\n \"GT_n_heads\": 4,\n\n \"residual\": true,\n \"readout\": \"sum\",\n \"in_feat_dropout\": 0.0,\n \"dropout\": 0.03,\n \"layer_norm\": false,\n \"batch_norm\": true\n }\n}\n" }, { "path": "configs/MOLHIV/ablation/full/1e-7/node", "content": "{\n \"gpu\": {\n \"use\": true,\n \"id\": 0\n },\n\n \"model\": \"GraphTransformer\",\n \"dataset\": \"MOL-HIV\",\n\n \"out_dir\": \"out/MOLHIV/full/1e-7/node/\",\n\n \"params\": {\n \"seed\": 41,\n \"epochs\": 1000,\n \"batch_size\": 128,\n \"init_lr\": 0.0001,\n \"lr_reduce_factor\": 0.5,\n \"lr_schedule_patience\": 20,\n \"min_lr\": 1e-5,\n \"weight_decay\": 0,\n \"print_epoch_interval\": 5,\n \"max_time\": 24\n },\n\n \"net_params\": {\n \"full_graph\": true,\n \"gamma\": 1e-7,\n \"m\": 10,\n\n \"LPE\": \"node\",\n \"LPE_layers\": 2,\n \"LPE_dim\": 16,\n \"LPE_n_heads\": 4,\n\n \"GT_layers\": 6,\n \"GT_hidden_dim\": 72,\n \"GT_out_dim\": 72,\n \"GT_n_heads\": 4,\n\n \"residual\": true,\n \"readout\": \"sum\",\n \"in_feat_dropout\": 0.0,\n \"dropout\": 0.03,\n \"layer_norm\": false,\n \"batch_norm\": true\n }\n}\n" }, { "path": "configs/MOLHIV/ablation/full/1e-8/node", "content": "{\n \"gpu\": {\n \"use\": true,\n \"id\": 0\n },\n\n \"model\": \"GraphTransformer\",\n \"dataset\": \"MOL-HIV\",\n\n \"out_dir\": \"out/MOLHIV/full/1e-8/node/\",\n\n \"params\": {\n \"seed\": 41,\n \"epochs\": 1000,\n \"batch_size\": 128,\n \"init_lr\": 0.0001,\n \"lr_reduce_factor\": 0.5,\n \"lr_schedule_patience\": 20,\n \"min_lr\": 1e-5,\n \"weight_decay\": 0,\n \"print_epoch_interval\": 5,\n \"max_time\": 24\n },\n\n \"net_params\": {\n \"full_graph\": true,\n \"gamma\": 1e-8,\n \"m\": 10,\n\n \"LPE\": \"node\",\n \"LPE_layers\": 2,\n \"LPE_dim\": 16,\n \"LPE_n_heads\": 4,\n\n \"GT_layers\": 6,\n \"GT_hidden_dim\": 72,\n \"GT_out_dim\": 72,\n \"GT_n_heads\": 4,\n\n \"residual\": true,\n \"readout\": \"sum\",\n \"in_feat_dropout\": 0.0,\n \"dropout\": 0.03,\n \"layer_norm\": false,\n \"batch_norm\": true\n }\n}\n" }, { "path": "configs/MOLHIV/ablation/full/none", "content": "{\n \"gpu\": {\n \"use\": true,\n \"id\": 0\n },\n\n \"model\": \"GraphTransformer\",\n \"dataset\": \"MOL-HIV\",\n\n \"out_dir\": \"out/MOLHIV/full/none/\",\n\n \"params\": {\n \"seed\": 41,\n \"epochs\": 1000,\n \"batch_size\": 64,\n \"init_lr\": 0.0001,\n \"lr_reduce_factor\": 0.5,\n \"lr_schedule_patience\": 20,\n \"min_lr\": 1e-5,\n \"weight_decay\": 0,\n \"print_epoch_interval\": 5,\n \"max_time\": 24\n },\n\n \"net_params\": {\n \"full_graph\": true,\n \"gamma\": 1e-6,\n \"m\": 10,\n\n \"LPE\": \"none\",\n \"LPE_layers\": 2,\n \"LPE_dim\": 16,\n \"LPE_n_heads\": 4,\n\n \"GT_layers\": 6,\n \"GT_hidden_dim\": 80,\n \"GT_out_dim\": 80,\n \"GT_n_heads\": 4,\n\n \"residual\": true,\n \"readout\": \"sum\",\n \"in_feat_dropout\": 0.0,\n \"dropout\": 0.03,\n \"layer_norm\": false,\n \"batch_norm\": true\n }\n}\n" }, { "path": "configs/MOLHIV/ablation/sparse/node", "content": "{\n \"gpu\": {\n \"use\": true,\n \"id\": 0\n },\n\n \"model\": \"GraphTransformer\",\n \"dataset\": \"MOL-HIV\",\n\n \"out_dir\": \"out/MOLHIV/sparse/node\",\n\n \"params\": {\n \"seed\": 41,\n \"epochs\": 1000,\n \"batch_size\": 64,\n \"init_lr\": 0.0001,\n \"lr_reduce_factor\": 0.5,\n \"lr_schedule_patience\": 20,\n \"min_lr\": 1e-5,\n \"weight_decay\": 0,\n \"print_epoch_interval\": 5,\n \"max_time\": 24\n },\n\n \"net_params\": {\n \"full_graph\": false,\n \"gamma\": 1e-5,\n \"m\": 10,\n\n \"LPE\": \"node\",\n \"LPE_layers\": 2,\n \"LPE_dim\": 16,\n \"LPE_n_heads\": 4,\n\n \"GT_layers\": 6,\n \"GT_hidden_dim\": 80,\n \"GT_out_dim\": 80,\n \"GT_n_heads\": 4,\n\n \"residual\": true,\n \"readout\": \"sum\",\n \"in_feat_dropout\": 0.0,\n \"dropout\": 0.03,\n \"layer_norm\": false,\n \"batch_norm\": true\n }\n}\n" }, { "path": "configs/MOLHIV/ablation/sparse/none", "content": "{\n \"gpu\": {\n \"use\": true,\n \"id\": 0\n },\n\n \"model\": \"GraphTransformer\",\n \"dataset\": \"MOL-HIV\",\n\n \"out_dir\": \"out/MOLHIV/sparse/none\",\n\n \"params\": {\n \"seed\": 41,\n \"epochs\": 1000,\n \"batch_size\": 64,\n \"init_lr\": 0.0001,\n \"lr_reduce_factor\": 0.5,\n \"lr_schedule_patience\": 20,\n \"min_lr\": 1e-5,\n \"weight_decay\": 0,\n \"print_epoch_interval\": 5,\n \"max_time\": 24\n },\n\n \"net_params\": {\n \"full_graph\": false,\n \"gamma\": 1e-5,\n \"m\": 10,\n\n \"LPE\": \"none\",\n \"LPE_layers\": 2,\n \"LPE_dim\": 16,\n \"LPE_n_heads\": 4,\n\n \"GT_layers\": 6,\n \"GT_hidden_dim\": 96,\n \"GT_out_dim\": 96,\n \"GT_n_heads\": 4,\n\n \"residual\": true,\n \"readout\": \"sum\",\n \"in_feat_dropout\": 0.0,\n \"dropout\": 0.03,\n \"layer_norm\": false,\n \"batch_norm\": true\n }\n}\n" }, { "path": "configs/MOLHIV/optimized", "content": "{\n \"gpu\": {\n \"use\": true,\n \"id\": 0\n },\n\n \"model\": \"GraphTransformer\",\n \"dataset\": \"MOL-HIV\",\n\n \"out_dir\": \"out/MOLHIV/full/1e-6/node/\",\n\n \"params\": {\n \"seed\": 41,\n \"epochs\": 1000,\n \"batch_size\": 64,\n \"init_lr\": 0.0001,\n \"lr_reduce_factor\": 0.5,\n \"lr_schedule_patience\": 20,\n \"min_lr\": 1e-5,\n \"weight_decay\": 0,\n \"print_epoch_interval\": 5,\n \"max_time\": 24\n },\n\n \"net_params\": {\n \"full_graph\": true,\n \"gamma\": 1e-6,\n \"m\": 10,\n\n \"LPE\": \"node\",\n \"LPE_layers\": 3,\n \"LPE_dim\": 16,\n \"LPE_n_heads\": 4,\n\n \"GT_layers\": 10,\n \"GT_hidden_dim\": 64,\n \"GT_out_dim\": 64,\n \"GT_n_heads\": 4,\n\n \"residual\": true,\n \"readout\": \"sum\",\n \"in_feat_dropout\": 0.0,\n \"dropout\": 0.01,\n \"layer_norm\": false,\n \"batch_norm\": true\n }\n}\n" }, { "path": "configs/MOLPCBA/ablation/full/1e-3/node", "content": "{\n \"gpu\": {\n \"use\": true,\n \"id\": 0\n },\n\n \"model\": \"GraphTransformer\",\n \"dataset\": \"MOL-HIV\",\n\n \"out_dir\": \"out/MOLHIV/full/1e-3/node/\",\n\n \"params\": {\n \"seed\": 41,\n \"epochs\": 1000,\n \"batch_size\": 128,\n \"init_lr\": 0.0001,\n \"lr_reduce_factor\": 0.5,\n \"lr_schedule_patience\": 20,\n \"min_lr\": 1e-5,\n \"weight_decay\": 0,\n \"print_epoch_interval\": 5,\n \"max_time\": 24\n },\n\n \"net_params\": {\n \"full_graph\": true,\n \"gamma\": 1e-3,\n \"m\": 10,\n\n \"LPE\": \"node\",\n \"LPE_layers\": 2,\n \"LPE_dim\": 16,\n \"LPE_n_heads\": 4,\n\n \"GT_layers\": 6,\n \"GT_hidden_dim\": 72,\n \"GT_out_dim\": 72,\n \"GT_n_heads\": 4,\n\n \"residual\": true,\n \"readout\": \"sum\",\n \"in_feat_dropout\": 0.0,\n \"dropout\": 0.03,\n \"layer_norm\": false,\n \"batch_norm\": true\n }\n}\n" }, { "path": "configs/MOLPCBA/ablation/full/1e-4/node", "content": "{\n \"gpu\": {\n \"use\": true,\n \"id\": 0\n },\n\n \"model\": \"GraphTransformer\",\n \"dataset\": \"MOL-HIV\",\n\n \"out_dir\": \"out/MOLHIV/full/1e-4/node/\",\n\n \"params\": {\n \"seed\": 41,\n \"epochs\": 1000,\n \"batch_size\": 128,\n \"init_lr\": 0.0001,\n \"lr_reduce_factor\": 0.5,\n \"lr_schedule_patience\": 20,\n \"min_lr\": 1e-5,\n \"weight_decay\": 0,\n \"print_epoch_interval\": 5,\n \"max_time\": 24\n },\n\n \"net_params\": {\n \"full_graph\": true,\n \"gamma\": 1e-4,\n \"m\": 10,\n\n \"LPE\": \"node\",\n \"LPE_layers\": 2,\n \"LPE_dim\": 16,\n \"LPE_n_heads\": 4,\n\n \"GT_layers\": 6,\n \"GT_hidden_dim\": 72,\n \"GT_out_dim\": 72,\n \"GT_n_heads\": 4,\n\n \"residual\": true,\n \"readout\": \"sum\",\n \"in_feat_dropout\": 0.0,\n \"dropout\": 0.03,\n \"layer_norm\": false,\n \"batch_norm\": true\n }\n}\n" }, { "path": "configs/MOLPCBA/ablation/full/1e-5/node", "content": "{\n \"gpu\": {\n \"use\": true,\n \"id\": 0\n },\n\n \"model\": \"GraphTransformer\",\n \"dataset\": \"MOL-HIV\",\n\n \"out_dir\": \"out/MOLHIV/full/1e-5/node/\",\n\n \"params\": {\n \"seed\": 41,\n \"epochs\": 1000,\n \"batch_size\": 128,\n \"init_lr\": 0.0001,\n \"lr_reduce_factor\": 0.5,\n \"lr_schedule_patience\": 20,\n \"min_lr\": 1e-5,\n \"weight_decay\": 0,\n \"print_epoch_interval\": 5,\n \"max_time\": 24\n },\n\n \"net_params\": {\n \"full_graph\": true,\n \"gamma\": 1e-5,\n \"m\": 10,\n\n \"LPE\": \"node\",\n \"LPE_layers\": 2,\n \"LPE_dim\": 16,\n \"LPE_n_heads\": 4,\n\n \"GT_layers\": 6,\n \"GT_hidden_dim\": 72,\n \"GT_out_dim\": 72,\n \"GT_n_heads\": 4,\n\n \"residual\": true,\n \"readout\": \"sum\",\n \"in_feat_dropout\": 0.0,\n \"dropout\": 0.03,\n \"layer_norm\": false,\n \"batch_norm\": true\n }\n}\n" }, { "path": "configs/MOLPCBA/ablation/full/1e-6/node", "content": "{\n \"gpu\": {\n \"use\": true,\n \"id\": 0\n },\n\n \"model\": \"GraphTransformer\",\n \"dataset\": \"MOL-HIV\",\n\n \"out_dir\": \"out/MOLHIV/full/1e-6/node/\",\n\n \"params\": {\n \"seed\": 41,\n \"epochs\": 1000,\n \"batch_size\": 128,\n \"init_lr\": 0.0001,\n \"lr_reduce_factor\": 0.5,\n \"lr_schedule_patience\": 20,\n \"min_lr\": 1e-5,\n \"weight_decay\": 0,\n \"print_epoch_interval\": 5,\n \"max_time\": 24\n },\n\n \"net_params\": {\n \"full_graph\": true,\n \"gamma\": 1e-6,\n \"m\": 10,\n\n \"LPE\": \"node\",\n \"LPE_layers\": 2,\n \"LPE_dim\": 16,\n \"LPE_n_heads\": 4,\n\n \"GT_layers\": 6,\n \"GT_hidden_dim\": 72,\n \"GT_out_dim\": 72,\n \"GT_n_heads\": 4,\n\n \"residual\": true,\n \"readout\": \"sum\",\n \"in_feat_dropout\": 0.0,\n \"dropout\": 0.03,\n \"layer_norm\": false,\n \"batch_norm\": true\n }\n}\n" }, { "path": "configs/MOLPCBA/ablation/full/1e-7/node", "content": "{\n \"gpu\": {\n \"use\": true,\n \"id\": 0\n },\n\n \"model\": \"GraphTransformer\",\n \"dataset\": \"MOL-HIV\",\n\n \"out_dir\": \"out/MOLHIV/full/1e-7/node/\",\n\n \"params\": {\n \"seed\": 41,\n \"epochs\": 1000,\n \"batch_size\": 128,\n \"init_lr\": 0.0001,\n \"lr_reduce_factor\": 0.5,\n \"lr_schedule_patience\": 20,\n \"min_lr\": 1e-5,\n \"weight_decay\": 0,\n \"print_epoch_interval\": 5,\n \"max_time\": 24\n },\n\n \"net_params\": {\n \"full_graph\": true,\n \"gamma\": 1e-7,\n \"m\": 10,\n\n \"LPE\": \"node\",\n \"LPE_layers\": 2,\n \"LPE_dim\": 16,\n \"LPE_n_heads\": 4,\n\n \"GT_layers\": 6,\n \"GT_hidden_dim\": 72,\n \"GT_out_dim\": 72,\n \"GT_n_heads\": 4,\n\n \"residual\": true,\n \"readout\": \"sum\",\n \"in_feat_dropout\": 0.0,\n \"dropout\": 0.03,\n \"layer_norm\": false,\n \"batch_norm\": true\n }\n}\n" }, { "path": "configs/MOLPCBA/ablation/full/1e-8/node", "content": "{\n \"gpu\": {\n \"use\": true,\n \"id\": 0\n },\n\n \"model\": \"GraphTransformer\",\n \"dataset\": \"MOL-HIV\",\n\n \"out_dir\": \"out/MOLHIV/full/1e-8/node/\",\n\n \"params\": {\n \"seed\": 41,\n \"epochs\": 1000,\n \"batch_size\": 128,\n \"init_lr\": 0.0001,\n \"lr_reduce_factor\": 0.5,\n \"lr_schedule_patience\": 20,\n \"min_lr\": 1e-5,\n \"weight_decay\": 0,\n \"print_epoch_interval\": 5,\n \"max_time\": 24\n },\n\n \"net_params\": {\n \"full_graph\": true,\n \"gamma\": 1e-8,\n \"m\": 10,\n\n \"LPE\": \"node\",\n \"LPE_layers\": 2,\n \"LPE_dim\": 16,\n \"LPE_n_heads\": 4,\n\n \"GT_layers\": 6,\n \"GT_hidden_dim\": 72,\n \"GT_out_dim\": 72,\n \"GT_n_heads\": 4,\n\n \"residual\": true,\n \"readout\": \"sum\",\n \"in_feat_dropout\": 0.0,\n \"dropout\": 0.03,\n \"layer_norm\": false,\n \"batch_norm\": true\n }\n}\n" }, { "path": "configs/MOLPCBA/ablation/full/none", "content": "{\n \"gpu\": {\n \"use\": true,\n \"id\": 0\n },\n\n \"model\": \"GraphTransformer\",\n \"dataset\": \"MOL-HIV\",\n\n \"out_dir\": \"out/MOLHIV/full/none/\",\n\n \"params\": {\n \"seed\": 41,\n \"epochs\": 1000,\n \"batch_size\": 64,\n \"init_lr\": 0.0001,\n \"lr_reduce_factor\": 0.5,\n \"lr_schedule_patience\": 20,\n \"min_lr\": 1e-5,\n \"weight_decay\": 0,\n \"print_epoch_interval\": 5,\n \"max_time\": 24\n },\n\n \"net_params\": {\n \"full_graph\": true,\n \"gamma\": 1e-6,\n \"m\": 10,\n\n \"LPE\": \"none\",\n \"LPE_layers\": 2,\n \"LPE_dim\": 16,\n \"LPE_n_heads\": 4,\n\n \"GT_layers\": 6,\n \"GT_hidden_dim\": 80,\n \"GT_out_dim\": 80,\n \"GT_n_heads\": 4,\n\n \"residual\": true,\n \"readout\": \"sum\",\n \"in_feat_dropout\": 0.0,\n \"dropout\": 0.03,\n \"layer_norm\": false,\n \"batch_norm\": true\n }\n}\n" }, { "path": "configs/MOLPCBA/ablation/sparse/node", "content": "{\n \"gpu\": {\n \"use\": true,\n \"id\": 0\n },\n\n \"model\": \"GraphTransformer\",\n \"dataset\": \"MOL-HIV\",\n\n \"out_dir\": \"out/MOLHIV/sparse/node\",\n\n \"params\": {\n \"seed\": 41,\n \"epochs\": 1000,\n \"batch_size\": 64,\n \"init_lr\": 0.0001,\n \"lr_reduce_factor\": 0.5,\n \"lr_schedule_patience\": 20,\n \"min_lr\": 1e-5,\n \"weight_decay\": 0,\n \"print_epoch_interval\": 5,\n \"max_time\": 24\n },\n\n \"net_params\": {\n \"full_graph\": false,\n \"gamma\": 1e-5,\n \"m\": 10,\n\n \"LPE\": \"node\",\n \"LPE_layers\": 2,\n \"LPE_dim\": 16,\n \"LPE_n_heads\": 4,\n\n \"GT_layers\": 6,\n \"GT_hidden_dim\": 80,\n \"GT_out_dim\": 80,\n \"GT_n_heads\": 4,\n\n \"residual\": true,\n \"readout\": \"sum\",\n \"in_feat_dropout\": 0.0,\n \"dropout\": 0.03,\n \"layer_norm\": false,\n \"batch_norm\": true\n }\n}\n" }, { "path": "configs/MOLPCBA/ablation/sparse/none", "content": "{\n \"gpu\": {\n \"use\": true,\n \"id\": 0\n },\n\n \"model\": \"GraphTransformer\",\n \"dataset\": \"MOL-HIV\",\n\n \"out_dir\": \"out/MOLHIV/sparse/none\",\n\n \"params\": {\n \"seed\": 41,\n \"epochs\": 1000,\n \"batch_size\": 64,\n \"init_lr\": 0.0001,\n \"lr_reduce_factor\": 0.5,\n \"lr_schedule_patience\": 20,\n \"min_lr\": 1e-5,\n \"weight_decay\": 0,\n \"print_epoch_interval\": 5,\n \"max_time\": 24\n },\n\n \"net_params\": {\n \"full_graph\": false,\n \"gamma\": 1e-5,\n \"m\": 10,\n\n \"LPE\": \"none\",\n \"LPE_layers\": 2,\n \"LPE_dim\": 16,\n \"LPE_n_heads\": 4,\n\n \"GT_layers\": 6,\n \"GT_hidden_dim\": 96,\n \"GT_out_dim\": 96,\n \"GT_n_heads\": 4,\n\n \"residual\": true,\n \"readout\": \"sum\",\n \"in_feat_dropout\": 0.0,\n \"dropout\": 0.03,\n \"layer_norm\": false,\n \"batch_norm\": true\n }\n}\n" }, { "path": "configs/MOLPCBA/optimized", "content": "{\n \"gpu\": {\n \"use\": true,\n \"id\": 0\n },\n\n \"model\": \"GraphTransformer\",\n \"dataset\": \"MOL-PCBA\",\n\n \"out_dir\": \"out/MOLPCBA/full/1e-6/node/\",\n\n \"params\": {\n \"seed\": 41,\n \"epochs\": 1000,\n \"batch_size\": 512,\n \"init_lr\": 0.0003,\n \"lr_reduce_factor\": 0.5,\n \"lr_schedule_patience\": 20,\n \"min_lr\": 1e-5,\n \"weight_decay\": 0,\n \"print_epoch_interval\": 5,\n \"max_time\": 24,\n \"batch_accumulation\": 2\n },\n\n \"net_params\": {\n \"full_graph\": true,\n \"gamma\": 1e-6,\n \"m\": 10,\n \n \"extra_mlp\": False,\n\n \"LPE\": \"node\",\n \"LPE_layers\": 3,\n \"LPE_dim\": 16,\n \"LPE_n_heads\": 4,\n\n \"GT_layers\": 5,\n \"GT_hidden_dim\": 304,\n \"GT_out_dim\": 304,\n \"GT_n_heads\": 4,\n\n \"residual\": true,\n \"readout\": \"sum\",\n \"in_feat_dropout\": 0.0,\n \"dropout\": 0.2,\n \"layer_norm\": false,\n \"batch_norm\": true\n }\n}\n" }, { "path": "configs/PATTERN/ablation/full/1e-1/node", "content": "{\n \"gpu\": {\n \"use\": true,\n \"id\": 0\n },\n \n \"model\": \"GraphTransformer\",\n \"dataset\": \"SBM_PATTERN\",\n \n \"out_dir\": \"out/SBM_PATTERN/full/1e-1/node/\",\n \n \"params\": {\n \"seed\": 41,\n \"epochs\": 1000,\n \"batch_size\": 16,\n \"init_lr\": 0.0005,\n \"lr_reduce_factor\": 0.5,\n \"lr_schedule_patience\": 10,\n \"min_lr\": 1e-6,\n \"weight_decay\": 0.0,\n \"print_epoch_interval\": 5,\n \"max_time\": 24\n },\n \n \"net_params\": {\n \"full_graph\": true,\n \"gamma\": 1e-1,\n \"m\": 10,\n \n \"LPE\": \"node\",\n \"LPE_layers\": 3,\n \"LPE_dim\": 16,\n \"LPE_n_heads\": 4,\n\n \"GT_layers\": 6,\n \"GT_hidden_dim\": 64,\n \"GT_out_dim\": 64,\n \"GT_n_heads\": 8,\n\n \"residual\": true,\n \"readout\": \"mean\",\n \"in_feat_dropout\": 0.0,\n \"dropout\": 0.0,\n \"layer_norm\": false,\n \"batch_norm\": true\n }\n}\n" }, { "path": "configs/PATTERN/ablation/full/1e-2/node", "content": "{\n \"gpu\": {\n \"use\": true,\n \"id\": 0\n },\n \n \"model\": \"GraphTransformer\",\n \"dataset\": \"SBM_PATTERN\",\n \n \"out_dir\": \"out/SBM_PATTERN/full/1e-2/node/\",\n \n \"params\": {\n \"seed\": 41,\n \"epochs\": 1000,\n \"batch_size\": 16,\n \"init_lr\": 0.0005,\n \"lr_reduce_factor\": 0.5,\n \"lr_schedule_patience\": 10,\n \"min_lr\": 1e-6,\n \"weight_decay\": 0.0,\n \"print_epoch_interval\": 5,\n \"max_time\": 24\n },\n \n \"net_params\": {\n \"full_graph\": true,\n \"gamma\": 1e-2,\n \"m\": 10,\n \n \"LPE\": \"node\",\n \"LPE_layers\": 3,\n \"LPE_dim\": 16,\n \"LPE_n_heads\": 4,\n\n \"GT_layers\": 6,\n \"GT_hidden_dim\": 64,\n \"GT_out_dim\": 64,\n \"GT_n_heads\": 8,\n\n \"residual\": true,\n \"readout\": \"mean\",\n \"in_feat_dropout\": 0.0,\n \"dropout\": 0.0,\n \"layer_norm\": false,\n \"batch_norm\": true\n }\n}\n" }, { "path": "configs/PATTERN/ablation/full/1e-3/node", "content": "{\n \"gpu\": {\n \"use\": true,\n \"id\": 0\n },\n \n \"model\": \"GraphTransformer\",\n \"dataset\": \"SBM_PATTERN\",\n \n \"out_dir\": \"out/SBM_PATTERN/full/1e-3/node/\",\n \n \"params\": {\n \"seed\": 41,\n \"epochs\": 1000,\n \"batch_size\": 16,\n \"init_lr\": 0.0005,\n \"lr_reduce_factor\": 0.5,\n \"lr_schedule_patience\": 10,\n \"min_lr\": 1e-6,\n \"weight_decay\": 0.0,\n \"print_epoch_interval\": 5,\n \"max_time\": 24\n },\n \n \"net_params\": {\n \"full_graph\": true,\n \"gamma\": 1e-3,\n \"m\": 10,\n \n \"LPE\": \"node\",\n \"LPE_layers\": 3,\n \"LPE_dim\": 16,\n \"LPE_n_heads\": 4,\n\n \"GT_layers\": 6,\n \"GT_hidden_dim\": 64,\n \"GT_out_dim\": 64,\n \"GT_n_heads\": 8,\n\n \"residual\": true,\n \"readout\": \"mean\",\n \"in_feat_dropout\": 0.0,\n \"dropout\": 0.0,\n \"layer_norm\": false,\n \"batch_norm\": true\n }\n}\n" }, { "path": "configs/PATTERN/ablation/full/1e-4/node", "content": "{\n \"gpu\": {\n \"use\": true,\n \"id\": 0\n },\n \n \"model\": \"GraphTransformer\",\n \"dataset\": \"SBM_PATTERN\",\n \n \"out_dir\": \"out/SBM_PATTERN/full/1e-4/node/\",\n \n \"params\": {\n \"seed\": 41,\n \"epochs\": 1000,\n \"batch_size\": 16,\n \"init_lr\": 0.0005,\n \"lr_reduce_factor\": 0.5,\n \"lr_schedule_patience\": 10,\n \"min_lr\": 1e-6,\n \"weight_decay\": 0.0,\n \"print_epoch_interval\": 5,\n \"max_time\": 24\n },\n \n \"net_params\": {\n \"full_graph\": true,\n \"gamma\": 1e-4,\n \"m\": 10,\n \n \"LPE\": \"node\",\n \"LPE_layers\": 3,\n \"LPE_dim\": 16,\n \"LPE_n_heads\": 4,\n\n \"GT_layers\": 6,\n \"GT_hidden_dim\": 64,\n \"GT_out_dim\": 64,\n \"GT_n_heads\": 8,\n\n \"residual\": true,\n \"readout\": \"mean\",\n \"in_feat_dropout\": 0.0,\n \"dropout\": 0.0,\n \"layer_norm\": false,\n \"batch_norm\": true\n }\n}\n" }, { "path": "configs/PATTERN/ablation/full/1e-5/node", "content": "{\n \"gpu\": {\n \"use\": true,\n \"id\": 0\n },\n \n \"model\": \"GraphTransformer\",\n \"dataset\": \"SBM_PATTERN\",\n \n \"out_dir\": \"out/SBM_PATTERN/full/1e-5/node/\",\n \n \"params\": {\n \"seed\": 41,\n \"epochs\": 1000,\n \"batch_size\": 16,\n \"init_lr\": 0.0005,\n \"lr_reduce_factor\": 0.5,\n \"lr_schedule_patience\": 10,\n \"min_lr\": 1e-6,\n \"weight_decay\": 0.0,\n \"print_epoch_interval\": 5,\n \"max_time\": 24\n },\n \n \"net_params\": {\n \"full_graph\": true,\n \"gamma\": 1e-5,\n \"m\": 10,\n \n \"LPE\": \"node\",\n \"LPE_layers\": 3,\n \"LPE_dim\": 16,\n \"LPE_n_heads\": 4,\n\n \"GT_layers\": 6,\n \"GT_hidden_dim\": 64,\n \"GT_out_dim\": 64,\n \"GT_n_heads\": 8,\n\n \"residual\": true,\n \"readout\": \"mean\",\n \"in_feat_dropout\": 0.0,\n \"dropout\": 0.0,\n \"layer_norm\": false,\n \"batch_norm\": true\n }\n}\n" }, { "path": "configs/PATTERN/ablation/full/1e-6/node", "content": "{\n \"gpu\": {\n \"use\": true,\n \"id\": 0\n },\n \n \"model\": \"GraphTransformer\",\n \"dataset\": \"SBM_PATTERN\",\n \n \"out_dir\": \"out/SBM_PATTERN/full/1e-6/node/\",\n \n \"params\": {\n \"seed\": 41,\n \"epochs\": 1000,\n \"batch_size\": 16,\n \"init_lr\": 0.0005,\n \"lr_reduce_factor\": 0.5,\n \"lr_schedule_patience\": 10,\n \"min_lr\": 1e-6,\n \"weight_decay\": 0.0,\n \"print_epoch_interval\": 5,\n \"max_time\": 24\n },\n \n \"net_params\": {\n \"full_graph\": true,\n \"gamma\": 1e-6,\n \"m\": 10,\n \n \"LPE\": \"node\",\n \"LPE_layers\": 3,\n \"LPE_dim\": 16,\n \"LPE_n_heads\": 4,\n\n \"GT_layers\": 6,\n \"GT_hidden_dim\": 64,\n \"GT_out_dim\": 64,\n \"GT_n_heads\": 8,\n\n \"residual\": true,\n \"readout\": \"mean\",\n \"in_feat_dropout\": 0.0,\n \"dropout\": 0.0,\n \"layer_norm\": false,\n \"batch_norm\": true\n }\n}\n" }, { "path": "configs/PATTERN/ablation/full/1e-7/node", "content": "{\n \"gpu\": {\n \"use\": true,\n \"id\": 0\n },\n \n \"model\": \"GraphTransformer\",\n \"dataset\": \"SBM_PATTERN\",\n \n \"out_dir\": \"out/SBM_PATTERN/full/1e-7/node/\",\n \n \"params\": {\n \"seed\": 41,\n \"epochs\": 1000,\n \"batch_size\": 16,\n \"init_lr\": 0.0005,\n \"lr_reduce_factor\": 0.5,\n \"lr_schedule_patience\": 10,\n \"min_lr\": 1e-6,\n \"weight_decay\": 0.0,\n \"print_epoch_interval\": 5,\n \"max_time\": 24\n },\n \n \"net_params\": {\n \"full_graph\": true,\n \"gamma\": 1e-7,\n \"m\": 10,\n \n \"LPE\": \"node\",\n \"LPE_layers\": 3,\n \"LPE_dim\": 16,\n \"LPE_n_heads\": 4,\n\n \"GT_layers\": 6,\n \"GT_hidden_dim\": 64,\n \"GT_out_dim\": 64,\n \"GT_n_heads\": 8,\n\n \"residual\": true,\n \"readout\": \"mean\",\n \"in_feat_dropout\": 0.0,\n \"dropout\": 0.0,\n \"layer_norm\": false,\n \"batch_norm\": true\n }\n}\n" }, { "path": "configs/PATTERN/ablation/full/1e-8/node", "content": "{\n \"gpu\": {\n \"use\": true,\n \"id\": 0\n },\n \n \"model\": \"GraphTransformer\",\n \"dataset\": \"SBM_PATTERN\",\n \n \"out_dir\": \"out/SBM_PATTERN/full/1e-8/node/\",\n \n \"params\": {\n \"seed\": 41,\n \"epochs\": 1000,\n \"batch_size\": 16,\n \"init_lr\": 0.0005,\n \"lr_reduce_factor\": 0.5,\n \"lr_schedule_patience\": 10,\n \"min_lr\": 1e-6,\n \"weight_decay\": 0.0,\n \"print_epoch_interval\": 5,\n \"max_time\": 24\n },\n \n \"net_params\": {\n \"full_graph\": true,\n \"gamma\": 1e-8,\n \"m\": 10,\n \n \"LPE\": \"node\",\n \"LPE_layers\": 3,\n \"LPE_dim\": 16,\n \"LPE_n_heads\": 4,\n\n \"GT_layers\": 6,\n \"GT_hidden_dim\": 64,\n \"GT_out_dim\": 64,\n \"GT_n_heads\": 8,\n\n \"residual\": true,\n \"readout\": \"mean\",\n \"in_feat_dropout\": 0.0,\n \"dropout\": 0.0,\n \"layer_norm\": false,\n \"batch_norm\": true\n }\n}\n" }, { "path": "configs/PATTERN/ablation/full/none", "content": "{\n \"gpu\": {\n \"use\": true,\n \"id\": 0\n },\n \n \"model\": \"GraphTransformer\",\n \"dataset\": \"SBM_PATTERN\",\n \n \"out_dir\": \"out/SBM_PATTERN/full/none/\",\n \n \"params\": {\n \"seed\": 41,\n \"epochs\": 1000,\n \"batch_size\": 16,\n \"init_lr\": 0.0005,\n \"lr_reduce_factor\": 0.5,\n \"lr_schedule_patience\": 10,\n \"min_lr\": 1e-6,\n \"weight_decay\": 0.0,\n \"print_epoch_interval\": 5,\n \"max_time\": 24\n },\n \n \"net_params\": {\n \"full_graph\": true,\n \"gamma\": 1e-1,\n \"m\": 10,\n \n \"LPE\": \"none\",\n \"LPE_layers\": 3,\n \"LPE_dim\": 16,\n \"LPE_n_heads\": 4,\n\n \"GT_layers\": 6,\n \"GT_hidden_dim\": 80,\n \"GT_out_dim\": 80,\n \"GT_n_heads\": 8,\n\n \"residual\": true,\n \"readout\": \"mean\",\n \"in_feat_dropout\": 0.0,\n \"dropout\": 0.0,\n \"layer_norm\": false,\n \"batch_norm\": true\n }\n}\n" }, { "path": "configs/PATTERN/ablation/sparse/node", "content": "{\n \"gpu\": {\n \"use\": true,\n \"id\": 0\n },\n \n \"model\": \"GraphTransformer\",\n \"dataset\": \"SBM_PATTERN\",\n \n \"out_dir\": \"out/SBM_PATTERN/sparse/node/\",\n \n \"params\": {\n \"seed\": 41,\n \"epochs\": 1000,\n \"batch_size\": 16,\n \"init_lr\": 0.0005,\n \"lr_reduce_factor\": 0.5,\n \"lr_schedule_patience\": 10,\n \"min_lr\": 1e-6,\n \"weight_decay\": 0.0,\n \"print_epoch_interval\": 5,\n \"max_time\": 24\n },\n \n \"net_params\": {\n \"full_graph\": false,\n \"gamma\": 1e-2,\n \"m\": 10,\n \n \"LPE\": \"node\",\n \"LPE_layers\": 3,\n \"LPE_dim\": 16,\n \"LPE_n_heads\": 4,\n\n \"GT_layers\": 6,\n \"GT_hidden_dim\": 72,\n \"GT_out_dim\": 72,\n \"GT_n_heads\": 8,\n\n \"residual\": true,\n \"readout\": \"mean\",\n \"in_feat_dropout\": 0.0,\n \"dropout\": 0.0,\n \"layer_norm\": false,\n \"batch_norm\": true\n }\n}\n" }, { "path": "configs/PATTERN/ablation/sparse/none", "content": "{\n \"gpu\": {\n \"use\": true,\n \"id\": 0\n },\n \n \"model\": \"GraphTransformer\",\n \"dataset\": \"SBM_PATTERN\",\n \n \"out_dir\": \"out/SBM_PATTERN/sparse/none/\",\n \n \"params\": {\n \"seed\": 41,\n \"epochs\": 1000,\n \"batch_size\": 16,\n \"init_lr\": 0.0005,\n \"lr_reduce_factor\": 0.5,\n \"lr_schedule_patience\": 10,\n \"min_lr\": 1e-6,\n \"weight_decay\": 0.0,\n \"print_epoch_interval\": 5,\n \"max_time\": 24\n },\n \n \"net_params\": {\n \"full_graph\": false,\n \"gamma\": 1e-2,\n \"m\": 10,\n \n \"LPE\": \"none\",\n \"LPE_layers\": 3,\n \"LPE_dim\": 16,\n \"LPE_n_heads\": 4,\n\n \"GT_layers\": 6,\n \"GT_hidden_dim\": 96,\n \"GT_out_dim\": 96,\n \"GT_n_heads\": 8,\n\n \"residual\": true,\n \"readout\": \"mean\",\n \"in_feat_dropout\": 0.0,\n \"dropout\": 0.0,\n \"layer_norm\": false,\n \"batch_norm\": true\n }\n}\n" }, { "path": "configs/PATTERN/optimized", "content": "{\n \"gpu\": {\n \"use\": true,\n \"id\": 0\n },\n \n \"model\": \"GraphTransformer\",\n \"dataset\": \"SBM_PATTERN\",\n \n \"out_dir\": \"out/SBM_PATTERN/full/1e-2/node/\",\n \n \"params\": {\n \"seed\": 41,\n \"epochs\": 1000,\n \"batch_size\": 16,\n \"init_lr\": 0.0005,\n \"lr_reduce_factor\": 0.5,\n \"lr_schedule_patience\": 10,\n \"min_lr\": 1e-6,\n \"weight_decay\": 0.0,\n \"print_epoch_interval\": 5,\n \"max_time\": 24\n },\n \n \"net_params\": {\n \"full_graph\": true,\n \"gamma\": 1e-2,\n \"m\": 10,\n \n \"LPE\": \"node\",\n \"LPE_layers\": 3,\n \"LPE_dim\": 16,\n \"LPE_n_heads\": 4,\n\n \"GT_layers\": 4,\n \"GT_hidden_dim\": 80,\n \"GT_out_dim\": 80,\n \"GT_n_heads\": 10,\n\n \"residual\": true,\n \"readout\": \"mean\",\n \"in_feat_dropout\": 0.0,\n \"dropout\": 0.0,\n \"layer_norm\": false,\n \"batch_norm\": true\n }\n}\n" }, { "path": "configs/ZINC/ablation/full/1e-2/node", "content": "{\n \"gpu\": {\n \"use\": true,\n \"id\": 0\n },\n \n \"model\": \"GraphTransformer\",\n \"dataset\": \"ZINC\",\n \n \"out_dir\": \"out/ZINC/full/1e-2/node/\",\n \n \"params\": {\n \"seed\": 41,\n \"epochs\": 1000,\n \"batch_size\": 32,\n \"init_lr\": 0.0007,\n \"lr_reduce_factor\": 0.5,\n \"lr_schedule_patience\": 25,\n \"min_lr\": 1e-6,\n \"weight_decay\": 0.0,\n \"print_epoch_interval\": 5,\n \"max_time\": 24\n },\n \n \"net_params\": {\n \"full_graph\": true,\n \"gamma\": 1e-2,\n\n \"LPE\": \"node\",\n \"m\": 30,\n \"LPE_layers\": 3,\n \"LPE_dim\": 16,\n \"LPE_n_heads\": 4,\n\n \"GT_layers\": 6,\n \"GT_hidden_dim\": 64,\n \"GT_out_dim\": 64,\n \"GT_n_heads\": 8,\n\n \"residual\": true,\n \"readout\": \"mean\",\n \"in_feat_dropout\": 0.0,\n \"dropout\": 0.0,\n \"layer_norm\": false,\n \"batch_norm\": true\n }\n}\n" }, { "path": "configs/ZINC/ablation/full/1e-3/node", "content": "{\n \"gpu\": {\n \"use\": true,\n \"id\": 0\n },\n \n \"model\": \"GraphTransformer\",\n \"dataset\": \"ZINC\",\n \n \"out_dir\": \"out/ZINC/full/1e-3/node/\",\n \n \"params\": {\n \"seed\": 41,\n \"epochs\": 1000,\n \"batch_size\": 32,\n \"init_lr\": 0.0007,\n \"lr_reduce_factor\": 0.5,\n \"lr_schedule_patience\": 25,\n \"min_lr\": 1e-6,\n \"weight_decay\": 0.0,\n \"print_epoch_interval\": 5,\n \"max_time\": 24\n },\n \n \"net_params\": {\n \"full_graph\": true,\n \"gamma\": 1e-3,\n\n \"LPE\": \"node\",\n \"m\": 30,\n \"LPE_layers\": 3,\n \"LPE_dim\": 16,\n \"LPE_n_heads\": 4,\n\n \"GT_layers\": 6,\n \"GT_hidden_dim\": 64,\n \"GT_out_dim\": 64,\n \"GT_n_heads\": 8,\n\n \"residual\": true,\n \"readout\": \"mean\",\n \"in_feat_dropout\": 0.0,\n \"dropout\": 0.0,\n \"layer_norm\": false,\n \"batch_norm\": true\n }\n}\n" }, { "path": "configs/ZINC/ablation/full/1e-4/node", "content": "{\n \"gpu\": {\n \"use\": true,\n \"id\": 0\n },\n \n \"model\": \"GraphTransformer\",\n \"dataset\": \"ZINC\",\n \n \"out_dir\": \"out/ZINC/full/1e-4/node/\",\n \n \"params\": {\n \"seed\": 41,\n \"epochs\": 1000,\n \"batch_size\": 32,\n \"init_lr\": 0.0007,\n \"lr_reduce_factor\": 0.5,\n \"lr_schedule_patience\": 25,\n \"min_lr\": 1e-6,\n \"weight_decay\": 0.0,\n \"print_epoch_interval\": 5,\n \"max_time\": 24\n },\n \n \"net_params\": {\n \"full_graph\": true,\n \"gamma\": 1e-4,\n\n \"LPE\": \"node\",\n \"m\": 30,\n \"LPE_layers\": 3,\n \"LPE_dim\": 16,\n \"LPE_n_heads\": 4,\n\n \"GT_layers\": 6,\n \"GT_hidden_dim\": 64,\n \"GT_out_dim\": 64,\n \"GT_n_heads\": 8,\n\n \"residual\": true,\n \"readout\": \"mean\",\n \"in_feat_dropout\": 0.0,\n \"dropout\": 0.0,\n \"layer_norm\": false,\n \"batch_norm\": true\n }\n}\n" }, { "path": "configs/ZINC/ablation/full/1e-5/node", "content": "{\n \"gpu\": {\n \"use\": true,\n \"id\": 0\n },\n \n \"model\": \"GraphTransformer\",\n \"dataset\": \"ZINC\",\n \n \"out_dir\": \"out/ZINC/full/1e-5/node/\",\n \n \"params\": {\n \"seed\": 41,\n \"epochs\": 1000,\n \"batch_size\": 32,\n \"init_lr\": 0.0007,\n \"lr_reduce_factor\": 0.5,\n \"lr_schedule_patience\": 25,\n \"min_lr\": 1e-6,\n \"weight_decay\": 0.0,\n \"print_epoch_interval\": 5,\n \"max_time\": 24\n },\n \n \"net_params\": {\n \"full_graph\": true,\n \"gamma\": 1e-5,\n\n \"LPE\": \"node\",\n \"m\": 30,\n \"LPE_layers\": 3,\n \"LPE_dim\": 16,\n \"LPE_n_heads\": 4,\n\n \"GT_layers\": 6,\n \"GT_hidden_dim\": 64,\n \"GT_out_dim\": 64,\n \"GT_n_heads\": 8,\n\n \"residual\": true,\n \"readout\": \"mean\",\n \"in_feat_dropout\": 0.0,\n \"dropout\": 0.0,\n \"layer_norm\": false,\n \"batch_norm\": true\n }\n}\n" }, { "path": "configs/ZINC/ablation/full/1e-6/node", "content": "{\n \"gpu\": {\n \"use\": true,\n \"id\": 0\n },\n \n \"model\": \"GraphTransformer\",\n \"dataset\": \"ZINC\",\n \n \"out_dir\": \"out/ZINC/full/1e-6/node/\",\n \n \"params\": {\n \"seed\": 41,\n \"epochs\": 1000,\n \"batch_size\": 32,\n \"init_lr\": 0.0007,\n \"lr_reduce_factor\": 0.5,\n \"lr_schedule_patience\": 25,\n \"min_lr\": 1e-6,\n \"weight_decay\": 0.0,\n \"print_epoch_interval\": 5,\n \"max_time\": 24\n },\n \n \"net_params\": {\n \"full_graph\": true,\n \"gamma\": 1e-6,\n\n \"LPE\": \"node\",\n \"m\": 30,\n \"LPE_layers\": 3,\n \"LPE_dim\": 16,\n \"LPE_n_heads\": 4,\n\n \"GT_layers\": 6,\n \"GT_hidden_dim\": 64,\n \"GT_out_dim\": 64,\n \"GT_n_heads\": 8,\n\n \"residual\": true,\n \"readout\": \"mean\",\n \"in_feat_dropout\": 0.0,\n \"dropout\": 0.0,\n \"layer_norm\": false,\n \"batch_norm\": true\n }\n}\n" }, { "path": "configs/ZINC/ablation/full/1e-7/node", "content": "{\n \"gpu\": {\n \"use\": true,\n \"id\": 0\n },\n \n \"model\": \"GraphTransformer\",\n \"dataset\": \"ZINC\",\n \n \"out_dir\": \"out/ZINC/full/1e-7/node/\",\n \n \"params\": {\n \"seed\": 41,\n \"epochs\": 1000,\n \"batch_size\": 32,\n \"init_lr\": 0.0007,\n \"lr_reduce_factor\": 0.5,\n \"lr_schedule_patience\": 25,\n \"min_lr\": 1e-6,\n \"weight_decay\": 0.0,\n \"print_epoch_interval\": 5,\n \"max_time\": 24\n },\n \n \"net_params\": {\n \"full_graph\": true,\n \"gamma\": 1e-7,\n\n \"LPE\": \"node\",\n \"m\": 30,\n \"LPE_layers\": 3,\n \"LPE_dim\": 16,\n \"LPE_n_heads\": 4,\n\n \"GT_layers\": 6,\n \"GT_hidden_dim\": 64,\n \"GT_out_dim\": 64,\n \"GT_n_heads\": 8,\n\n \"residual\": true,\n \"readout\": \"mean\",\n \"in_feat_dropout\": 0.0,\n \"dropout\": 0.0,\n \"layer_norm\": false,\n \"batch_norm\": true\n }\n}\n" }, { "path": "configs/ZINC/ablation/full/1e-8/node", "content": "{\n \"gpu\": {\n \"use\": true,\n \"id\": 0\n },\n \n \"model\": \"GraphTransformer\",\n \"dataset\": \"ZINC\",\n \n \"out_dir\": \"out/ZINC/full/1e-8/node/\",\n \n \"params\": {\n \"seed\": 41,\n \"epochs\": 1000,\n \"batch_size\": 32,\n \"init_lr\": 0.0007,\n \"lr_reduce_factor\": 0.5,\n \"lr_schedule_patience\": 25,\n \"min_lr\": 1e-6,\n \"weight_decay\": 0.0,\n \"print_epoch_interval\": 5,\n \"max_time\": 24\n },\n \n \"net_params\": {\n \"full_graph\": true,\n \"gamma\": 1e-8,\n\n \"LPE\": \"node\",\n \"m\": 30,\n \"LPE_layers\": 3,\n \"LPE_dim\": 16,\n \"LPE_n_heads\": 4,\n\n \"GT_layers\": 6,\n \"GT_hidden_dim\": 64,\n \"GT_out_dim\": 64,\n \"GT_n_heads\": 8,\n\n \"residual\": true,\n \"readout\": \"mean\",\n \"in_feat_dropout\": 0.0,\n \"dropout\": 0.0,\n \"layer_norm\": false,\n \"batch_norm\": true\n }\n}\n" }, { "path": "configs/ZINC/ablation/full/none", "content": "{\n \"gpu\": {\n \"use\": true,\n \"id\": 0\n },\n \n \"model\": \"GraphTransformer\",\n \"dataset\": \"ZINC\",\n \n \"out_dir\": \"out/ZINC/full/none/\",\n \n \"params\": {\n \"seed\": 41,\n \"epochs\": 1000,\n \"batch_size\": 32,\n \"init_lr\": 0.0007,\n \"lr_reduce_factor\": 0.5,\n \"lr_schedule_patience\": 25,\n \"min_lr\": 1e-6,\n \"weight_decay\": 0.0,\n \"print_epoch_interval\": 5,\n \"max_time\": 24\n },\n \n \"net_params\": {\n \"full_graph\": true,\n \"gamma\": 1e-5,\n\n \"LPE\": \"none\",\n \"m\": 30,\n \"LPE_layers\": 3,\n \"LPE_dim\": 16,\n \"LPE_n_heads\": 4,\n\n \"GT_layers\": 6,\n \"GT_hidden_dim\": 80,\n \"GT_out_dim\": 80,\n \"GT_n_heads\": 8,\n\n \"residual\": true,\n \"readout\": \"mean\",\n \"in_feat_dropout\": 0.0,\n \"dropout\": 0.0,\n \"layer_norm\": false,\n \"batch_norm\": true\n }\n}\n" }, { "path": "configs/ZINC/ablation/sparse/node", "content": "{\n \"gpu\": {\n \"use\": true,\n \"id\": 0\n },\n \n \"model\": \"GraphTransformer\",\n \"dataset\": \"ZINC\",\n \n \"out_dir\": \"out/ZINC/sparse/node/\",\n \n \"params\": {\n \"seed\": 41,\n \"epochs\": 1000,\n \"batch_size\": 32,\n \"init_lr\": 0.0007,\n \"lr_reduce_factor\": 0.5,\n \"lr_schedule_patience\": 25,\n \"min_lr\": 1e-6,\n \"weight_decay\": 0.0,\n \"print_epoch_interval\": 5,\n \"max_time\": 24\n },\n \n \"net_params\": {\n \"full_graph\": false,\n \"gamma\": 1e-5,\n\n \"LPE\": \"node\",\n \"m\": 30,\n \"LPE_layers\": 3,\n \"LPE_dim\": 16,\n \"LPE_n_heads\": 4,\n\n \"GT_layers\": 6,\n \"GT_hidden_dim\": 72,\n \"GT_out_dim\": 72,\n \"GT_n_heads\": 8,\n\n \"residual\": true,\n \"readout\": \"mean\",\n \"in_feat_dropout\": 0.0,\n \"dropout\": 0.0,\n \"layer_norm\": false,\n \"batch_norm\": true\n }\n}\n" }, { "path": "configs/ZINC/ablation/sparse/none", "content": "{\n \"gpu\": {\n \"use\": true,\n \"id\": 0\n },\n \n \"model\": \"GraphTransformer\",\n \"dataset\": \"ZINC\",\n \n \"out_dir\": \"out/ZINC/sparse/none/\",\n \n \"params\": {\n \"seed\": 41,\n \"epochs\": 1000,\n \"batch_size\": 64,\n \"init_lr\": 0.0007,\n \"lr_reduce_factor\": 0.5,\n \"lr_schedule_patience\": 25,\n \"min_lr\": 1e-6,\n \"weight_decay\": 0.0,\n \"print_epoch_interval\": 5,\n \"max_time\": 24\n },\n \n \"net_params\": {\n \"full_graph\": false,\n \"gamma\": 1e-5,\n\n \"LPE\": \"none\",\n \"m\": 30,\n \"LPE_layers\": 3,\n \"LPE_dim\": 16,\n \"LPE_n_heads\": 4,\n\n \"GT_layers\": 6,\n \"GT_hidden_dim\": 96,\n \"GT_out_dim\": 96,\n \"GT_n_heads\": 8,\n\n \"residual\": true,\n \"readout\": \"mean\",\n \"in_feat_dropout\": 0.0,\n \"dropout\": 0.0,\n \"layer_norm\": false,\n \"batch_norm\": true\n }\n}\n" }, { "path": "configs/ZINC/optimized", "content": "{\n \"gpu\": {\n \"use\": true,\n \"id\": 0\n },\n \n \"model\": \"GraphTransformer\",\n \"dataset\": \"ZINC\",\n \n \"out_dir\": \"out/ZINC/full/1e-5/node/\",\n \n \"params\": {\n \"seed\": 41,\n \"epochs\": 1000,\n \"batch_size\": 32,\n \"init_lr\": 0.0007,\n \"lr_reduce_factor\": 0.5,\n \"lr_schedule_patience\": 25,\n \"min_lr\": 1e-6,\n \"weight_decay\": 0.0,\n \"print_epoch_interval\": 5,\n \"max_time\": 24\n },\n \n \"net_params\": {\n \"full_graph\": true,\n \"gamma\": 1e-5,\n\n \"LPE\": \"node\",\n \"m\": 10,\n \"LPE_layers\": 2,\n \"LPE_dim\": 8,\n \"LPE_n_heads\": 4,\n\n \"GT_layers\": 10,\n \"GT_hidden_dim\": 56,\n \"GT_out_dim\": 56,\n \"GT_n_heads\": 8,\n\n \"residual\": true,\n \"readout\": \"sum\",\n \"in_feat_dropout\": 0.0,\n \"dropout\": 0.0,\n \"layer_norm\": false,\n \"batch_norm\": true\n }\n}\n" }, { "path": "data/SBMs.py", "content": "\nimport time\nimport os\nimport pickle\nimport numpy as np\n\nimport dgl\nimport torch\nimport torch.nn.functional as F\n\nfrom scipy import sparse as sp\nimport numpy as np\nimport networkx as nx\n\nimport hashlib\n\n\nclass load_SBMsDataSetDGL(torch.utils.data.Dataset):\n\n def __init__(self,\n data_dir,\n name,\n split):\n\n self.split = split\n self.is_test = split.lower() in ['test', 'val'] \n with open(os.path.join(data_dir, name + '_%s.pkl' % self.split), 'rb') as f:\n self.dataset = pickle.load(f)\n self.node_labels = []\n self.graph_lists = []\n self.n_samples = len(self.dataset)\n self._prepare()\n \n\n def _prepare(self):\n\n print(\"preparing %d graphs for the %s set...\" % (self.n_samples, self.split.upper()))\n\n for data in self.dataset:\n\n node_features = data.node_feat\n edge_list = (data.W != 0).nonzero() # converting adj matrix to edge_list\n\n # Create the DGL Graph\n g = dgl.DGLGraph()\n g.add_nodes(node_features.size(0))\n g.ndata['feat'] = node_features.long()\n for src, dst in edge_list:\n g.add_edges(src.item(), dst.item())\n\n # adding edge features for Residual Gated ConvNet\n #edge_feat_dim = g.ndata['feat'].size(1) # dim same as node feature dim\n edge_feat_dim = 1 # dim same as node feature dim\n g.edata['feat'] = torch.ones(g.number_of_edges(), edge_feat_dim)\n\n self.graph_lists.append(g)\n self.node_labels.append(data.node_label)\n\n\n def __len__(self):\n \"\"\"Return the number of graphs in the dataset.\"\"\"\n return self.n_samples\n\n def __getitem__(self, idx):\n \"\"\"\n Get the idx^th sample.\n Parameters\n ---------\n idx : int\n The sample index.\n Returns\n -------\n (dgl.DGLGraph, int)\n DGLGraph with node feature stored in `feat` field\n And its label.\n \"\"\"\n return self.graph_lists[idx], self.node_labels[idx]\n\n\nclass SBMsDatasetDGL(torch.utils.data.Dataset):\n\n def __init__(self, name):\n \"\"\"\n TODO\n \"\"\"\n start = time.time()\n print(\"[I] Loading data ...\")\n self.name = name\n data_dir = 'data/SBMs'\n self.train = load_SBMsDataSetDGL(data_dir, name, split='train')\n self.test = load_SBMsDataSetDGL(data_dir, name, split='test')\n self.val = load_SBMsDataSetDGL(data_dir, name, split='val')\n print(\"[I] Finished loading.\")\n print(\"[I] Data load time: {:.4f}s\".format(time.time()-start))\n\n\n\ndef laplace_decomp(g, max_freqs):\n\n\n # Laplacian\n n = g.number_of_nodes()\n A = g.adjacency_matrix_scipy(return_edge_ids=False).astype(float)\n N = sp.diags(dgl.backend.asnumpy(g.in_degrees()).clip(1) ** -0.5, dtype=float)\n L = sp.eye(g.number_of_nodes()) - N * A * N\n\n # Eigenvectors with numpy\n EigVals, EigVecs = np.linalg.eigh(L.toarray())\n EigVals, EigVecs = EigVals[: max_freqs], EigVecs[:, :max_freqs] # Keep up to the maximum desired number of frequencies\n\n # Normalize and pad EigenVectors\n EigVecs = torch.from_numpy(EigVecs).float()\n EigVecs = F.normalize(EigVecs, p=2, dim=1, eps=1e-12, out=None)\n \n if n.index\n with open(data_dir + \"/%s.index\" % self.split,\"r\") as f:\n data_idx = [list(map(int, idx)) for idx in csv.reader(f)]\n self.data = [ self.data[i] for i in data_idx[0] ]\n\n assert len(self.data)==num_graphs, \"Sample num_graphs again; available idx: train/val/test => 10k/1k/1k\"\n \n \"\"\"\n data is a list of Molecule dict objects with following attributes\n \n molecule = data[idx]\n ; molecule['num_atom'] : nb of atoms, an integer (N)\n ; molecule['atom_type'] : tensor of size N, each element is an atom type, an integer between 0 and num_atom_type\n ; molecule['bond_type'] : tensor of size N x N, each element is a bond type, an integer between 0 and num_bond_type\n ; molecule['logP_SA_cycle_normalized'] : the chemical property to regress, a float variable\n \"\"\"\n \n self.graph_lists = []\n self.graph_labels = []\n self.n_samples = len(self.data)\n self._prepare()\n \n def _prepare(self):\n print(\"preparing %d graphs for the %s set...\" % (self.num_graphs, self.split.upper()))\n \n for molecule in self.data:\n node_features = molecule['atom_type'].long()\n \n adj = molecule['bond_type']\n edge_list = (adj != 0).nonzero() # converting adj matrix to edge_list\n \n edge_idxs_in_adj = edge_list.split(1, dim=1)\n edge_features = adj[edge_idxs_in_adj].reshape(-1).long()\n \n # Create the DGL Graph\n g = dgl.DGLGraph()\n g.add_nodes(molecule['num_atom'])\n g.ndata['feat'] = node_features\n \n for src, dst in edge_list:\n g.add_edges(src.item(), dst.item())\n g.edata['feat'] = edge_features\n \n self.graph_lists.append(g)\n self.graph_labels.append(molecule['logP_SA_cycle_normalized'])\n \n def __len__(self):\n \"\"\"Return the number of graphs in the dataset.\"\"\"\n return self.n_samples\n\n def __getitem__(self, idx):\n \"\"\"\n Get the idx^th sample.\n Parameters\n ---------\n idx : int\n The sample index.\n Returns\n -------\n (dgl.DGLGraph, int)\n DGLGraph with node feature stored in `feat` field\n And its label.\n \"\"\"\n return self.graph_lists[idx], self.graph_labels[idx]\n \n \nclass MoleculeDatasetDGL(torch.utils.data.Dataset):\n def __init__(self, name='Zinc'):\n t0 = time.time()\n self.name = name\n \n self.num_atom_type = 28 # known meta-info about the zinc dataset; can be calculated as well\n self.num_bond_type = 4 # known meta-info about the zinc dataset; can be calculated as well\n \n data_dir='./data/molecules'\n \n if self.name == 'ZINC-full':\n data_dir='./data/molecules/zinc_full'\n self.train = MoleculeDGL(data_dir, 'train', num_graphs=220011)\n self.val = MoleculeDGL(data_dir, 'val', num_graphs=24445)\n self.test = MoleculeDGL(data_dir, 'test', num_graphs=5000)\n else: \n self.train = MoleculeDGL(data_dir, 'train', num_graphs=10000)\n self.val = MoleculeDGL(data_dir, 'val', num_graphs=1000)\n self.test = MoleculeDGL(data_dir, 'test', num_graphs=1000)\n print(\"Time taken: {:.4f}s\".format(time.time()-t0))\n\n\n\ndef laplace_decomp(g, max_freqs):\n\n\n # Laplacian\n n = g.number_of_nodes()\n A = g.adjacency_matrix_scipy(return_edge_ids=False).astype(float)\n N = sp.diags(dgl.backend.asnumpy(g.in_degrees()).clip(1) ** -0.5, dtype=float)\n L = sp.eye(g.number_of_nodes()) - N * A * N\n\n # Eigenvectors with numpy\n EigVals, EigVecs = np.linalg.eigh(L.toarray())\n EigVals, EigVecs = EigVals[: max_freqs], EigVecs[:, :max_freqs] # Keep up to the maximum desired number of frequencies\n\n # Normalize and pad EigenVectors\n EigVecs = torch.from_numpy(EigVecs).float()\n EigVecs = F.normalize(EigVecs, p=2, dim=1, eps=1e-12, out=None)\n \n if n params['max_time']*3600:\n print('-' * 89)\n print(\"Max_time for training elapsed {:.2f} hours, so stopping\".format(params['max_time']))\n break\n \n except KeyboardInterrupt:\n print('-' * 89)\n print('Exiting from training early because of KeyboardInterrupt')\n \n #Return test and train metrics at best val metric\n index = epoch_val_accs.index(max(epoch_val_accs))\n \n test_acc = epoch_test_accs[index]\n train_acc = epoch_train_accs[index]\n \n print(\"Test Accuracy: {:.4f}\".format(test_acc))\n print(\"Train Accuracy: {:.4f}\".format(train_acc))\n print(\"Convergence Time (Epochs): {:.4f}\".format(epoch))\n print(\"TOTAL TIME TAKEN: {:.4f}s\".format(time.time()-start0))\n print(\"AVG TIME PER EPOCH: {:.4f}s\".format(np.mean(per_epoch_time)))\n\n writer.close()\n\n \"\"\"\n Write the results in out_dir/results folder\n \"\"\"\n with open(write_file_name + '.txt', 'w') as f:\n f.write(\"\"\"Dataset: {},\\nModel: {}\\n\\nparams={}\\n\\nnet_params={}\\n\\n{}\\n\\nTotal Parameters: {}\\n\\n\n FINAL RESULTS\\nTEST ACCURACY: {:.4f}\\nTRAIN ACCURACY: {:.4f}\\n\\n\n Convergence Time (Epochs): {:.4f}\\nTotal Time Taken: {:.4f} hrs\\nAverage Time Per Epoch: {:.4f} s\\n\\n\\n\"\"\"\\\n .format(DATASET_NAME, MODEL_NAME, params, net_params, model, net_params['total_param'],\n test_acc, train_acc, epoch, (time.time()-start0)/3600, np.mean(per_epoch_time)))\n\n \n\n\n\n\ndef main(): \n \"\"\"\n USER CONTROLS\n \"\"\"\n \n parser = argparse.ArgumentParser()\n parser.add_argument('--config', help=\"Please give a config.json file with training/model/data/param details\")\n parser.add_argument('--gpu_id', help=\"Please give a value for gpu id\")\n parser.add_argument('--model', help=\"Please give a value for model name\")\n parser.add_argument('--dataset', help=\"Please give a value for dataset name\")\n parser.add_argument('--out_dir', help=\"Please give a value for out_dir\")\n parser.add_argument('--seed', help=\"Please give a value for seed\")\n parser.add_argument('--epochs', help=\"Please give a value for epochs\")\n parser.add_argument('--batch_size', help=\"Please give a value for batch_size\")\n parser.add_argument('--init_lr', help=\"Please give a value for init_lr\")\n parser.add_argument('--lr_reduce_factor', help=\"Please give a value for lr_reduce_factor\")\n parser.add_argument('--lr_schedule_patience', help=\"Please give a value for lr_schedule_patience\")\n parser.add_argument('--min_lr', help=\"Please give a value for min_lr\")\n parser.add_argument('--weight_decay', help=\"Please give a value for weight_decay\")\n parser.add_argument('--print_epoch_interval', help=\"Please give a value for print_epoch_interval\")\n parser.add_argument('--max_time', help=\"Please give a value for max_time\")\n \n #Model details\n parser.add_argument('--full_graph', help=\"Please give a value for full_graph\") \n parser.add_argument('--gamma', help=\"Please give a value for gamma\")\n parser.add_argument('--m', help=\"Please give a value for m\")\n \n parser.add_argument('--LPE', help=\"Please give a value for LPE\")\n parser.add_argument('--LPE_layers', help=\"Please give a value for LPE_layers\")\n parser.add_argument('--LPE_dim', help=\"Please give a value for LPE_dim\")\n parser.add_argument('--LPE_n_heads', help=\"Please give a value for LPE_n_heads\")\n \n parser.add_argument('--GT_layers', help=\"Please give a value for GT_layers\")\n parser.add_argument('--GT_hidden_dim', help=\"Please give a value for GT_hidden_dim\")\n parser.add_argument('--GT_out_dim', help=\"Please give a value for GT_out_dim\")\n parser.add_argument('--GT_n_heads', help=\"Please give a value for GT_n_heads\")\n \n parser.add_argument('--residual', help=\"Please give a value for readout\")\n parser.add_argument('--readout', help=\"Please give a value for readout\")\n parser.add_argument('--in_feat_dropout', help=\"Please give a value for in_feat_dropout\")\n parser.add_argument('--dropout', help=\"Please give a value for dropout\")\n parser.add_argument('--layer_norm', help=\"Please give a value for layer_norm\")\n parser.add_argument('--batch_norm', help=\"Please give a value for batch_norm\")\n args = parser.parse_args()\n \n with open(args.config) as f:\n config = json.load(f)\n \n # device\n if args.gpu_id is not None:\n config['gpu']['id'] = int(args.gpu_id)\n config['gpu']['use'] = True\n device = gpu_setup(config['gpu']['use'], config['gpu']['id'])\n # model, dataset, out_dir\n if args.model is not None:\n MODEL_NAME = args.model\n else:\n MODEL_NAME = config['model']\n if args.dataset is not None:\n DATASET_NAME = args.dataset\n else:\n DATASET_NAME = config['dataset']\n dataset = LoadData(DATASET_NAME)\n if args.out_dir is not None:\n out_dir = args.out_dir\n else:\n out_dir = config['out_dir']\n # parameters\n params = config['params']\n if args.seed is not None:\n params['seed'] = int(args.seed)\n if args.epochs is not None:\n params['epochs'] = int(args.epochs)\n if args.batch_size is not None:\n params['batch_size'] = int(args.batch_size)\n if args.init_lr is not None:\n params['init_lr'] = float(args.init_lr)\n if args.lr_reduce_factor is not None:\n params['lr_reduce_factor'] = float(args.lr_reduce_factor)\n if args.lr_schedule_patience is not None:\n params['lr_schedule_patience'] = int(args.lr_schedule_patience)\n if args.min_lr is not None:\n params['min_lr'] = float(args.min_lr)\n if args.weight_decay is not None:\n params['weight_decay'] = float(args.weight_decay)\n if args.print_epoch_interval is not None:\n params['print_epoch_interval'] = int(args.print_epoch_interval)\n if args.max_time is not None:\n params['max_time'] = float(args.max_time)\n \n \n # model parameters\n net_params = config['net_params']\n net_params['device'] = device\n net_params['gpu_id'] = config['gpu']['id']\n net_params['batch_size'] = params['batch_size']\n \n \n if args.full_graph is not None:\n net_params['full_graph'] = True if args.full_graph=='True' else False\n if args.gamma is not None:\n net_params['gamma'] = float(args.gamma)\n if args.m is not None:\n net_params['m'] = int(args.m)\n \n \n if args.LPE is not None:\n net_params['LPE'] = args.LPE\n \n \n if net_params['LPE'] not in ['node', 'edge', 'none']:\n print('[!] User did not provide a valid input argument for \\'LPE\\'. Valid inputs are \\'node\\', \\'edge\\', and \\'none\\'.')\n exit()\n \n if args.LPE_layers is not None:\n net_params['LPE_layers'] = int(args.LPE_layers)\n if args.LPE_dim is not None:\n net_params['LPE_dim'] = int(args.LPE_dim)\n if args.LPE_n_heads is not None:\n net_params['LPE_n_heads'] = int(args.LPE_n_heads) \n \n if args.GT_layers is not None:\n net_params['GT_layers'] = int(args.GT_layers)\n if args.GT_hidden_dim is not None:\n net_params['GT_hidden_dim'] = int(args.GT_hidden_dim)\n if args.GT_out_dim is not None:\n net_params['GT_out_dim'] = int(args.GT_out_dim) \n if args.GT_n_heads is not None:\n net_params['GT_n_heads'] = int(args.GT_n_heads) \n \n if args.residual is not None:\n net_params['residual'] = True if args.residual=='True' else False\n if args.readout is not None:\n net_params['readout'] = args.readout\n if args.in_feat_dropout is not None:\n net_params['in_feat_dropout'] = float(args.in_feat_dropout)\n if args.dropout is not None:\n net_params['dropout'] = float(args.dropout)\n if args.layer_norm is not None:\n net_params['layer_norm'] = True if args.layer_norm=='True' else False\n if args.batch_norm is not None:\n net_params['batch_norm'] = True if args.batch_norm=='True' else False\n \n # SBM\n\n net_params['in_dim'] = torch.unique(dataset.train[0][0].ndata['feat'],dim=0).size(0) # node_dim (feat is an integer)\n \n net_params['n_classes'] = torch.unique(dataset.train[0][1],dim=0).size(0)\n \n\n root_log_dir = out_dir + 'logs/' + MODEL_NAME + \"_\" + DATASET_NAME + \"_GPU\" + str(config['gpu']['id']) + \"_\" + time.strftime('%Hh%Mm%Ss_on_%b_%d_%Y')\n root_ckpt_dir = out_dir + 'checkpoints/' + MODEL_NAME + \"_\" + DATASET_NAME + \"_GPU\" + str(config['gpu']['id']) + \"_\" + time.strftime('%Hh%Mm%Ss_on_%b_%d_%Y')\n write_file_name = out_dir + 'results/result_' + MODEL_NAME + \"_\" + DATASET_NAME + \"_GPU\" + str(config['gpu']['id']) + \"_\" + time.strftime('%Hh%Mm%Ss_on_%b_%d_%Y')\n write_config_file = out_dir + 'configs/config_' + MODEL_NAME + \"_\" + DATASET_NAME + \"_GPU\" + str(config['gpu']['id']) + \"_\" + time.strftime('%Hh%Mm%Ss_on_%b_%d_%Y')\n dirs = root_log_dir, root_ckpt_dir, write_file_name, write_config_file\n\n if not os.path.exists(out_dir + 'results'):\n os.makedirs(out_dir + 'results')\n \n if not os.path.exists(out_dir + 'configs'):\n os.makedirs(out_dir + 'configs')\n\n train_val_pipeline(MODEL_NAME, dataset, params, net_params, dirs)\n\n \nmain() \n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n" }, { "path": "main_ZINC_graph_regression.py", "content": "\"\"\"\n IMPORTING LIBS\n\"\"\"\nimport dgl\n\nimport numpy as np\nimport os\nimport socket\nimport time\nimport random\nimport glob\nimport argparse, json\nimport pickle\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader\n\nfrom tensorboardX import SummaryWriter\nfrom tqdm import tqdm\n\n\n\n\"\"\"\n IMPORTING CUSTOM MODULES/METHODS\n\"\"\"\nfrom nets.ZINC_graph_regression.load_net import gnn_model \nfrom data.data import LoadData \n\n\n\n\"\"\"\n GPU Setup\n\"\"\"\ndef gpu_setup(use_gpu, gpu_id):\n os.environ[\"CUDA_DEVICE_ORDER\"] = \"PCI_BUS_ID\"\n os.environ[\"CUDA_VISIBLE_DEVICES\"] = str(gpu_id) \n\n if torch.cuda.is_available() and use_gpu:\n print('cuda available with GPU:',torch.cuda.get_device_name(0))\n device = torch.device(\"cuda\")\n else:\n print('cuda not available')\n device = torch.device(\"cpu\")\n return device\n\n\n\n\"\"\"\n VIEWING ENCODING TYPE AND NUM PARAMS\n\"\"\"\ndef view_model_param(LPE, net_params):\n model = gnn_model(LPE, net_params)\n total_param = 0\n print(\"MODEL DETAILS:\\n\")\n for param in model.parameters():\n total_param += np.prod(list(param.data.size()))\n \n if LPE == 'edge':\n print('Encoding Type/Total parameters:', 'Edge Laplace Encoding/', total_param)\n elif LPE == 'node':\n print('Encoding Type/Total parameters:', 'Node Laplace Encoding', total_param)\n else:\n print('Encoding Type/Total parameters:', 'None', total_param)\n return total_param\n\n\n\"\"\"\n TRAINING CODE\n\"\"\"\n\ndef train_val_pipeline(MODEL_NAME, dataset, params, net_params, dirs):\n start0 = time.time()\n per_epoch_time = []\n \n DATASET_NAME = dataset.name\n \n if net_params['LPE'] in ['edge', 'node']:\n st = time.time()\n print(\"[!] Computing Laplace Decompositions..\")\n dataset._laplace_decomp(net_params['m'])\n print('Time taken to decompose Laplacians: ',time.time()-st)\n \n if net_params['full_graph']:\n st = time.time()\n print(\"[!] Adding full graph connectivity..\")\n dataset._make_full_graph()\n print('Time taken to add full graph connectivity: ',time.time()-st)\n \n if net_params['LPE'] == 'edge':\n st = time.time()\n print(\"[!] Computing edge Laplace features..\")\n dataset._add_edge_laplace_feats()\n print('Time taken to compute edge Laplace features: ',time.time()-st)\n \n trainset, valset, testset = dataset.train, dataset.val, dataset.test\n \n net_params['total_param'] = view_model_param(net_params['LPE'], net_params)\n \n root_log_dir, root_ckpt_dir, write_file_name, write_config_file = dirs\n device = net_params['device']\n \n # Write the network and optimization hyper-parameters in folder config/\n with open(write_config_file + '.txt', 'w') as f:\n f.write(\"\"\"Dataset: {},\\nModel: {}\\n\\nparams={}\\n\\nnet_params={}\\n\\n\\nTotal Parameters: {}\\n\\n\"\"\" .format(DATASET_NAME, MODEL_NAME, params, net_params, net_params['total_param']))\n \n log_dir = os.path.join(root_log_dir, \"RUN_\" + str(0))\n writer = SummaryWriter(log_dir=log_dir)\n\n # setting seeds\n random.seed(params['seed'])\n np.random.seed(params['seed'])\n torch.manual_seed(params['seed'])\n if device.type == 'cuda':\n torch.cuda.manual_seed(params['seed'])\n \n print(\"Training Graphs: \", len(trainset))\n print(\"Validation Graphs: \", len(valset))\n print(\"Test Graphs: \", len(testset))\n\n model = gnn_model(net_params['LPE'], net_params)\n model = model.to(device)\n\n optimizer = optim.Adam(model.parameters(), lr=params['init_lr'], weight_decay=params['weight_decay'])\n scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min',\n factor=params['lr_reduce_factor'],\n patience=params['lr_schedule_patience'],\n verbose=True)\n \n epoch_train_losses, epoch_val_losses = [], []\n epoch_train_MAEs, epoch_val_MAEs, epoch_test_MAEs = [], [], []\n \n \n\n # import train and evaluate functions\n from train.train_ZINC_graph_regression import train_epoch, evaluate_network\n\n \n train_loader = DataLoader(trainset, num_workers=4, batch_size=params['batch_size'], shuffle=True, collate_fn=dataset.collate)\n val_loader = DataLoader(valset, num_workers=4, batch_size=params['batch_size'], shuffle=False, collate_fn=dataset.collate)\n test_loader = DataLoader(testset, num_workers=4, batch_size=params['batch_size'], shuffle=False, collate_fn=dataset.collate)\n \n # At any point you can hit Ctrl + C to break out of training early.\n try:\n with tqdm(range(params['epochs'])) as t:\n for epoch in t:\n\n t.set_description('Epoch %d' % epoch)\n\n start = time.time()\n\n epoch_train_loss, epoch_train_mae, optimizer = train_epoch(model, optimizer, device, train_loader, epoch, net_params['LPE'])\n \n epoch_val_loss, epoch_val_mae = evaluate_network(model, device, val_loader, epoch, net_params['LPE'])\n _, epoch_test_mae = evaluate_network(model, device, test_loader, epoch, net_params['LPE'])\n \n epoch_train_losses.append(epoch_train_loss)\n epoch_val_losses.append(epoch_val_loss)\n epoch_train_MAEs.append(epoch_train_mae)\n epoch_val_MAEs.append(epoch_val_mae)\n \n epoch_test_MAEs.append(epoch_test_mae)\n\n writer.add_scalar('train/_loss', epoch_train_loss, epoch)\n writer.add_scalar('val/_loss', epoch_val_loss, epoch)\n writer.add_scalar('train/_mae', epoch_train_mae, epoch)\n writer.add_scalar('val/_mae', epoch_val_mae, epoch)\n writer.add_scalar('test/_mae', epoch_test_mae, epoch)\n writer.add_scalar('learning_rate', optimizer.param_groups[0]['lr'], epoch)\n\n \n t.set_postfix(time=time.time()-start, lr=optimizer.param_groups[0]['lr'],\n train_loss=epoch_train_loss, val_loss=epoch_val_loss,\n train_MAE=epoch_train_mae, val_MAE=epoch_val_mae,\n test_MAE=epoch_test_mae)\n\n\n per_epoch_time.append(time.time()-start)\n\n # Saving checkpoint\n ckpt_dir = os.path.join(root_ckpt_dir, \"RUN_\")\n if not os.path.exists(ckpt_dir):\n os.makedirs(ckpt_dir)\n torch.save(model.state_dict(), '{}.pkl'.format(ckpt_dir + \"/epoch_\" + str(epoch)))\n\n files = glob.glob(ckpt_dir + '/*.pkl')\n for file in files:\n epoch_nb = file.split('_')[-1]\n epoch_nb = int(epoch_nb.split('.')[0])\n if epoch_nb < epoch-1:\n os.remove(file)\n\n scheduler.step(epoch_val_loss)\n\n if optimizer.param_groups[0]['lr'] < params['min_lr']:\n print(\"\\n!! LR EQUAL TO MIN LR SET.\")\n break\n \n # Stop training after params['max_time'] hours\n if time.time()-start0 > params['max_time']*3600:\n print('-' * 89)\n print(\"Max_time for training elapsed {:.2f} hours, so stopping\".format(params['max_time']))\n break\n \n except KeyboardInterrupt:\n print('-' * 89)\n print('Exiting from training early because of KeyboardInterrupt')\n \n #Return test and train metrics at best val metric\n index = epoch_val_MAEs.index(min(epoch_val_MAEs))\n \n test_mae = epoch_test_MAEs[index]\n train_mae = epoch_train_MAEs[index]\n \n print(\"Test MAE: {:.4f}\".format(test_mae))\n print(\"Train MAE: {:.4f}\".format(train_mae))\n print(\"Convergence Time (Epochs): {:.4f}\".format(epoch))\n print(\"TOTAL TIME TAKEN: {:.4f}s\".format(time.time()-start0))\n print(\"AVG TIME PER EPOCH: {:.4f}s\".format(np.mean(per_epoch_time)))\n\n writer.close()\n\n \"\"\"\n Write the results in out_dir/results folder\n \"\"\"\n with open(write_file_name + '.txt', 'w') as f:\n f.write(\"\"\"Dataset: {},\\nModel: {}\\n\\nparams={}\\n\\nnet_params={}\\n\\n{}\\n\\nTotal Parameters: {}\\n\\n\n FINAL RESULTS\\nTEST MAE: {:.4f}\\nTRAIN MAE: {:.4f}\\n\\n\n Convergence Time (Epochs): {:.4f}\\nTotal Time Taken: {:.4f} hrs\\nAverage Time Per Epoch: {:.4f} s\\n\\n\\n\"\"\"\\\n .format(DATASET_NAME, MODEL_NAME, params, net_params, model, net_params['total_param'],\n test_mae, train_mae, epoch, (time.time()-start0)/3600, np.mean(per_epoch_time)))\n \n\n\n\n\ndef main(): \n \"\"\"\n USER CONTROLS\n \"\"\"\n \n \n parser = argparse.ArgumentParser()\n parser.add_argument('--config', help=\"Please give a config.json file with training/model/data/param details\")\n parser.add_argument('--gpu_id', help=\"Please give a value for gpu id\")\n parser.add_argument('--model', help=\"Please give a value for model name\")\n parser.add_argument('--dataset', help=\"Please give a value for dataset name\")\n parser.add_argument('--out_dir', help=\"Please give a value for out_dir\")\n parser.add_argument('--seed', help=\"Please give a value for seed\")\n parser.add_argument('--epochs', help=\"Please give a value for epochs\")\n parser.add_argument('--batch_size', help=\"Please give a value for batch_size\")\n parser.add_argument('--init_lr', help=\"Please give a value for init_lr\")\n parser.add_argument('--lr_reduce_factor', help=\"Please give a value for lr_reduce_factor\")\n parser.add_argument('--lr_schedule_patience', help=\"Please give a value for lr_schedule_patience\")\n parser.add_argument('--min_lr', help=\"Please give a value for min_lr\")\n parser.add_argument('--weight_decay', help=\"Please give a value for weight_decay\")\n parser.add_argument('--print_epoch_interval', help=\"Please give a value for print_epoch_interval\")\n parser.add_argument('--max_time', help=\"Please give a value for max_time\")\n \n #Model details\n parser.add_argument('--full_graph', help=\"Please give a value for full_graph\") \n parser.add_argument('--gamma', help=\"Please give a value for gamma\")\n parser.add_argument('--m', help=\"Please give a value for m\")\n \n parser.add_argument('--LPE', help=\"Please give a value for LPE\")\n parser.add_argument('--LPE_layers', help=\"Please give a value for LPE_layers\")\n parser.add_argument('--LPE_dim', help=\"Please give a value for LPE_dim\")\n parser.add_argument('--LPE_n_heads', help=\"Please give a value for LPE_n_heads\")\n \n parser.add_argument('--GT_layers', help=\"Please give a value for GT_layers\")\n parser.add_argument('--GT_hidden_dim', help=\"Please give a value for GT_hidden_dim\")\n parser.add_argument('--GT_out_dim', help=\"Please give a value for GT_out_dim\")\n parser.add_argument('--GT_n_heads', help=\"Please give a value for GT_n_heads\")\n \n parser.add_argument('--residual', help=\"Please give a value for readout\")\n parser.add_argument('--readout', help=\"Please give a value for readout\")\n parser.add_argument('--in_feat_dropout', help=\"Please give a value for in_feat_dropout\")\n parser.add_argument('--dropout', help=\"Please give a value for dropout\")\n parser.add_argument('--layer_norm', help=\"Please give a value for layer_norm\")\n parser.add_argument('--batch_norm', help=\"Please give a value for batch_norm\")\n args = parser.parse_args()\n \n with open(args.config) as f:\n config = json.load(f)\n \n # device\n if args.gpu_id is not None:\n config['gpu']['id'] = int(args.gpu_id)\n config['gpu']['use'] = True\n device = gpu_setup(config['gpu']['use'], config['gpu']['id'])\n # model, dataset, out_dir\n if args.model is not None:\n MODEL_NAME = args.model\n else:\n MODEL_NAME = config['model']\n if args.dataset is not None:\n DATASET_NAME = args.dataset\n else:\n DATASET_NAME = config['dataset']\n dataset = LoadData(DATASET_NAME)\n if args.out_dir is not None:\n out_dir = args.out_dir\n else:\n out_dir = config['out_dir']\n # parameters\n params = config['params']\n if args.seed is not None:\n params['seed'] = int(args.seed)\n if args.epochs is not None:\n params['epochs'] = int(args.epochs)\n if args.batch_size is not None:\n params['batch_size'] = int(args.batch_size)\n if args.init_lr is not None:\n params['init_lr'] = float(args.init_lr)\n if args.lr_reduce_factor is not None:\n params['lr_reduce_factor'] = float(args.lr_reduce_factor)\n if args.lr_schedule_patience is not None:\n params['lr_schedule_patience'] = int(args.lr_schedule_patience)\n if args.min_lr is not None:\n params['min_lr'] = float(args.min_lr)\n if args.weight_decay is not None:\n params['weight_decay'] = float(args.weight_decay)\n if args.print_epoch_interval is not None:\n params['print_epoch_interval'] = int(args.print_epoch_interval)\n if args.max_time is not None:\n params['max_time'] = float(args.max_time)\n \n \n # model parameters\n net_params = config['net_params']\n net_params['device'] = device\n net_params['gpu_id'] = config['gpu']['id']\n net_params['batch_size'] = params['batch_size']\n \n \n if args.full_graph is not None:\n net_params['full_graph'] = True if args.full_graph=='True' else False\n if args.gamma is not None:\n net_params['gamma'] = float(args.gamma)\n if args.m is not None:\n net_params['m'] = int(args.m)\n \n \n if args.LPE is not None:\n net_params['LPE'] = args.LPE\n \n \n if net_params['LPE'] not in ['node', 'edge', 'none']:\n print('[!] User did not provide a valid input argument for \\'LPE\\'. Valid inputs are \\'node\\', \\'edge\\', and \\'none\\'.')\n exit()\n \n if args.LPE_layers is not None:\n net_params['LPE_layers'] = int(args.LPE_layers)\n if args.LPE_dim is not None:\n net_params['LPE_dim'] = int(args.LPE_dim)\n if args.LPE_n_heads is not None:\n net_params['LPE_n_heads'] = int(args.LPE_n_heads) \n \n if args.GT_layers is not None:\n net_params['GT_layers'] = int(args.GT_layers)\n if args.GT_hidden_dim is not None:\n net_params['GT_hidden_dim'] = int(args.GT_hidden_dim)\n if args.GT_out_dim is not None:\n net_params['GT_out_dim'] = int(args.GT_out_dim) \n if args.GT_n_heads is not None:\n net_params['GT_n_heads'] = int(args.GT_n_heads) \n \n if args.residual is not None:\n net_params['residual'] = True if args.residual=='True' else False\n if args.readout is not None:\n net_params['readout'] = args.readout\n if args.in_feat_dropout is not None:\n net_params['in_feat_dropout'] = float(args.in_feat_dropout)\n if args.dropout is not None:\n net_params['dropout'] = float(args.dropout)\n if args.layer_norm is not None:\n net_params['layer_norm'] = True if args.layer_norm=='True' else False\n if args.batch_norm is not None:\n net_params['batch_norm'] = True if args.batch_norm=='True' else False\n\n # ZINC\n net_params['num_atom_type'] = dataset.num_atom_type\n net_params['num_bond_type'] = dataset.num_bond_type\n \n #If using full graph, need to add a possible edge type (fake edge)\n if net_params['full_graph']:\n net_params['num_bond_type']+=1\n \n\n root_log_dir = out_dir + 'logs/' + MODEL_NAME + \"_\" + DATASET_NAME + \"_GPU\" + str(config['gpu']['id']) + \"_\" + time.strftime('%Hh%Mm%Ss_on_%b_%d_%Y')\n root_ckpt_dir = out_dir + 'checkpoints/' + MODEL_NAME + \"_\" + DATASET_NAME + \"_GPU\" + str(config['gpu']['id']) + \"_\" + time.strftime('%Hh%Mm%Ss_on_%b_%d_%Y')\n write_file_name = out_dir + 'results/result_' + MODEL_NAME + \"_\" + DATASET_NAME + \"_GPU\" + str(config['gpu']['id']) + \"_\" + time.strftime('%Hh%Mm%Ss_on_%b_%d_%Y')\n write_config_file = out_dir + 'configs/config_' + MODEL_NAME + \"_\" + DATASET_NAME + \"_GPU\" + str(config['gpu']['id']) + \"_\" + time.strftime('%Hh%Mm%Ss_on_%b_%d_%Y')\n dirs = root_log_dir, root_ckpt_dir, write_file_name, write_config_file\n\n if not os.path.exists(out_dir + 'results'):\n os.makedirs(out_dir + 'results')\n \n if not os.path.exists(out_dir + 'configs'):\n os.makedirs(out_dir + 'configs')\n\n train_val_pipeline(MODEL_NAME, dataset, params, net_params, dirs)\n\n \n \n \nmain() \n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n" }, { "path": "main_molhiv.py", "content": "\"\"\"\n IMPORTING LIBS\n\"\"\"\nimport dgl\n\nimport numpy as np\nimport os\nimport socket\nimport time\nimport random\nimport glob\nimport argparse, json\nimport pickle\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader\n\nfrom tensorboardX import SummaryWriter\nfrom tqdm import tqdm\n\n\n\"\"\"\n IMPORTING CUSTOM MODULES/METHODS\n\"\"\"\nfrom nets.molhiv_graph_regression.load_net import gnn_model\nfrom data.data import LoadData\n\n\n\n\"\"\"\n GPU Setup\n\"\"\"\ndef gpu_setup(use_gpu, gpu_id):\n os.environ[\"CUDA_DEVICE_ORDER\"] = \"PCI_BUS_ID\"\n os.environ[\"CUDA_VISIBLE_DEVICES\"] = str(gpu_id)\n\n if torch.cuda.is_available() and use_gpu:\n print('cuda available with GPU:',torch.cuda.get_device_name(0))\n device = torch.device(\"cuda\")\n else:\n print('cuda not available')\n device = torch.device(\"cpu\")\n return device\n\n\n\n\"\"\"\n VIEWING ENCODING TYPE AND NUM PARAMS\n\"\"\"\ndef view_model_param(LPE, net_params):\n model = gnn_model(LPE, net_params)\n total_param = 0\n print(\"MODEL DETAILS:\\n\")\n for param in model.parameters():\n total_param += np.prod(list(param.data.size()))\n\n if LPE == 'edge':\n print('Encoding Type/Total parameters:', 'Edge Laplace Encoding/', total_param)\n elif LPE == 'node':\n print('Encoding Type/Total parameters:', 'Node Laplace Encoding', total_param)\n else:\n print('Encoding Type/Total parameters:', 'None', total_param)\n return total_param\n\n\n\"\"\"\n TRAINING CODE\n\"\"\"\n\ndef train_val_pipeline(MODEL_NAME, dataset, params, net_params, dirs):\n t0 = time.time()\n per_epoch_time = []\n\n DATASET_NAME = dataset.name\n\n if net_params['LPE'] in ['edge', 'node']:\n st = time.time()\n print(\"[!] Computing Laplace Decompositions..\")\n dataset._laplace_decomp(net_params['m'])\n print('Time LapPE:',time.time()-st)\n\n if net_params['full_graph']:\n st = time.time()\n print(\"[!] Adding full graph connectivity..\")\n dataset._make_full_graph()\n print('Time taken to convert to full graphs:',time.time()-st)\n\n if net_params['LPE'] == 'edge':\n st = time.time()\n print(\"[!] Computing edge Laplace features..\")\n dataset._add_edge_laplace_feats()\n print('Time taken to compute edge Laplace features: ',time.time()-st)\n\n\n net_params['total_param'] = view_model_param(net_params['LPE'], net_params)\n\n trainset, valset, testset = dataset.train, dataset.val, dataset.test\n\n\n root_log_dir, root_ckpt_dir, write_file_name, write_config_file = dirs\n device = net_params['device']\n\n # Write the network and optimization hyper-parameters in folder config/\n with open(write_config_file + '.txt', 'w') as f:\n f.write(\"\"\"Dataset: {},\\nModel: {}\\n\\nparams={}\\n\\nnet_params={}\\n\\n\\nTotal Parameters: {}\\n\\n\"\"\" .format(DATASET_NAME, MODEL_NAME, params, net_params, net_params['total_param']))\n\n log_dir = os.path.join(root_log_dir, \"RUN_\" + str(0))\n writer = SummaryWriter(log_dir=log_dir)\n\n # setting seeds\n random.seed(params['seed'])\n np.random.seed(params['seed'])\n torch.manual_seed(params['seed'])\n if device.type == 'cuda':\n torch.cuda.manual_seed(params['seed'])\n\n print(\"Training Graphs: \", len(trainset))\n print(\"Validation Graphs: \", len(valset))\n print(\"Test Graphs: \", len(testset))\n\n model = gnn_model(net_params['LPE'], net_params)\n model = model.to(device)\n\n optimizer = optim.Adam(model.parameters(), lr=params['init_lr'], weight_decay=params['weight_decay'])\n scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min',\n factor=params['lr_reduce_factor'],\n patience=params['lr_schedule_patience'],\n verbose=True)\n\n epoch_train_losses, epoch_val_losses = [], []\n epoch_train_AUCs, epoch_val_AUCs, epoch_test_AUCs = [], [], []\n\n\n from train.train_molhiv import train_epoch, evaluate_network\n\n\n train_loader = DataLoader(trainset, batch_size=params['batch_size'], shuffle=True, collate_fn=dataset.collate)\n val_loader = DataLoader(valset, batch_size=params['batch_size'], shuffle=False, collate_fn=dataset.collate)\n test_loader = DataLoader(testset, batch_size=params['batch_size'], shuffle=False, collate_fn=dataset.collate)\n\n prev_lr = params['init_lr']\n\n # At any point you can hit Ctrl + C to break out of training early.\n try:\n with tqdm(range(params['epochs'])) as t:\n for epoch in t:\n\n t.set_description('Epoch %d' % epoch)\n\n start = time.time()\n\n epoch_train_loss, epoch_train_auc, optimizer = train_epoch(model, optimizer, device, train_loader, epoch, net_params['LPE'])\n\n epoch_val_loss, epoch_val_auc = evaluate_network(model, device, val_loader, epoch, net_params['LPE'])\n _, epoch_test_auc = evaluate_network(model, device, test_loader, epoch, net_params['LPE'])\n\n epoch_train_losses.append(epoch_train_loss)\n epoch_val_losses.append(epoch_val_loss)\n epoch_train_AUCs.append(epoch_train_auc)\n epoch_val_AUCs.append(epoch_val_auc)\n\n epoch_test_AUCs.append(epoch_test_auc)\n\n writer.add_scalar('train/_loss', epoch_train_loss, epoch)\n writer.add_scalar('val/_loss', epoch_val_loss, epoch)\n writer.add_scalar('train/_auc', epoch_train_auc, epoch)\n writer.add_scalar('val/_auc', epoch_val_auc, epoch)\n writer.add_scalar('test/_auc', epoch_test_auc, epoch)\n writer.add_scalar('learning_rate', optimizer.param_groups[0]['lr'], epoch)\n\n\n t.set_postfix(time=time.time()-start, lr=optimizer.param_groups[0]['lr'],\n train_loss=epoch_train_loss, val_loss=epoch_val_loss,\n train_AUC=epoch_train_auc, val_AUC=epoch_val_auc,\n test_AUC=epoch_test_auc)\n\n\n per_epoch_time.append(time.time()-start)\n\n # Saving checkpoint\n ckpt_dir = os.path.join(root_ckpt_dir, \"RUN_\")\n if not os.path.exists(ckpt_dir):\n os.makedirs(ckpt_dir)\n torch.save(model.state_dict(), '{}.pkl'.format(ckpt_dir + \"/epoch_\" + str(epoch)))\n\n files = glob.glob(ckpt_dir + '/*.pkl')\n for file in files:\n epoch_nb = file.split('_')[-1]\n epoch_nb = int(epoch_nb.split('.')[0])\n if epoch_nb < epoch-1:\n os.remove(file)\n\n scheduler.step(epoch_val_loss)\n\n current_lr = optimizer.param_groups[0]['lr']\n if current_lr < prev_lr:\n print(f\"Learning rate dropped to {current_lr}\")\n prev_lr = current_lr\n\n if current_lr < params['min_lr']:\n print(\"\\n!! LR EQUAL TO MIN LR SET.\")\n break\n\n\n # Stop training after params['max_time'] hours\n if time.time()-t0 > params['max_time']*3600:\n print('-' * 89)\n print(\"Max_time for training elapsed {:.2f} hours, so stopping\".format(params['max_time']))\n break\n\n except KeyboardInterrupt:\n print('-' * 89)\n print('Exiting from training early because of KeyboardInterrupt')\n\n #Return test and train metrics at best val metric\n index = epoch_val_AUCs.index(max(epoch_val_AUCs))\n\n test_auc = epoch_test_AUCs[index]\n train_auc = epoch_train_AUCs[index]\n\n print(\"Test AUC: {:.4f}\".format(test_auc))\n print(\"Train AUC: {:.4f}\".format(train_auc))\n print(\"Convergence Time (Epochs): {:.4f}\".format(epoch))\n print(\"TOTAL TIME TAKEN: {:.4f}s\".format(time.time()-t0))\n print(\"AVG TIME PER EPOCH: {:.4f}s\".format(np.mean(per_epoch_time)))\n\n writer.close()\n\n \"\"\"\n Write the results in out_dir/results folder\n \"\"\"\n with open(write_file_name + '.txt', 'w') as f:\n f.write(\"\"\"Dataset: {},\\nModel: {}\\n\\nparams={}\\n\\nnet_params={}\\n\\n{}\\n\\nTotal Parameters: {}\\n\\n\n FINAL RESULTS\\nTEST AUC: {:.4f}\\nTRAIN AUC: {:.4f}\\n\\n\n Convergence Time (Epochs): {:.4f}\\nTotal Time Taken: {:.4f} hrs\\nAverage Time Per Epoch: {:.4f} s\\n\\n\\n\"\"\"\\\n .format(DATASET_NAME, MODEL_NAME, params, net_params, model, net_params['total_param'],\n test_auc, train_auc, epoch, (time.time()-t0)/3600, np.mean(per_epoch_time)))\n\n\n\n\n\ndef main():\n \"\"\"\n USER CONTROLS\n \"\"\"\n\n\n parser = argparse.ArgumentParser()\n parser.add_argument('--config', help=\"Please give a config.json file with training/model/data/param details\")\n parser.add_argument('--gpu_id', help=\"Please give a value for gpu id\")\n parser.add_argument('--model', help=\"Please give a value for model name\")\n parser.add_argument('--dataset', help=\"Please give a value for dataset name\")\n parser.add_argument('--out_dir', help=\"Please give a value for out_dir\")\n parser.add_argument('--seed', help=\"Please give a value for seed\")\n parser.add_argument('--epochs', help=\"Please give a value for epochs\")\n parser.add_argument('--batch_size', help=\"Please give a value for batch_size\")\n parser.add_argument('--init_lr', help=\"Please give a value for init_lr\")\n parser.add_argument('--lr_reduce_factor', help=\"Please give a value for lr_reduce_factor\")\n parser.add_argument('--lr_schedule_patience', help=\"Please give a value for lr_schedule_patience\")\n parser.add_argument('--min_lr', help=\"Please give a value for min_lr\")\n parser.add_argument('--weight_decay', help=\"Please give a value for weight_decay\")\n parser.add_argument('--print_epoch_interval', help=\"Please give a value for print_epoch_interval\")\n parser.add_argument('--max_time', help=\"Please give a value for max_time\")\n\n #Model details\n parser.add_argument('--full_graph', help=\"Please give a value for full_graph\")\n parser.add_argument('--gamma', help=\"Please give a value for gamma\")\n parser.add_argument('--m', help=\"Please give a value for m\")\n\n parser.add_argument('--LPE', help=\"Please give a value for LPE\")\n parser.add_argument('--LPE_layers', help=\"Please give a value for LPE_layers\")\n parser.add_argument('--LPE_dim', help=\"Please give a value for LPE_dim\")\n parser.add_argument('--LPE_n_heads', help=\"Please give a value for LPE_n_heads\")\n\n parser.add_argument('--GT_layers', help=\"Please give a value for GT_layers\")\n parser.add_argument('--GT_hidden_dim', help=\"Please give a value for GT_hidden_dim\")\n parser.add_argument('--GT_out_dim', help=\"Please give a value for GT_out_dim\")\n parser.add_argument('--GT_n_heads', help=\"Please give a value for GT_n_heads\")\n\n parser.add_argument('--residual', help=\"Please give a value for readout\")\n parser.add_argument('--readout', help=\"Please give a value for readout\")\n parser.add_argument('--in_feat_dropout', help=\"Please give a value for in_feat_dropout\")\n parser.add_argument('--dropout', help=\"Please give a value for dropout\")\n parser.add_argument('--layer_norm', help=\"Please give a value for layer_norm\")\n parser.add_argument('--batch_norm', help=\"Please give a value for batch_norm\")\n\n args = parser.parse_args()\n\n with open(args.config) as f:\n config = json.load(f)\n\n # device\n if args.gpu_id is not None:\n config['gpu']['id'] = int(args.gpu_id)\n config['gpu']['use'] = True\n device = gpu_setup(config['gpu']['use'], config['gpu']['id'])\n # model, dataset, out_dir\n if args.model is not None:\n MODEL_NAME = args.model\n else:\n MODEL_NAME = config['model']\n if args.dataset is not None:\n DATASET_NAME = args.dataset\n else:\n DATASET_NAME = config['dataset']\n dataset = LoadData(DATASET_NAME)\n\n\n if args.out_dir is not None:\n out_dir = args.out_dir\n else:\n out_dir = config['out_dir']\n # parameters\n params = config['params']\n if args.seed is not None:\n params['seed'] = int(args.seed)\n if args.epochs is not None:\n params['epochs'] = int(args.epochs)\n if args.batch_size is not None:\n params['batch_size'] = int(args.batch_size)\n if args.init_lr is not None:\n params['init_lr'] = float(args.init_lr)\n if args.lr_reduce_factor is not None:\n params['lr_reduce_factor'] = float(args.lr_reduce_factor)\n if args.lr_schedule_patience is not None:\n params['lr_schedule_patience'] = int(args.lr_schedule_patience)\n if args.min_lr is not None:\n params['min_lr'] = float(args.min_lr)\n if args.weight_decay is not None:\n params['weight_decay'] = float(args.weight_decay)\n if args.print_epoch_interval is not None:\n params['print_epoch_interval'] = int(args.print_epoch_interval)\n if args.max_time is not None:\n params['max_time'] = float(args.max_time)\n\n\n # model parameters\n net_params = config['net_params']\n net_params['device'] = device\n net_params['gpu_id'] = config['gpu']['id']\n net_params['batch_size'] = params['batch_size']\n\n\n if args.full_graph is not None:\n net_params['full_graph'] = True if args.full_graph=='True' else False\n if args.gamma is not None:\n net_params['gamma'] = float(args.gamma)\n if args.m is not None:\n net_params['m'] = int(args.m)\n\n\n if args.LPE is not None:\n net_params['LPE'] = args.LPE\n\n\n if net_params['LPE'] not in ['node', 'edge', 'none']:\n print('[!] User did not provide a valid input argument for \\'LPE\\'. Valid inputs are \\'node\\', \\'edge\\', and \\'none\\'.')\n exit()\n\n if args.LPE_layers is not None:\n net_params['LPE_layers'] = int(args.LPE_layers)\n if args.LPE_dim is not None:\n net_params['LPE_dim'] = int(args.LPE_dim)\n if args.LPE_n_heads is not None:\n net_params['LPE_n_heads'] = int(args.LPE_n_heads)\n\n if args.GT_layers is not None:\n net_params['GT_layers'] = int(args.GT_layers)\n if args.GT_hidden_dim is not None:\n net_params['GT_hidden_dim'] = int(args.GT_hidden_dim)\n if args.GT_out_dim is not None:\n net_params['GT_out_dim'] = int(args.GT_out_dim)\n if args.GT_n_heads is not None:\n net_params['GT_n_heads'] = int(args.GT_n_heads)\n\n\n if args.residual is not None:\n net_params['residual'] = True if args.residual=='True' else False\n if args.readout is not None:\n net_params['readout'] = args.readout\n if args.in_feat_dropout is not None:\n net_params['in_feat_dropout'] = float(args.in_feat_dropout)\n if args.dropout is not None:\n net_params['dropout'] = float(args.dropout)\n if args.layer_norm is not None:\n net_params['layer_norm'] = True if args.layer_norm=='True' else False\n if args.batch_norm is not None:\n net_params['batch_norm'] = True if args.batch_norm=='True' else False\n\n\n root_log_dir = out_dir + 'logs/' + MODEL_NAME + \"_\" + DATASET_NAME + \"_GPU\" + str(config['gpu']['id']) + \"_\" + time.strftime('%Hh%Mm%Ss_on_%b_%d_%Y')\n root_ckpt_dir = out_dir + 'checkpoints/' + MODEL_NAME + \"_\" + DATASET_NAME + \"_GPU\" + str(config['gpu']['id']) + \"_\" + time.strftime('%Hh%Mm%Ss_on_%b_%d_%Y')\n write_file_name = out_dir + 'results/result_' + MODEL_NAME + \"_\" + DATASET_NAME + \"_GPU\" + str(config['gpu']['id']) + \"_\" + time.strftime('%Hh%Mm%Ss_on_%b_%d_%Y')\n write_config_file = out_dir + 'configs/config_' + MODEL_NAME + \"_\" + DATASET_NAME + \"_GPU\" + str(config['gpu']['id']) + \"_\" + time.strftime('%Hh%Mm%Ss_on_%b_%d_%Y')\n dirs = root_log_dir, root_ckpt_dir, write_file_name, write_config_file\n\n if not os.path.exists(out_dir + 'results'):\n os.makedirs(out_dir + 'results')\n\n if not os.path.exists(out_dir + 'configs'):\n os.makedirs(out_dir + 'configs')\n\n train_val_pipeline(MODEL_NAME, dataset, params, net_params, dirs)\n\n\n\n\n\n\n\n\nmain()\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n" }, { "path": "main_molpcba.py", "content": "\"\"\"\n IMPORTING LIBS\n\"\"\"\nimport dgl\n\nimport numpy as np\nimport os\nimport socket\nimport time\nimport random\nimport glob\nimport argparse, json\nimport pickle\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader\n\nfrom tensorboardX import SummaryWriter\nfrom tqdm import tqdm\n\n\n\"\"\"\n IMPORTING CUSTOM MODULES/METHODS\n\"\"\"\nfrom nets.molpcba.load_net import gnn_model\nfrom data.data import LoadData\n\ntorch.set_default_dtype(torch.float32)\n\n\"\"\"\n GPU Setup\n\"\"\"\ndef gpu_setup(use_gpu, gpu_id):\n os.environ[\"CUDA_DEVICE_ORDER\"] = \"PCI_BUS_ID\"\n os.environ[\"CUDA_VISIBLE_DEVICES\"] = str(gpu_id)\n\n if torch.cuda.is_available() and use_gpu:\n print('cuda available with GPU:',torch.cuda.get_device_name(0))\n device = torch.device(\"cuda\")\n else:\n print('cuda not available')\n device = torch.device(\"cpu\")\n return device\n\n\n\n\"\"\"\n VIEWING ENCODING TYPE AND NUM PARAMS\n\"\"\"\ndef view_model_param(LPE, net_params):\n model = gnn_model(LPE, net_params)\n total_param = 0\n print(\"MODEL DETAILS:\\n\")\n for param in model.parameters():\n total_param += np.prod(list(param.data.size()))\n\n if LPE == 'edge':\n print('Encoding Type/Total parameters:', 'Edge Laplace Encoding/', total_param)\n elif LPE == 'node':\n print('Encoding Type/Total parameters:', 'Node Laplace Encoding', total_param)\n else:\n print('Encoding Type/Total parameters:', 'None', total_param)\n return total_param\n\n\n\"\"\"\n TRAINING CODE\n\"\"\"\n\ndef train_val_pipeline(MODEL_NAME, dataset, params, net_params, dirs):\n t0 = time.time()\n per_epoch_time = []\n\n DATASET_NAME = dataset.name\n\n if net_params['LPE'] in ['edge', 'node']:\n st = time.time()\n print(\"[!] Computing Laplace Decompositions..\")\n dataset._laplace_decomp(net_params['m'])\n print('Time LapPE:',time.time()-st)\n\n if net_params['full_graph']:\n st = time.time()\n print(\"[!] Adding full graph connectivity..\")\n dataset._make_full_graph()\n print('Time taken to convert to full graphs:',time.time()-st)\n\n if net_params['LPE'] == 'edge':\n st = time.time()\n print(\"[!] Computing edge Laplace features..\")\n dataset._add_edge_laplace_feats()\n print('Time taken to compute edge Laplace features: ',time.time()-st)\n\n\n net_params['total_param'] = view_model_param(net_params['LPE'], net_params)\n\n trainset, valset, testset = dataset.train, dataset.val, dataset.test\n\n\n root_log_dir, root_ckpt_dir, write_file_name, write_config_file = dirs\n device = net_params['device']\n\n # Write the network and optimization hyper-parameters in folder config/\n with open(write_config_file + '.txt', 'w') as f:\n f.write(\"\"\"Dataset: {},\\nModel: {}\\n\\nparams={}\\n\\nnet_params={}\\n\\n\\nTotal Parameters: {}\\n\\n\"\"\" .format(DATASET_NAME, MODEL_NAME, params, net_params, net_params['total_param']))\n\n log_dir = os.path.join(root_log_dir, \"RUN_\" + str(0))\n writer = SummaryWriter(log_dir=log_dir)\n\n # setting seeds\n random.seed(params['seed'])\n np.random.seed(params['seed'])\n torch.manual_seed(params['seed'])\n if device.type == 'cuda':\n torch.cuda.manual_seed(params['seed'])\n\n print(\"Training Graphs: \", len(trainset))\n print(\"Validation Graphs: \", len(valset))\n print(\"Test Graphs: \", len(testset))\n\n model = gnn_model(net_params['LPE'], net_params)\n model = model.to(device=device)\n\n optimizer = optim.Adam(model.parameters(), lr=params['init_lr'], weight_decay=params['weight_decay'])\n scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min',\n factor=params['lr_reduce_factor'],\n patience=params['lr_schedule_patience'],\n verbose=True)\n\n epoch_train_losses, epoch_val_losses = [], []\n epoch_train_APs, epoch_val_APs, epoch_test_APs = [], [], []\n\n\n from train.train_molpcba import train_epoch, evaluate_network\n\n\n train_loader = DataLoader(trainset, batch_size=params['batch_size'], shuffle=True, collate_fn=dataset.collate)\n val_loader = DataLoader(valset, batch_size=params['batch_size'], shuffle=False, collate_fn=dataset.collate)\n test_loader = DataLoader(testset, batch_size=params['batch_size'], shuffle=False, collate_fn=dataset.collate)\n\n # At any point you can hit Ctrl + C to break out of training early.\n try:\n with tqdm(range(params['epochs'])) as t:\n for epoch in t:\n\n t.set_description('Epoch %d' % epoch)\n\n start = time.time()\n\n epoch_train_loss, epoch_train_ap, optimizer = train_epoch(model, optimizer, device, train_loader, epoch, net_params['LPE'], params[\"batch_accumulation\"])\n\n epoch_val_loss, epoch_val_ap = evaluate_network(model, device, val_loader, epoch, net_params['LPE'])\n _, epoch_test_ap = evaluate_network(model, device, test_loader, epoch, net_params['LPE'])\n\n epoch_train_losses.append(epoch_train_loss)\n epoch_val_losses.append(epoch_val_loss)\n epoch_train_APs.append(epoch_train_ap)\n epoch_val_APs.append(epoch_val_ap)\n\n epoch_test_APs.append(epoch_test_ap)\n\n writer.add_scalar('train/_loss', epoch_train_loss, epoch)\n writer.add_scalar('val/_loss', epoch_val_loss, epoch)\n writer.add_scalar('train/_AP', epoch_train_ap, epoch)\n writer.add_scalar('val/_AP', epoch_val_ap, epoch)\n writer.add_scalar('test/_AP', epoch_test_ap, epoch)\n writer.add_scalar('learning_rate', optimizer.param_groups[0]['lr'], epoch)\n\n\n t.set_postfix(time=time.time()-start, lr=optimizer.param_groups[0]['lr'],\n train_loss=epoch_train_loss, val_loss=epoch_val_loss,\n train_AP=epoch_train_ap, val_AP=epoch_val_ap,\n test_AP=epoch_test_ap)\n\n\n per_epoch_time.append(time.time()-start)\n\n # Saving checkpoint\n ckpt_dir = os.path.join(root_ckpt_dir, \"RUN_\")\n if not os.path.exists(ckpt_dir):\n os.makedirs(ckpt_dir)\n torch.save(model.state_dict(), '{}.pkl'.format(ckpt_dir + \"/epoch_\" + str(epoch)))\n\n files = glob.glob(ckpt_dir + '/*.pkl')\n for file in files:\n epoch_nb = file.split('_')[-1]\n epoch_nb = int(epoch_nb.split('.')[0])\n if epoch_nb < epoch-1:\n os.remove(file)\n\n scheduler.step(-epoch_val_ap)\n\n if optimizer.param_groups[0]['lr'] < params['min_lr']:\n print(\"\\n!! LR EQUAL TO MIN LR SET.\")\n break\n\n # Stop training after params['max_time'] hours\n if time.time()-t0 > params['max_time']*3600:\n print('-' * 89)\n print(\"Max_time for training elapsed {:.2f} hours, so stopping\".format(params['max_time']))\n break\n\n except Exception as e: # Sometimes there's out of memory error after many epochs\n print('-' * 89)\n print(f'Exiting from training early Exception: {e}')\n\n except KeyboardInterrupt: # Sometimes there's out of memory error after many epochs\n print('-' * 89)\n print(f'Exiting from training keyboard interrupt')\n\n\n #Return test and train metrics at best val metric\n index = epoch_val_APs.index(max(epoch_val_APs))\n\n test_ap = epoch_test_APs[index]\n val_ap = epoch_val_APs[index]\n train_ap = epoch_train_APs[index]\n\n print(\"Test AP: {:.4f}\".format(test_ap))\n print(\"Val AP: {:.4f}\".format(val_ap))\n print(\"Train AP: {:.4f}\".format(train_ap))\n print(\"Best epoch index: {:.4f}\".format(index))\n print(\"Convergence Time (Epochs): {:.4f}\".format(epoch))\n print(\"TOTAL TIME TAKEN: {:.4f}s\".format(time.time()-t0))\n print(\"AVG TIME PER EPOCH: {:.4f}s\".format(np.mean(per_epoch_time)))\n\n writer.close()\n\n \"\"\"\n Write the results in out_dir/results folder\n \"\"\"\n with open(write_file_name + '.txt', 'w') as f:\n f.write(\"\"\"Dataset: {},\\nModel: {}\\n\\nparams={}\\n\\nnet_params={}\\n\\n{}\\n\\nTotal Parameters: {}\\n\\n\n FINAL RESULTS\\nTEST AP: {:.4f}\\nTRAIN AP: {:.4f}\\n\\n\n Convergence Time (Epochs): {:.4f}\\nTotal Time Taken: {:.4f} hrs\\nAverage Time Per Epoch: {:.4f} s\\n\\n\\n\"\"\"\\\n .format(DATASET_NAME, MODEL_NAME, params, net_params, model, net_params['total_param'],\n test_ap, train_ap, epoch, (time.time()-t0)/3600, np.mean(per_epoch_time)))\n\n\n\n\n\ndef main():\n \"\"\"\n USER CONTROLS\n \"\"\"\n\n\n parser = argparse.ArgumentParser()\n parser.add_argument('--config', default=\"configs/MOLPCBA/optimized\", help=\"Please give a config.json file with training/model/data/param details\")\n parser.add_argument('--gpu_id', help=\"Please give a value for gpu id\")\n parser.add_argument('--model', help=\"Please give a value for model name\")\n parser.add_argument('--dataset', help=\"Please give a value for dataset name\")\n parser.add_argument('--out_dir', help=\"Please give a value for out_dir\")\n parser.add_argument('--seed', help=\"Please give a value for seed\")\n parser.add_argument('--epochs', help=\"Please give a value for epochs\")\n parser.add_argument('--batch_size', help=\"Please give a value for batch_size\")\n parser.add_argument('--init_lr', help=\"Please give a value for init_lr\")\n parser.add_argument('--lr_reduce_factor', help=\"Please give a value for lr_reduce_factor\")\n parser.add_argument('--lr_schedule_patience', help=\"Please give a value for lr_schedule_patience\")\n parser.add_argument('--min_lr', help=\"Please give a value for min_lr\")\n parser.add_argument('--weight_decay', help=\"Please give a value for weight_decay\")\n parser.add_argument('--print_epoch_interval', help=\"Please give a value for print_epoch_interval\")\n parser.add_argument('--max_time', help=\"Please give a value for max_time\")\n\n #Model details\n parser.add_argument('--full_graph', help=\"Please give a value for full_graph\")\n parser.add_argument('--gamma', help=\"Please give a value for gamma\")\n parser.add_argument('--m', help=\"Please give a value for m\")\n\n parser.add_argument('--LPE', help=\"Please give a value for LPE\")\n parser.add_argument('--LPE_layers', help=\"Please give a value for LPE_layers\")\n parser.add_argument('--LPE_dim', help=\"Please give a value for LPE_dim\")\n parser.add_argument('--LPE_n_heads', help=\"Please give a value for LPE_n_heads\")\n \n parser.add_argument('--extra_mlp', help=\"Please give a value for extra_mlp\")\n\n parser.add_argument('--GT_layers', help=\"Please give a value for GT_layers\")\n parser.add_argument('--GT_hidden_dim', help=\"Please give a value for GT_hidden_dim\")\n parser.add_argument('--GT_out_dim', help=\"Please give a value for GT_out_dim\")\n parser.add_argument('--GT_n_heads', help=\"Please give a value for GT_n_heads\")\n\n parser.add_argument('--residual', help=\"Please give a value for readout\")\n parser.add_argument('--readout', help=\"Please give a value for readout\")\n parser.add_argument('--in_feat_dropout', help=\"Please give a value for in_feat_dropout\")\n parser.add_argument('--dropout', help=\"Please give a value for dropout\")\n parser.add_argument('--layer_norm', help=\"Please give a value for layer_norm\")\n parser.add_argument('--batch_norm', help=\"Please give a value for batch_norm\")\n\n args = parser.parse_args()\n\n with open(args.config) as f:\n config = json.load(f)\n\n # device\n if args.gpu_id is not None:\n config['gpu']['id'] = int(args.gpu_id)\n config['gpu']['use'] = True\n device = gpu_setup(config['gpu']['use'], config['gpu']['id'])\n # model, dataset, out_dir\n if args.model is not None:\n MODEL_NAME = args.model\n else:\n MODEL_NAME = config['model']\n if args.dataset is not None:\n DATASET_NAME = args.dataset\n else:\n DATASET_NAME = config['dataset']\n dataset = LoadData(DATASET_NAME)\n\n\n if args.out_dir is not None:\n out_dir = args.out_dir\n else:\n out_dir = config['out_dir']\n # parameters\n params = config['params']\n if args.seed is not None:\n params['seed'] = int(args.seed)\n if args.epochs is not None:\n params['epochs'] = int(args.epochs)\n if args.batch_size is not None:\n params['batch_size'] = int(args.batch_size)\n if args.init_lr is not None:\n params['init_lr'] = float(args.init_lr)\n if args.lr_reduce_factor is not None:\n params['lr_reduce_factor'] = float(args.lr_reduce_factor)\n if args.lr_schedule_patience is not None:\n params['lr_schedule_patience'] = int(args.lr_schedule_patience)\n if args.min_lr is not None:\n params['min_lr'] = float(args.min_lr)\n if args.weight_decay is not None:\n params['weight_decay'] = float(args.weight_decay)\n if args.print_epoch_interval is not None:\n params['print_epoch_interval'] = int(args.print_epoch_interval)\n if args.max_time is not None:\n params['max_time'] = float(args.max_time)\n\n\n # model parameters\n net_params = config['net_params']\n net_params['device'] = device\n net_params['gpu_id'] = config['gpu']['id']\n net_params['batch_size'] = params['batch_size']\n\n\n if args.full_graph is not None:\n net_params['full_graph'] = True if args.full_graph=='True' else False\n if args.gamma is not None:\n net_params['gamma'] = float(args.gamma)\n if args.m is not None:\n net_params['m'] = int(args.m)\n\n\n if args.LPE is not None:\n net_params['LPE'] = args.LPE\n \n if args.extra_mlp is not None:\n net_params['extra_mlp'] = args.extra_mlp\n\n\n if net_params['LPE'] not in ['node', 'edge', 'none']:\n print('[!] User did not provide a valid input argument for \\'LPE\\'. Valid inputs are \\'node\\', \\'edge\\', and \\'none\\'.')\n exit()\n\n if args.LPE_layers is not None:\n net_params['LPE_layers'] = int(args.LPE_layers)\n if args.LPE_dim is not None:\n net_params['LPE_dim'] = int(args.LPE_dim)\n if args.LPE_n_heads is not None:\n net_params['LPE_n_heads'] = int(args.LPE_n_heads)\n\n if args.GT_layers is not None:\n net_params['GT_layers'] = int(args.GT_layers)\n if args.GT_hidden_dim is not None:\n net_params['GT_hidden_dim'] = int(args.GT_hidden_dim)\n if args.GT_out_dim is not None:\n net_params['GT_out_dim'] = int(args.GT_out_dim)\n if args.GT_n_heads is not None:\n net_params['GT_n_heads'] = int(args.GT_n_heads)\n\n\n if args.residual is not None:\n net_params['residual'] = True if args.residual=='True' else False\n if args.readout is not None:\n net_params['readout'] = args.readout\n if args.in_feat_dropout is not None:\n net_params['in_feat_dropout'] = float(args.in_feat_dropout)\n if args.dropout is not None:\n net_params['dropout'] = float(args.dropout)\n if args.layer_norm is not None:\n net_params['layer_norm'] = True if args.layer_norm=='True' else False\n if args.batch_norm is not None:\n net_params['batch_norm'] = True if args.batch_norm=='True' else False\n\n\n root_log_dir = out_dir + 'logs/' + MODEL_NAME + \"_\" + DATASET_NAME + \"_GPU\" + str(config['gpu']['id']) + \"_\" + time.strftime('%Hh%Mm%Ss_on_%b_%d_%Y')\n root_ckpt_dir = out_dir + 'checkpoints/' + MODEL_NAME + \"_\" + DATASET_NAME + \"_GPU\" + str(config['gpu']['id']) + \"_\" + time.strftime('%Hh%Mm%Ss_on_%b_%d_%Y')\n write_file_name = out_dir + 'results/result_' + MODEL_NAME + \"_\" + DATASET_NAME + \"_GPU\" + str(config['gpu']['id']) + \"_\" + time.strftime('%Hh%Mm%Ss_on_%b_%d_%Y')\n write_config_file = out_dir + 'configs/config_' + MODEL_NAME + \"_\" + DATASET_NAME + \"_GPU\" + str(config['gpu']['id']) + \"_\" + time.strftime('%Hh%Mm%Ss_on_%b_%d_%Y')\n dirs = root_log_dir, root_ckpt_dir, write_file_name, write_config_file\n\n if not os.path.exists(out_dir + 'results'):\n os.makedirs(out_dir + 'results')\n\n if not os.path.exists(out_dir + 'configs'):\n os.makedirs(out_dir + 'configs')\n\n train_val_pipeline(MODEL_NAME, dataset, params, net_params, dirs)\n\nmain()\n" }, { "path": "misc/download_datasets.md", "content": "# Download datasets\n\nAll the datasets work with DGL 0.5.x or later. Please update the environment using the yml files in the root directory if the use of these datasets throw error(s).\n\n\n\n
\n\n## 1. ZINC molecular dataset\nZINC size is 58.9MB. \n\n```\n# At the root of the project\ncd data/ \nbash script_download_molecules.sh\n```\nScript [script_download_molecules.sh](../data/script_download_molecules.sh) is located here. Refer to [benchmarking-gnns repo](https://github.com/graphdeeplearning/benchmarking-gnns) for details on preparation.\n\n\n
\n\n## 2. PATTERN/CLUSTER SBM datasets\nPATTERN size is 1.98GB and CLUSTER size is 1.26GB.\n\n```\n# At the root of the project\ncd data/ \nbash script_download_SBMs.sh\n```\nScript [script_download_SBMs.sh](../data/script_download_SBMs.sh) is located here. Refer to [benchmarking-gnns repo](https://github.com/graphdeeplearning/benchmarking-gnns) for details on preparation.\n\n
\n\n## 3. All BGNN datasets\n\n```\n# At the root of the project\ncd data/ \nbash script_download_all_datasets.sh\n```\n\nScript [script_download_all_datasets.sh](../data/script_download_all_datasets.sh) is located here. \n\n
\n\n## 4. MolHIV OGB dataset\n\n```\n# Ensure OGB is installed: \npip install ogb\n```\nIf properly installed, the dataset will automatically be downloaded and saved to the ```dataset/``` folder after running a MolHIV experiment.\n\n


\n" }, { "path": "misc/env_installation.md", "content": "# Benchmark installation\n\n\n\n
\n\n## 1. Setup Conda\n\n```\n# Conda installation\n\n# For Linux\ncurl -o ~/miniconda.sh -O https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh\n\n# For OSX\ncurl -o ~/miniconda.sh -O https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh\n\nchmod +x ~/miniconda.sh \n./miniconda.sh \n\nsource ~/.bashrc # For Linux\nsource ~/.bash_profile # For OSX\n```\n\n\n
\n\n## 2. Setup Python environment for CPU\n\n```\n# Clone GitHub repo\nconda install git\ngit clone https://github.com/DevinKreuzer/SAN.git\ncd SAN\n\n# Install python environment\n# using pip\npip install -r requirements.txt\n\n# using Conda\nconda create --name --file requirements.txt\n```\n\n\n\n
\n\n## 3. Setup Python environment for GPU\n\nDGL 0.5.x requires CUDA **10.2**.\n\nFor Ubuntu **18.04**\n\n```\n# Setup CUDA 10.2 on Ubuntu 18.04\nsudo apt-get --purge remove \"*cublas*\" \"cuda*\"\nsudo apt --purge remove \"nvidia*\"\nsudo apt autoremove\nwget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/cuda-repo-ubuntu1804_10.2.89-1_amd64.deb\nsudo dpkg -i cuda-repo-ubuntu1804_10.2.89-1_amd64.deb\nsudo apt-key adv --fetch-keys http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/7fa2af80.pub\nsudo apt update\nsudo apt install -y cuda-10-2\nsudo reboot\ncat /usr/local/cuda/version.txt # Check CUDA version is 10.2\n\n# Clone GitHub repo\nconda install git\ngit clone https://github.com/DevinKreuzer/SAN.git\ncd SAN\n\n# Install python environment\n# using pip\npip install -r requirements.txt\n\n# using Conda\nconda create --name --file requirements.txt\n```\n\n\n\n\n\n\n


\n" }, { "path": "nets/SBMs_node_classification/SAN.py", "content": "import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nimport dgl\nimport numpy as np\n\n\"\"\"\n Graph Transformer\n \n\"\"\"\nfrom layers.graph_transformer_layer import GraphTransformerLayer\nfrom layers.mlp_readout_layer import MLPReadout\n\nclass SAN(nn.Module):\n\n def __init__(self, net_params):\n super().__init__()\n\n\n in_dim_node = net_params['in_dim'] # node_dim (feat is an integer)\n self.n_classes = net_params['n_classes']\n \n full_graph = net_params['full_graph']\n gamma = net_params['gamma']\n \n GT_layers = net_params['GT_layers']\n GT_hidden_dim = net_params['GT_hidden_dim']\n GT_out_dim = net_params['GT_out_dim']\n GT_n_heads = net_params['GT_n_heads']\n \n self.residual = net_params['residual']\n self.readout = net_params['readout']\n in_feat_dropout = net_params['in_feat_dropout']\n dropout = net_params['dropout']\n\n self.readout = net_params['readout']\n self.layer_norm = net_params['layer_norm']\n self.batch_norm = net_params['batch_norm']\n\n self.device = net_params['device']\n self.in_feat_dropout = nn.Dropout(in_feat_dropout)\n \n self.embedding_h = nn.Embedding(in_dim_node, GT_hidden_dim)\n self.embedding_e = nn.Embedding(2, GT_hidden_dim)\n\n \n self.layers = nn.ModuleList([ GraphTransformerLayer(gamma, GT_hidden_dim, GT_hidden_dim, GT_n_heads, full_graph, dropout, self.layer_norm, self.batch_norm, self.residual) for _ in range(GT_layers-1) ])\n \n self.layers.append(GraphTransformerLayer(gamma, GT_hidden_dim, GT_out_dim, GT_n_heads, full_graph, dropout, self.layer_norm, self.batch_norm, self.residual))\n\n self.MLP_layer = MLPReadout(GT_out_dim, self.n_classes)\n\n\n def forward(self, g, h, e):\n \n # input embedding\n h=self.embedding_h(h)\n h = self.in_feat_dropout(h)\n e = self.embedding_e(e) \n \n \n # GraphTransformer Layers\n for conv in self.layers:\n h, e = conv(g, h, e)\n \n # output\n h_out = self.MLP_layer(h)\n\n return h_out\n \n \n def loss(self, pred, label):\n\n # calculating label weights for weighted loss computation\n V = label.size(0)\n label_count = torch.bincount(label)\n label_count = label_count[label_count.nonzero()].squeeze()\n cluster_sizes = torch.zeros(self.n_classes).long().to(self.device)\n cluster_sizes[torch.unique(label)] = label_count\n weight = (V - cluster_sizes).float() / V\n weight *= (cluster_sizes>0).float()\n \n # weighted cross-entropy for unbalanced classes\n criterion = nn.CrossEntropyLoss(weight=weight)\n loss = criterion(pred, label)\n\n return loss\n\n\n\n \n" }, { "path": "nets/SBMs_node_classification/SAN_EdgeLPE.py", "content": "import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nimport dgl\nimport numpy as np\n\n\"\"\"\n Graph Transformer\n \n\"\"\"\nfrom layers.graph_transformer_layer import GraphTransformerLayer\nfrom layers.mlp_readout_layer import MLPReadout\n\nclass SAN_EdgeLPE(nn.Module):\n\n def __init__(self, net_params):\n super().__init__()\n\n in_dim_node = net_params['in_dim'] # node_dim (feat is an integer)\n self.n_classes = net_params['n_classes']\n \n full_graph = net_params['full_graph']\n gamma = net_params['gamma']\n \n LPE_layers = net_params['LPE_layers']\n LPE_dim = net_params['LPE_dim']\n LPE_n_heads = net_params['LPE_n_heads']\n \n GT_layers = net_params['GT_layers']\n GT_hidden_dim = net_params['GT_hidden_dim']\n GT_out_dim = net_params['GT_out_dim']\n GT_n_heads = net_params['GT_n_heads']\n \n self.residual = net_params['residual']\n self.readout = net_params['readout']\n in_feat_dropout = net_params['in_feat_dropout']\n dropout = net_params['dropout']\n\n self.readout = net_params['readout']\n self.layer_norm = net_params['layer_norm']\n self.batch_norm = net_params['batch_norm']\n\n self.device = net_params['device']\n self.in_feat_dropout = nn.Dropout(in_feat_dropout)\n \n self.embedding_h = nn.Embedding(in_dim_node, GT_hidden_dim)\n self.embedding_e = nn.Embedding(2, GT_hidden_dim-LPE_dim)#Remove some embedding dimensions to make room for concatenating laplace encoding\n self.linear_A = nn.Linear(3, LPE_dim)\n \n encoder_layer = nn.TransformerEncoderLayer(d_model=LPE_dim, nhead=LPE_n_heads)\n self.PE_Transformer = nn.TransformerEncoder(encoder_layer, num_layers=LPE_layers)\n \n self.layers = nn.ModuleList([ GraphTransformerLayer(gamma, GT_hidden_dim, GT_hidden_dim, GT_n_heads, full_graph, dropout, self.layer_norm, self.batch_norm, self.residual) for _ in range(GT_layers-1) ])\n \n self.layers.append(GraphTransformerLayer(gamma, GT_hidden_dim, GT_out_dim, GT_n_heads, full_graph, dropout, self.layer_norm, self.batch_norm, self.residual))\n self.MLP_layer = MLPReadout(GT_out_dim, self.n_classes)\n\n\n def forward(self, g, h, e, diff, product, EigVals):\n \n # input embedding\n h = self.embedding_h(h)\n h = self.in_feat_dropout(h)\n e = self.embedding_e(e) \n \n PosEnc = torch.cat((diff, product, EigVals), 2) # (Num edges) x (Num Eigenvectors) x 3\n empty_mask = torch.isnan(PosEnc) # (Num edges) x (Num Eigenvectors) x 3\n PosEnc[empty_mask] = 0 # (Num edges) x (Num Eigenvectors) x 3\n\n PosEnc = torch.transpose(PosEnc, 0 ,1).float() # (Num Eigenvectors) x (Num edges) x 3\n PosEnc = self.linear_A(PosEnc) # (Num Eigenvectors) x (Num edges) x PE_dim\n \n #1st Transformer: Learned PE\n PosEnc = self.PE_Transformer(src=PosEnc, src_key_padding_mask=empty_mask[:,:,0]) \n \n #remove masked sequences\n PosEnc[torch.transpose(empty_mask, 0 ,1)[:,:,0]] = float('nan') \n\n #Sum pooling\n PosEnc = torch.nansum(PosEnc, 0, keepdim=False) # (Num edge) x PE_dim\n\n #Concatenate learned PE to input embedding\n e = torch.cat((e, PosEnc), 1)\n \n \n # GraphTransformer Layers\n for conv in self.layers:\n h, e = conv(g, h, e)\n \n # output\n h_out = self.MLP_layer(h)\n\n return h_out\n \n \n def loss(self, pred, label):\n\n # calculating label weights for weighted loss computation\n V = label.size(0)\n label_count = torch.bincount(label)\n label_count = label_count[label_count.nonzero()].squeeze()\n cluster_sizes = torch.zeros(self.n_classes).long().to(self.device)\n cluster_sizes[torch.unique(label)] = label_count\n weight = (V - cluster_sizes).float() / V\n weight *= (cluster_sizes>0).float()\n \n # weighted cross-entropy for unbalanced classes\n criterion = nn.CrossEntropyLoss(weight=weight)\n loss = criterion(pred, label)\n\n return loss\n\n\n\n \n" }, { "path": "nets/SBMs_node_classification/SAN_NodeLPE.py", "content": "import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nimport dgl\nimport numpy as np\n\n\"\"\"\n Graph Transformer\n \n\"\"\"\nfrom layers.graph_transformer_layer import GraphTransformerLayer\nfrom layers.mlp_readout_layer import MLPReadout\n\nclass SAN_NodeLPE(nn.Module):\n\n def __init__(self, net_params):\n super().__init__()\n\n in_dim_node = net_params['in_dim'] # node_dim (feat is an integer)\n self.n_classes = net_params['n_classes']\n \n full_graph = net_params['full_graph']\n gamma = net_params['gamma']\n \n LPE_layers = net_params['LPE_layers']\n LPE_dim = net_params['LPE_dim']\n LPE_n_heads = net_params['LPE_n_heads']\n \n GT_layers = net_params['GT_layers']\n GT_hidden_dim = net_params['GT_hidden_dim']\n GT_out_dim = net_params['GT_out_dim']\n GT_n_heads = net_params['GT_n_heads']\n \n self.residual = net_params['residual']\n self.readout = net_params['readout']\n in_feat_dropout = net_params['in_feat_dropout']\n dropout = net_params['dropout']\n\n self.readout = net_params['readout']\n self.layer_norm = net_params['layer_norm']\n self.batch_norm = net_params['batch_norm']\n\n self.device = net_params['device']\n self.in_feat_dropout = nn.Dropout(in_feat_dropout)\n \n self.embedding_h = nn.Embedding(in_dim_node, GT_hidden_dim-LPE_dim)#Remove some embedding dimensions to make room for concatenating laplace encoding\n self.embedding_e = nn.Embedding(2, GT_hidden_dim)\n self.linear_A = nn.Linear(2, LPE_dim)\n \n encoder_layer = nn.TransformerEncoderLayer(d_model=LPE_dim, nhead=LPE_n_heads)\n self.PE_Transformer = nn.TransformerEncoder(encoder_layer, num_layers=LPE_layers)\n \n self.layers = nn.ModuleList([ GraphTransformerLayer(gamma, GT_hidden_dim, GT_hidden_dim, GT_n_heads, full_graph, dropout, self.layer_norm, self.batch_norm, self.residual) for _ in range(GT_layers-1) ])\n \n self.layers.append(GraphTransformerLayer(gamma, GT_hidden_dim, GT_out_dim, GT_n_heads, full_graph, dropout, self.layer_norm, self.batch_norm, self.residual))\n\n self.MLP_layer = MLPReadout(GT_out_dim, self.n_classes)\n\n\n def forward(self, g, h, e, EigVecs, EigVals):\n \n # input embedding\n h = self.embedding_h(h)\n e = self.embedding_e(e) \n \n PosEnc = torch.cat((EigVecs.unsqueeze(2), EigVals), dim=2).float() # (Num nodes) x (Num Eigenvectors) x 2\n empty_mask = torch.isnan(PosEnc) # (Num nodes) x (Num Eigenvectors) x 2\n \n PosEnc[empty_mask] = 0 # (Num nodes) x (Num Eigenvectors) x 2\n PosEnc = torch.transpose(PosEnc, 0 ,1).float() # (Num Eigenvectors) x (Num nodes) x 2\n PosEnc = self.linear_A(PosEnc) # (Num Eigenvectors) x (Num nodes) x PE_dim\n \n \n #1st Transformer: Learned PE\n PosEnc = self.PE_Transformer(src=PosEnc, src_key_padding_mask=empty_mask[:,:,0]) \n \n #remove masked sequences\n PosEnc[torch.transpose(empty_mask, 0 ,1)[:,:,0]] = float('nan') \n \n #Sum pooling\n PosEnc = torch.nansum(PosEnc, 0, keepdim=False)\n \n #Concatenate learned PE to input embedding\n h = torch.cat((h, PosEnc), 1)\n \n h = self.in_feat_dropout(h)\n \n # GraphTransformer Layers\n for conv in self.layers:\n h, e = conv(g, h, e)\n \n # output\n h_out = self.MLP_layer(h)\n\n return h_out\n \n \n def loss(self, pred, label):\n\n # calculating label weights for weighted loss computation\n V = label.size(0)\n label_count = torch.bincount(label)\n label_count = label_count[label_count.nonzero()].squeeze()\n cluster_sizes = torch.zeros(self.n_classes).long().to(self.device)\n cluster_sizes[torch.unique(label)] = label_count\n weight = (V - cluster_sizes).float() / V\n weight *= (cluster_sizes>0).float()\n \n # weighted cross-entropy for unbalanced classes\n criterion = nn.CrossEntropyLoss(weight=weight)\n loss = criterion(pred, label)\n\n return loss\n\n\n\n \n" }, { "path": "nets/SBMs_node_classification/load_net.py", "content": "\"\"\"\n Utility file to select GraphNN model as\n selected by the user\n\"\"\"\n\nfrom nets.SBMs_node_classification.SAN_NodeLPE import SAN_NodeLPE\nfrom nets.SBMs_node_classification.SAN_EdgeLPE import SAN_EdgeLPE\nfrom nets.SBMs_node_classification.SAN import SAN\n\n\ndef NodeLPE(net_params):\n return SAN_NodeLPE(net_params)\n\ndef EdgeLPE(net_params):\n return SAN_EdgeLPE(net_params)\n\ndef NoLPE(net_params):\n return SAN(net_params)\n\ndef gnn_model(LPE, net_params):\n model = {\n 'edge': EdgeLPE,\n 'node': NodeLPE,\n 'none': NoLPE\n }\n \n return model[LPE](net_params)" }, { "path": "nets/ZINC_graph_regression/SAN.py", "content": "import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nimport dgl\nimport numpy as np\n\n\"\"\"\n Graph Transformer with edge features\n \n\"\"\"\nfrom layers.graph_transformer_layer import GraphTransformerLayer\nfrom layers.mlp_readout_layer import MLPReadout\n\nclass SAN(nn.Module):\n def __init__(self, net_params):\n super().__init__()\n \n num_atom_type = net_params['num_atom_type']\n num_bond_type = net_params['num_bond_type']\n \n full_graph = net_params['full_graph']\n gamma = net_params['gamma']\n \n GT_layers = net_params['GT_layers']\n GT_hidden_dim = net_params['GT_hidden_dim']\n GT_out_dim = net_params['GT_out_dim']\n GT_n_heads = net_params['GT_n_heads']\n \n self.residual = net_params['residual']\n self.readout = net_params['readout']\n in_feat_dropout = net_params['in_feat_dropout']\n dropout = net_params['dropout']\n\n self.readout = net_params['readout']\n self.layer_norm = net_params['layer_norm']\n self.batch_norm = net_params['batch_norm']\n\n self.device = net_params['device']\n self.in_feat_dropout = nn.Dropout(in_feat_dropout)\n \n self.embedding_h = nn.Embedding(num_atom_type, GT_hidden_dim)\n self.embedding_e = nn.Embedding(num_bond_type, GT_hidden_dim)\n \n self.layers = nn.ModuleList([ GraphTransformerLayer(gamma, GT_hidden_dim, GT_hidden_dim, GT_n_heads, full_graph, dropout, self.layer_norm, self.batch_norm, self.residual) for _ in range(GT_layers-1) ])\n \n self.layers.append(GraphTransformerLayer(gamma, GT_hidden_dim, GT_out_dim, GT_n_heads, full_graph, dropout, self.layer_norm, self.batch_norm, self.residual))\n self.MLP_layer = MLPReadout(GT_out_dim, 1) # 1 out dim since regression problem \n \n \n def forward(self, g, h, e):\n \n # input embedding\n h = self.embedding_h(h)\n h = self.in_feat_dropout(h)\n e = self.embedding_e(e) \n \n # GNN\n for conv in self.layers:\n h, e = conv(g, h, e)\n g.ndata['h'] = h\n \n if self.readout == \"sum\":\n hg = dgl.sum_nodes(g, 'h')\n elif self.readout == \"max\":\n hg = dgl.max_nodes(g, 'h')\n elif self.readout == \"mean\":\n hg = dgl.mean_nodes(g, 'h')\n else:\n hg = dgl.mean_nodes(g, 'h') # default readout is mean nodes\n \n return self.MLP_layer(hg)\n \n def loss(self, scores, targets):\n\n loss = nn.L1Loss()(scores, targets)\n \n return loss\n" }, { "path": "nets/ZINC_graph_regression/SAN_EdgeLPE.py", "content": "import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nimport dgl\nimport numpy as np\n\n\"\"\"\n Graph Transformer with edge features\n \n\"\"\"\nfrom layers.graph_transformer_layer import GraphTransformerLayer\nfrom layers.mlp_readout_layer import MLPReadout\n\nclass SAN_EdgeLPE(nn.Module):\n def __init__(self, net_params):\n super().__init__()\n \n num_atom_type = net_params['num_atom_type']\n num_bond_type = net_params['num_bond_type']\n \n full_graph = net_params['full_graph']\n gamma = net_params['gamma']\n \n LPE_layers = net_params['LPE_layers']\n LPE_dim = net_params['LPE_dim']\n LPE_n_heads = net_params['LPE_n_heads']\n \n GT_layers = net_params['GT_layers']\n GT_hidden_dim = net_params['GT_hidden_dim']\n GT_out_dim = net_params['GT_out_dim']\n GT_n_heads = net_params['GT_n_heads']\n \n self.residual = net_params['residual']\n self.readout = net_params['readout']\n in_feat_dropout = net_params['in_feat_dropout']\n dropout = net_params['dropout']\n\n self.readout = net_params['readout']\n self.layer_norm = net_params['layer_norm']\n self.batch_norm = net_params['batch_norm']\n\n self.device = net_params['device']\n self.in_feat_dropout = nn.Dropout(in_feat_dropout)\n \n self.embedding_h = nn.Embedding(num_atom_type, GT_hidden_dim)\n self.embedding_e = nn.Embedding(num_bond_type, GT_hidden_dim-LPE_dim)#Remove some embedding dimensions to make room for concatenating laplace encoding\n self.linear_A = nn.Linear(3, LPE_dim)\n \n encoder_layer = nn.TransformerEncoderLayer(d_model=LPE_dim, nhead=LPE_n_heads)\n self.PE_Transformer = nn.TransformerEncoder(encoder_layer, num_layers=LPE_layers)\n \n self.layers = nn.ModuleList([ GraphTransformerLayer(gamma, GT_hidden_dim, GT_hidden_dim, GT_n_heads, full_graph, dropout, self.layer_norm, self.batch_norm, self.residual) for _ in range(GT_layers-1) ])\n \n self.layers.append(GraphTransformerLayer(gamma, GT_hidden_dim, GT_out_dim, GT_n_heads, full_graph, dropout, self.layer_norm, self.batch_norm, self.residual))\n self.MLP_layer = MLPReadout(GT_out_dim, 1) # 1 out dim since regression problem \n \n \n def forward(self, g, h, e, diff, product, EigVals):\n \n # input embedding\n h = self.embedding_h(h)\n h = self.in_feat_dropout(h)\n e = self.embedding_e(e) \n \n PosEnc = torch.cat((diff, product, EigVals), 2) # (Num edges) x (Num Eigenvectors) x 3\n empty_mask = torch.isnan(PosEnc) # (Num edges) x (Num Eigenvectors) x 3\n PosEnc[empty_mask] = 0 # (Num edges) x (Num Eigenvectors) x 3\n\n PosEnc = torch.transpose(PosEnc, 0 ,1).float() # (Num Eigenvectors) x (Num edges) x 3\n PosEnc = self.linear_A(PosEnc) # (Num Eigenvectors) x (Num edges) x PE_dim\n \n #1st Transformer: Learned PE\n PosEnc = self.PE_Transformer(src=PosEnc, src_key_padding_mask=empty_mask[:,:,0]) \n \n #remove masked sequences\n PosEnc[torch.transpose(empty_mask, 0 ,1)[:,:,0]] = float('nan') \n\n #Sum pooling\n PosEnc = torch.nansum(PosEnc, 0, keepdim=False) # (Num edge) x PE_dim\n\n #Concatenate learned PE to input embedding\n e = torch.cat((e, PosEnc), 1)\n \n \n # GNN\n for conv in self.layers:\n h, e = conv(g, h, e)\n g.ndata['h'] = h\n \n if self.readout == \"sum\":\n hg = dgl.sum_nodes(g, 'h')\n elif self.readout == \"max\":\n hg = dgl.max_nodes(g, 'h')\n elif self.readout == \"mean\":\n hg = dgl.mean_nodes(g, 'h')\n else:\n hg = dgl.mean_nodes(g, 'h') # default readout is mean nodes\n \n return self.MLP_layer(hg)\n \n def loss(self, scores, targets):\n\n loss = nn.L1Loss()(scores, targets)\n \n return loss\n" }, { "path": "nets/ZINC_graph_regression/SAN_NodeLPE.py", "content": "import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nimport dgl\nimport numpy as np\n\n\"\"\"\n Graph Transformer with edge features\n \n\"\"\"\nfrom layers.graph_transformer_layer import GraphTransformerLayer\nfrom layers.mlp_readout_layer import MLPReadout\n\nclass SAN_NodeLPE(nn.Module):\n def __init__(self, net_params):\n super().__init__()\n \n num_atom_type = net_params['num_atom_type']\n num_bond_type = net_params['num_bond_type']\n \n full_graph = net_params['full_graph']\n gamma = net_params['gamma']\n \n LPE_layers = net_params['LPE_layers']\n LPE_dim = net_params['LPE_dim']\n LPE_n_heads = net_params['LPE_n_heads']\n \n GT_layers = net_params['GT_layers']\n GT_hidden_dim = net_params['GT_hidden_dim']\n GT_out_dim = net_params['GT_out_dim']\n GT_n_heads = net_params['GT_n_heads']\n \n self.residual = net_params['residual']\n self.readout = net_params['readout']\n in_feat_dropout = net_params['in_feat_dropout']\n dropout = net_params['dropout']\n\n self.readout = net_params['readout']\n self.layer_norm = net_params['layer_norm']\n self.batch_norm = net_params['batch_norm']\n\n self.device = net_params['device']\n self.in_feat_dropout = nn.Dropout(in_feat_dropout)\n \n self.embedding_h = nn.Embedding(num_atom_type, GT_hidden_dim-LPE_dim)#Remove some embedding dimensions to make room for concatenating laplace encoding\n self.embedding_e = nn.Embedding(num_bond_type, GT_hidden_dim)\n self.linear_A = nn.Linear(2, LPE_dim)\n \n encoder_layer = nn.TransformerEncoderLayer(d_model=LPE_dim, nhead=LPE_n_heads)\n self.PE_Transformer = nn.TransformerEncoder(encoder_layer, num_layers=LPE_layers)\n \n self.layers = nn.ModuleList([ GraphTransformerLayer(gamma, GT_hidden_dim, GT_hidden_dim, GT_n_heads, full_graph, dropout, self.layer_norm, self.batch_norm, self.residual) for _ in range(GT_layers-1) ])\n \n self.layers.append(GraphTransformerLayer(gamma, GT_hidden_dim, GT_out_dim, GT_n_heads, full_graph, dropout, self.layer_norm, self.batch_norm, self.residual))\n self.MLP_layer = MLPReadout(GT_out_dim, 1) # 1 out dim since regression problem \n \n \n def forward(self, g, h, e, EigVecs, EigVals):\n \n # input embedding\n h = self.embedding_h(h)\n e = self.embedding_e(e) \n \n PosEnc = torch.cat((EigVecs.unsqueeze(2), EigVals), dim=2).float() # (Num nodes) x (Num Eigenvectors) x 2\n empty_mask = torch.isnan(PosEnc) # (Num nodes) x (Num Eigenvectors) x 2\n \n PosEnc[empty_mask] = 0 # (Num nodes) x (Num Eigenvectors) x 2\n PosEnc = torch.transpose(PosEnc, 0 ,1).float() # (Num Eigenvectors) x (Num nodes) x 2\n PosEnc = self.linear_A(PosEnc) # (Num Eigenvectors) x (Num nodes) x PE_dim\n \n \n #1st Transformer: Learned PE\n PosEnc = self.PE_Transformer(src=PosEnc, src_key_padding_mask=empty_mask[:,:,0]) \n \n #remove masked sequences\n PosEnc[torch.transpose(empty_mask, 0 ,1)[:,:,0]] = float('nan') \n \n #Sum pooling\n PosEnc = torch.nansum(PosEnc, 0, keepdim=False)\n \n #Concatenate learned PE to input embedding\n h = torch.cat((h, PosEnc), 1)\n \n h = self.in_feat_dropout(h)\n \n \n # GNN\n for conv in self.layers:\n h, e = conv(g, h, e)\n g.ndata['h'] = h\n \n if self.readout == \"sum\":\n hg = dgl.sum_nodes(g, 'h')\n elif self.readout == \"max\":\n hg = dgl.max_nodes(g, 'h')\n elif self.readout == \"mean\":\n hg = dgl.mean_nodes(g, 'h')\n else:\n hg = dgl.mean_nodes(g, 'h') # default readout is mean nodes\n \n return self.MLP_layer(hg)\n \n def loss(self, scores, targets):\n\n loss = nn.L1Loss()(scores, targets)\n \n return loss\n" }, { "path": "nets/ZINC_graph_regression/load_net.py", "content": "\nfrom nets.ZINC_graph_regression.SAN_NodeLPE import SAN_NodeLPE\nfrom nets.ZINC_graph_regression.SAN_EdgeLPE import SAN_EdgeLPE\nfrom nets.ZINC_graph_regression.SAN import SAN\n\n\ndef NodeLPE(net_params):\n return SAN_NodeLPE(net_params)\n\ndef EdgeLPE(net_params):\n return SAN_EdgeLPE(net_params)\n\ndef NoLPE(net_params):\n return SAN(net_params)\n\ndef gnn_model(LPE, net_params):\n model = {\n 'edge': EdgeLPE,\n 'node': NodeLPE,\n 'none': NoLPE\n }\n \n return model[LPE](net_params)" }, { "path": "nets/molhiv_graph_regression/SAN.py", "content": "import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport numpy as np\n\nimport dgl\n\nfrom ogb.graphproppred.mol_encoder import AtomEncoder, BondEncoder\n\n\"\"\"\n Graph Transformer with edge features\n \n\"\"\"\nfrom layers.graph_transformer_layer import GraphTransformerLayer\nfrom layers.mlp_readout_layer import MLPReadout\n\nclass SAN(nn.Module):\n def __init__(self, net_params):\n super().__init__()\n \n full_graph = net_params['full_graph']\n gamma = net_params['gamma']\n \n GT_layers = net_params['GT_layers']\n GT_hidden_dim = net_params['GT_hidden_dim']\n GT_out_dim = net_params['GT_out_dim']\n GT_n_heads = net_params['GT_n_heads']\n \n self.residual = net_params['residual']\n self.readout = net_params['readout']\n in_feat_dropout = net_params['in_feat_dropout']\n dropout = net_params['dropout']\n\n self.readout = net_params['readout']\n self.layer_norm = net_params['layer_norm']\n self.batch_norm = net_params['batch_norm']\n\n self.device = net_params['device']\n self.in_feat_dropout = nn.Dropout(in_feat_dropout)\n \n self.embedding_h = AtomEncoder(emb_dim = GT_hidden_dim)\n self.embedding_e = BondEncoder(emb_dim = GT_hidden_dim)\n\n \n self.layers = nn.ModuleList([ GraphTransformerLayer(gamma, GT_hidden_dim, GT_hidden_dim, GT_n_heads, full_graph, dropout, self.layer_norm, self.batch_norm, self.residual) for _ in range(GT_layers-1) ]) \n \n self.layers.append(GraphTransformerLayer(gamma, GT_hidden_dim, GT_out_dim, GT_n_heads, full_graph, dropout, self.layer_norm, self.batch_norm, self.residual))\n self.MLP_layer = MLPReadout(GT_out_dim, 1) # out dim for probability \n \n \n def forward(self, g, h, e):\n \n # input embedding\n h = self.embedding_h(h)\n h = self.in_feat_dropout(h)\n e = self.embedding_e(e) \n \n # Second Transformer\n for conv in self.layers:\n h, e = conv(g, h, e)\n g.ndata['h'] = h\n \n if self.readout == \"sum\":\n hg = dgl.sum_nodes(g, 'h')\n elif self.readout == \"max\":\n hg = dgl.max_nodes(g, 'h')\n elif self.readout == \"mean\":\n hg = dgl.mean_nodes(g, 'h')\n else:\n hg = dgl.mean_nodes(g, 'h') # default readout is mean nodes\n \n sig = nn.Sigmoid()\n \n return sig(self.MLP_layer(hg))\n \n def loss(self, scores, targets):\n \n loss = nn.BCELoss()\n \n l = loss(scores.float(), targets.float())\n \n return l\n" }, { "path": "nets/molhiv_graph_regression/SAN_EdgeLPE.py", "content": "import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport numpy as np\n\nimport dgl\n\nfrom ogb.graphproppred.mol_encoder import AtomEncoder, BondEncoder\n\n\"\"\"\n Graph Transformer with edge features\n \n\"\"\"\nfrom layers.graph_transformer_layer import GraphTransformerLayer\nfrom layers.mlp_readout_layer import MLPReadout\n\nclass SAN_EdgeLPE(nn.Module):\n def __init__(self, net_params):\n super().__init__()\n \n full_graph = net_params['full_graph']\n gamma = net_params['gamma']\n \n LPE_layers = net_params['LPE_layers']\n LPE_dim = net_params['LPE_dim']\n LPE_n_heads = net_params['LPE_n_heads']\n \n GT_layers = net_params['GT_layers']\n GT_hidden_dim = net_params['GT_hidden_dim']\n GT_out_dim = net_params['GT_out_dim']\n GT_n_heads = net_params['GT_n_heads']\n \n self.residual = net_params['residual']\n self.readout = net_params['readout']\n in_feat_dropout = net_params['in_feat_dropout']\n dropout = net_params['dropout']\n\n self.readout = net_params['readout']\n self.layer_norm = net_params['layer_norm']\n self.batch_norm = net_params['batch_norm']\n\n self.device = net_params['device']\n self.in_feat_dropout = nn.Dropout(in_feat_dropout)\n \n self.embedding_h = AtomEncoder(emb_dim = GT_hidden_dim)\n self.embedding_e = BondEncoder(emb_dim = GT_hidden_dim-LPE_dim)#Remove some embedding dimensions to make room for concatenating laplace encoding\n self.linear_A = nn.Linear(3, LPE_dim)\n \n encoder_layer = nn.TransformerEncoderLayer(d_model=LPE_dim, nhead=LPE_n_heads)\n self.PE_Transformer = nn.TransformerEncoder(encoder_layer, num_layers=LPE_layers)\n \n self.layers = nn.ModuleList([ GraphTransformerLayer(gamma, GT_hidden_dim, GT_hidden_dim, GT_n_heads, full_graph, dropout, self.layer_norm, self.batch_norm, self.residual) for _ in range(GT_layers-1) ])\n self.layers.append(GraphTransformerLayer(gamma, GT_hidden_dim, GT_out_dim, GT_n_heads, full_graph, dropout, self.layer_norm, self.batch_norm, self.residual)) \n self.MLP_layer = MLPReadout(GT_out_dim, 1) # out dim for probability \n \n \n\n def forward(self, g, h, e, diff, product, EigVals):\n \n # input embedding\n h = self.embedding_h(h)\n h = self.in_feat_dropout(h)\n e = self.embedding_e(e) \n \n PosEnc = torch.cat((diff, product, EigVals), 2) # (Num edges) x (Num Eigenvectors) x 3\n empty_mask = torch.isnan(PosEnc) # (Num edges) x (Num Eigenvectors) x 3\n PosEnc[empty_mask] = 0 # (Num edges) x (Num Eigenvectors) x 3\n\n PosEnc = torch.transpose(PosEnc, 0 ,1).float() # (Num Eigenvectors) x (Num edges) x 3\n PosEnc = self.linear_A(PosEnc) # (Num Eigenvectors) x (Num edges) x PE_dim\n \n #1st Transformer: Learned PE\n PosEnc = self.PE_Transformer(src=PosEnc, src_key_padding_mask=empty_mask[:,:,0]) \n \n #remove masked sequences\n PosEnc[torch.transpose(empty_mask, 0 ,1)[:,:,0]] = float('nan') \n\n #Sum pooling\n PosEnc = torch.nansum(PosEnc, 0, keepdim=False) # (Num edge) x PE_dim\n\n #Concatenate learned PE to input embedding\n e = torch.cat((e, PosEnc), 1)\n \n \n # GNN\n for conv in self.layers:\n h, e = conv(g, h, e)\n g.ndata['h'] = h\n \n \n if self.readout == \"sum\":\n hg = dgl.sum_nodes(g, 'h')\n elif self.readout == \"max\":\n hg = dgl.max_nodes(g, 'h')\n elif self.readout == \"mean\":\n hg = dgl.mean_nodes(g, 'h')\n else:\n hg = dgl.mean_nodes(g, 'h') # default readout is mean nodes\n \n sig = nn.Sigmoid()\n \n return sig(self.MLP_layer(hg))\n \n def loss(self, scores, targets):\n \n loss = nn.BCELoss()\n \n l = loss(scores.float(), targets.float())\n \n \n return l\n" }, { "path": "nets/molhiv_graph_regression/SAN_NodeLPE.py", "content": "import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport numpy as np\n\nimport dgl\n\nfrom ogb.graphproppred.mol_encoder import AtomEncoder, BondEncoder\n\n\"\"\"\n Graph Transformer with edge features\n \n\"\"\"\nfrom layers.graph_transformer_layer import GraphTransformerLayer\nfrom layers.mlp_readout_layer import MLPReadout\n\nclass SAN_NodeLPE(nn.Module):\n def __init__(self, net_params):\n super().__init__()\n \n full_graph = net_params['full_graph']\n gamma = net_params['gamma']\n \n LPE_layers = net_params['LPE_layers']\n LPE_dim = net_params['LPE_dim']\n LPE_n_heads = net_params['LPE_n_heads']\n \n GT_layers = net_params['GT_layers']\n GT_hidden_dim = net_params['GT_hidden_dim']\n GT_out_dim = net_params['GT_out_dim']\n GT_n_heads = net_params['GT_n_heads']\n \n self.residual = net_params['residual']\n self.readout = net_params['readout']\n in_feat_dropout = net_params['in_feat_dropout']\n dropout = net_params['dropout']\n\n self.readout = net_params['readout']\n self.layer_norm = net_params['layer_norm']\n self.batch_norm = net_params['batch_norm']\n\n self.device = net_params['device']\n self.in_feat_dropout = nn.Dropout(in_feat_dropout)\n \n self.embedding_h = AtomEncoder(emb_dim = GT_hidden_dim-LPE_dim) #Remove some embedding dimensions to make room for concatenating LPE\n self.embedding_e_real = BondEncoder(emb_dim = GT_hidden_dim)\n self.embedding_e_fake = nn.Embedding(1, GT_hidden_dim)\n self.linear_A = nn.Linear(2, LPE_dim)\n \n encoder_layer = nn.TransformerEncoderLayer(d_model=LPE_dim, nhead=LPE_n_heads)\n self.PE_Transformer = nn.TransformerEncoder(encoder_layer, num_layers=LPE_layers)\n \n self.layers = nn.ModuleList([ GraphTransformerLayer(gamma, GT_hidden_dim, GT_hidden_dim, GT_n_heads, full_graph, dropout, self.layer_norm, self.batch_norm, self.residual) for _ in range(GT_layers-1) ])\n \n self.layers.append(GraphTransformerLayer(gamma, GT_hidden_dim, GT_out_dim, GT_n_heads, full_graph, dropout, self.layer_norm, self.batch_norm, self.residual))\n self.MLP_layer = MLPReadout(GT_out_dim, 1) # 1 out dim for probability \n \n\n def forward(self, g, h, e, EigVecs, EigVals):\n \n\n # input embedding\n h = self.embedding_h(h)\n e = self.embedding_e_real(e)\n \n PosEnc = torch.cat((EigVecs.unsqueeze(2), EigVals), dim=2).float() # (Num nodes) x (Num Eigenvectors) x 2\n empty_mask = torch.isnan(PosEnc) # (Num nodes) x (Num Eigenvectors) x 2\n \n PosEnc[empty_mask] = 0 # (Num nodes) x (Num Eigenvectors) x 2\n PosEnc = torch.transpose(PosEnc, 0 ,1).float() # (Num Eigenvectors) x (Num nodes) x 2\n PosEnc = self.linear_A(PosEnc) # (Num Eigenvectors) x (Num nodes) x PE_dim\n \n \n #1st Transformer: Learned PE\n PosEnc = self.PE_Transformer(src=PosEnc, src_key_padding_mask=empty_mask[:,:,0]) \n \n #remove masked sequences\n PosEnc[torch.transpose(empty_mask, 0 ,1)[:,:,0]] = float('nan') \n \n #Sum pooling\n PosEnc = torch.nansum(PosEnc, 0, keepdim=False)\n \n #Concatenate learned PE to input embedding\n h = torch.cat((h, PosEnc), 1)\n\n h = self.in_feat_dropout(h)\n \n # Second Transformer\n for conv in self.layers:\n h, e = conv(g, h, e)\n g.ndata['h'] = h\n \n if self.readout == \"sum\":\n hg = dgl.sum_nodes(g, 'h')\n elif self.readout == \"max\":\n hg = dgl.max_nodes(g, 'h')\n elif self.readout == \"mean\":\n hg = dgl.mean_nodes(g, 'h')\n else:\n hg = dgl.mean_nodes(g, 'h') # default readout is mean nodes\n \n sig = nn.Sigmoid()\n \n return sig(self.MLP_layer(hg))\n \n def loss(self, scores, targets):\n \n loss = nn.BCELoss()\n \n l = loss(scores.float(), targets.float())\n \n return l\n" }, { "path": "nets/molhiv_graph_regression/load_net.py", "content": "\nfrom nets.molhiv_graph_regression.SAN_NodeLPE import SAN_NodeLPE\nfrom nets.molhiv_graph_regression.SAN_EdgeLPE import SAN_EdgeLPE\nfrom nets.molhiv_graph_regression.SAN import SAN\n\ndef NodeLPE(net_params):\n return SAN_NodeLPE(net_params)\n\ndef EdgeLPE(net_params):\n return SAN_EdgeLPE(net_params)\n\ndef NoLPE(net_params):\n return SAN(net_params)\n\ndef gnn_model(LPE, net_params):\n model = {\n 'edge': EdgeLPE,\n 'node': NodeLPE,\n 'none': NoLPE\n }\n \n return model[LPE](net_params)\n" }, { "path": "nets/molpcba/SAN.py", "content": "import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport numpy as np\n\nimport dgl\n\nfrom ogb.graphproppred.mol_encoder import AtomEncoder, BondEncoder\n\n\"\"\"\n Graph Transformer with edge features\n \n\"\"\"\nfrom layers.graph_transformer_layer import GraphTransformerLayer\nfrom layers.mlp_readout_layer import MLPReadout\n\nclass SAN(nn.Module):\n def __init__(self, net_params):\n super().__init__()\n \n full_graph = net_params['full_graph']\n gamma = net_params['gamma']\n \n GT_layers = net_params['GT_layers']\n GT_hidden_dim = net_params['GT_hidden_dim']\n GT_out_dim = net_params['GT_out_dim']\n GT_n_heads = net_params['GT_n_heads']\n \n self.residual = net_params['residual']\n self.readout = net_params['readout']\n in_feat_dropout = net_params['in_feat_dropout']\n dropout = net_params['dropout']\n\n self.readout = net_params['readout']\n self.layer_norm = net_params['layer_norm']\n self.batch_norm = net_params['batch_norm']\n\n self.device = net_params['device']\n self.in_feat_dropout = nn.Dropout(in_feat_dropout)\n \n self.embedding_h = AtomEncoder(emb_dim = GT_hidden_dim)\n self.embedding_e = BondEncoder(emb_dim = GT_hidden_dim)\n\n \n self.layers = nn.ModuleList([ GraphTransformerLayer(gamma, GT_hidden_dim, GT_hidden_dim, GT_n_heads, full_graph, dropout, self.layer_norm, self.batch_norm, self.residual) for _ in range(GT_layers-1) ]) \n \n self.layers.append(GraphTransformerLayer(gamma, GT_hidden_dim, GT_out_dim, GT_n_heads, full_graph, dropout, self.layer_norm, self.batch_norm, self.residual))\n self.MLP_layer = MLPReadout(GT_out_dim, 128) # out dim for probability \n \n \n def forward(self, g, h, e):\n \n # input embedding\n h = self.embedding_h(h)\n h = self.in_feat_dropout(h)\n e = self.embedding_e(e) \n \n # Second Transformer\n for conv in self.layers:\n h, e = conv(g, h, e)\n g.ndata['h'] = h\n \n if self.readout == \"sum\":\n hg = dgl.sum_nodes(g, 'h')\n elif self.readout == \"max\":\n hg = dgl.max_nodes(g, 'h')\n elif self.readout == \"mean\":\n hg = dgl.mean_nodes(g, 'h')\n else:\n hg = dgl.mean_nodes(g, 'h') # default readout is mean nodes\n \n sig = nn.Sigmoid()\n \n return sig(self.MLP_layer(hg))\n \n def loss(self, scores, targets):\n \n loss = nn.BCELoss()\n \n l = loss(scores.float(), targets.float())\n \n return l\n" }, { "path": "nets/molpcba/SAN_EdgeLPE.py", "content": "import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport numpy as np\n\nimport dgl\n\nfrom ogb.graphproppred.mol_encoder import AtomEncoder, BondEncoder\n\n\"\"\"\n Graph Transformer with edge features\n \n\"\"\"\nfrom layers.graph_transformer_layer import GraphTransformerLayer\nfrom layers.mlp_readout_layer import MLPReadout\n\nclass SAN_EdgeLPE(nn.Module):\n def __init__(self, net_params):\n super().__init__()\n \n full_graph = net_params['full_graph']\n gamma = net_params['gamma']\n \n LPE_layers = net_params['LPE_layers']\n LPE_dim = net_params['LPE_dim']\n LPE_n_heads = net_params['LPE_n_heads']\n \n GT_layers = net_params['GT_layers']\n GT_hidden_dim = net_params['GT_hidden_dim']\n GT_out_dim = net_params['GT_out_dim']\n GT_n_heads = net_params['GT_n_heads']\n \n self.residual = net_params['residual']\n self.readout = net_params['readout']\n in_feat_dropout = net_params['in_feat_dropout']\n dropout = net_params['dropout']\n\n self.readout = net_params['readout']\n self.layer_norm = net_params['layer_norm']\n self.batch_norm = net_params['batch_norm']\n\n self.device = net_params['device']\n self.in_feat_dropout = nn.Dropout(in_feat_dropout)\n \n self.embedding_h = AtomEncoder(emb_dim = GT_hidden_dim)\n self.embedding_e = BondEncoder(emb_dim = GT_hidden_dim-LPE_dim)#Remove some embedding dimensions to make room for concatenating laplace encoding\n self.linear_A = nn.Linear(3, LPE_dim)\n \n encoder_layer = nn.TransformerEncoderLayer(d_model=LPE_dim, nhead=LPE_n_heads)\n self.PE_Transformer = nn.TransformerEncoder(encoder_layer, num_layers=LPE_layers)\n \n self.layers = nn.ModuleList([ GraphTransformerLayer(gamma, GT_hidden_dim, GT_hidden_dim, GT_n_heads, full_graph, dropout, self.layer_norm, self.batch_norm, self.residual) for _ in range(GT_layers-1) ])\n self.layers.append(GraphTransformerLayer(gamma, GT_hidden_dim, GT_out_dim, GT_n_heads, full_graph, dropout, self.layer_norm, self.batch_norm, self.residual)) \n self.MLP_layer = MLPReadout(GT_out_dim, 128) # out dim for probability \n \n \n\n def forward(self, g, h, e, diff, product, EigVals):\n \n # input embedding\n h = self.embedding_h(h)\n h = self.in_feat_dropout(h)\n e = self.embedding_e(e) \n \n PosEnc = torch.cat((diff, product, EigVals), 2) # (Num edges) x (Num Eigenvectors) x 3\n empty_mask = torch.isnan(PosEnc) # (Num edges) x (Num Eigenvectors) x 3\n PosEnc[empty_mask] = 0 # (Num edges) x (Num Eigenvectors) x 3\n\n PosEnc = torch.transpose(PosEnc, 0 ,1).float() # (Num Eigenvectors) x (Num edges) x 3\n PosEnc = self.linear_A(PosEnc) # (Num Eigenvectors) x (Num edges) x PE_dim\n \n #1st Transformer: Learned PE\n PosEnc = self.PE_Transformer(src=PosEnc, src_key_padding_mask=empty_mask[:,:,0]) \n \n #remove masked sequences\n PosEnc[torch.transpose(empty_mask, 0 ,1)[:,:,0]] = float('nan') \n\n #Sum pooling\n PosEnc = torch.nansum(PosEnc, 0, keepdim=False) # (Num edge) x PE_dim\n\n #Concatenate learned PE to input embedding\n e = torch.cat((e, PosEnc), 1)\n \n \n # GNN\n for conv in self.layers:\n h, e = conv(g, h, e)\n g.ndata['h'] = h\n \n \n if self.readout == \"sum\":\n hg = dgl.sum_nodes(g, 'h')\n elif self.readout == \"max\":\n hg = dgl.max_nodes(g, 'h')\n elif self.readout == \"mean\":\n hg = dgl.mean_nodes(g, 'h')\n else:\n hg = dgl.mean_nodes(g, 'h') # default readout is mean nodes\n \n sig = nn.Sigmoid()\n \n return sig(self.MLP_layer(hg))\n \n def loss(self, scores, targets):\n \n loss = nn.BCELoss()\n \n l = loss(scores.float(), targets.float())\n \n \n return l\n" }, { "path": "nets/molpcba/SAN_NodeLPE.py", "content": "import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport numpy as np\n\nimport dgl\n\nfrom ogb.graphproppred.mol_encoder import AtomEncoder, BondEncoder\n\n\"\"\"\n Graph Transformer with edge features\n\n\"\"\"\nfrom layers.graph_transformer_layer import GraphTransformerLayer\nfrom layers.mlp_readout_layer import MLPReadout\n\nclass SAN_NodeLPE(nn.Module):\n def __init__(self, net_params):\n super().__init__()\n\n full_graph = net_params['full_graph']\n gamma = net_params['gamma']\n\n LPE_layers = net_params['LPE_layers']\n LPE_dim = net_params['LPE_dim']\n LPE_n_heads = net_params['LPE_n_heads']\n\n GT_layers = net_params['GT_layers']\n GT_hidden_dim = net_params['GT_hidden_dim']\n GT_out_dim = net_params['GT_out_dim']\n GT_n_heads = net_params['GT_n_heads']\n\n self.residual = net_params['residual']\n self.readout = net_params['readout']\n in_feat_dropout = net_params['in_feat_dropout']\n dropout = net_params['dropout']\n\n self.readout = net_params['readout']\n self.layer_norm = net_params['layer_norm']\n self.batch_norm = net_params['batch_norm']\n\n self.device = net_params['device']\n self.in_feat_dropout = nn.Dropout(in_feat_dropout)\n\n self.embedding_h = AtomEncoder(emb_dim = GT_hidden_dim-LPE_dim) #Remove some embedding dimensions to make room for concatenating LPE\n self.embedding_e_real = BondEncoder(emb_dim = GT_hidden_dim)\n \n #Optional extra MLP at beginning\n self.extra_mlp = net_params['extra_mlp']\n \n if self.extra_mlp:\n self.norm_node = nn.BatchNorm1d(GT_hidden_dim-LPE_dim)\n self.norm_edge = nn.BatchNorm1d(GT_hidden_dim)\n self.relu = nn.ReLU()\n self.linear_init_node = nn.Linear(GT_hidden_dim-LPE_dim, GT_hidden_dim-LPE_dim)\n self.linear_init_edge = nn.Linear(GT_hidden_dim, GT_hidden_dim)\n\n self.linear_A = nn.Linear(2, LPE_dim)\n\n encoder_layer = nn.TransformerEncoderLayer(d_model=LPE_dim, nhead=LPE_n_heads)\n self.PE_Transformer = nn.TransformerEncoder(encoder_layer, num_layers=LPE_layers)\n\n self.layers = nn.ModuleList([ GraphTransformerLayer(gamma, GT_hidden_dim, GT_hidden_dim, GT_n_heads, full_graph, dropout, self.layer_norm, self.batch_norm, self.residual) for _ in range(GT_layers-1) ])\n\n self.layers.append(GraphTransformerLayer(gamma, GT_hidden_dim, GT_out_dim, GT_n_heads, full_graph, dropout, self.layer_norm, self.batch_norm, self.residual))\n self.MLP_layer = MLPReadout(GT_out_dim, 128) # 1 out dim for probability\n\n\n def forward(self, g, h, e, EigVecs, EigVals):\n\n # input embedding\n h = self.embedding_h(h)\n e = self.embedding_e_real(e)\n \n if self.extra_mlp:\n h = self.norm_node(h)\n h = self.relu(h)\n h = self.linear_init_node(h)\n \n e = self.norm_edge(e)\n e = self.relu(e)\n e = self.linear_init_edge(e)\n\n EigVecs = EigVecs.to(dtype=h.dtype)\n EigVals = EigVals.to(dtype=h.dtype)\n PosEnc = torch.cat((EigVecs.unsqueeze(2), EigVals), dim=2) # (Num nodes) x (Num Eigenvectors) x 2\n empty_mask = torch.isnan(PosEnc) # (Num nodes) x (Num Eigenvectors) x 2\n\n PosEnc[empty_mask] = 0 # (Num nodes) x (Num Eigenvectors) x 2\n PosEnc = torch.transpose(PosEnc, 0 ,1) # (Num Eigenvectors) x (Num nodes) x 2\n PosEnc = self.linear_A(PosEnc) # (Num Eigenvectors) x (Num nodes) x PE_dim\n\n\n #1st Transformer: Learned PE\n PosEnc = self.PE_Transformer(src=PosEnc, src_key_padding_mask=empty_mask[:,:,0])\n\n #remove masked sequences\n PosEnc[torch.transpose(empty_mask, 0 ,1)[:,:,0]] = float('nan')\n\n #Sum pooling\n PosEnc = torch.nansum(PosEnc, 0, keepdim=False)\n\n #Concatenate learned PE to input embedding\n h = torch.cat((h, PosEnc), 1)\n\n h = self.in_feat_dropout(h)\n\n # Second Transformer\n for conv in self.layers:\n h, e = conv(g, h, e)\n g.ndata['h'] = h\n\n if self.readout == \"sum\":\n hg = dgl.sum_nodes(g, 'h')\n elif self.readout == \"max\":\n hg = dgl.max_nodes(g, 'h')\n elif self.readout == \"mean\":\n hg = dgl.mean_nodes(g, 'h')\n else:\n hg = dgl.mean_nodes(g, 'h') # default readout is mean nodes\n\n sig = nn.Sigmoid()\n\n return sig(self.MLP_layer(hg))\n\n def loss(self, scores, targets):\n\n loss = nn.BCELoss()\n\n l = loss(scores, targets.to(dtype=scores.dtype))\n\n return l\n" }, { "path": "nets/molpcba/load_net.py", "content": "\nfrom nets.molpcba.SAN_NodeLPE import SAN_NodeLPE\nfrom nets.molpcba.SAN_EdgeLPE import SAN_EdgeLPE\nfrom nets.molpcba.SAN import SAN\n\ndef NodeLPE(net_params):\n return SAN_NodeLPE(net_params)\n\ndef EdgeLPE(net_params):\n return SAN_EdgeLPE(net_params)\n\ndef NoLPE(net_params):\n return SAN(net_params)\n\ndef gnn_model(LPE, net_params):\n model = {\n 'edge': EdgeLPE,\n 'node': NodeLPE,\n 'none': NoLPE\n }\n \n return model[LPE](net_params)\n" }, { "path": "requirements.txt", "content": "absl-py==0.11.0\nargon2-cffi==20.1.0\nase==3.20.1\nastunparse==1.6.3\nasync-generator==1.10\nattrs==20.3.0\nbackcall==0.2.0\nbleach==3.2.1\ncachetools==4.1.1\ncertifi==2020.11.8\ncffi==1.14.4\nchardet==3.0.4\ncycler==0.10.0\ndataclasses==0.8\ndecorator==4.4.2\ndefusedxml==0.6.0\ndgl==0.5.3\ndgl-cu102==0.5.3\nentrypoints==0.3\nfsspec==0.8.4\nfuture==0.18.2\ngast==0.3.3\ngoogle-auth==1.23.0\ngoogle-auth-oauthlib==0.4.2\ngoogle-pasta==0.2.0\ngoogledrivedownloader==0.4\ngrpcio==1.33.2\nh5py==2.10.0\nidna==2.10\nimportlib-metadata==3.1.0\nipykernel==5.3.4\nipython==7.16.1\nipython-genutils==0.2.0\nipywidgets==7.6.3\nisodate==0.6.0\njedi==0.17.2\nJinja2==2.11.2\njoblib==0.17.0\njson5==0.9.5\njsonschema==3.2.0\njupyter-client==6.1.7\njupyter-core==4.7.0\njupyterlab==2.2.9\njupyterlab-pygments==0.1.2\njupyterlab-server==1.2.0\njupyterlab-widgets==1.0.0\nKeras==2.4.3\nKeras-Preprocessing==1.1.2\nkiwisolver==1.3.1\nlittleutils==0.2.2\nllvmlite==0.34.0\nMarkdown==3.3.3\nMarkupSafe==1.1.1\nmatplotlib==3.3.3\nmistune==0.8.4\nnbclient==0.5.1\nnbconvert==6.0.7\nnbformat==5.0.8\nnest-asyncio==1.4.3\nnetworkx==2.5\nnotebook==6.1.5\nnumba==0.51.2\nnumpy==1.18.5\noauthlib==3.1.0\nogb==1.3.0\nopt-einsum==3.3.0\noutdated==0.2.1\npackaging==20.4\npandas==1.1.4\npandocfilters==1.4.3\nparso==0.7.1\npexpect==4.8.0\npickleshare==0.7.5\nPillow==8.0.1\nprometheus-client==0.9.0\nprompt-toolkit==3.0.8\nprotobuf==3.14.0\nptyprocess==0.6.0\npyasn1==0.4.8\npyasn1-modules==0.2.8\npycparser==2.20\nPygments==2.7.2\npyparsing==2.4.7\npyrsistent==0.17.3\npython-dateutil==2.8.1\npytorch-lightning==1.0.8\npytz==2020.4\nPyYAML==5.3.1\npyzmq==20.0.0\nrdflib==5.0.0\nrequests==2.25.0\nrequests-oauthlib==1.3.0\nrsa==4.6\nscikit-learn==0.23.2\nscipy==1.5.4\nseaborn==0.11.0\nSend2Trash==1.5.0\nsix==1.15.0\ntensorboard==2.4.0\ntensorboard-plugin-wit==1.7.0\ntensorboardX==2.1\ntensorflow-estimator==2.3.0\ntensorflow-gpu==2.3.1\ntermcolor==1.1.0\nterminado==0.9.1\ntestpath==0.4.4\nthreadpoolctl==2.1.0\ntorch==1.7.0\ntorch-cluster==1.5.8\ntorch-geometric==1.6.1\ntorch-scatter==2.0.5\ntorch-sparse==0.6.8\ntorch-spline-conv==1.2.0\ntorchvision==0.8.1\ntornado==6.1\ntqdm==4.52.0\ntraitlets==4.3.3\ntyping-extensions==3.7.4.3\nurllib3==1.26.2\nwcwidth==0.2.5\nwebencodings==0.5.1\nWerkzeug==1.0.1\nwidgetsnbextension==3.5.1\nwrapt==1.12.1\nzipp==3.4.0\n" }, { "path": "scripts/CLUSTER/ablation/full_node_1e-1", "content": "#!/bin/bash\n\ncd ~/SAN/\n\npython main_SBMs_node_classification.py --dataset SBM_CLUSTER --seed 41 --config 'configs/CLUSTER/ablation/full/1e-1/node'\npython main_SBMs_node_classification.py --dataset SBM_CLUSTER --seed 55 --config 'configs/CLUSTER/ablation/full/1e-1/node'\npython main_SBMs_node_classification.py --dataset SBM_CLUSTER --seed 13 --config 'configs/CLUSTER/ablation/full/1e-1/node'\npython main_SBMs_node_classification.py --dataset SBM_CLUSTER --seed 88 --config 'configs/CLUSTER/ablation/full/1e-1/node'\n\n\n\n" }, { "path": "scripts/CLUSTER/ablation/full_node_1e-2", "content": "#!/bin/bash\ncd ~/SAN/\n\npython main_SBMs_node_classification.py --dataset SBM_CLUSTER --seed 41 --config 'configs/CLUSTER/ablation/full/1e-2/node'\npython main_SBMs_node_classification.py --dataset SBM_CLUSTER --seed 55 --config 'configs/CLUSTER/ablation/full/1e-2/node'\npython main_SBMs_node_classification.py --dataset SBM_CLUSTER --seed 13 --config 'configs/CLUSTER/ablation/full/1e-2/node'\npython main_SBMs_node_classification.py --dataset SBM_CLUSTER --seed 88 --config 'configs/CLUSTER/ablation/full/1e-2/node'\n\n\n\n" }, { "path": "scripts/CLUSTER/ablation/full_node_1e-3", "content": "#!/bin/bash\n\ncd ~/SAN/\n\npython main_SBMs_node_classification.py --dataset SBM_CLUSTER --seed 41 --config 'configs/CLUSTER/ablation/full/1e-3/node'\npython main_SBMs_node_classification.py --dataset SBM_CLUSTER --seed 55 --config 'configs/CLUSTER/ablation/full/1e-3/node'\npython main_SBMs_node_classification.py --dataset SBM_CLUSTER --seed 13 --config 'configs/CLUSTER/ablation/full/1e-3/node'\npython main_SBMs_node_classification.py --dataset SBM_CLUSTER --seed 88 --config 'configs/CLUSTER/ablation/full/1e-3/node'\n\n\n\n" }, { "path": "scripts/CLUSTER/ablation/full_node_1e-4", "content": "#!/bin/bash\n\ncd ~/SAN/\n\npython main_SBMs_node_classification.py --dataset SBM_CLUSTER --seed 41 --config 'configs/CLUSTER/ablation/full/1e-4/node'\npython main_SBMs_node_classification.py --dataset SBM_CLUSTER --seed 55 --config 'configs/CLUSTER/ablation/full/1e-4/node'\npython main_SBMs_node_classification.py --dataset SBM_CLUSTER --seed 13 --config 'configs/CLUSTER/ablation/full/1e-4/node'\npython main_SBMs_node_classification.py --dataset SBM_CLUSTER --seed 88 --config 'configs/CLUSTER/ablation/full/1e-4/node'\n\n\n\n" }, { "path": "scripts/CLUSTER/ablation/full_node_1e-5", "content": "#!/bin/bash\n\ncd ~/SAN/\n\npython main_SBMs_node_classification.py --dataset SBM_CLUSTER --seed 41 --config 'configs/CLUSTER/ablation/full/1e-5/node'\npython main_SBMs_node_classification.py --dataset SBM_CLUSTER --seed 55 --config 'configs/CLUSTER/ablation/full/1e-5/node'\npython main_SBMs_node_classification.py --dataset SBM_CLUSTER --seed 13 --config 'configs/CLUSTER/ablation/full/1e-5/node'\npython main_SBMs_node_classification.py --dataset SBM_CLUSTER --seed 88 --config 'configs/CLUSTER/ablation/full/1e-5/node'\n\n\n\n" }, { "path": "scripts/CLUSTER/ablation/full_node_1e-6", "content": "#!/bin/bash\n\ncd ~/SAN/\n\npython main_SBMs_node_classification.py --dataset SBM_CLUSTER --seed 41 --config 'configs/CLUSTER/ablation/full/1e-6/node'\npython main_SBMs_node_classification.py --dataset SBM_CLUSTER --seed 55 --config 'configs/CLUSTER/ablation/full/1e-6/node'\npython main_SBMs_node_classification.py --dataset SBM_CLUSTER --seed 13 --config 'configs/CLUSTER/ablation/full/1e-6/node'\npython main_SBMs_node_classification.py --dataset SBM_CLUSTER --seed 88 --config 'configs/CLUSTER/ablation/full/1e-6/node'\n\n\n\n" }, { "path": "scripts/CLUSTER/ablation/full_node_1e-7", "content": "#!/bin/bash\n\ncd ~/SAN/\n\npython main_SBMs_node_classification.py --dataset SBM_CLUSTER --seed 41 --config 'configs/CLUSTER/ablation/full/1e-7/node'\npython main_SBMs_node_classification.py --dataset SBM_CLUSTER --seed 55 --config 'configs/CLUSTER/ablation/full/1e-7/node'\npython main_SBMs_node_classification.py --dataset SBM_CLUSTER --seed 13 --config 'configs/CLUSTER/ablation/full/1e-7/node'\npython main_SBMs_node_classification.py --dataset SBM_CLUSTER --seed 88 --config 'configs/CLUSTER/ablation/full/1e-7/node'\n\n\n\n" }, { "path": "scripts/CLUSTER/ablation/full_node_1e-8", "content": "#!/bin/bash\n\ncd ~/SAN/\n\npython main_SBMs_node_classification.py --dataset SBM_CLUSTER --seed 41 --config 'configs/CLUSTER/ablation/full/1e-8/node'\npython main_SBMs_node_classification.py --dataset SBM_CLUSTER --seed 55 --config 'configs/CLUSTER/ablation/full/1e-8/node'\npython main_SBMs_node_classification.py --dataset SBM_CLUSTER --seed 13 --config 'configs/CLUSTER/ablation/full/1e-8/node'\npython main_SBMs_node_classification.py --dataset SBM_CLUSTER --seed 88 --config 'configs/CLUSTER/ablation/full/1e-8/node'\n\n\n\n" }, { "path": "scripts/CLUSTER/ablation/full_none", "content": "#!/bin/bash\n\ncd ~/SAN/\n\npython main_SBMs_node_classification.py --dataset SBM_CLUSTER --seed 41 --config 'configs/CLUSTER/ablation/full/none'\npython main_SBMs_node_classification.py --dataset SBM_CLUSTER --seed 55 --config 'configs/CLUSTER/ablation/full/none'\npython main_SBMs_node_classification.py --dataset SBM_CLUSTER --seed 13 --config 'configs/CLUSTER/ablation/full/none'\npython main_SBMs_node_classification.py --dataset SBM_CLUSTER --seed 88 --config 'configs/CLUSTER/ablation/full/none'\n" }, { "path": "scripts/CLUSTER/ablation/sparse_node", "content": "#!/bin/bash\n\ncd ~/SAN/\n\npython main_SBMs_node_classification.py --dataset SBM_CLUSTER --seed 41 --config 'configs/CLUSTER/ablation/sparse/node'\npython main_SBMs_node_classification.py --dataset SBM_CLUSTER --seed 55 --config 'configs/CLUSTER/ablation/sparse/node'\npython main_SBMs_node_classification.py --dataset SBM_CLUSTER --seed 13 --config 'configs/CLUSTER/ablation/sparse/node'\npython main_SBMs_node_classification.py --dataset SBM_CLUSTER --seed 88 --config 'configs/CLUSTER/ablation/sparse/node'\n" }, { "path": "scripts/CLUSTER/ablation/sparse_none", "content": "#!/bin/bash\n\ncd ~/SAN/\n\npython main_SBMs_node_classification.py --dataset SBM_CLUSTER --seed 41 --config 'configs/CLUSTER/ablation/sparse/none'\npython main_SBMs_node_classification.py --dataset SBM_CLUSTER --seed 55 --config 'configs/CLUSTER/ablation/sparse/none'\npython main_SBMs_node_classification.py --dataset SBM_CLUSTER --seed 13 --config 'configs/CLUSTER/ablation/sparse/none'\npython main_SBMs_node_classification.py --dataset SBM_CLUSTER --seed 88 --config 'configs/CLUSTER/ablation/sparse/none'\n" }, { "path": "scripts/CLUSTER/optimized/cluster_optimized", "content": "#!/bin/bash\n\ncd ~/SAN/\n\npython main_SBMs_node_classification.py --dataset SBM_CLUSTER --seed 41 --config 'configs/CLUSTER/optimized'\npython main_SBMs_node_classification.py --dataset SBM_CLUSTER --seed 55 --config 'configs/CLUSTER/optimized'\npython main_SBMs_node_classification.py --dataset SBM_CLUSTER --seed 13 --config 'configs/CLUSTER/optimized'\npython main_SBMs_node_classification.py --dataset SBM_CLUSTER --seed 88 --config 'configs/CLUSTER/optimized'\n\n\n\n" }, { "path": "scripts/MOLHIV/ablation/full_node_1e-3", "content": "#!/bin/bash\n\ncd ~/SAN/\n\npython main_molhiv.py --config 'configs/MOLHIV/ablation/full/1e-3/node'\npython main_molhiv.py --config 'configs/MOLHIV/ablation/full/1e-3/node'\npython main_molhiv.py --config 'configs/MOLHIV/ablation/full/1e-3/node'\npython main_molhiv.py --config 'configs/MOLHIV/ablation/full/1e-3/node'\n\n\n\n" }, { "path": "scripts/MOLHIV/ablation/full_node_1e-4", "content": "#!/bin/bash\n\ncd ~/SAN/\n\npython main_molhiv.py --config 'configs/MOLHIV/ablation/full/1e-4/node'\npython main_molhiv.py --config 'configs/MOLHIV/ablation/full/1e-4/node'\npython main_molhiv.py --config 'configs/MOLHIV/ablation/full/1e-4/node'\npython main_molhiv.py --config 'configs/MOLHIV/ablation/full/1e-4/node'\n\n\n\n" }, { "path": "scripts/MOLHIV/ablation/full_node_1e-5", "content": "#!/bin/bash\n\ncd ~/SAN/\n\npython main_molhiv.py --config 'configs/MOLHIV/ablation/full/1e-5/node'\npython main_molhiv.py --config 'configs/MOLHIV/ablation/full/1e-5/node'\npython main_molhiv.py --config 'configs/MOLHIV/ablation/full/1e-5/node'\npython main_molhiv.py --config 'configs/MOLHIV/ablation/full/1e-5/node'\n\n\n\n" }, { "path": "scripts/MOLHIV/ablation/full_node_1e-6", "content": "#!/bin/bash\n\ncd ~/SAN/\n\npython main_molhiv.py --config 'configs/MOLHIV/ablation/full/1e-6/node'\npython main_molhiv.py --config 'configs/MOLHIV/ablation/full/1e-6/node'\npython main_molhiv.py --config 'configs/MOLHIV/ablation/full/1e-6/node'\npython main_molhiv.py --config 'configs/MOLHIV/ablation/full/1e-6/node'\n\n\n\n" }, { "path": "scripts/MOLHIV/ablation/full_node_1e-7", "content": "#!/bin/bash\n\ncd ~/SAN/\n\npython main_molhiv.py --config 'configs/MOLHIV/ablation/full/1e-7/node'\npython main_molhiv.py --config 'configs/MOLHIV/ablation/full/1e-7/node'\npython main_molhiv.py --config 'configs/MOLHIV/ablation/full/1e-7/node'\npython main_molhiv.py --config 'configs/MOLHIV/ablation/full/1e-7/node'\n\n\n\n" }, { "path": "scripts/MOLHIV/ablation/full_node_1e-8", "content": "#!/bin/bash\n\ncd ~/SAN/\n\npython main_molhiv.py --config 'configs/MOLHIV/ablation/full/1e-8/node'\npython main_molhiv.py --config 'configs/MOLHIV/ablation/full/1e-8/node'\npython main_molhiv.py --config 'configs/MOLHIV/ablation/full/1e-8/node'\npython main_molhiv.py --config 'configs/MOLHIV/ablation/full/1e-8/node'\n\n\n\n" }, { "path": "scripts/MOLHIV/ablation/full_none", "content": "#!/bin/bash\n\ncd ~/SAN/\n\npython main_molhiv.py --config 'configs/MOLHIV/ablation/full/none'\npython main_molhiv.py --config 'configs/MOLHIV/ablation/full/none'\npython main_molhiv.py --config 'configs/MOLHIV/ablation/full/none'\npython main_molhiv.py --config 'configs/MOLHIV/ablation/full/none'\n" }, { "path": "scripts/MOLHIV/ablation/sparse_node", "content": "#!/bin/bash\n\ncd ~/SAN/\n\npython main_molhiv.py --config 'configs/MOLHIV/ablation/sparse/node'\npython main_molhiv.py --config 'configs/MOLHIV/ablation/sparse/node'\npython main_molhiv.py --config 'configs/MOLHIV/ablation/sparse/node'\npython main_molhiv.py --config 'configs/MOLHIV/ablation/sparse/node'\n" }, { "path": "scripts/MOLHIV/ablation/sparse_none", "content": "#!/bin/bash\n\ncd ~/SAN/\n\npython main_molhiv.py --config 'configs/MOLHIV/ablation/sparse/none'\npython main_molhiv.py --config 'configs/MOLHIV/ablation/sparse/none'\npython main_molhiv.py --config 'configs/MOLHIV/ablation/sparse/none'\npython main_molhiv.py --config 'configs/MOLHIV/ablation/sparse/none'\n" }, { "path": "scripts/MOLHIV/optimized/molhiv_optimized", "content": "#!/bin/bash\n\ncd ~/SAN/\n\npython main_molhiv.py --config 'configs/MOLHIV/optimized'\npython main_molhiv.py --config 'configs/MOLHIV/optimized'\npython main_molhiv.py --config 'configs/MOLHIV/optimized'\npython main_molhiv.py --config 'configs/MOLHIV/optimized'\n\n\n\n" }, { "path": "scripts/MOLPCBA/ablation/full_node_1e-3", "content": "#!/bin/bash\n\ncd ~/SAN/\n\npython main_molhiv.py --config 'configs/MOLHIV/ablation/full/1e-3/node'\npython main_molhiv.py --config 'configs/MOLHIV/ablation/full/1e-3/node'\npython main_molhiv.py --config 'configs/MOLHIV/ablation/full/1e-3/node'\npython main_molhiv.py --config 'configs/MOLHIV/ablation/full/1e-3/node'\n\n\n\n" }, { "path": "scripts/MOLPCBA/ablation/full_node_1e-4", "content": "#!/bin/bash\n\ncd ~/SAN/\n\npython main_molhiv.py --config 'configs/MOLHIV/ablation/full/1e-4/node'\npython main_molhiv.py --config 'configs/MOLHIV/ablation/full/1e-4/node'\npython main_molhiv.py --config 'configs/MOLHIV/ablation/full/1e-4/node'\npython main_molhiv.py --config 'configs/MOLHIV/ablation/full/1e-4/node'\n\n\n\n" }, { "path": "scripts/MOLPCBA/ablation/full_node_1e-5", "content": "#!/bin/bash\n\ncd ~/SAN/\n\npython main_molhiv.py --config 'configs/MOLHIV/ablation/full/1e-5/node'\npython main_molhiv.py --config 'configs/MOLHIV/ablation/full/1e-5/node'\npython main_molhiv.py --config 'configs/MOLHIV/ablation/full/1e-5/node'\npython main_molhiv.py --config 'configs/MOLHIV/ablation/full/1e-5/node'\n\n\n\n" }, { "path": "scripts/MOLPCBA/ablation/full_node_1e-6", "content": "#!/bin/bash\n\ncd ~/SAN/\n\npython main_molhiv.py --config 'configs/MOLHIV/ablation/full/1e-6/node'\npython main_molhiv.py --config 'configs/MOLHIV/ablation/full/1e-6/node'\npython main_molhiv.py --config 'configs/MOLHIV/ablation/full/1e-6/node'\npython main_molhiv.py --config 'configs/MOLHIV/ablation/full/1e-6/node'\n\n\n\n" }, { "path": "scripts/MOLPCBA/ablation/full_node_1e-7", "content": "#!/bin/bash\n\ncd ~/SAN/\n\npython main_molhiv.py --config 'configs/MOLHIV/ablation/full/1e-7/node'\npython main_molhiv.py --config 'configs/MOLHIV/ablation/full/1e-7/node'\npython main_molhiv.py --config 'configs/MOLHIV/ablation/full/1e-7/node'\npython main_molhiv.py --config 'configs/MOLHIV/ablation/full/1e-7/node'\n\n\n\n" }, { "path": "scripts/MOLPCBA/ablation/full_node_1e-8", "content": "#!/bin/bash\n\ncd ~/SAN/\n\npython main_molhiv.py --config 'configs/MOLHIV/ablation/full/1e-8/node'\npython main_molhiv.py --config 'configs/MOLHIV/ablation/full/1e-8/node'\npython main_molhiv.py --config 'configs/MOLHIV/ablation/full/1e-8/node'\npython main_molhiv.py --config 'configs/MOLHIV/ablation/full/1e-8/node'\n\n\n\n" }, { "path": "scripts/MOLPCBA/ablation/full_none", "content": "#!/bin/bash\n\ncd ~/SAN/\n\npython main_molhiv.py --config 'configs/MOLHIV/ablation/full/none'\npython main_molhiv.py --config 'configs/MOLHIV/ablation/full/none'\npython main_molhiv.py --config 'configs/MOLHIV/ablation/full/none'\npython main_molhiv.py --config 'configs/MOLHIV/ablation/full/none'\n" }, { "path": "scripts/MOLPCBA/ablation/sparse_node", "content": "#!/bin/bash\n\ncd ~/SAN/\n\npython main_molhiv.py --config 'configs/MOLHIV/ablation/sparse/node'\npython main_molhiv.py --config 'configs/MOLHIV/ablation/sparse/node'\npython main_molhiv.py --config 'configs/MOLHIV/ablation/sparse/node'\npython main_molhiv.py --config 'configs/MOLHIV/ablation/sparse/node'\n" }, { "path": "scripts/MOLPCBA/ablation/sparse_none", "content": "#!/bin/bash\n\ncd ~/SAN/\n\npython main_molhiv.py --config 'configs/MOLHIV/ablation/sparse/none'\npython main_molhiv.py --config 'configs/MOLHIV/ablation/sparse/none'\npython main_molhiv.py --config 'configs/MOLHIV/ablation/sparse/none'\npython main_molhiv.py --config 'configs/MOLHIV/ablation/sparse/none'\n" }, { "path": "scripts/MOLPCBA/optimized/molpcba_optimized", "content": "#!/bin/bash\n\ncd ~/SAN/\n\npython main_molpcba.py --config 'configs/MOLPCBA/optimized'\npython main_molpcba.py --config 'configs/MOLPCBA/optimized'\npython main_molpcba.py --config 'configs/MOLPCBA/optimized'\npython main_molpcba.py --config 'configs/MOLPCBA/optimized'\n\n\n\n" }, { "path": "scripts/PATTERN/ablation/full_node_1e-1", "content": "#!/bin/bash\n\ncd ~/SAN/\n\npython main_SBMs_node_classification.py --dataset SBM_PATTERN --seed 41 --config 'configs/PATTERN/ablation/full/1e-1/node'\npython main_SBMs_node_classification.py --dataset SBM_PATTERN --seed 55 --config 'configs/PATTERN/ablation/full/1e-1/node'\npython main_SBMs_node_classification.py --dataset SBM_PATTERN --seed 13 --config 'configs/PATTERN/ablation/full/1e-1/node'\npython main_SBMs_node_classification.py --dataset SBM_PATTERN --seed 88 --config 'configs/PATTERN/ablation/full/1e-1/node'\n\n\n\n" }, { "path": "scripts/PATTERN/ablation/full_node_1e-2", "content": "#!/bin/bash\n\ncd ~/SAN/\n\npython main_SBMs_node_classification.py --dataset SBM_PATTERN --seed 41 --config 'configs/PATTERN/ablation/full/1e-2/node'\npython main_SBMs_node_classification.py --dataset SBM_PATTERN --seed 55 --config 'configs/PATTERN/ablation/full/1e-2/node'\npython main_SBMs_node_classification.py --dataset SBM_PATTERN --seed 13 --config 'configs/PATTERN/ablation/full/1e-2/node'\npython main_SBMs_node_classification.py --dataset SBM_PATTERN --seed 88 --config 'configs/PATTERN/ablation/full/1e-2/node'\n\n\n\n" }, { "path": "scripts/PATTERN/ablation/full_node_1e-3", "content": "#!/bin/bash\n\ncd ~/SAN/\n\npython main_SBMs_node_classification.py --dataset SBM_PATTERN --seed 41 --config 'configs/PATTERN/ablation/full/1e-3/node'\npython main_SBMs_node_classification.py --dataset SBM_PATTERN --seed 55 --config 'configs/PATTERN/ablation/full/1e-3/node'\npython main_SBMs_node_classification.py --dataset SBM_PATTERN --seed 13 --config 'configs/PATTERN/ablation/full/1e-3/node'\npython main_SBMs_node_classification.py --dataset SBM_PATTERN --seed 88 --config 'configs/PATTERN/ablation/full/1e-3/node'\n\n\n\n" }, { "path": "scripts/PATTERN/ablation/full_node_1e-4", "content": "#!/bin/bash\n\ncd ~/SAN/\n\npython main_SBMs_node_classification.py --dataset SBM_PATTERN --seed 41 --config 'configs/PATTERN/ablation/full/1e-4/node'\npython main_SBMs_node_classification.py --dataset SBM_PATTERN --seed 55 --config 'configs/PATTERN/ablation/full/1e-4/node'\npython main_SBMs_node_classification.py --dataset SBM_PATTERN --seed 13 --config 'configs/PATTERN/ablation/full/1e-4/node'\npython main_SBMs_node_classification.py --dataset SBM_PATTERN --seed 88 --config 'configs/PATTERN/ablation/full/1e-4/node'\n\n\n\n" }, { "path": "scripts/PATTERN/ablation/full_node_1e-5", "content": "#!/bin/bash\n\ncd ~/SAN/\n\npython main_SBMs_node_classification.py --dataset SBM_PATTERN --seed 41 --config 'configs/PATTERN/ablation/full/1e-5/node'\npython main_SBMs_node_classification.py --dataset SBM_PATTERN --seed 55 --config 'configs/PATTERN/ablation/full/1e-5/node'\npython main_SBMs_node_classification.py --dataset SBM_PATTERN --seed 13 --config 'configs/PATTERN/ablation/full/1e-5/node'\npython main_SBMs_node_classification.py --dataset SBM_PATTERN --seed 88 --config 'configs/PATTERN/ablation/full/1e-5/node'\n\n\n\n" }, { "path": "scripts/PATTERN/ablation/full_node_1e-6", "content": "#!/bin/bash\n\ncd ~/SAN/\n\npython main_SBMs_node_classification.py --dataset SBM_PATTERN --seed 41 --config 'configs/PATTERN/ablation/full/1e-6/node'\npython main_SBMs_node_classification.py --dataset SBM_PATTERN --seed 55 --config 'configs/PATTERN/ablation/full/1e-6/node'\npython main_SBMs_node_classification.py --dataset SBM_PATTERN --seed 13 --config 'configs/PATTERN/ablation/full/1e-6/node'\npython main_SBMs_node_classification.py --dataset SBM_PATTERN --seed 88 --config 'configs/PATTERN/ablation/full/1e-6/node'\n\n\n\n" }, { "path": "scripts/PATTERN/ablation/full_node_1e-7", "content": "#!/bin/bash\n\ncd ~/SAN/\n\npython main_SBMs_node_classification.py --dataset SBM_PATTERN --seed 41 --config 'configs/PATTERN/ablation/full/1e-7/node'\npython main_SBMs_node_classification.py --dataset SBM_PATTERN --seed 55 --config 'configs/PATTERN/ablation/full/1e-7/node'\npython main_SBMs_node_classification.py --dataset SBM_PATTERN --seed 13 --config 'configs/PATTERN/ablation/full/1e-7/node'\npython main_SBMs_node_classification.py --dataset SBM_PATTERN --seed 88 --config 'configs/PATTERN/ablation/full/1e-7/node'\n\n\n\n" }, { "path": "scripts/PATTERN/ablation/full_node_1e-8", "content": "#!/bin/bash\n\ncd ~/SAN/\n\npython main_SBMs_node_classification.py --dataset SBM_PATTERN --seed 41 --config 'configs/PATTERN/ablation/full/1e-8/node'\npython main_SBMs_node_classification.py --dataset SBM_PATTERN --seed 55 --config 'configs/PATTERN/ablation/full/1e-8/node'\npython main_SBMs_node_classification.py --dataset SBM_PATTERN --seed 13 --config 'configs/PATTERN/ablation/full/1e-8/node'\npython main_SBMs_node_classification.py --dataset SBM_PATTERN --seed 88 --config 'configs/PATTERN/ablation/full/1e-8/node'\n\n\n\n" }, { "path": "scripts/PATTERN/ablation/full_none", "content": "#!/bin/bash\ncd ~/SAN/\n\npython main_SBMs_node_classification.py --dataset SBM_PATTERN --seed 41 --config 'configs/PATTERN/ablation/full/none'\npython main_SBMs_node_classification.py --dataset SBM_PATTERN --seed 55 --config 'configs/PATTERN/ablation/full/none'\npython main_SBMs_node_classification.py --dataset SBM_PATTERN --seed 13 --config 'configs/PATTERN/ablation/full/none'\npython main_SBMs_node_classification.py --dataset SBM_PATTERN --seed 88 --config 'configs/PATTERN/ablation/full/none'\n" }, { "path": "scripts/PATTERN/ablation/sparse_node", "content": "#!/bin/bash\n\ncd ~/SAN/\n\npython main_SBMs_node_classification.py --dataset SBM_PATTERN --seed 41 --config 'configs/PATTERN/ablation/sparse/node'\npython main_SBMs_node_classification.py --dataset SBM_PATTERN --seed 55 --config 'configs/PATTERN/ablation/sparse/node'\npython main_SBMs_node_classification.py --dataset SBM_PATTERN --seed 13 --config 'configs/PATTERN/ablation/sparse/node'\npython main_SBMs_node_classification.py --dataset SBM_PATTERN --seed 88 --config 'configs/PATTERN/ablation/sparse/node'\n" }, { "path": "scripts/PATTERN/ablation/sparse_none", "content": "#!/bin/bash\n\ncd ~/SAN/\n\npython main_SBMs_node_classification.py --dataset SBM_PATTERN --seed 41 --config 'configs/PATTERN/ablation/sparse/none'\npython main_SBMs_node_classification.py --dataset SBM_PATTERN --seed 55 --config 'configs/PATTERN/ablation/sparse/none'\npython main_SBMs_node_classification.py --dataset SBM_PATTERN --seed 13 --config 'configs/PATTERN/ablation/sparse/none'\npython main_SBMs_node_classification.py --dataset SBM_PATTERN --seed 88 --config 'configs/PATTERN/ablation/sparse/none'\n" }, { "path": "scripts/PATTERN/optimized/pattern_optimized", "content": "#!/bin/bash\n\ncd ~/SAN/\n\npython main_SBMs_node_classification.py --dataset SBM_PATTERN --seed 41 --config 'configs/PATTERN/optimized'\npython main_SBMs_node_classification.py --dataset SBM_PATTERN --seed 55 --config 'configs/PATTERN/optimized'\npython main_SBMs_node_classification.py --dataset SBM_PATTERN --seed 13 --config 'configs/PATTERN/optimized'\npython main_SBMs_node_classification.py --dataset SBM_PATTERN --seed 88 --config 'configs/PATTERN/optimized'\n\n\n\n" }, { "path": "scripts/ZINC/ablation/full_node_1e-2", "content": "#!/bin/bash\n\ncd ~/SAN/\n\npython main_ZINC_graph_regression.py --seed 41 --config 'configs/ZINC/ablation/full/1e-2/node'\npython main_ZINC_graph_regression.py --seed 55 --config 'configs/ZINC/ablation/full/1e-2/node'\npython main_ZINC_graph_regression.py --seed 13 --config 'configs/ZINC/ablation/full/1e-2/node'\npython main_ZINC_graph_regression.py --seed 88 --config 'configs/ZINC/ablation/full/1e-2/node'\n\n\n\n" }, { "path": "scripts/ZINC/ablation/full_node_1e-3", "content": "#!/bin/bash\n\ncd ~/SAN/\n\npython main_ZINC_graph_regression.py --seed 41 --config 'configs/ZINC/ablation/full/1e-3/node'\npython main_ZINC_graph_regression.py --seed 55 --config 'configs/ZINC/ablation/full/1e-3/node'\npython main_ZINC_graph_regression.py --seed 13 --config 'configs/ZINC/ablation/full/1e-3/node'\npython main_ZINC_graph_regression.py --seed 88 --config 'configs/ZINC/ablation/full/1e-3/node'\n\n\n\n" }, { "path": "scripts/ZINC/ablation/full_node_1e-4", "content": "#!/bin/bash\n\ncd ~/SAN/\n\npython main_ZINC_graph_regression.py --seed 41 --config 'configs/ZINC/ablation/full/1e-4/node'\npython main_ZINC_graph_regression.py --seed 55 --config 'configs/ZINC/ablation/full/1e-4/node'\npython main_ZINC_graph_regression.py --seed 13 --config 'configs/ZINC/ablation/full/1e-4/node'\npython main_ZINC_graph_regression.py --seed 88 --config 'configs/ZINC/ablation/full/1e-4/node'\n\n\n\n" }, { "path": "scripts/ZINC/ablation/full_node_1e-5", "content": "#!/bin/bash\ncd ~/SAN/\n\npython main_ZINC_graph_regression.py --seed 41 --config 'configs/ZINC/ablation/full/1e-5/node'\npython main_ZINC_graph_regression.py --seed 55 --config 'configs/ZINC/ablation/full/1e-5/node'\npython main_ZINC_graph_regression.py --seed 13 --config 'configs/ZINC/ablation/full/1e-5/node'\npython main_ZINC_graph_regression.py --seed 88 --config 'configs/ZINC/ablation/full/1e-5/node'\n\n\n\n" }, { "path": "scripts/ZINC/ablation/full_node_1e-6", "content": "#!/bin/bash\n\ncd ~/SAN/\n\npython main_ZINC_graph_regression.py --seed 41 --config 'configs/ZINC/ablation/full/1e-6/node'\npython main_ZINC_graph_regression.py --seed 55 --config 'configs/ZINC/ablation/full/1e-6/node'\npython main_ZINC_graph_regression.py --seed 13 --config 'configs/ZINC/ablation/full/1e-6/node'\npython main_ZINC_graph_regression.py --seed 88 --config 'configs/ZINC/ablation/full/1e-6/node'\n\n\n\n" }, { "path": "scripts/ZINC/ablation/full_node_1e-7", "content": "#!/bin/bash\n\ncd ~/SAN/\n\npython main_ZINC_graph_regression.py --seed 41 --config 'configs/ZINC/ablation/full/1e-7/node'\npython main_ZINC_graph_regression.py --seed 55 --config 'configs/ZINC/ablation/full/1e-7/node'\npython main_ZINC_graph_regression.py --seed 13 --config 'configs/ZINC/ablation/full/1e-7/node'\npython main_ZINC_graph_regression.py --seed 88 --config 'configs/ZINC/ablation/full/1e-7/node'\n\n\n\n" }, { "path": "scripts/ZINC/ablation/full_node_1e-8", "content": "#!/bin/bash\n\ncd ~/SAN/\n\npython main_ZINC_graph_regression.py --seed 41 --config 'configs/ZINC/ablation/full/1e-8/node'\npython main_ZINC_graph_regression.py --seed 55 --config 'configs/ZINC/ablation/full/1e-8/node'\npython main_ZINC_graph_regression.py --seed 13 --config 'configs/ZINC/ablation/full/1e-8/node'\npython main_ZINC_graph_regression.py --seed 88 --config 'configs/ZINC/ablation/full/1e-8/node'\n\n\n\n" }, { "path": "scripts/ZINC/ablation/full_none", "content": "#!/bin/bash\n\ncd ~/SAN/\n\npython main_ZINC_graph_regression.py --seed 41 --config 'configs/ZINC/ablation/full/none'\npython main_ZINC_graph_regression.py --seed 55 --config 'configs/ZINC/ablation/full/none'\npython main_ZINC_graph_regression.py --seed 13 --config 'configs/ZINC/ablation/full/none'\npython main_ZINC_graph_regression.py --seed 88 --config 'configs/ZINC/ablation/full/none'\n" }, { "path": "scripts/ZINC/ablation/sparse_node", "content": "#!/bin/bash\n\ncd ~/SAN/\n\npython main_ZINC_graph_regression.py --seed 41 --config 'configs/ZINC/ablation/sparse/node'\npython main_ZINC_graph_regression.py --seed 55 --config 'configs/ZINC/ablation/sparse/node'\npython main_ZINC_graph_regression.py --seed 13 --config 'configs/ZINC/ablation/sparse/node'\npython main_ZINC_graph_regression.py --seed 88 --config 'configs/ZINC/ablation/sparse/node'\n" }, { "path": "scripts/ZINC/ablation/sparse_none", "content": "#!/bin/bash\n\ncd ~/SAN/\n\npython main_ZINC_graph_regression.py --seed 41 --config 'configs/ZINC/ablation/sparse/none'\npython main_ZINC_graph_regression.py --seed 55 --config 'configs/ZINC/ablation/sparse/none'\npython main_ZINC_graph_regression.py --seed 13 --config 'configs/ZINC/ablation/sparse/none'\npython main_ZINC_graph_regression.py --seed 88 --config 'configs/ZINC/ablation/sparse/none'\n" }, { "path": "scripts/ZINC/optimized/zinc_optimized", "content": "#!/bin/bash\ncd ~/SAN/\n\npython main_ZINC_graph_regression.py --seed 41 --config 'configs/ZINC/optimized'\npython main_ZINC_graph_regression.py --seed 55 --config 'configs/ZINC/optimized'\npython main_ZINC_graph_regression.py --seed 13 --config 'configs/ZINC/optimized'\npython main_ZINC_graph_regression.py --seed 88 --config 'configs/ZINC/optimized'\n\n\n\n" }, { "path": "scripts/reproduce.md", "content": "# Reproducibility\n\n\n
\n\n\nAll outputs will be sent to the ```/out``` directory from the root of the project in the folders described in the respective ```configs/``` files.\n\n\n## 1. ZINC\n\n\n```\n# SOTA comparison results\nbash ZINC/optimized/zinc_optimized\n\n# Ablation results\nbash ZINC/ablation/sparse_none\nbash ZINC/ablation/sparse_node\nbash ZINC/ablation/full_none\nbash ZINC/ablation/full_node_1e-2\nbash ZINC/ablation/full_node_1e-3\nbash ZINC/ablation/full_node_1e-4\nbash ZINC/ablation/full_node_1e-5\nbash ZINC/ablation/full_node_1e-6\nbash ZINC/ablation/full_node_1e-7\nbash ZINC/ablation/full_node_1e-8\n```\n
\n\n## 2. PATTERN\n\n\n```\n# SOTA comparison results\nbash PATTERN/optimized/pattern_optimized\n\n# Ablation results\nbash PATTERN/ablation/sparse_none\nbash PATTERN/ablation/sparse_node\nbash PATTERN/ablation/full_none\nbash PATTERN/ablation/full_node_1e-1\nbash PATTERN/ablation/full_node_1e-2\nbash PATTERN/ablation/full_node_1e-3\nbash PATTERN/ablation/full_node_1e-4\nbash PATTERN/ablation/full_node_1e-5\nbash PATTERN/ablation/full_node_1e-6\nbash PATTERN/ablation/full_node_1e-7\nbash PATTERN/ablation/full_node_1e-8\n```\n
\n \n ## 3. CLUSTER\n\n\n```\n# SOTA comparison results\nbash CLUSTER/optimized/cluster_optimized\n\n# Ablation results\nbash CLUSTER/ablation/sparse_none\nbash CLUSTER/ablation/sparse_node\nbash CLUSTER/ablation/full_none\nbash CLUSTER/ablation/full_node_1e-1\nbash CLUSTER/ablation/full_node_1e-2\nbash CLUSTER/ablation/full_node_1e-3\nbash CLUSTER/ablation/full_node_1e-4\nbash CLUSTER/ablation/full_node_1e-5\nbash CLUSTER/ablation/full_node_1e-6\nbash CLUSTER/ablation/full_node_1e-7\nbash CLUSTER/ablation/full_node_1e-8\n```\n
\n \n ## 4. MolHIV\n\n\n```\n# SOTA comparison results\nbash MOLHIV/optimized/molhiv_optimized\n\n# Ablation results\nbash MOLHIV/ablation/sparse_none\nbash MOLHIV/ablation/sparse_node\nbash MOLHIV/ablation/full_none\nbash MOLHIV/ablation/full_node_1e-3\nbash MOLHIV/ablation/full_node_1e-4\nbash MOLHIV/ablation/full_node_1e-5\nbash MOLHIV/ablation/full_node_1e-6\nbash MOLHIV/ablation/full_node_1e-7\nbash MOLHIV/ablation/full_node_1e-8\n```\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n


\n" }, { "path": "train/MetricWrapper.py", "content": "from typing import Union, Callable, Optional, Dict, Any\nfrom copy import deepcopy\nimport torch\nfrom torch import Tensor\n\nclass MetricWrapper:\n r\"\"\"\n Allows to initialize a metric from a name or Callable, and initialize the\n `Thresholder` in case the metric requires a threshold.\n \"\"\"\n\n def __init__(\n self,\n metric: Union[str, Callable],\n target_nan_mask: Optional[Union[str, int]] = None,\n **kwargs,\n ):\n r\"\"\"\n Parameters\n metric:\n The metric to use. See `METRICS_DICT`\n\n target_nan_mask:\n\n - None: Do not change behaviour if there are NaNs\n\n - int, float: Value used to replace NaNs. For example, if `target_nan_mask==0`, then\n all NaNs will be replaced by zeros\n\n - 'ignore-flatten': The Tensor will be reduced to a vector without the NaN values.\n\n - 'ignore-mean-label': NaNs will be ignored when computing the loss. Note that each column\n has a different number of NaNs, so the metric will be computed separately\n on each column, and the metric result will be averaged over all columns.\n *This option might slowdown the computation if there are too many labels*\n\n kwargs:\n Other arguments to call with the metric\n \"\"\"\n\n self.metric = metric\n self.target_nan_mask = target_nan_mask\n self.kwargs = kwargs\n\n def compute(self, preds: torch.Tensor, target: torch.Tensor) -> torch.Tensor:\n r\"\"\"\n Compute the metric, apply the thresholder if provided, and manage the NaNs\n \"\"\"\n\n if preds.ndim == 1:\n preds = preds.unsqueeze(-1)\n\n if target.ndim == 1:\n target = target.unsqueeze(-1)\n\n target_nans = torch.isnan(target)\n\n # Manage the NaNs\n if self.target_nan_mask is None:\n pass\n elif isinstance(self.target_nan_mask, (int, float)):\n target = target.clone()\n target[torch.isnan(target)] = self.target_nan_mask\n elif self.target_nan_mask == \"ignore-flatten\":\n target = target[~target_nans]\n preds = preds[~target_nans]\n elif self.target_nan_mask == \"ignore-mean-label\":\n target_list = [target[..., ii][~target_nans[..., ii]] for ii in range(target.shape[-1])]\n preds_list = [preds[..., ii][~target_nans[..., ii]] for ii in range(preds.shape[-1])]\n target = target_list\n preds = preds_list\n else:\n raise ValueError(f\"Invalid option `{self.target_nan_mask}`\")\n\n if self.target_nan_mask == \"ignore-mean-label\":\n\n # Compute the metric for each column, and output nan if there's an error on a given column\n metric_val = []\n for ii in range(len(target)):\n try:\n metric_val.append(self.metric(preds[ii], target[ii], **self.kwargs))\n except:\n pass\n\n # Average the metric\n \n metric_val = self.nan_mean(torch.stack(metric_val))\n\n else:\n metric_val = self.metric(preds, target, **self.kwargs)\n return metric_val\n\n def __call__(self, preds: torch.Tensor, target: torch.Tensor) -> torch.Tensor:\n r\"\"\"\n Compute the metric with the method `self.compute`\n \"\"\"\n return self.compute(preds, target)\n\n def __repr__(self):\n r\"\"\"\n Control how the class is printed\n \"\"\"\n full_str = f\"{self.metric.__name__}\"\n\n return full_str\n \n def nan_mean(self, input: Tensor, **kwargs) -> Tensor:\n sum = torch.nansum(input, **kwargs)\n num = torch.sum(~torch.isnan(input), **kwargs)\n mean = sum / num\n return mean\n\n\n" }, { "path": "train/metrics.py", "content": "import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.metrics import f1_score\nimport numpy as np\n\n\ndef MAE(scores, targets):\n MAE = F.l1_loss(scores, targets)\n MAE = MAE.detach().item()\n return MAE\n\n\ndef accuracy_TU(scores, targets):\n scores = scores.detach().argmax(dim=1)\n acc = (scores==targets).float().sum().item()\n return acc\n\n\ndef accuracy_MNIST_CIFAR(scores, targets):\n scores = scores.detach().argmax(dim=1)\n acc = (scores==targets).float().sum().item()\n return acc\n\ndef accuracy_CITATION_GRAPH(scores, targets):\n scores = scores.detach().argmax(dim=1)\n acc = (scores==targets).float().sum().item()\n acc = acc / len(targets)\n return acc\n\n\ndef accuracy_SBM(scores, targets):\n S = targets.cpu().numpy()\n C = np.argmax( torch.nn.Softmax(dim=1)(scores).cpu().detach().numpy() , axis=1 )\n CM = confusion_matrix(S,C).astype(np.float32)\n nb_classes = CM.shape[0]\n targets = targets.cpu().detach().numpy()\n nb_non_empty_classes = 0\n pr_classes = np.zeros(nb_classes)\n for r in range(nb_classes):\n cluster = np.where(targets==r)[0]\n if cluster.shape[0] != 0:\n pr_classes[r] = CM[r,r]/ float(cluster.shape[0])\n if CM[r,r]>0:\n nb_non_empty_classes += 1\n else:\n pr_classes[r] = 0.0\n acc = 100.* np.sum(pr_classes)/ float(nb_classes)\n return acc\n\n\ndef binary_f1_score(scores, targets):\n \"\"\"Computes the F1 score using scikit-learn for binary class labels. \n \n Returns the F1 score for the positive class, i.e. labelled '1'.\n \"\"\"\n y_true = targets.cpu().numpy()\n y_pred = scores.argmax(dim=1).cpu().numpy()\n return f1_score(y_true, y_pred, average='binary')\n\n \ndef accuracy_VOC(scores, targets):\n scores = scores.detach().argmax(dim=1).cpu()\n targets = targets.cpu().detach().numpy()\n acc = f1_score(scores, targets, average='weighted')\n return acc\n" }, { "path": "train/train_SBMs_node_classification.py", "content": "\"\"\"\n Utility function for training one epoch \n and evaluating one epoch\n\"\"\"\nimport torch\nimport torch.nn as nn\nimport math\nimport dgl\n\nfrom train.metrics import accuracy_SBM as accuracy\n\ndef train_epoch(model, optimizer, device, data_loader, epoch, LPE):\n model.train()\n epoch_loss = 0\n epoch_train_acc = 0\n \n for iter, (batch_graphs, batch_labels) in enumerate(data_loader):\n\n batch_graphs = batch_graphs.to(device)\n batch_x = batch_graphs.ndata['feat'].to(device)\n batch_e = batch_graphs.edata['feat'].flatten().long().to(device)\n\n batch_labels = batch_labels.to(device)\n optimizer.zero_grad() \n \n if LPE == 'node':\n batch_EigVecs = batch_graphs.ndata['EigVecs'].to(device)\n #random sign flipping\n sign_flip = torch.rand(batch_EigVecs.size(1)).to(device)\n sign_flip[sign_flip>=0.5] = 1.0; sign_flip[sign_flip<0.5] = -1.0\n \n batch_EigVals = batch_graphs.ndata['EigVals'].to(device)\n batch_scores = model.forward(batch_graphs, batch_x, batch_e, batch_EigVecs, batch_EigVals)\n\n elif LPE == 'edge':\n batch_diff = batch_graphs.edata['diff'].to(device)\n batch_prod = batch_graphs.edata['product'].to(device)\n batch_EigVals = batch_graphs.edata['EigVals'].to(device)\n batch_scores = model.forward(batch_graphs, batch_x, batch_e, batch_diff, batch_prod, batch_EigVals)\n \n else:\n batch_scores = model.forward(batch_graphs, batch_x, batch_e)\n \n loss = model.loss(batch_scores, batch_labels)\n loss.backward()\n optimizer.step()\n epoch_loss += loss.detach().item()\n epoch_train_acc += accuracy(batch_scores, batch_labels)\n epoch_loss /= (iter + 1)\n epoch_train_acc /= (iter + 1)\n \n return epoch_loss, epoch_train_acc, optimizer\n\n\ndef evaluate_network(model, device, data_loader, epoch, LPE):\n \n model.eval()\n epoch_test_loss = 0\n epoch_test_acc = 0\n\n with torch.no_grad():\n for iter, (batch_graphs, batch_labels) in enumerate(data_loader):\n batch_graphs = batch_graphs.to(device)\n batch_x = batch_graphs.ndata['feat'].to(device)\n batch_e = batch_graphs.edata['feat'].flatten().long().to(device)\n batch_labels = batch_labels.to(device)\n\n if LPE == 'node':\n batch_EigVecs = batch_graphs.ndata['EigVecs'].to(device)\n batch_EigVals = batch_graphs.ndata['EigVals'].to(device)\n batch_scores = model.forward(batch_graphs, batch_x, batch_e, batch_EigVecs, batch_EigVals)\n\n elif LPE == 'edge':\n batch_diff = batch_graphs.edata['diff'].to(device)\n batch_prod = batch_graphs.edata['product'].to(device)\n batch_EigVals = batch_graphs.edata['EigVals'].to(device)\n batch_scores = model.forward(batch_graphs, batch_x, batch_e, batch_diff, batch_prod, batch_EigVals)\n \n else:\n batch_scores = model.forward(batch_graphs, batch_x, batch_e)\n \n loss = model.loss(batch_scores, batch_labels)\n epoch_test_loss += loss.detach().item()\n epoch_test_acc += accuracy(batch_scores, batch_labels)\n \n epoch_test_loss /= (iter + 1)\n epoch_test_acc /= (iter + 1)\n \n return epoch_test_loss, epoch_test_acc\n\n\n" }, { "path": "train/train_ZINC_graph_regression.py", "content": "\"\"\"\n Utility function for training one epoch \n and evaluating one epoch\n\"\"\"\nimport torch\nimport torch.nn as nn\nimport math\n\nfrom train.metrics import MAE\n\ndef train_epoch(model, optimizer, device, data_loader, epoch, LPE):\n model.train()\n epoch_loss = 0\n epoch_train_mae = 0\n \n for iter, (batch_graphs, batch_targets) in enumerate(data_loader):\n\n batch_graphs = batch_graphs.to(device)\n batch_x = batch_graphs.ndata['feat'].to(device)\n batch_e = batch_graphs.edata['feat'].to(device)\n\n batch_targets = batch_targets.to(device)\n optimizer.zero_grad() \n \n if LPE == 'node':\n batch_EigVecs = batch_graphs.ndata['EigVecs'].to(device)\n #random sign flipping\n sign_flip = torch.rand(batch_EigVecs.size(1)).to(device)\n sign_flip[sign_flip>=0.5] = 1.0; sign_flip[sign_flip<0.5] = -1.0\n \n batch_EigVals = batch_graphs.ndata['EigVals'].to(device)\n batch_scores = model.forward(batch_graphs, batch_x, batch_e, batch_EigVecs, batch_EigVals)\n\n elif LPE == 'edge':\n batch_diff = batch_graphs.edata['diff'].to(device)\n batch_prod = batch_graphs.edata['product'].to(device)\n batch_EigVals = batch_graphs.edata['EigVals'].to(device)\n batch_scores = model.forward(batch_graphs, batch_x, batch_e, batch_diff, batch_prod, batch_EigVals)\n \n else:\n batch_scores = model.forward(batch_graphs, batch_x, batch_e)\n \n loss = model.loss(batch_scores, batch_targets)\n loss.backward()\n optimizer.step()\n epoch_loss += loss.detach().item()\n epoch_train_mae += MAE(batch_scores, batch_targets)\n epoch_loss /= (iter + 1)\n epoch_train_mae /= (iter + 1)\n \n return epoch_loss, epoch_train_mae, optimizer\n\ndef evaluate_network(model, device, data_loader, epoch, LPE):\n model.eval()\n epoch_test_loss = 0\n epoch_test_mae = 0\n\n with torch.no_grad():\n for iter, (batch_graphs, batch_targets) in enumerate(data_loader):\n batch_graphs = batch_graphs.to(device)\n batch_x = batch_graphs.ndata['feat'].to(device)\n batch_e = batch_graphs.edata['feat'].to(device)\n batch_targets = batch_targets.to(device)\n\n if LPE == 'node':\n batch_EigVecs = batch_graphs.ndata['EigVecs'].to(device)\n batch_EigVals = batch_graphs.ndata['EigVals'].to(device)\n batch_scores = model.forward(batch_graphs, batch_x, batch_e, batch_EigVecs, batch_EigVals)\n\n elif LPE == 'edge':\n batch_diff = batch_graphs.edata['diff'].to(device)\n batch_prod = batch_graphs.edata['product'].to(device)\n batch_EigVals = batch_graphs.edata['EigVals'].to(device)\n batch_scores = model.forward(batch_graphs, batch_x, batch_e, batch_diff, batch_prod, batch_EigVals)\n \n else:\n batch_scores = model.forward(batch_graphs, batch_x, batch_e)\n \n loss = model.loss(batch_scores, batch_targets)\n epoch_test_loss += loss.detach().item()\n epoch_test_mae += MAE(batch_scores, batch_targets)\n\n epoch_test_loss /= (iter + 1)\n epoch_test_mae /= (iter + 1)\n \n return epoch_test_loss, epoch_test_mae\n\n" }, { "path": "train/train_molhiv.py", "content": "\"\"\"\n Utility functions for training one epoch \n and evaluating one epoch\n\"\"\"\nimport torch\nimport torch.nn as nn\nimport math\n\nfrom ogb.graphproppred import Evaluator\n\ndef train_epoch(model, optimizer, device, data_loader, epoch, LPE):\n model.train()\n evaluator = Evaluator(name = \"ogbg-molhiv\")\n \n epoch_loss = 0\n epoch_train_auc = 0\n\n targets=torch.tensor([]).to(device)\n scores=torch.tensor([]).to(device)\n \n for iter, (batch_graphs, batch_targets) in enumerate(data_loader):\n \n batch_graphs = batch_graphs.to(device)\n batch_x = batch_graphs.ndata['feat'].to(device) # num x feat\n batch_e = batch_graphs.edata['feat'].to(device)\n \n batch_targets = batch_targets.to(device)\n optimizer.zero_grad() \n \n if LPE == 'node':\n \n batch_EigVecs = batch_graphs.ndata['EigVecs'].to(device)\n #random sign flipping\n sign_flip = torch.rand(batch_EigVecs.size(1)).to(device)\n sign_flip[sign_flip>=0.5] = 1.0; sign_flip[sign_flip<0.5] = -1.0\n batch_EigVecs = batch_EigVecs * sign_flip.unsqueeze(0)\n \n batch_EigVals = batch_graphs.ndata['EigVals'].to(device)\n batch_scores = model.forward(batch_graphs, batch_x, batch_e, batch_EigVecs, batch_EigVals)\n\n elif LPE == 'edge':\n \n batch_diff = batch_graphs.edata['diff'].to(device)\n batch_prod = batch_graphs.edata['product'].to(device)\n batch_EigVals = batch_graphs.edata['EigVals'].to(device)\n batch_scores = model.forward(batch_graphs, batch_x, batch_e, batch_diff, batch_prod, batch_EigVals)\n \n else:\n batch_scores = model.forward(batch_graphs, batch_x, batch_e)\n \n\n targets = torch.cat((targets, batch_targets), 0)\n scores = torch.cat((scores, batch_scores), 0)\n \n loss = model.loss(batch_scores, batch_targets)\n loss.backward()\n optimizer.step()\n epoch_loss += loss.detach().item()\n \n \n input_dict = {\"y_true\": targets, \"y_pred\": scores}\n epoch_train_auc = evaluator.eval(input_dict)['rocauc'] \n\n epoch_loss /= (iter + 1)\n \n return epoch_loss, epoch_train_auc, optimizer\n\ndef evaluate_network(model, device, data_loader, epoch, LPE):\n model.eval()\n evaluator = Evaluator(name = \"ogbg-molhiv\")\n \n epoch_test_loss = 0\n epoch_test_auc = 0\n \n targets=torch.tensor([]).to(device)\n scores=torch.tensor([]).to(device)\n \n with torch.no_grad():\n for iter, (batch_graphs, batch_targets) in enumerate(data_loader):\n batch_graphs = batch_graphs.to(device)\n batch_x = batch_graphs.ndata['feat'].to(device) # num x feat\n batch_e = batch_graphs.edata['feat'].to(device)\n batch_targets = batch_targets.to(device)\n \n if LPE == 'node':\n batch_EigVecs = batch_graphs.ndata['EigVecs'].to(device)\n batch_EigVals = batch_graphs.ndata['EigVals'].to(device)\n batch_scores = model.forward(batch_graphs, batch_x, batch_e, batch_EigVecs, batch_EigVals)\n\n elif LPE == 'edge':\n batch_diff = batch_graphs.edata['diff'].to(device)\n batch_prod = batch_graphs.edata['product'].to(device)\n batch_EigVals = batch_graphs.edata['EigVals'].to(device)\n batch_scores = model.forward(batch_graphs, batch_x, batch_e, batch_diff, batch_prod, batch_EigVals)\n \n else:\n batch_scores = model.forward(batch_graphs, batch_x, batch_e)\n \n\n targets = torch.cat((targets, batch_targets), 0)\n scores = torch.cat((scores, batch_scores), 0) \n \n loss = model.loss(batch_scores, batch_targets)\n epoch_test_loss += loss.detach().item()\n\n \n input_dict = {\"y_true\": targets, \"y_pred\": scores}\n epoch_test_auc = evaluator.eval(input_dict)['rocauc']\n \n epoch_test_loss /= (iter + 1)\n\n return epoch_test_loss, epoch_test_auc\n\n" }, { "path": "train/train_molpcba.py", "content": "\"\"\"\n Utility functions for training one epoch\n and evaluating one epoch\n\"\"\"\nimport torch\nfrom torch._C import dtype\nimport torch.nn as nn\nimport math\n\nfrom ogb.graphproppred import Evaluator\nfrom train.MetricWrapper import MetricWrapper\n\n\ndef train_epoch(model, optimizer, device, data_loader, epoch, LPE, batch_accumulation):\n model.train()\n evaluator = Evaluator(name = \"ogbg-molpcba\")\n\n epoch_loss = 0\n\n targets=torch.tensor([])\n scores=torch.tensor([])\n\n wrapped_loss_fun = MetricWrapper(metric=model.loss, target_nan_mask=\"ignore-flatten\")\n\n for iter, (batch_graphs, batch_targets) in enumerate(data_loader):\n # print(iter, torch.cuda.memory_allocated(0))\n batch_graphs = batch_graphs.to(device=device)\n batch_x = batch_graphs.ndata['feat']\n batch_e = batch_graphs.edata['feat']\n\n batch_targets = batch_targets.to(device)\n\n if LPE == 'node':\n\n batch_EigVecs = batch_graphs.ndata['EigVecs']\n #random sign flipping\n sign_flip = torch.rand(batch_EigVecs.size(1), device=device)\n sign_flip[sign_flip>=0.5] = 1.0; sign_flip[sign_flip<0.5] = -1.0\n batch_EigVecs = batch_EigVecs * sign_flip.unsqueeze(0)\n\n batch_EigVals = batch_graphs.ndata['EigVals']\n batch_scores = model.forward(batch_graphs, batch_x, batch_e, batch_EigVecs, batch_EigVals)\n\n elif LPE == 'edge':\n\n batch_diff = batch_graphs.edata['diff']\n batch_prod = batch_graphs.edata['product']\n batch_EigVals = batch_graphs.edata['EigVals']\n batch_scores = model.forward(batch_graphs, batch_x, batch_e, batch_diff, batch_prod, batch_EigVals)\n\n else:\n batch_scores = model.forward(batch_graphs, batch_x, batch_e)\n\n loss = wrapped_loss_fun(batch_scores, batch_targets)\n loss = loss / batch_accumulation\n loss.backward()\n\n # weights update\n if ((iter + 1) % batch_accumulation == 0) or (iter + 1 == len(data_loader)):\n optimizer.step()\n optimizer.zero_grad()\n\n epoch_loss += loss.detach().item()\n\n targets = torch.cat((targets, batch_targets.detach().cpu()), 0)\n scores = torch.cat((scores, batch_scores.detach().cpu()), 0)\n\n\n input_dict = {\"y_true\": targets, \"y_pred\": scores}\n epoch_train_ap = evaluator.eval(input_dict)['ap']\n\n epoch_loss /= (iter + 1)\n\n return epoch_loss, epoch_train_ap, optimizer\n\ndef evaluate_network(model, device, data_loader, epoch, LPE):\n model.eval()\n evaluator = Evaluator(name = \"ogbg-molpcba\")\n\n epoch_test_loss = 0\n\n targets=torch.tensor([])\n scores=torch.tensor([])\n\n wrapped_loss_fun = MetricWrapper(metric=model.loss, target_nan_mask=\"ignore-flatten\")\n\n with torch.no_grad():\n for iter, (batch_graphs, batch_targets) in enumerate(data_loader):\n batch_graphs = batch_graphs.to(device=device)\n batch_x = batch_graphs.ndata['feat']\n batch_e = batch_graphs.edata['feat']\n batch_targets = batch_targets.to(device=device)\n\n if LPE == 'node':\n batch_EigVecs = batch_graphs.ndata['EigVecs']\n batch_EigVals = batch_graphs.ndata['EigVals']\n batch_scores = model.forward(batch_graphs, batch_x, batch_e, batch_EigVecs, batch_EigVals)\n\n elif LPE == 'edge':\n batch_diff = batch_graphs.edata['diff']\n batch_prod = batch_graphs.edata['product']\n batch_EigVals = batch_graphs.edata['EigVals']\n batch_scores = model.forward(batch_graphs, batch_x, batch_e, batch_diff, batch_prod, batch_EigVals)\n\n else:\n batch_scores = model.forward(batch_graphs, batch_x, batch_e)\n\n loss = wrapped_loss_fun(batch_scores, batch_targets)\n epoch_test_loss += loss.detach().item()\n\n targets = torch.cat((targets, batch_targets.detach().cpu()), 0)\n scores = torch.cat((scores, batch_scores.detach().cpu()), 0)\n\n\n input_dict = {\"y_true\": targets, \"y_pred\": scores}\n epoch_test_ap = evaluator.eval(input_dict)['ap']\n\n epoch_test_loss /= (iter + 1)\n\n return epoch_test_loss, epoch_test_ap\n\n" } ]